From 45c7fc68e11f1f85466321e376454d064f03681e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:26:03 +0000 Subject: [PATCH 1/2] g2 local --- crates/asap-aware-mapping/src/replacement.rs | 71 ++++- .../src/physical_planner/promql_rows.rs | 76 +----- .../tests/planspace_series_identity_heap.rs | 250 ++++++++++++++++++ crates/types/src/pre_asap/schema.rs | 65 +++++ .../physical-planning-and-deployment.md | 7 + docs/develop_docs/library-api.md | 22 ++ 6 files changed, 418 insertions(+), 73 deletions(-) create mode 100644 crates/asap-physical-operators/tests/planspace_series_identity_heap.rs diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index f8f55a1d..fb697251 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -560,6 +560,12 @@ pub enum ReplacementProvenance { /// [`Replacement::ExactComposition`] with /// [`OperationPlacement::Maintenance`] (issue #171). ValueOperationAtIngestionTime, + /// A finalized whole-query result over rows carrying the PromQL series + /// identity, which the logical root does not expose (see + /// [`ReplacementStrategy::propose_for_root`]). Default selection never + /// commits it, because its readout must be validated and priced by + /// deployment; otherwise it would silently replace the logical plan. + RootPhysicalRealization, } /// A candidate a strategy considered for a target but refused to propose on @@ -634,6 +640,15 @@ pub trait ReplacementStrategy { domain_error: None, } } + + /// Whole-query logical alternatives for a workload root under its + /// end-to-end `target`. These may need input rows the root does not expose + /// (for example, the PromQL series identity), so + /// [`search_workload_with_targets`] asks only workload roots, once each. + /// They decide what to compute, never placement. Default: none. + fn propose_for_root(&self, _root: &Rc, _target: &AccuracyTarget) -> Proposals { + Proposals::default() + } } // ── Realization: how one AggIntent may be realised ─────────────────────── @@ -1705,6 +1720,37 @@ impl ReplacementStrategy for SketchAlgorithmStrategy<'_> { fn propose(&self, target: &TargetSubDAG<'_>) -> Proposals { self.propose_with(target.root, None) } + + /// Heap realizations of an instant-vector ranking (current-series TopK). + /// They rank rows that carry the complete PromQL series identity, which + /// the logical root does not expose, so each is a finalized query result + /// for the identity-carrying root. Placement variants (for example, + /// fixed-window or query-time Rate aggregation) are not listed here: the + /// lifecycle assigns timing and the physical compiler reads it. + fn propose_for_root(&self, root: &Rc, target: &AccuracyTarget) -> Proposals { + let Ok(typed) = asap_types::pre_asap::schema::with_promql_series_identity(root) else { + return Proposals::default(); + }; + let typed = Rc::new(typed); + let mut proposals = self.current_series_topk_candidates(&typed, target); + for mut candidate in std::mem::take(&mut proposals.candidates) { + let Replacement::Summary(node) = candidate.replacement else { + continue; + }; + let Ok(node) = finalize_query_candidate(node, &typed) else { + continue; + }; + let duplicate = proposals.candidates.iter().any(|existing| { + matches!(&existing.replacement, Replacement::Summary(other) if *other == node) + }); + if !duplicate { + candidate.replacement = Replacement::Summary(node); + candidate.provenance = ReplacementProvenance::RootPhysicalRealization; + proposals.candidates.push(candidate); + } + } + proposals + } } /// A human-readable rationale for one candidate `Realization`, for @@ -5966,7 +6012,8 @@ fn is_cse_candidate(candidate: &ReplacementSubDAG) -> bool { } fn is_automatically_selectable(candidate: &ReplacementSubDAG, cost_model: &dyn CostModel) -> bool { - !candidate.has_missing_accuracy_evidence() + candidate.provenance != ReplacementProvenance::RootPhysicalRealization + && !candidate.has_missing_accuracy_evidence() && candidate.runtime_support_evidence(cost_model) != Some(false) } @@ -6468,6 +6515,28 @@ pub fn search_workload_with_targets<'s, Id>( .zip(targets) .filter_map(|((_, root), target)| target.map(|t| (Rc::as_ptr(root), t))) .collect(); + // Whole-root proposals join the root group before its target check. + for (index, (ptr, target)) in root_ptrs.iter().enumerate() { + if root_ptrs[..index].contains(&(*ptr, target.clone())) { + continue; + } + let group = space.groups.get_mut(ptr).expect("every root has a group"); + let root = Rc::clone(&group.target); + for strategy in strategies { + let name = strategy.name(); + let proposals = strategy.propose_for_root(&root, target); + for mut candidate in proposals.candidates { + candidate.strategy = name; + group.add_candidate(candidate); + } + group + .rejected + .extend(proposals.rejected.into_iter().map(|mut rejection| { + rejection.strategy = name; + rejection + })); + } + } let mut composition_targets: HashMap<_, Vec<_>> = HashMap::new(); for (ptr, target) in root_ptrs { composition_targets diff --git a/crates/asap-physical-operators/src/physical_planner/promql_rows.rs b/crates/asap-physical-operators/src/physical_planner/promql_rows.rs index cb680096..8b887580 100644 --- a/crates/asap-physical-operators/src/physical_planner/promql_rows.rs +++ b/crates/asap-physical-operators/src/physical_planner/promql_rows.rs @@ -1,7 +1,7 @@ //! A bounded PromQL source row carries the entire label set, not just labels //! mentioned by the query. The source adapter owns this lossless encoding. use super::*; -use planner_types::pre_asap::{Column, DataType, Source as LogicalSource}; +use planner_types::pre_asap::DataType; use std::rc::Rc; /// Not a legal PromQL label name, so it cannot shadow a user label. @@ -22,78 +22,10 @@ pub fn decode_series_identity(encoded: &str) -> Result, Ok(labels) } -/// Resolve the row representation before candidate search. `closed` describes -/// physical columns here: the final column contains every dynamic source label. -/// It does not assert that the query's projected labels are the full label set. -/// -/// This realization supports explicit `by` grouping and per-series computation. -/// Operators that rewrite or implicitly match dynamic label sets require their -/// own realization; they must not accidentally treat the opaque identity as a -/// user label or silently discard it. +/// Resolve the row representation before candidate search; see +/// [`planner_types::pre_asap::schema::with_promql_series_identity`]. pub fn with_series_identity(root: &QueryExpr) -> Result { - let mut root = root.clone(); - fn visit(node: &mut QueryExpr) -> Result<(), Error> { - use planner_types::pre_asap::Reduction; - match node { - QueryExpr::Scan { - source: LogicalSource::TimeSeries { .. }, - schema, - .. - } => { - if schema - .columns - .iter() - .any(|column| column.name == SERIES_IDENTITY_COLUMN) - { - return Err(invalid( - "source already contains a physical series identity", - )); - } - if schema.closed { - return Err(invalid( - "dynamic series identity requires an open PromQL source", - )); - } - schema - .columns - .push(Column::new(SERIES_IDENTITY_COLUMN, DataType::Utf8, false)); - schema.closed = true; - Ok(()) - } - QueryExpr::TimeRange { child, .. } | QueryExpr::Limit { child, .. } => { - visit(Rc::make_mut(child)) - } - QueryExpr::Aggregate { - child, reduction, .. - } => { - if matches!(reduction, Reduction::Reduce(keys) if keys.is_without()) { - return Err(invalid( - "dynamic without grouping requires label-set projection", - )); - } - visit(Rc::make_mut(child)) - } - QueryExpr::Sort { - child, - partition_by, - .. - } => { - if partition_by.is_without() { - return Err(invalid( - "dynamic without ranking requires label-set projection", - )); - } - visit(Rc::make_mut(child)) - } - _ => Err(invalid( - "operator has no dynamic series-identity realization", - )), - } - } - visit(&mut root)?; - root.output_schema() - .map_err(|error| invalid(error.to_string()))?; - Ok(root) + planner_types::pre_asap::schema::with_promql_series_identity(root).map_err(invalid) } /// Construct source rows only from full identities. The named label columns diff --git a/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs b/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs new file mode 100644 index 00000000..2f8d5249 --- /dev/null +++ b/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs @@ -0,0 +1,250 @@ +//! Logical heap alternatives that need the PromQL series identity are part of +//! Planner's search space: `enumerate_candidate_dags_for_root` lists +//! current-series TopK heaps without a caller-side series-identity pass, cost +//! ranking, or workload Cartesian expansion. Placement variants are not listed. +use asap_aware_mapping::{ + accuracy::{AccuracyEvidenceProvider, DefaultAccuracyModel, PropagationStats}, + cost_model::DefaultCostModel, + replacement::{default_strategies_with_evidence, ReplacementProvenance}, + search_workload_with_targets, Proposals, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, +}; +use asap_physical_operators::physical_planner::promql_rows::{ + compile_current_series_readout, SERIES_IDENTITY_COLUMN, +}; +use planner_types::{ + post_asap::*, + pre_asap::QueryExpr, + types::AccuracyTarget, + workload::{ + AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence as WorkloadEvidence, + PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, + TimeSelection, + }, +}; +use std::rc::Rc; + +struct Evidence; +impl AccuracyEvidenceProvider for Evidence { + fn topk_max_distinct_items(&self, _: &QueryExpr) -> Option { + Some(1000) + } + fn propagation_stats( + &self, + op: &CompositionOperator, + _: &SummaryFamilyType, + _: Option<&SketchQuery>, + ) -> PropagationStats { + if matches!(op, CompositionOperator::TopKSelection) { + PropagationStats { + topk_selected_lower_bound: Some(101.), + topk_excluded_upper_bound: Some(100.), + topk_interval_failure_probability: Some(0.001), + ..Default::default() + } + } else { + Default::default() + } + } +} + +/// Forwards everything except whole-root proposals: the pre-change search. +struct LogicalOnly(Box); +impl ReplacementStrategy for LogicalOnly { + fn name(&self) -> &'static str { + self.0.name() + } + fn matches(&self, target: &TargetSubDAG<'_>) -> bool { + self.0.matches(target) + } + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + self.0.replacements(target) + } + fn propose(&self, target: &TargetSubDAG<'_>) -> Proposals { + self.0.propose(target) + } +} + +fn lower(query: &str, accuracy: &AccuracyTarget) -> Rc { + let workload = PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![BatchEntry { + query: Query(query.into()), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit(accuracy.clone()), + ..Default::default() + }, + predictability: Predictability::Unknown, + invocations: 1, + execute_at: None, + time_selection: TimeSelection::default(), + }]), + repeating_queries: None, + }, + data_workload: Some(DataWorkload { + data_ingestion_interval: WorkloadEvidence { + value: Some(DurationMs(1_000)), + ..Default::default() + }, + ..Default::default() + }), + }; + Rc::new( + asap_frontend_promql::lower_promql_workload(&workload, 0) + .unwrap() + .remove(0), + ) +} + +type Dag = Vec<(usize, Rc)>; + +/// Candidate DAGs for query 1 of a two-query workload, with and without +/// whole-root proposals. Query 0 is a bystander that must not multiply them. +fn inventories(query: &str, accuracy: AccuracyTarget) -> (Vec, Vec) { + let roots = vec![ + ( + 0, + lower("sum by(job)(m)", &AccuracyTarget::Exact), + Some(AccuracyTarget::Exact), + ), + (1, lower(query, &accuracy), Some(accuracy)), + ]; + let full = default_strategies_with_evidence(&DefaultCostModel, &Evidence); + let logical: Vec> = + default_strategies_with_evidence(&DefaultCostModel, &Evidence) + .into_iter() + .map(|strategy| Box::new(LogicalOnly(strategy)) as Box) + .collect(); + let enumerate = |strategies: &[Box]| { + search_workload_with_targets(roots.clone(), strategies, &DefaultAccuracyModel) + .enumerate_candidate_dags_for_root(&1, 65_536) + .unwrap() + .candidates + }; + (enumerate(&full), enumerate(&logical)) +} + +fn carries_identity(dag: &Dag) -> bool { + dag.iter().any(|(_, root)| { + compile_post_asap_dag(root) + .unwrap() + .nodes + .iter() + .any(|node| { + node.output_schema + .fields + .iter() + .any(|field| field.name == SERIES_IDENTITY_COLUMN) + }) + }) +} + +/// Shared acceptance checks; returns the added identity-carrying alternatives. +fn added_alternatives(query: &str, accuracy: AccuracyTarget) -> Vec> { + let (full, logical) = inventories(query, accuracy); + for (index, dag) in full.iter().enumerate() { + assert_eq!(dag.len(), 1, "one root per candidate, no workload product"); + assert!( + !full[..index].contains(dag), + "{query}: identical DAG listed twice" + ); + } + let (added, kept): (Vec<_>, Vec<_>) = full.into_iter().partition(carries_identity); + assert_eq!( + kept, logical, + "{query}: existing candidates must be unchanged" + ); + added.into_iter().map(|mut dag| dag.remove(0).1).collect() +} + +const CURRENT_SERIES_TOPK: &str = "topk by(job)(1, m)"; + +// Instant-vector TopK lists finalized current-series heap readouts. +#[test] +fn current_series_topk_lists_heap_readouts() { + let added = added_alternatives(CURRENT_SERIES_TOPK, AccuracyTarget::Epsilon(0.1)); + assert!(!added.is_empty()); + for root in added { + assert!(!matches!(root.expr, SummaryExpr::SummaryAgg { .. })); + assert!( + compile_current_series_readout(&root).is_ok(), + "unbindable alternative {root:?}" + ); + } +} + +// Rate queries gain no fixed-window or query-time placement variants. +#[test] +fn rate_placement_variants_are_not_listed() { + for (query, accuracy) in [ + ("sum by(job)(rate(m[1m]))", AccuracyTarget::Exact), + ("topk by(job)(2, rate(m[1m]))", AccuracyTarget::Epsilon(0.1)), + ("rate(m[1m])", AccuracyTarget::Exact), + ] { + assert!(added_alternatives(query, accuracy).is_empty(), "{query}"); + } +} + +// Queries without a current-series heap realization are unchanged. +#[test] +fn unrelated_queries_keep_their_inventory() { + for (query, accuracy) in [ + ("sum by(job)(m)", AccuracyTarget::Exact), + ( + "quantile_over_time(0.9, m[1m])", + AccuracyTarget::Epsilon(0.05), + ), + ("max_over_time(m[1m])", AccuracyTarget::Exact), + ] { + assert!(added_alternatives(query, accuracy).is_empty(), "{query}"); + } +} + +// Default cost-based selection keeps the logical plan; deployment prices heaps. +#[test] +fn global_selection_never_commits_a_series_identity_heap() { + let accuracy = AccuracyTarget::Epsilon(0.1); + let root = lower(CURRENT_SERIES_TOPK, &accuracy); + let strategies = default_strategies_with_evidence(&DefaultCostModel, &Evidence); + let space = search_workload_with_targets( + vec![(0, root, Some(accuracy))], + &strategies, + &DefaultAccuracyModel, + ); + let selected = space + .global_selection(&DefaultCostModel) + .assemble_selected_dag(&space.roots[0].1) + .unwrap() + .unwrap(); + assert!(!carries_identity(&vec![(0, selected)])); +} + +// A query repeated in the workload is proposed once, not once per copy. +#[test] +fn repeated_roots_do_not_duplicate_alternatives() { + let accuracy = AccuracyTarget::Epsilon(0.1); + let strategies = default_strategies_with_evidence(&DefaultCostModel, &Evidence); + let count = |copies: usize| { + let roots = (0..copies) + .map(|id| { + ( + id, + lower(CURRENT_SERIES_TOPK, &accuracy), + Some(accuracy.clone()), + ) + }) + .collect(); + let space = search_workload_with_targets(roots, &strategies, &DefaultAccuracyModel); + space + .candidates_for_target(&space.roots[0].1) + .unwrap() + .candidates + .iter() + .filter(|candidate| { + candidate.provenance == ReplacementProvenance::RootPhysicalRealization + }) + .count() + }; + assert!(count(1) > 0); + assert_eq!(count(2), count(1)); +} diff --git a/crates/types/src/pre_asap/schema.rs b/crates/types/src/pre_asap/schema.rs index fbcb9d67..938b9b7f 100644 --- a/crates/types/src/pre_asap/schema.rs +++ b/crates/types/src/pre_asap/schema.rs @@ -158,6 +158,71 @@ pub struct Schema { /// label map. `$` cannot occur in a user PromQL label name. pub const PROMQL_SERIES_IDENTITY: &str = "$promql_series_identity"; +/// Resolve a PromQL root to rows carrying [`PROMQL_SERIES_IDENTITY`] before +/// candidate search. `closed` describes physical columns here: the final +/// column contains every dynamic source label. It does not assert that the +/// query's projected labels are the full label set. +/// +/// This realization supports explicit `by` grouping and per-series computation. +/// Operators that rewrite or implicitly match dynamic label sets require their +/// own realization; they must not accidentally treat the opaque identity as a +/// user label or silently discard it. +pub fn with_promql_series_identity(root: &super::QueryExpr) -> Result { + use super::{QueryExpr, Reduction, Source}; + use std::rc::Rc; + fn visit(node: &mut QueryExpr) -> Result<(), String> { + match node { + QueryExpr::Scan { + source: Source::TimeSeries { .. }, + schema, + .. + } => { + if schema + .columns + .iter() + .any(|column| column.name == PROMQL_SERIES_IDENTITY) + { + return Err("source already contains a physical series identity".into()); + } + if schema.closed { + return Err("dynamic series identity requires an open PromQL source".into()); + } + schema + .columns + .push(Column::new(PROMQL_SERIES_IDENTITY, DataType::Utf8, false)); + schema.closed = true; + Ok(()) + } + QueryExpr::TimeRange { child, .. } | QueryExpr::Limit { child, .. } => { + visit(Rc::make_mut(child)) + } + QueryExpr::Aggregate { + child, reduction, .. + } => { + if matches!(reduction, Reduction::Reduce(keys) if keys.is_without()) { + return Err("dynamic without grouping requires label-set projection".into()); + } + visit(Rc::make_mut(child)) + } + QueryExpr::Sort { + child, + partition_by, + .. + } => { + if partition_by.is_without() { + return Err("dynamic without ranking requires label-set projection".into()); + } + visit(Rc::make_mut(child)) + } + _ => Err("operator has no dynamic series-identity realization".into()), + } + } + let mut root = root.clone(); + visit(&mut root)?; + root.output_schema().map_err(|error| error.to_string())?; + Ok(root) +} + impl Schema { pub fn has_promql_series_identity(&self) -> bool { self.closed diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index fe71c8c3..7e7b3bd0 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -79,6 +79,13 @@ sample values does not preserve instant-vector semantics. Replacement, rank decrease, expiry, grouping and the required approximation guarantee must be validated before admitting that physical candidate. +Planner's candidate space decides what to compute, not placement. For an +instant-vector PromQL TopK, Planner resolves rows that carry the complete series +identity and lists the current-series heap realizations per root with the other +candidates, unranked. Precompute or query-time placement of Rate and grouped Sum +is not a separate Planner candidate: the summary maintenance lifecycle assigns +each node's timing, and the physical compiler reads it. + This is the target ownership contract. A backend path that still reconstructs operators from logical candidates has not completed this integration. diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index bfeea034..92288067 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -267,6 +267,28 @@ they are not necessarily a globally sortable physical-cost scalar. Unavailable cost alternatives may remain for explanation. Inspect eligibility and evidence before physical selection; do not treat their presence as deployment permission. +### Enumerate candidate DAGs per root + +```text +PlanSpace::enumerate_candidate_dags_for_root(&self, id: &Id, expansion_limit: usize) + -> Result, RealizationError> +``` + +Returns every distinct finalized DAG for one root, unranked; other roots' +choices are not multiplied in. Exceeding `expansion_limit` is an error, never a +partial inventory. + +For PromQL roots that carry a target, `search_workload_with_targets` also asks +each strategy's `ReplacementStrategy::propose_for_root`. `SketchAlgorithmStrategy` +answers an instant-vector TopK with current-series heap realizations over rows +carrying the complete series identity (`$promql_series_identity`). They are +finalized, deduplicated, and marked `ReplacementProvenance::RootPhysicalRealization`. +Callers do not apply `with_series_identity` themselves. Compile each with +`promql_rows::compile_current_series_readout`; other queries keep their previous +inventory. `global_selection` never commits these candidates; the backend +compiles and prices them. PlanSpace lists no placement variants: node timing +comes from the summary maintenance lifecycle. + ## Choose strategies and models ### Strategy options From 5dd3d2bdc170f800cfc17b9d90937df59174790f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:27:10 +0000 Subject: [PATCH 2/2] g3 local --- crates/asap-aware-mapping/src/lib.rs | 15 +- .../src/maintained_population.rs | 31 +- crates/asap-aware-mapping/src/replacement.rs | 193 +-- .../src/summary_maintenance_dag_export.rs | 2 +- .../src/summary_maintenance_lifecycle.rs | 1495 +++++++++++++++-- .../src/physical_planner/candidates.rs | 268 ++- .../src/physical_planner/compiled.rs | 56 +- .../src/physical_planner/mod.rs | 26 +- .../src/physical_planner/promql_rows.rs | 15 +- .../tests/precompute_candidates.rs | 184 +- .../tests/weighted_topk_binding.rs | 178 +- .../summary_maintenance_lifecycle_e2e.rs | 536 ++++++ .../src/post_asap/execution_data_state.rs | 11 +- .../architecture/input-output-workflow.md | 4 +- .../physical-planning-and-deployment.md | 115 +- .../maintained-populations.md | 7 +- .../design_docs/proposals/operator-sharing.md | 2 +- docs/develop_docs/library-api.md | 48 + 18 files changed, 2752 insertions(+), 434 deletions(-) diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 0349db1e..6b99dbf9 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -221,13 +221,14 @@ pub use summary_maintenance_dag_export::{ }; pub use summary_maintenance_lifecycle::{ assemble_selected_dag_with_summary_maintenance_lifecycles, - global_selection_with_summary_maintenance_lifecycles, plan_summary_maintenance_lifecycles, - SummaryMaintenanceCapabilities, SummaryMaintenanceDeployment, - SummaryMaintenanceLifecycleAlternative, SummaryMaintenanceLifecycleAssemblyError, - SummaryMaintenanceLifecycleCapabilities, SummaryMaintenanceLifecycleCostInputs, - SummaryMaintenanceLifecyclePlan, SummaryMaintenanceLifecyclePlanError, - SummaryMaintenanceLifecycleRejection, SummaryMaintenanceLifecycleSelectionError, - WorkloadDemand, + enumerate_summary_maintenance_lifecycles, global_selection_with_summary_maintenance_lifecycles, + plan_summary_maintenance_lifecycles, SummaryMaintenanceCapabilities, + SummaryMaintenanceDeployment, SummaryMaintenanceLifecycleAlternative, + SummaryMaintenanceLifecycleAssemblyError, SummaryMaintenanceLifecycleCandidates, + SummaryMaintenanceLifecycleCapabilities, SummaryMaintenanceLifecycleChoiceError, + SummaryMaintenanceLifecycleCostInputs, SummaryMaintenanceLifecyclePlan, + SummaryMaintenanceLifecyclePlanError, SummaryMaintenanceLifecycleRejection, + SummaryMaintenanceLifecycleSelectionError, SummaryMaintenanceTimingError, WorkloadDemand, }; pub use topk_reuse::TopKLimitReuseStrategy; diff --git a/crates/asap-aware-mapping/src/maintained_population.rs b/crates/asap-aware-mapping/src/maintained_population.rs index 504c8100..29cd3c54 100644 --- a/crates/asap-aware-mapping/src/maintained_population.rs +++ b/crates/asap-aware-mapping/src/maintained_population.rs @@ -258,11 +258,15 @@ impl MaintainedPopulationStrategy { schema: input_schema.clone(), guarantee: Some(ResultGuarantee::exact("source samples")), }); + // Query time is only the initial layout: whether the population is + // retained at ingestion or rebuilt per query is its lifecycle choice + // (`SummaryMaintenanceLifecyclePlan::execution_timed_dag`). The readout + // and projection above it are query-time by construction. let maintained = Rc::new(SummaryNode { expr: SummaryExpr::ValueOperation { child: scan, operation: ValueOperation::MaintainPopulation { population }, - timing: ExecutionTiming::IngestionTime, + timing: ExecutionTiming::QueryTime, }, schema: input_schema, guarantee: Some(ResultGuarantee::exact( @@ -435,6 +439,31 @@ mod tests { assert_eq!(p.grouping, ["instance"]); assert_eq!(p.matchers[0].operation, CurrentSeriesMatch::Regex); } + // Population timing is a lifecycle choice: a retained or rebuilt + // population both validate, while its readout must stay at query time. + #[test] + fn population_timing_is_not_structural() { + let root = lower("topk(5,a)"); + let candidate = MaintainedPopulationStrategy::new(std::slice::from_ref(&root)) + .candidate(&root) + .unwrap(); + let with_timings = |population: ExecutionTiming, readout: ExecutionTiming| { + let mut node = (*candidate).clone(); + let SummaryExpr::ValueOperation { child, timing, .. } = &mut node.expr else { + unreachable!() + }; + *timing = readout; + let SummaryExpr::ValueOperation { timing, .. } = &mut Rc::make_mut(child).expr else { + unreachable!() + }; + *timing = population; + compile_post_asap_dag(&Rc::new(node)) + }; + use ExecutionTiming::{IngestionTime, QueryTime}; + assert!(with_timings(IngestionTime, QueryTime).is_ok()); + assert!(with_timings(QueryTime, QueryTime).is_ok()); + assert!(with_timings(IngestionTime, IngestionTime).is_err()); + } // A readout cannot reinterpret arbitrary rows as maintained state or exceed its producer's contract. #[test] fn malformed_population_dags_fail_closed() { diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index fb697251..8c826b37 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -1356,114 +1356,6 @@ impl<'a> SketchAlgorithmStrategy<'a> { self.propose_with(&ranked, None) } - /// Fixed-window maintenance can finalize each series' counter state and - /// build a fresh heap or grouped Sum for that evaluation window. Deployment must provide - /// a complete, synchronized population and bind the matching window; this - /// candidate never incrementally adds one window's rates to another. - pub fn fixed_window_rate_candidates(&self, root: &Rc) -> Proposals { - fn place(node: &Rc) -> Option> { - let mut next = node.as_ref().clone(); - match &mut next.expr { - SummaryExpr::ValueOperation { - child, - operation: ValueOperation::FinalizeExactAccumulator, - timing, - } if matches!(&child.expr, SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Rate, _), - reduction: Reduction::PerEntity, child: source, .. - } if matches!(&source.expr, SummaryExpr::KeepPreAsap(source) if matches!(source.as_ref(), QueryExpr::TimeRange { .. }))) => - { - *timing = ExecutionTiming::IngestionTime; - } - SummaryExpr::ValueOperation { child, .. } - | SummaryExpr::SummaryAgg { child, .. } => *child = place(child)?, - SummaryExpr::SummaryEstimate { summary_input, .. } => { - *summary_input = place(summary_input)? - } - _ => return None, - } - Some(Rc::new(next)) - } - let mut proposals = self.propose_with(root, None); - proposals.candidates.retain_mut(|candidate| { - let Replacement::Summary(node) = &candidate.replacement else { - return false; - }; - let Ok(dag) = asap_types::post_asap::compile_post_asap_dag(node) else { - return false; - }; - if !dag.nodes.iter().any(|node| match &node.payload { - asap_types::post_asap::PostAsapOperatorPayload::SummaryAgg { - family: SummaryFamilyType::Sketch(kind, _), - .. - } => matches!( - kind.algorithm(), - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap - ), - asap_types::post_asap::PostAsapOperatorPayload::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), - .. - } => true, - _ => false, - }) { - return false; - } - let Some(placed) = place(node) else { - return false; - }; - if asap_types::post_asap::compile_post_asap_dag(&placed).is_err() { - return false; - } - let Ok(placed) = finalize_query_candidate(placed, root) else { - return false; - }; - candidate.replacement = Replacement::Summary(placed); - candidate - .rationale - .push_str("; fixed-window precompute over complete per-series counter states"); - true - }); - proposals - } - - /// Retain grouped Sum after a per-series Rate readout as a query-time - /// candidate alongside its complete-window maintenance placement. - pub fn query_time_rate_aggregation_candidates(&self, root: &Rc) -> Proposals { - fn query_time(node: &Rc) -> Rc { - let mut next = node.as_ref().clone(); - match &mut next.expr { - SummaryExpr::ValueOperation { - child, - operation: ValueOperation::FinalizeExactAccumulator, - timing, - } if matches!( - &child.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Rate, _), - .. - } - ) => - { - *timing = ExecutionTiming::QueryTime; - } - SummaryExpr::ValueOperation { child, .. } - | SummaryExpr::SummaryAgg { child, .. } => *child = query_time(child), - _ => {} - } - Rc::new(next) - } - let mut proposals = self.fixed_window_rate_candidates(root); - proposals.candidates.retain_mut(|candidate| { - let Replacement::Summary(node) = &candidate.replacement else { return false }; - if !matches!(&node.expr, SummaryExpr::ValueOperation { child, operation: ValueOperation::FinalizeExactAccumulator, .. } - if matches!(&child.expr, SummaryExpr::SummaryAgg { family: SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), .. })) { return false; } - candidate.replacement = Replacement::Summary(query_time(node)); - candidate.rationale = "query-time grouped Sum over complete per-series Rate readouts".into(); - true - }); - proposals - } - pub(crate) fn from_planning_inputs(planning_inputs: CandidatePlanningInputs<'a>) -> Self { Self { planning_inputs } } @@ -2764,7 +2656,9 @@ fn realize_physical_summary_input( /// Emit `SummaryAgg` (recursively binding the child), plus the /// `SummaryEstimate` readout when `estimate` is set. // Retain the exact expression and schema while placing its value production -// on the update path. Read-time consumers keep their original shared nodes. +// on the update path. This is the initial layout for values feeding a summary; +// lifecycle timing is authoritative. Read-time consumers keep their original +// shared nodes. fn maintenance_exact_values(node: Rc) -> Option> { let expr = match &node.expr { // These guards can fall back at read time, but cannot recover a parent @@ -2993,8 +2887,9 @@ fn construct_summary_agg( }; Rc::clone(child) } else if snapshot_weighted { - // A fresh query-time summary consumes this evaluation's finalized rates. - // Moving rate snapshots must never accumulate across evaluations. + // Each evaluation's finalized rates feed a fresh summary; rate snapshots + // must never accumulate across evaluations. Query time is only the + // initial layout; a retained summary's lifecycle moves it to ingestion. finalize_query_candidate(bound_child, &input.child)? } else { let child = finalize_exact_accumulator_at( @@ -5249,6 +5144,9 @@ impl<'a> GlobalSelection<'a> { if let Some(node) = self.assembled_nodes.borrow().get(&ptr) { return Ok(Rc::clone(node)); } + // A selected summary that realizes its inner aggregate, instead of + // hiding it in `KeepPreAsap`, is kept; lifecycle assignment decides + // whether it runs in precompute or at query time. let selected_composed_summary = self .groups .get(&ptr) @@ -5256,7 +5154,7 @@ impl<'a> GlobalSelection<'a> { .is_some_and(|candidate| matches!(&candidate.replacement, Replacement::Summary(node) if matches!(&node.expr, SummaryExpr::SummaryAgg { child, .. } - if matches!(&child.expr, SummaryExpr::KeepPreAsap(raw) if !contains_aggregate(raw))))); + if !matches!(&child.expr, SummaryExpr::KeepPreAsap(raw) if contains_aggregate(raw))))); let node = if query_time_nested_sum(target) && !selected_composed_summary { self.assemble_residual(target)? } else { @@ -6955,6 +6853,77 @@ mod tests { use asap_types::types::AccuracyTarget; use std::collections::HashMap; + // Candidate shape without execution timing: what is computed, not where. + fn timing_free_shape(node: &Rc) -> serde_json::Value { + fn strip(value: &mut serde_json::Value) { + match value { + serde_json::Value::Object(fields) => { + fields.remove("timing"); + fields.values_mut().for_each(strip); + } + serde_json::Value::Array(values) => values.iter_mut().for_each(strip), + _ => {} + } + } + let mut shape = + serde_json::to_value(asap_types::post_asap::compile_post_asap_dag(node).unwrap()) + .unwrap(); + strip(&mut shape); + shape + } + + // Rate inventories never offer two candidates that differ only in timing. + #[test] + fn rate_candidate_inventories_have_no_timing_only_duplicates() { + for (query, accuracy) in [ + ("sum by(job)(rate(m[1m]))", AccuracyTarget::Exact), + ("topk by(job)(2, rate(m[1m]))", AccuracyTarget::Epsilon(0.1)), + ] { + let root = Rc::new(lower_promql(query, accuracy)); + let inventory = search_workload(vec![(0usize, root)]) + .enumerate_candidate_dags(4096) + .unwrap(); + let shapes = inventory + .candidates + .iter() + .map(|forest| timing_free_shape(&forest[0].1)) + .collect::>(); + for (i, shape) in shapes.iter().enumerate() { + assert!(!shapes[..i].contains(shape), "{query}: duplicate {i}"); + } + } + } + + // Grouped Sum over Rate readouts stays a summary state in the inventory, + // so lifecycle assignment can place it in precompute or at query time. + #[test] + fn grouped_rate_sum_inventory_keeps_sum_state_for_lifecycle_placement() { + let root = Rc::new(lower_promql( + "sum by(job)(rate(m[1m]))", + AccuracyTarget::Exact, + )); + let inventory = search_workload(vec![(0usize, root)]) + .enumerate_candidate_dags(4096) + .unwrap(); + let is_exact = |node: &SummaryNode, kind: ExactKind| { + matches!(&node.expr, SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(k, _), .. + } if *k == kind) + }; + assert!(inventory.candidates.iter().any(|forest| { + let SummaryExpr::ValueOperation { child: sum, .. } = &forest[0].1.expr else { + return false; + }; + let SummaryExpr::SummaryAgg { child: rate, .. } = &sum.expr else { + return false; + }; + is_exact(sum, ExactKind::Sum) + && matches!(&rate.expr, SummaryExpr::ValueOperation { + child, operation: ValueOperation::FinalizeExactAccumulator, .. + } if is_exact(child, ExactKind::Rate)) + })); + } + // Every exposed query result has a readout; internal accumulator frontiers stay states. #[test] fn query_candidate_roots_do_not_leak_exact_accumulator_state() { diff --git a/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs b/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs index efc8cf13..129d06ed 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs @@ -117,7 +117,7 @@ pub fn export_summary_maintenance_plan( } /// Walk in the same post-order as `dag_export::export_summary` and attach a -/// deployment directly to every flattened occurrence of its `SummaryAgg`. +/// deployment directly to every flattened occurrence of its state node. /// This makes the decision visible to graph consumers without asking them to /// reconstruct pointer identity from graph position. fn annotate_lifecycle_deployments( diff --git a/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs index b0d9d79f..cb1dd9db 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs @@ -11,7 +11,8 @@ //! //! This module enumerates and costs `Ephemeral`, `Prepared`, `Shared`, and //! `ContinuouslyMaintained` alternatives for every unique `SummaryAgg` in a -//! materialized plan. [`SummaryMaintenanceMode`] is an orthogonal detail of +//! materialized plan, and for every maintained population (`MaintainPopulation`) +//! that is not an input of a `SummaryAgg`. [`SummaryMaintenanceMode`] is an orthogonal detail of //! the selected deployment: state is either built directly or updated //! incrementally. Unknown evidence stays unknown and therefore cannot make a //! long-lived alternative win. @@ -21,9 +22,10 @@ use std::rc::Rc; use asap_types::post_asap::{ compile_post_asap_dag_with_node_ids, EvaluationSchedule, ExecutionDataStateError, - OutputRepresentation, PostAsapNodeId, ResultGuarantee, SummaryExpr, - SummaryMaintenanceLifecycle, SummaryMaintenanceLifecycleGuarantee, SummaryMaintenanceMode, - SummaryNode, SummaryWindowFramework, + ExecutionTiming, OutputRepresentation, PostAsapDag, PostAsapDagValidationError, PostAsapNodeId, + ResultGuarantee, SummaryExpr, SummaryMaintenanceLifecycle, + SummaryMaintenanceLifecycleGuarantee, SummaryMaintenanceMode, SummaryNode, + SummaryWindowFramework, ValueOperation, }; use asap_types::pre_asap::QueryExpr; use asap_types::types::AccuracyTarget; @@ -158,14 +160,16 @@ impl SummaryMaintenanceLifecycleAlternative { } } -/// One unique summary-state deployment. Shared `Rc` nodes are emitted once. +/// One unique retained-state deployment. Shared `Rc` nodes are emitted once. #[derive(Debug, Clone)] pub struct SummaryMaintenanceDeployment { /// Identity of this summary in the exported post-ASAP semantic DAG. /// It is scoped to one plan version and is not a summary definition or /// summary instance identity. pub post_asap_node_id: PostAsapNodeId, - /// The unique materialized `SummaryAgg` represented by this deployment. + /// The unique materialized `SummaryAgg`, or maintained population + /// (`MaintainPopulation`) not consumed by a `SummaryAgg`, represented by + /// this deployment. Cost-model lifecycle hooks receive this node. pub summary: Rc, /// Lifecycle, evaluation, and representation commitment selected for this /// state, or `None` when no alternative is selectable. @@ -183,8 +187,9 @@ pub struct SummaryMaintenanceDeployment { pub struct SummaryMaintenanceLifecyclePlan { /// Root of the materialized post-ASAP DAG being deployed. pub root: Rc, - /// One entry per unique reachable `SummaryAgg`; shared `Rc` nodes appear - /// only once. + /// One entry per unique reachable `SummaryAgg`, then per unique + /// maintained population outside any `SummaryAgg`'s inputs; shared `Rc` + /// nodes appear only once. pub deployments: Vec, /// Caller-supplied optimization horizon used to turn rates into total /// costs. `None` keeps horizon-dependent alternatives unselectable. @@ -211,6 +216,89 @@ pub struct SummaryMaintenanceLifecyclePlan { pub raw_recompute_total_cost: Option, } +/// Why a lifecycle plan cannot assign execution timing to its DAG. +#[derive(Debug, thiserror::Error, PartialEq)] +pub enum SummaryMaintenanceTimingError { + #[error(transparent)] + InvalidPostAsapDag(#[from] ExecutionDataStateError), + #[error("summary {0:?} has no selected lifecycle")] + UnselectedLifecycle(PostAsapNodeId), + /// A maintained population outside any `SummaryAgg`'s inputs has no + /// deployment, so its timing would be guessed. Enumeration always emits + /// one; this arises only for a plan whose root or deployments were edited. + #[error("node {0:?} maintains state that has no summary-maintenance lifecycle")] + UnplannedMaintainedState(PostAsapNodeId), + #[error(transparent)] + InvalidPhases(#[from] PostAsapDagValidationError), +} + +impl SummaryMaintenanceLifecyclePlan { + /// The post-ASAP DAG of [`Self::root`] with every node's timing derived + /// from the selected lifecycles, so physical compilation places it. + /// + /// A retained (non-`Ephemeral`) state outlives one query, so it and every + /// input it consumes run at ingestion time. Every other node runs at query + /// time: readouts and consumers of retained state, and each `Ephemeral` + /// state not consumed by retained state together with its inputs, whose + /// raw data the deployment must supply as a query source. This applies to + /// maintained populations as to `SummaryAgg` states; a population feeding + /// a `SummaryAgg` is one of its inputs. Timings already on the root are + /// ignored. + pub fn execution_timed_dag(&self) -> Result { + let compiled = compile_post_asap_dag_with_node_ids(&self.root)?; + let dag = compiled.dag; + for population in &standalone_populations(&self.root) { + let id = compiled + .node_ids + .node_id(population) + .expect("collected population belongs to the compiled DAG"); + if !self + .deployments + .iter() + .any(|deployment| deployment.post_asap_node_id == id) + { + return Err(SummaryMaintenanceTimingError::UnplannedMaintainedState(id)); + } + } + let mut pending = Vec::new(); + for deployment in &self.deployments { + let guarantee = deployment + .summary_maintenance_lifecycle_guarantee + .as_ref() + .ok_or(SummaryMaintenanceTimingError::UnselectedLifecycle( + deployment.post_asap_node_id, + ))?; + if guarantee.summary_maintenance_lifecycle != SummaryMaintenanceLifecycle::Ephemeral { + pending.push(deployment.post_asap_node_id); + } + } + let mut ingestion = HashSet::new(); + while let Some(id) = pending.pop() { + if ingestion.insert(id) { + pending.extend( + dag.edges + .iter() + .filter(|edge| edge.consumer == id) + .map(|edge| edge.producer), + ); + } + } + let phases = dag + .nodes + .iter() + .map(|node| { + let timing = if ingestion.contains(&node.id) { + ExecutionTiming::IngestionTime + } else { + ExecutionTiming::QueryTime + }; + (node.id, timing) + }) + .collect(); + Ok(dag.with_execution_phases(&phases)?) + } +} + /// Explicit association between a materialized target and the normalized /// workload entries whose demand consumes it. /// @@ -287,6 +375,171 @@ pub enum SummaryMaintenanceLifecycleSelectionError { SummaryMaintenance(#[from] SummaryMaintenanceLifecyclePlanError), } +/// Every lifecycle alternative for each unique retained state of one fixed +/// root, before any lifecycle is chosen. +/// +/// Planner selection ([`plan_summary_maintenance_lifecycles`]) and a +/// deployment's explicit choice ([`Self::select`]) both finish from this value, +/// so they produce the same [`SummaryMaintenanceLifecyclePlan`] shape. +pub struct SummaryMaintenanceLifecycleCandidates<'a> { + /// Unselected plan: deployments carry alternatives but no guarantee or + /// window framework. + plan: SummaryMaintenanceLifecyclePlan, + components: Vec, + arrival: DataArrival, + required_accuracy: Vec, + cost_model: &'a dyn CostModel, + comparison_target: Option<&'a QueryExpr>, +} + +/// Why an explicit per-state lifecycle choice cannot be bound. +#[derive(Debug, thiserror::Error, PartialEq)] +pub enum SummaryMaintenanceLifecycleChoiceError { + #[error("summary {0:?} is not a deployment of this root")] + UnknownSummary(PostAsapNodeId), + #[error("summary {0:?} is chosen more than once")] + DuplicateChoice(PostAsapNodeId), + #[error("summary {0:?} has no chosen lifecycle")] + MissingChoice(PostAsapNodeId), + #[error("chosen lifecycle is not an enumerated alternative of summary {0:?}")] + NotAnAlternative(PostAsapNodeId), + #[error("chosen lifecycle of summary {post_asap_node_id:?} is rejected: {rejection:?}")] + Rejected { + post_asap_node_id: PostAsapNodeId, + rejection: Option, + }, + #[error("summary states on one maintenance path have different evaluation schedules")] + IncompatibleEvaluationSchedules, + #[error("the cost model supplied no complete estimate for the chosen combination")] + NoCompleteEstimate, +} + +impl SummaryMaintenanceLifecycleCandidates<'_> { + /// One entry per unique retained state (see + /// [`SummaryMaintenanceLifecyclePlan::deployments`]), with every + /// alternative and its rejection; no lifecycle or window framework is + /// selected. + pub fn deployments(&self) -> &[SummaryMaintenanceDeployment] { + &self.plan.deployments + } + + /// Guarantee that binding `lifecycle` would attach under this workload's + /// data arrival, so a caller can price an alternative before choosing it. + pub fn guarantee( + &self, + lifecycle: &SummaryMaintenanceLifecycle, + ) -> SummaryMaintenanceLifecycleGuarantee { + lifecycle_guarantee(lifecycle, self.arrival) + } + + fn context(&self) -> CompleteCostContext<'_> { + CompleteCostContext { + root: &self.plan.root, + components: &self.components, + cost_model: self.cost_model, + comparison_target: self.comparison_target, + horizon: self.plan.horizon, + expected_reads: self.plan.expected_reads, + required_accuracy: &self.required_accuracy, + } + } + + fn finish( + mut self, + estimate: Option, + ) -> SummaryMaintenanceLifecyclePlan { + if let Some(estimate) = estimate { + self.plan.summary_total_cost = Some(estimate.cost); + self.plan.selected_window_implementation_id = estimate.physical_plan_id; + self.plan.window_accuracy_guarantee = estimate.window_accuracy_guarantee; + } + self.plan + } + + /// Planner's choice: the cheapest complete combination of eligible + /// alternatives. + fn select_cheapest(mut self) -> SummaryMaintenanceLifecyclePlan { + let estimate = select_complete_lifecycle_combination( + &self.plan.root, + &mut self.plan.deployments, + &self.components, + self.arrival, + self.cost_model, + self.comparison_target, + self.plan.horizon, + self.plan.expected_reads, + &self.required_accuracy, + ); + self.finish(estimate) + } + + /// Bind one caller-chosen lifecycle per summary state. Each choice must be + /// an alternative Planner itself could select; the complete estimate is + /// then obtained exactly as for Planner selection, so window framework and + /// cost are the model's and unknown cost is never replaced by zero. + pub fn select( + mut self, + choices: &[(PostAsapNodeId, SummaryMaintenanceLifecycle)], + ) -> Result { + use SummaryMaintenanceLifecycleChoiceError as E; + let deployments = &self.plan.deployments; + let mut chosen: Vec> = + vec![None; deployments.len()]; + let context = self.context(); + for (id, lifecycle) in choices { + let index = deployments + .iter() + .position(|deployment| deployment.post_asap_node_id == *id) + .ok_or(E::UnknownSummary(*id))?; + if chosen[index].is_some() { + return Err(E::DuplicateChoice(*id)); + } + let alternative = deployments[index] + .alternatives + .iter() + .find(|alternative| alternative.summary_maintenance_lifecycle == *lifecycle) + .ok_or(E::NotAnAlternative(*id))?; + if !context.eligible(alternative) { + return Err(E::Rejected { + post_asap_node_id: *id, + rejection: alternative.rejection.clone(), + }); + } + chosen[index] = Some(alternative); + } + let selected = chosen + .into_iter() + .enumerate() + .map(|(index, alternative)| { + let alternative = + alternative.ok_or(E::MissingChoice(deployments[index].post_asap_node_id))?; + Ok(( + index, + lifecycle_guarantee(&alternative.summary_maintenance_lifecycle, self.arrival), + // Reached only for costed alternatives or when the + // complete hook is authoritative, matching Planner search. + alternative.total_cost.unwrap_or(Cost::ZERO), + )) + }) + .collect::, E>>()?; + if selected.is_empty() { + return Ok(self.finish(None)); + } + if !context.schedules_compatible(&selected) { + return Err(E::IncompatibleEvaluationSchedules); + } + let estimate = context + .estimate(deployments, &selected) + .ok_or(E::NoCompleteEstimate)?; + let guarantees = selected + .into_iter() + .map(|(index, guarantee, _)| (index, guarantee)) + .collect(); + apply_selection(&mut self.plan.deployments, guarantees, &estimate); + Ok(self.finish(Some(estimate))) + } +} + /// Workload-wide evidence derived specifically for summary-maintenance /// lifecycle enumeration and costing. /// @@ -333,7 +586,30 @@ pub fn plan_summary_maintenance_lifecycles( capabilities: SummaryMaintenanceLifecycleCapabilities, cost_model: &dyn CostModel, ) -> Result { - plan_summary_maintenance_lifecycles_with_profile( + Ok(enumerate_summary_maintenance_lifecycles( + root, + demand, + now_ms, + horizon, + capabilities, + cost_model, + )? + .select_cheapest()) +} + +/// Validate a materialized plan and enumerate lifecycle alternatives for each +/// unique summary state without choosing one. A deployment that prices the +/// alternatives itself binds its choice with +/// [`SummaryMaintenanceLifecycleCandidates::select`]. +pub fn enumerate_summary_maintenance_lifecycles<'a>( + root: Rc, + demand: WorkloadDemand<'_>, + now_ms: u64, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + cost_model: &'a dyn CostModel, +) -> Result, SummaryMaintenanceLifecyclePlanError> { + enumerate_with_profile( root, demand, now_ms, @@ -349,16 +625,16 @@ pub fn plan_summary_maintenance_lifecycles( /// eligibility and data-arrival facts; `profile` supplies effective uses after /// DAG path multiplicity has been propagated by `PlanSpace`. #[expect(clippy::too_many_arguments, reason = "internal bound planning context")] -fn plan_summary_maintenance_lifecycles_with_profile( +fn enumerate_with_profile<'a>( root: Rc, demand: WorkloadDemand<'_>, now_ms: u64, horizon: Option, capabilities: SummaryMaintenanceLifecycleCapabilities, - cost_model: &dyn CostModel, + cost_model: &'a dyn CostModel, profile: Option, - comparison_target: Option<&QueryExpr>, -) -> Result { + comparison_target: Option<&'a QueryExpr>, +) -> Result, SummaryMaintenanceLifecyclePlanError> { demand.workload.validate()?; if let Some(data) = demand.data_workload { data.validate()?; @@ -389,10 +665,16 @@ fn plan_summary_maintenance_lifecycles_with_profile( }; } let mut summaries = Vec::new(); - collect_summary_aggs(&root, &mut HashSet::new(), &mut summaries); + collect_states( + &root, + &mut HashSet::new(), + &mut summaries, + StateKind::SummaryAgg, + ); + summaries.extend(standalone_populations(&root)); let node_ids = compile_post_asap_dag_with_node_ids(&root)?.node_ids; let components = summary_state_components(&summaries); - let mut deployments: Vec = summaries + let deployments: Vec = summaries .into_iter() .map(|summary| { let alternatives = alternatives_for( @@ -413,37 +695,26 @@ fn plan_summary_maintenance_lifecycles_with_profile( } }) .collect(); - let complete_estimate = select_complete_lifecycle_combination( - &root, - &mut deployments, - &components, - facts.arrival, + let selected_raw_recompute = matches!(root.expr, SummaryExpr::KeepPreAsap(_)); + Ok(SummaryMaintenanceLifecycleCandidates { + plan: SummaryMaintenanceLifecyclePlan { + root, + deployments, + horizon, + evaluation_rate: facts.evaluation_rate, + update_rate: facts.update_rate, + expected_reads: facts.reads, + selected_raw_recompute, + selected_window_implementation_id: None, + summary_total_cost: None, + window_accuracy_guarantee: None, + raw_recompute_total_cost: None, + }, + components, + arrival: facts.arrival, + required_accuracy: facts.required_accuracy, cost_model, comparison_target, - horizon, - facts.reads, - &facts.required_accuracy, - ); - let summary_total_cost = complete_estimate.as_ref().map(|estimate| estimate.cost); - let selected_window_implementation_id = complete_estimate - .as_ref() - .and_then(|estimate| estimate.physical_plan_id.clone()); - let window_accuracy_guarantee = complete_estimate - .as_ref() - .and_then(|estimate| estimate.window_accuracy_guarantee.clone()); - let selected_raw_recompute = matches!(root.expr, SummaryExpr::KeepPreAsap(_)); - Ok(SummaryMaintenanceLifecyclePlan { - root, - deployments, - horizon, - evaluation_rate: facts.evaluation_rate, - update_rate: facts.update_rate, - expected_reads: facts.reads, - selected_raw_recompute, - selected_window_implementation_id, - summary_total_cost, - window_accuracy_guarantee, - raw_recompute_total_cost: None, }) } @@ -483,7 +754,7 @@ pub fn global_selection_with_summary_maintenance_lifecycles<'a, Id>( continue; }; costs.finalize_target(&group.target); - let plan = plan_summary_maintenance_lifecycles_with_profile( + let plan = enumerate_with_profile( Rc::clone(summary), WorkloadDemand { workload, @@ -496,7 +767,8 @@ pub fn global_selection_with_summary_maintenance_lifecycles<'a, Id>( cost_model, Some(profiles.for_target(&group.target)), Some(&group.target), - )?; + )? + .select_cheapest(); let raw = plan .expected_reads .and_then(|reads| cost_model.raw_query_recompute_total_cost(&group.target, reads)); @@ -529,7 +801,7 @@ pub fn assemble_selected_dag_with_summary_maintenance_lifecycles( selection .assemble_selected_dag(target)? .map(|root| { - let mut plan = plan_summary_maintenance_lifecycles_with_profile( + let mut plan = enumerate_with_profile( root, demand, now_ms, @@ -538,7 +810,8 @@ pub fn assemble_selected_dag_with_summary_maintenance_lifecycles( cost_model, None, Some(target), - )?; + )? + .select_cheapest(); plan.raw_recompute_total_cost = plan .expected_reads .and_then(|reads| cost_model.raw_query_recompute_total_cost(target, reads)); @@ -993,20 +1266,39 @@ fn rejected( } } -fn collect_summary_aggs( +#[derive(Clone, Copy, PartialEq)] +enum StateKind { + SummaryAgg, + Population, +} + +/// Collect every unique node of `kind` reachable from `node`. +fn collect_states( node: &Rc, seen: &mut HashSet<*const SummaryNode>, output: &mut Vec>, + kind: StateKind, ) { if !seen.insert(Rc::as_ptr(node)) { return; } match &node.expr { SummaryExpr::SummaryAgg { child, .. } => { - output.push(Rc::clone(node)); - collect_summary_aggs(child, seen, output); + if kind == StateKind::SummaryAgg { + output.push(Rc::clone(node)); + } + collect_states(child, seen, output, kind); + } + SummaryExpr::ValueOperation { + child, operation, .. + } => { + if kind == StateKind::Population + && matches!(operation, ValueOperation::MaintainPopulation { .. }) + { + output.push(Rc::clone(node)); + } + collect_states(child, seen, output, kind) } - SummaryExpr::ValueOperation { child, .. } => collect_summary_aggs(child, seen, output), SummaryExpr::SummaryJoin { outer, inner, .. } | SummaryExpr::RelationalJoin { left: outer, @@ -1022,22 +1314,50 @@ fn collect_summary_aggs( left: outer, right: inner, } => { - collect_summary_aggs(outer, seen, output); - collect_summary_aggs(inner, seen, output); + collect_states(outer, seen, output, kind); + collect_states(inner, seen, output, kind); } SummaryExpr::SummaryDelete { summary_input, .. } | SummaryExpr::SummaryEstimate { summary_input, .. } => { - collect_summary_aggs(summary_input, seen, output) + collect_states(summary_input, seen, output, kind) } SummaryExpr::SummaryMerge { children, .. } => { for child in children { - collect_summary_aggs(child, seen, output); + collect_states(child, seen, output, kind); } } SummaryExpr::KeepPreAsap(_) => {} } } +/// Maintained populations that are not an input of any `SummaryAgg`. A +/// population feeding summary state is on that state's maintenance path, so +/// that state's lifecycle times it, even when a readout also reads it directly. +fn standalone_populations(root: &Rc) -> Vec> { + let mut summaries = Vec::new(); + collect_states( + root, + &mut HashSet::new(), + &mut summaries, + StateKind::SummaryAgg, + ); + let mut nested = Vec::new(); + let mut seen = HashSet::new(); + for summary in &summaries { + collect_states(summary, &mut seen, &mut nested, StateKind::Population); + } + let nested: HashSet<_> = nested.iter().map(Rc::as_ptr).collect(); + let mut populations = Vec::new(); + collect_states( + root, + &mut HashSet::new(), + &mut populations, + StateKind::Population, + ); + populations.retain(|population| !nested.contains(&Rc::as_ptr(population))); + populations +} + pub(crate) fn evaluation_schedule( lifecycle: &SummaryMaintenanceLifecycle, arrival: DataArrival, @@ -1060,7 +1380,7 @@ pub(crate) fn evaluation_schedule( } /// Summary states composed on one maintenance path must be produced on the -/// same schedule. Return a component id for each collected `SummaryAgg`. +/// same schedule. Return a component id for each collected state. fn summary_state_components(summaries: &[Rc]) -> Vec { let indices: HashMap<_, _> = summaries .iter() @@ -1091,7 +1411,12 @@ fn summary_state_components(summaries: &[Rc]) -> Vec { continue; } let mut descendants = Vec::new(); - collect_summary_aggs(child, &mut HashSet::new(), &mut descendants); + collect_states( + child, + &mut HashSet::new(), + &mut descendants, + StateKind::SummaryAgg, + ); for descendant in descendants { let child_index = indices[&Rc::as_ptr(&descendant)]; let parent_root = find(&mut parents, parent_index); @@ -1104,6 +1429,99 @@ fn summary_state_components(summaries: &[Rc]) -> Vec { .collect() } +/// Inputs shared by every complete lifecycle-combination evaluation of one +/// root, whether Planner searches combinations or a caller supplies one. +struct CompleteCostContext<'a> { + root: &'a SummaryNode, + components: &'a [usize], + cost_model: &'a dyn CostModel, + comparison_target: Option<&'a QueryExpr>, + horizon: Option, + expected_reads: Option, + required_accuracy: &'a [AccuracyTarget], +} + +impl CompleteCostContext<'_> { + /// Planner's own admission rule for one alternative. Uncosted alternatives + /// are admitted only when the complete-candidate hook is authoritative. + fn eligible(&self, alternative: &SummaryMaintenanceLifecycleAlternative) -> bool { + alternative.selectable() + || (self + .cost_model + .complete_summary_candidate_estimate_covers_lifecycle_costs() + && alternative.rejection + == Some(SummaryMaintenanceLifecycleRejection::MissingCostEvidence)) + } + + /// `selected` holds one entry per deployment, in deployment order. + fn schedules_compatible( + &self, + selected: &[(usize, SummaryMaintenanceLifecycleGuarantee, Cost)], + ) -> bool { + !selected.iter().enumerate().any(|(left, (_, a, _))| { + selected.iter().enumerate().any(|(right, (_, b, _))| { + self.components[left] == self.components[right] + && a.evaluation_schedule != b.evaluation_schedule + }) + }) + } + + fn estimate( + &self, + deployments: &[SummaryMaintenanceDeployment], + selected: &[(usize, SummaryMaintenanceLifecycleGuarantee, Cost)], + ) -> Option { + if !self.schedules_compatible(selected) { + return None; + } + let costed: Vec<_> = selected + .iter() + .map(|(index, guarantee, cost)| CostedSummaryDeployment { + summary: &deployments[*index].summary, + guarantee, + selected_cost: *cost, + }) + .collect(); + let estimate = self.cost_model.complete_summary_candidate_estimate( + self.root, + self.comparison_target, + &costed, + self.horizon, + self.expected_reads, + self.required_accuracy, + )?; + (estimate.window_frameworks.len() == deployments.len()).then_some(estimate) + } +} + +fn lifecycle_guarantee( + lifecycle: &SummaryMaintenanceLifecycle, + arrival: DataArrival, +) -> SummaryMaintenanceLifecycleGuarantee { + SummaryMaintenanceLifecycleGuarantee { + summary_maintenance_mode: maintenance_mode(lifecycle, arrival), + evaluation_schedule: evaluation_schedule(lifecycle, arrival), + summary_maintenance_lifecycle: lifecycle.clone(), + output_representation: OutputRepresentation::SummaryState, + } +} + +fn apply_selection( + deployments: &mut [SummaryMaintenanceDeployment], + guarantees: Vec<(usize, SummaryMaintenanceLifecycleGuarantee)>, + estimate: &CompleteSummaryCandidateEstimate, +) { + for (index, guarantee) in guarantees { + deployments[index].summary_maintenance_lifecycle_guarantee = Some(guarantee); + } + for (deployment, framework) in deployments + .iter_mut() + .zip(estimate.window_frameworks.iter().cloned()) + { + deployment.selected_window_framework = framework; + } +} + #[expect(clippy::too_many_arguments, reason = "complete combination context")] fn select_complete_lifecycle_combination( root: &SummaryNode, @@ -1120,82 +1538,48 @@ fn select_complete_lifecycle_combination( if deployments.is_empty() { return None; } + let context = CompleteCostContext { + root, + components, + cost_model, + comparison_target, + horizon, + expected_reads, + required_accuracy, + }; // The whole-candidate hook is intentionally arbitrary, so partial costs // cannot soundly prune the search. Bound exhaustive enumeration and fail // closed instead of allowing an adversarial DAG to consume exponential // planner time. - let complete_costing = cost_model.complete_summary_candidate_estimate_covers_lifecycle_costs(); - let eligible = |alternative: &SummaryMaintenanceLifecycleAlternative| { - alternative.selectable() - || (complete_costing - && alternative.rejection - == Some(SummaryMaintenanceLifecycleRejection::MissingCostEvidence)) - }; let combinations = deployments .iter() .try_fold(1_usize, |product, deployment| { let selectable = deployment .alternatives .iter() - .filter(|alternative| eligible(alternative)) + .filter(|alternative| context.eligible(alternative)) .count(); product.checked_mul(selectable) })?; if combinations == 0 || combinations > MAX_COMPLETE_LIFECYCLE_COMBINATIONS { return None; } - #[expect( - clippy::too_many_arguments, - reason = "recursive lifecycle-combination search state" - )] + type Best = Option<( + CompleteSummaryCandidateEstimate, + Vec<(usize, SummaryMaintenanceLifecycleGuarantee)>, + )>; fn visit( index: usize, - root: &SummaryNode, + context: &CompleteCostContext<'_>, deployments: &[SummaryMaintenanceDeployment], - components: &[usize], arrival: DataArrival, - cost_model: &dyn CostModel, - comparison_target: Option<&QueryExpr>, - horizon: Option, - expected_reads: Option, - required_accuracy: &[AccuracyTarget], - complete_costing: bool, selected: &mut Vec<(usize, SummaryMaintenanceLifecycleGuarantee, Cost)>, - best: &mut Option<( - CompleteSummaryCandidateEstimate, - Vec<(usize, SummaryMaintenanceLifecycleGuarantee)>, - )>, + best: &mut Best, ) { if index == deployments.len() { - if selected.iter().enumerate().any(|(left, (_, a, _))| { - selected.iter().enumerate().any(|(right, (_, b, _))| { - components[left] == components[right] - && a.evaluation_schedule != b.evaluation_schedule - }) - }) { - return; - } - let costed: Vec<_> = selected - .iter() - .map(|(index, guarantee, cost)| CostedSummaryDeployment { - summary: &deployments[*index].summary, - guarantee, - selected_cost: *cost, - }) - .collect(); - let Some(estimate) = cost_model.complete_summary_candidate_estimate( - root, - comparison_target, - &costed, - horizon, - expected_reads, - required_accuracy, - ) else { + let Some(estimate) = context.estimate(deployments, selected) else { return; }; - if estimate.window_frameworks.len() != deployments.len() { - return; - } if best .as_ref() .is_none_or(|(best_estimate, _)| estimate.cost.0 < best_estimate.cost.0) @@ -1213,40 +1597,14 @@ fn select_complete_lifecycle_combination( for alternative in deployments[index] .alternatives .iter() - .filter(|alternative| { - alternative.selectable() - || (complete_costing - && alternative.rejection - == Some(SummaryMaintenanceLifecycleRejection::MissingCostEvidence)) - }) + .filter(|alternative| context.eligible(alternative)) { - let lifecycle = alternative.summary_maintenance_lifecycle.clone(); - let guarantee = SummaryMaintenanceLifecycleGuarantee { - summary_maintenance_mode: maintenance_mode(&lifecycle, arrival), - evaluation_schedule: evaluation_schedule(&lifecycle, arrival), - summary_maintenance_lifecycle: lifecycle, - output_representation: OutputRepresentation::SummaryState, - }; selected.push(( index, - guarantee, + lifecycle_guarantee(&alternative.summary_maintenance_lifecycle, arrival), alternative.total_cost.unwrap_or(Cost::ZERO), )); - visit( - index + 1, - root, - deployments, - components, - arrival, - cost_model, - comparison_target, - horizon, - expected_reads, - required_accuracy, - complete_costing, - selected, - best, - ); + visit(index + 1, context, deployments, arrival, selected, best); selected.pop(); } } @@ -1254,29 +1612,14 @@ fn select_complete_lifecycle_combination( let mut best = None; visit( 0, - root, + &context, deployments, - components, arrival, - cost_model, - comparison_target, - horizon, - expected_reads, - required_accuracy, - complete_costing, &mut Vec::new(), &mut best, ); let (estimate, guarantees) = best?; - for (index, guarantee) in guarantees { - deployments[index].summary_maintenance_lifecycle_guarantee = Some(guarantee); - } - for (deployment, framework) in deployments - .iter_mut() - .zip(estimate.window_frameworks.iter().cloned()) - { - deployment.selected_window_framework = framework; - } + apply_selection(deployments, guarantees, &estimate); Some(estimate) } @@ -1323,8 +1666,8 @@ mod tests { } use super::*; use asap_types::post_asap::{ - ExactKind, ExactParams, GroupingStrategy, ResultGuarantee, SketchAlgorithm, - SummaryFamilyType, SummaryField, SummarySchema, + ExactKind, ExactParams, GroupingStrategy, PostAsapOperatorPayload, ResultGuarantee, + SketchAlgorithm, SummaryFamilyType, SummaryField, SummarySchema, }; use asap_types::pre_asap::AggIntent; use asap_types::pre_asap::{Column, ColumnRef, DataType, QueryExpr, Reduction, Schema, Source}; @@ -2402,4 +2745,842 @@ mod tests { assert_eq!(batch.evaluation_rate, None); assert_eq!(batch.one_shot_consumers, 1); } + + fn continuous_candidates<'a>( + workload: &QueryWorkload, + data: &DataWorkload, + model: &'a dyn CostModel, + ) -> SummaryMaintenanceLifecycleCandidates<'a> { + enumerate_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new_with_data(workload, data, &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities { + supports_shared: false, + ..SummaryMaintenanceLifecycleCapabilities::ALL + }, + model, + ) + .unwrap() + } + + fn choose( + candidates: &SummaryMaintenanceLifecycleCandidates<'_>, + lifecycle: SummaryMaintenanceLifecycle, + ) -> Vec<(PostAsapNodeId, SummaryMaintenanceLifecycle)> { + candidates + .deployments() + .iter() + .map(|deployment| (deployment.post_asap_node_id, lifecycle.clone())) + .collect() + } + + // Enumeration reports all four lifecycle kinds with their rejections and + // selects nothing. + #[test] + fn enumeration_exposes_every_lifecycle_without_selecting() { + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let candidates = continuous_candidates(&workload, &data, &UnitCosts); + let [deployment] = candidates.deployments() else { + panic!("one summary state"); + }; + assert_eq!(deployment.summary_maintenance_lifecycle_guarantee, None); + assert_eq!(deployment.selected_window_framework, None); + let outcome: Vec<_> = deployment + .alternatives + .iter() + .map(|alternative| { + ( + &alternative.summary_maintenance_lifecycle, + alternative.rejection.clone(), + alternative.total_cost.is_some(), + ) + }) + .collect(); + assert!(matches!( + outcome.as_slice(), + [ + (SummaryMaintenanceLifecycle::Ephemeral, None, true), + ( + SummaryMaintenanceLifecycle::Prepared { .. }, + Some(SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery), + false + ), + ( + SummaryMaintenanceLifecycle::Shared { .. }, + Some(SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime), + false + ), + ( + SummaryMaintenanceLifecycle::ContinuouslyMaintained, + None, + true + ), + ] + )); + let guarantee = candidates.guarantee(&SummaryMaintenanceLifecycle::ContinuouslyMaintained); + assert_eq!( + guarantee.summary_maintenance_mode, + SummaryMaintenanceMode::Incremental + ); + assert_eq!(guarantee.evaluation_schedule, EvaluationSchedule::PerUpdate); + } + + // Explicitly choosing Planner's own selection reproduces Planner's plan. + #[test] + fn explicit_choice_of_planner_selection_reproduces_planner_plan() { + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let planned = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new_with_data(&workload, &data, &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities { + supports_shared: false, + ..SummaryMaintenanceLifecycleCapabilities::ALL + }, + &UnitCosts, + ) + .unwrap(); + let candidates = continuous_candidates(&workload, &data, &UnitCosts); + let choice = choose( + &candidates, + SummaryMaintenanceLifecycle::ContinuouslyMaintained, + ); + let chosen = candidates.select(&choice).unwrap(); + assert_eq!(format!("{chosen:?}"), format!("{planned:?}")); + } + + // A deployment may bind a legal alternative Planner's estimate does not + // prefer; the plan carries that alternative's guarantee and cost. + #[test] + fn explicit_choice_may_bind_a_costlier_legal_alternative() { + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let candidates = continuous_candidates(&workload, &data, &UnitCosts); + let ephemeral_cost = candidates.deployments()[0].alternatives[0].total_cost; + let choice = choose(&candidates, SummaryMaintenanceLifecycle::Ephemeral); + let plan = candidates.select(&choice).unwrap(); + assert_eq!( + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(&SummaryMaintenanceLifecycle::Ephemeral) + ); + assert_eq!(plan.summary_total_cost, ephemeral_cost); + } + + // Choices that Planner could not select, or that do not cover exactly the + // enumerated states, are refused rather than bound. + #[test] + fn explicit_choice_rejects_illegal_or_incomplete_choices() { + use SummaryMaintenanceLifecycleChoiceError as E; + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let select = |model: &dyn CostModel, choice: &dyn Fn(PostAsapNodeId) -> Vec<_>| { + let candidates = continuous_candidates(&workload, &data, model); + let id = candidates.deployments()[0].post_asap_node_id; + (id, candidates.select(&choice(id)).unwrap_err()) + }; + let shared = SummaryMaintenanceLifecycle::Shared { + retention: DurationMs(10_000), + }; + let (id, error) = select(&UnitCosts, &|id| vec![(id, shared.clone())]); + assert_eq!( + error, + E::Rejected { + post_asap_node_id: id, + rejection: Some(SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime), + } + ); + let continuous = SummaryMaintenanceLifecycle::ContinuouslyMaintained; + let (id, error) = select(&crate::cost_model::DefaultCostModel, &|id| { + vec![(id, SummaryMaintenanceLifecycle::Ephemeral)] + }); + assert_eq!( + error, + E::Rejected { + post_asap_node_id: id, + rejection: Some(SummaryMaintenanceLifecycleRejection::MissingCostEvidence), + } + ); + let (id, error) = select(&UnitCosts, &|id| { + vec![( + id, + SummaryMaintenanceLifecycle::Shared { + retention: DurationMs(1), + }, + )] + }); + assert_eq!(error, E::NotAnAlternative(id)); + let (id, error) = select(&UnitCosts, &|_| vec![]); + assert_eq!(error, E::MissingChoice(id)); + let (id, error) = select(&UnitCosts, &|id| { + vec![(id, continuous.clone()), (id, continuous.clone())] + }); + assert_eq!(error, E::DuplicateChoice(id)); + let (_, error) = select(&UnitCosts, &|_| { + vec![(PostAsapNodeId(u32::MAX), continuous.clone())] + }); + assert_eq!(error, E::UnknownSummary(PostAsapNodeId(u32::MAX))); + } + + // Nested states on one maintenance path must share an evaluation schedule. + #[test] + fn explicit_choice_rejects_incompatible_nested_schedules() { + let workload = workload(vec![], vec![repeating()], continuous(1_000, 20_000)); + let candidates = enumerate_summary_maintenance_lifecycles( + nested_summary(), + WorkloadDemand::new_with_data(&workload, &continuous(1_000, 20_000), &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &IncompatibleNestedCosts, + ) + .unwrap(); + let [outer, inner] = candidates.deployments() else { + panic!("two summary states"); + }; + let choice = vec![ + ( + outer.post_asap_node_id, + SummaryMaintenanceLifecycle::Ephemeral, + ), + ( + inner.post_asap_node_id, + SummaryMaintenanceLifecycle::ContinuouslyMaintained, + ), + ]; + assert_eq!( + candidates.select(&choice).unwrap_err(), + SummaryMaintenanceLifecycleChoiceError::IncompatibleEvaluationSchedules + ); + } + + // A multi-summary root yields one candidate entry per unique state, with + // a shared `Rc` state listed once. + #[test] + fn enumeration_lists_each_unique_summary_state_once() { + let shared = summary(); + let root = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryMerge { + timing: asap_types::post_asap::ExecutionTiming::IngestionTime, + children: vec![Rc::clone(&shared), Rc::clone(&shared), summary()], + }, + schema: shared.schema.clone(), + guarantee: None, + }); + let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); + let candidates = enumerate_summary_maintenance_lifecycles( + root, + WorkloadDemand::new_with_data(&workload, &at_rest(), &[0]), + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + let ids: HashSet<_> = candidates + .deployments() + .iter() + .map(|deployment| deployment.post_asap_node_id) + .collect(); + assert_eq!(candidates.deployments().len(), 2); + assert_eq!(ids.len(), 2); + assert!(candidates + .deployments() + .iter() + .any(|deployment| Rc::ptr_eq(&deployment.summary, &shared))); + } + + fn readout(state: &Rc) -> Rc { + Rc::new(SummaryNode { + expr: SummaryExpr::ValueOperation { + child: Rc::clone(state), + operation: ValueOperation::FinalizeExactAccumulator, + timing: ExecutionTiming::QueryTime, + }, + schema: SummarySchema { + fields: vec![SummaryField { + name: "value".into(), + dtype: SummaryFamilyType::Plain(DataType::Float64), + nullable: false, + }], + time_index: None, + }, + guarantee: Some(ResultGuarantee::exact("sum")), + }) + } + + fn lifecycle_matching( + alternatives: &[SummaryMaintenanceLifecycleAlternative], + kind: fn(&SummaryMaintenanceLifecycle) -> bool, + ) -> SummaryMaintenanceLifecycle { + alternatives + .iter() + .map(|alternative| &alternative.summary_maintenance_lifecycle) + .find(|lifecycle| kind(lifecycle)) + .expect("lifecycle kind is an alternative") + .clone() + } + + /// Bind the lifecycle `choose` picks for every state of `root`, then + /// derive the timed DAG. + fn timed_dag( + root: Rc, + workload: &QueryWorkload, + data: &DataWorkload, + horizon: Option, + choose: impl Fn(&SummaryMaintenanceDeployment) -> SummaryMaintenanceLifecycle, + ) -> PostAsapDag { + let candidates = enumerate_summary_maintenance_lifecycles( + root, + WorkloadDemand::new_with_data(workload, data, &[0]), + 1_000, + horizon, + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + let choice: Vec<_> = candidates + .deployments() + .iter() + .map(|deployment| (deployment.post_asap_node_id, choose(deployment))) + .collect(); + let dag = candidates + .select(&choice) + .unwrap() + .execution_timed_dag() + .unwrap(); + dag.validate().unwrap(); + dag + } + + /// Operator kinds in node-id order, each paired with its timing. + fn timings(dag: &PostAsapDag) -> Vec<(&'static str, ExecutionTiming)> { + dag.nodes + .iter() + .map(|node| { + let kind = match node.payload { + PostAsapOperatorPayload::Fallback { .. } => "raw", + PostAsapOperatorPayload::SummaryAgg { .. } => "state", + PostAsapOperatorPayload::Value { .. } => "readout", + PostAsapOperatorPayload::Binary { .. } => "binary", + _ => "other", + }; + (kind, node.output_state.timing) + }) + .collect() + } + + const INGEST: ExecutionTiming = ExecutionTiming::IngestionTime; + const QUERY: ExecutionTiming = ExecutionTiming::QueryTime; + + // Every retained lifecycle kind runs its state and inputs at ingestion + // time and its readout at query time. + #[test] + fn retained_lifecycles_time_state_and_inputs_at_ingestion() { + let mut scheduled = batch(Predictability::Predictable { + known_at: Some(TimestampMs(1_000)), + }); + scheduled.execute_at = Some(TimestampMs(11_000)); + type Case = ( + QueryWorkload, + DataWorkload, + Option, + fn(&SummaryMaintenanceLifecycle) -> bool, + ); + let cases: [Case; 3] = [ + ( + workload(vec![], vec![repeating()], continuous(1_000, 60_000)), + continuous(1_000, 60_000), + Some(Horizon(10.0)), + |lifecycle| { + matches!( + lifecycle, + SummaryMaintenanceLifecycle::ContinuouslyMaintained + ) + }, + ), + ( + workload(vec![], vec![repeating()], at_rest()), + at_rest(), + Some(Horizon(10.0)), + |lifecycle| matches!(lifecycle, SummaryMaintenanceLifecycle::Shared { .. }), + ), + ( + workload(vec![scheduled], vec![], at_rest()), + at_rest(), + None, + |lifecycle| matches!(lifecycle, SummaryMaintenanceLifecycle::Prepared { .. }), + ), + ]; + for (workload, data, horizon, kind) in cases { + let dag = timed_dag( + readout(&summary()), + &workload, + &data, + horizon, + |deployment| lifecycle_matching(&deployment.alternatives, kind), + ); + assert_eq!( + timings(&dag), + [("raw", INGEST), ("state", INGEST), ("readout", QUERY)] + ); + } + } + + // An Ephemeral state, its raw input, and its readout all run at query time. + #[test] + fn ephemeral_lifecycle_times_state_and_downstream_at_query() { + let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); + let dag = timed_dag(readout(&summary()), &workload, &at_rest(), None, |_| { + SummaryMaintenanceLifecycle::Ephemeral + }); + assert_eq!( + timings(&dag), + [("raw", QUERY), ("state", QUERY), ("readout", QUERY)] + ); + } + + // One state read by two consumers is one deployment; its timing follows + // that single choice while both consumers run at query time. + #[test] + fn shared_state_is_timed_once_for_all_consumers() { + let state = summary(); + let lhs = readout(&state); + let rhs = Rc::new(lhs.as_ref().clone()); + let root = Rc::new(SummaryNode { + expr: SummaryExpr::BinaryOp { + lhs, + rhs, + operator: asap_types::post_asap::BinaryOperator { + kind: asap_types::pre_asap::BinaryOpKind::Arithmetic( + asap_types::pre_asap::ArithmeticOpKind::Add, + ), + vector_match: None, + checked_relative_division: false, + checked_finite_division: false, + }, + timing: QUERY, + }, + schema: readout(&state).schema.clone(), + guarantee: None, + }); + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let dag = timed_dag(root, &workload, &data, Some(Horizon(10.0)), |deployment| { + assert!(Rc::ptr_eq(&deployment.summary, &state)); + SummaryMaintenanceLifecycle::ContinuouslyMaintained + }); + assert_eq!( + timings(&dag), + [ + ("raw", INGEST), + ("state", INGEST), + ("readout", QUERY), + ("readout", QUERY), + ("binary", QUERY), + ] + ); + } + + // An Ephemeral state consumed by retained state is built on the retained + // state's ingestion path; it is not retained, but cannot run at query time. + #[test] + fn ephemeral_state_feeding_retained_state_runs_at_ingestion() { + let mut scheduled = batch(Predictability::Predictable { + known_at: Some(TimestampMs(1_000)), + }); + scheduled.execute_at = Some(TimestampMs(11_000)); + let root = nested_summary(); + let workload = workload(vec![scheduled], vec![], at_rest()); + let dag = timed_dag( + Rc::clone(&root), + &workload, + &at_rest(), + None, + |deployment| { + if Rc::ptr_eq(&deployment.summary, &root) { + lifecycle_matching(&deployment.alternatives, |lifecycle| { + matches!(lifecycle, SummaryMaintenanceLifecycle::Prepared { .. }) + }) + } else { + SummaryMaintenanceLifecycle::Ephemeral + } + }, + ); + assert_eq!( + timings(&dag), + [("raw", INGEST), ("state", INGEST), ("state", INGEST)] + ); + } + + // Timing is not derived for a state without a selected lifecycle, and a + // raw-recompute plan runs entirely at query time. + #[test] + fn timing_requires_a_selected_lifecycle_for_every_state() { + let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); + let data = at_rest(); + let demand = WorkloadDemand::new_with_data(&workload, &data, &[0]); + let plan = plan_summary_maintenance_lifecycles( + readout(&summary()), + demand, + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &crate::cost_model::DefaultCostModel, + ) + .unwrap(); + assert_eq!( + plan.execution_timed_dag().unwrap_err(), + SummaryMaintenanceTimingError::UnselectedLifecycle( + plan.deployments[0].post_asap_node_id + ) + ); + let raw = plan_summary_maintenance_lifecycles( + crate::replacement::keep_pre_asap(&sum_query()).unwrap(), + demand, + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + assert_eq!( + timings(&raw.execution_timed_dag().unwrap()), + [("raw", QUERY)] + ); + } + + /// A strategy-built `sum(a)` over one maintained current-series population. + fn population_readout() -> Rc { + let target = Rc::new(crate::test_support::lower_promql( + "sum(a)", + AccuracyTarget::Exact, + )); + crate::maintained_population::MaintainedPopulationStrategy::new(std::slice::from_ref( + &target, + )) + .candidate(&target) + .unwrap() + } + + fn is_population(node: &SummaryNode) -> bool { + matches!( + node.expr, + SummaryExpr::ValueOperation { + operation: ValueOperation::MaintainPopulation { .. }, + .. + } + ) + } + + fn population_timings(dag: &PostAsapDag) -> Vec<(&'static str, ExecutionTiming)> { + dag.nodes + .iter() + .zip(timings(dag)) + .map(|(node, (kind, timing))| match node.payload { + PostAsapOperatorPayload::Value { + operation: ValueOperation::MaintainPopulation { .. }, + } => ("population", timing), + _ => (kind, timing), + }) + .collect() + } + + // A maintained population is enumerated as retained state, with costs + // from the caller's model for both the maintained and the rebuilt choice. + #[test] + fn enumeration_includes_maintained_population() { + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let candidates = enumerate_summary_maintenance_lifecycles( + population_readout(), + WorkloadDemand::new_with_data(&workload, &data, &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + let [deployment] = candidates.deployments() else { + panic!("one population state"); + }; + assert!(is_population(&deployment.summary)); + let cost = |lifecycle: SummaryMaintenanceLifecycle| { + deployment + .alternatives + .iter() + .find(|alternative| alternative.summary_maintenance_lifecycle == lifecycle) + .and_then(|alternative| alternative.total_cost) + }; + // Ephemeral: (build 10 + read 1 + retire 1) x 10 reads. Maintained over + // 10 s at 1 update/s: build 10 + updates 10 + reads 10 + retention 1 + retire 1. + assert_eq!( + cost(SummaryMaintenanceLifecycle::Ephemeral), + Some(Cost(120.0)) + ); + assert_eq!( + cost(SummaryMaintenanceLifecycle::ContinuouslyMaintained), + Some(Cost(32.0)) + ); + } + + // Without cost evidence a population's alternatives stay unknown: Planner + // selects none and timing is refused rather than guessed. + #[test] + fn population_without_cost_evidence_stays_unselected() { + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let plan = plan_summary_maintenance_lifecycles( + population_readout(), + WorkloadDemand::new_with_data(&workload, &data, &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &crate::cost_model::DefaultCostModel, + ) + .unwrap(); + let [deployment] = plan.deployments.as_slice() else { + panic!("one population state"); + }; + assert!(deployment + .alternatives + .iter() + .all(|alternative| alternative.total_cost.is_none())); + assert!(deployment.summary_maintenance_lifecycle_guarantee.is_none()); + assert_eq!( + plan.execution_timed_dag().unwrap_err(), + SummaryMaintenanceTimingError::UnselectedLifecycle(deployment.post_asap_node_id) + ); + } + + // A retained population and its raw input run at ingestion time; an + // Ephemeral population is rebuilt from raw input at query time. + #[test] + fn population_lifecycle_choice_decides_its_timing() { + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let timed = |lifecycle: SummaryMaintenanceLifecycle| { + population_timings(&timed_dag( + population_readout(), + &workload, + &data, + Some(Horizon(10.0)), + |_| lifecycle.clone(), + )) + }; + assert_eq!( + timed(SummaryMaintenanceLifecycle::ContinuouslyMaintained), + [("raw", INGEST), ("population", INGEST), ("readout", QUERY)] + ); + assert_eq!( + timed(SummaryMaintenanceLifecycle::Ephemeral), + [("raw", QUERY), ("population", QUERY), ("readout", QUERY)] + ); + } + + // When retaining is cheaper, Planner's own selection keeps the population + // maintained at ingestion time, as realization strategies placed it before + // population timing became a lifecycle decision. + #[test] + fn planner_selection_retains_population_at_ingestion() { + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let plan = plan_summary_maintenance_lifecycles( + population_readout(), + WorkloadDemand::new_with_data(&workload, &data, &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + // Shared and ContinuouslyMaintained tie at 32; the first wins. + assert!(matches!( + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(SummaryMaintenanceLifecycle::Shared { .. }) + )); + assert_eq!( + population_timings(&plan.execution_timed_dag().unwrap()), + [("raw", INGEST), ("population", INGEST), ("readout", QUERY)] + ); + } + + // A population feeding summary state is that state's input, not a separate + // deployment: the state's lifecycle times it. + #[test] + fn population_feeding_summary_state_follows_that_state() { + let SummaryExpr::ValueOperation { + child: population, .. + } = &population_readout().expr + else { + unreachable!() + }; + let state = summary(); + let SummaryExpr::SummaryAgg { + family, + input, + reduction, + grouping, + .. + } = &state.expr + else { + unreachable!() + }; + let state = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child: Rc::clone(population), + family: family.clone(), + input: input.clone(), + reduction: reduction.clone(), + grouping: grouping.clone(), + }, + ..state.as_ref().clone() + }); + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let timed = |lifecycle: SummaryMaintenanceLifecycle| { + population_timings(&timed_dag( + readout(&state), + &workload, + &data, + Some(Horizon(10.0)), + |deployment| { + assert!(Rc::ptr_eq(&deployment.summary, &state)); + lifecycle.clone() + }, + )) + }; + assert_eq!( + timed(SummaryMaintenanceLifecycle::ContinuouslyMaintained), + [ + ("raw", INGEST), + ("population", INGEST), + ("state", INGEST), + ("readout", QUERY) + ] + ); + assert_eq!( + timed(SummaryMaintenanceLifecycle::Ephemeral), + [ + ("raw", QUERY), + ("population", QUERY), + ("state", QUERY), + ("readout", QUERY) + ] + ); + } + + // A population both read directly and consumed by summary state is that + // state's input in either traversal order: not a separate deployment, and + // timed by the state's lifecycle. + #[test] + fn shared_population_follows_its_summary_consumer() { + let direct = population_readout(); + let SummaryExpr::ValueOperation { + child: population, .. + } = &direct.expr + else { + unreachable!() + }; + let state = summary(); + let SummaryExpr::SummaryAgg { + family, + input, + reduction, + grouping, + .. + } = &state.expr + else { + unreachable!() + }; + let state = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child: Rc::clone(population), + family: family.clone(), + input: input.clone(), + reduction: reduction.clone(), + grouping: grouping.clone(), + }, + ..state.as_ref().clone() + }); + let binary = |lhs: Rc, rhs: Rc| { + Rc::new(SummaryNode { + schema: lhs.schema.clone(), + expr: SummaryExpr::BinaryOp { + lhs, + rhs, + operator: asap_types::post_asap::BinaryOperator { + kind: asap_types::pre_asap::BinaryOpKind::Arithmetic( + asap_types::pre_asap::ArithmeticOpKind::Add, + ), + vector_match: None, + checked_relative_division: false, + checked_finite_division: false, + }, + timing: QUERY, + }, + guarantee: None, + }) + }; + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + for root in [ + binary(Rc::clone(&direct), readout(&state)), + binary(readout(&state), Rc::clone(&direct)), + ] { + for lifecycle in [ + SummaryMaintenanceLifecycle::ContinuouslyMaintained, + SummaryMaintenanceLifecycle::Ephemeral, + ] { + let dag = timed_dag( + Rc::clone(&root), + &workload, + &data, + Some(Horizon(10.0)), + |deployment| { + assert!(Rc::ptr_eq(&deployment.summary, &state)); + lifecycle.clone() + }, + ); + let expected = if lifecycle == SummaryMaintenanceLifecycle::Ephemeral { + QUERY + } else { + INGEST + }; + for (kind, timing) in population_timings(&dag) { + if matches!(kind, "raw" | "population" | "state") { + assert_eq!(timing, expected, "{kind}"); + } else { + assert_eq!(timing, QUERY, "{kind}"); + } + } + } + } + } + + // A plan whose population deployment was removed after enumeration is + // refused rather than timed by a guess. + #[test] + fn timing_refuses_population_without_deployment() { + let data = continuous(1_000, 60_000); + let workload = workload(vec![], vec![repeating()], data.clone()); + let mut plan = plan_summary_maintenance_lifecycles( + population_readout(), + WorkloadDemand::new_with_data(&workload, &data, &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + let id = plan.deployments.remove(0).post_asap_node_id; + assert_eq!( + plan.execution_timed_dag(), + Err(SummaryMaintenanceTimingError::UnplannedMaintainedState(id)) + ); + } } diff --git a/crates/asap-physical-operators/src/physical_planner/candidates.rs b/crates/asap-physical-operators/src/physical_planner/candidates.rs index a8f476f3..f5658a9a 100644 --- a/crates/asap-physical-operators/src/physical_planner/candidates.rs +++ b/crates/asap-physical-operators/src/physical_planner/candidates.rs @@ -25,24 +25,42 @@ pub fn compile_candidate( inputs: BTreeMap, roots: &[NodeId], frontier: &[NodeId], +) -> Result { + cut_candidate(&compile(dag, inputs, roots)?, frontier) +} + +/// Derive one frontier's candidate from a complete [`compile`] result by +/// partitioning its operators; nothing is lowered again. A deployment compiles +/// each query DAG once and derives every placement choice from that result. +/// The candidate is identical to [`compile_candidate`] for the same frontier. +pub fn cut_candidate( + compiled: &CompiledPhysicalDag, + frontier: &[NodeId], ) -> Result { if frontier.is_empty() { return Ok(PhysicalCandidate { precompute: None, - query: compile(dag, inputs, roots)?, + query: compiled.clone(), materialized_outputs: BTreeMap::new(), }); } let frontier_set: BTreeSet<_> = frontier.iter().copied().collect(); - if frontier_set.len() != frontier.len() || frontier.iter().any(|id| inputs.contains_key(id)) { + // `compile` retains only reachable nodes and numbers its helper operators + // above the u32 Planner ID range; only Planner outputs are boundaries. + if frontier_set.len() != frontier.len() + || frontier + .iter() + .any(|&id| !compiled.is_operator(id) || u32::try_from(id).is_err()) + { return Err(invalid("frontier must contain distinct computed outputs")); } - let full = compile(dag, inputs.clone(), roots)?; - let precompute = compile(dag, inputs.clone(), frontier)?; + let inputs: BTreeMap<_, _> = compiled + .input_contracts() + .map(|(id, contract)| (id, contract.clone())) + .collect(); + let precompute = compiled.cut(&inputs, frontier)?; let mut materialized_outputs = BTreeMap::new(); for &id in frontier { - // Also proves that the frontier is reachable from the requested roots. - full.output_contract(id)?; let mut output = precompute.output_contract(id)?; if output.properties.boundedness != Boundedness::Bounded { return Err(invalid("materialized output requires bounded execution")); @@ -54,7 +72,7 @@ pub fn compile_candidate( } let mut query_inputs = inputs; query_inputs.extend(materialized_outputs.clone()); - let query = compile(dag, query_inputs, roots)?; + let query = compiled.cut(&query_inputs, compiled.roots())?; let used: BTreeSet<_> = query.input_contracts().map(|(id, _)| id).collect(); if !frontier.iter().all(|id| used.contains(id)) { return Err(invalid( @@ -68,6 +86,41 @@ pub fn compile_candidate( }) } +/// Materialization frontier implied by lifecycle-assigned timing: ingestion-time +/// nodes read by a query-time node, plus the root when it is ingestion-timed. +/// `cut_candidate` of one [`compile`] result with this frontier realizes the +/// assignment, so different assignments are different cuts of one lowering. +/// That holds while timing-dependent lowering (an ingestion-time `Binary` +/// aligns by value column) has the same timing at compile time as here. +/// A query-time node feeding an ingestion-time node has no valid placement. +pub fn frontier_from_timing(dag: &PostAsapDag) -> Result, Error> { + use planner_types::post_asap::ExecutionTiming::IngestionTime; + let timing = dag + .nodes + .iter() + .map(|node| (node.id, node.output_state.timing)) + .collect::>(); + let mut frontier = BTreeSet::new(); + if timing.get(&dag.root) == Some(&IngestionTime) { + frontier.insert(u64::from(dag.root.0)); + } + for edge in &dag.edges { + let (Some(&producer), Some(&consumer)) = + (timing.get(&edge.producer), timing.get(&edge.consumer)) + else { + return Err(invalid("timed DAG edge names an unknown node")); + }; + match (producer == IngestionTime, consumer == IngestionTime) { + (true, false) => { + frontier.insert(u64::from(edge.producer.0)); + } + (false, true) => return Err(invalid("query-time node feeds an ingestion-time node")), + _ => {} + } + } + Ok(frontier.into_iter().collect()) +} + /// Enumerate bounded, reachable materialization frontiers above explicit inputs. /// Each frontier is an antichain: storing an output and its ancestor together /// would leave the ancestor unused by query execution. Lifecycle eligibility @@ -78,43 +131,38 @@ pub fn enumerate_frontiers( inputs: &BTreeMap, roots: &[NodeId], max_candidates: usize, +) -> Result>, Error> { + enumerate_compiled_frontiers(&compile(dag, inputs.clone(), roots)?, max_candidates) +} + +fn enumerate_compiled_frontiers( + compiled: &CompiledPhysicalDag, + max_candidates: usize, ) -> Result>, Error> { if max_candidates == 0 { return Err(invalid( "frontier search requires a positive candidate budget", )); } - let compiled = compile(dag, inputs.clone(), roots)?; let mut ancestors = BTreeMap::>::new(); let mut eligible = Vec::new(); - for node in &dag.nodes { - let id = u64::from(node.id.0); - if inputs.contains_key(&id) { - continue; - } - let Ok(contract) = compiled.output_contract(id) else { - continue; - }; - if contract.properties.boundedness != Boundedness::Bounded { + for (id, properties) in compiled.output_properties()? { + if !compiled.is_operator(id) + || u32::try_from(id).is_err() + || properties.boundedness != Boundedness::Bounded + { continue; } let mut seen = BTreeSet::new(); let mut pending = vec![id]; while let Some(current) = pending.pop() { - if !seen.insert(current) || inputs.contains_key(¤t) { - continue; + if seen.insert(current) { + pending.extend(compiled.dependencies(current)); } - pending.extend( - dag.edges - .iter() - .filter(|edge| u64::from(edge.consumer.0) == current) - .map(|edge| u64::from(edge.producer.0)), - ); } ancestors.insert(id, seen); eligible.push(id); } - eligible.sort_unstable(); let mut frontiers = vec![vec![]]; for id in eligible { let additions = frontiers @@ -142,16 +190,20 @@ pub fn enumerate_frontiers( /// Lower every maintenance candidate before feasibility/cost evaluation. Keep /// individual failures visible; do not substitute another computation on error. +/// The DAG is lowered once; each frontier is a [`cut_candidate`] of it. pub fn compile_candidates( dag: &PostAsapDag, inputs: BTreeMap, roots: &[NodeId], frontiers: &[Vec], ) -> Vec> { - frontiers - .iter() - .map(|frontier| compile_candidate(dag, inputs.clone(), roots, frontier)) - .collect() + match compile(dag, inputs, roots) { + Ok(compiled) => frontiers + .iter() + .map(|frontier| cut_candidate(&compiled, frontier)) + .collect(), + Err(error) => frontiers.iter().map(|_| Err(error.clone())).collect(), + } } /// Complete workload cost supplied by scoped optimizer/deployment evidence. @@ -273,3 +325,159 @@ impl PhysicalCandidate { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::workload::*; + + fn grouped_rate() -> (PostAsapDag, BTreeMap, NodeId) { + let workload = PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![BatchEntry { + query: Query("sum by(job)(rate(m[1m]))".into()), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit( + planner_types::types::AccuracyTarget::Exact, + ), + ..Default::default() + }, + predictability: Predictability::Unknown, + invocations: 1, + execute_at: None, + time_selection: TimeSelection::default(), + }]), + repeating_queries: None, + }, + data_workload: Some(DataWorkload { + data_ingestion_interval: Evidence { + value: Some(DurationMs(1000)), + ..Default::default() + }, + ..Default::default() + }), + }; + let root = asap_frontend_promql::lower_promql_workload(&workload, 0) + .unwrap() + .remove(0); + let root = std::rc::Rc::new(promql_rows::with_series_identity(&root).unwrap()); + let space = asap_aware_mapping::search_workload(vec![("q", root)]); + let selected = space + .global_selection(&asap_aware_mapping::cost_model::DefaultCostModel) + .assemble_selected_dag(&space.roots[0].1) + .unwrap() + .unwrap(); + let dag = planner_types::post_asap::compile_post_asap_dag(&selected).unwrap(); + let state = dag + .nodes + .iter() + .find(|node| matches!(node.payload, Payload::SummaryAgg { .. })) + .unwrap(); + let inputs = BTreeMap::from([( + u64::from(state.id.0), + InputContract::bounded(Arc::new(state.output_schema.clone())), + )]); + (dag.clone(), inputs, u64::from(dag.root.0)) + } + + /// Enumerating and cutting every frontier lowers each Planner node once. + #[test] + fn candidates_for_all_frontiers_share_one_lowering() { + let (dag, inputs, root) = grouped_rate(); + let lowered = || crate::physical_planner::LOWERED_NODES.with(|count| count.get()); + let before = lowered(); + let compiled = compile(&dag, inputs, &[root]).unwrap(); + let once = lowered() - before; + let frontiers = enumerate_compiled_frontiers(&compiled, 4096).unwrap(); + assert!(frontiers.len() >= 3, "{frontiers:?}"); + for frontier in &frontiers { + cut_candidate(&compiled, frontier).unwrap(); + } + assert!(once > 0); + assert_eq!(lowered() - before, once); + } + + fn with_timing( + dag: &PostAsapDag, + timing: impl Fn(&PostAsapDagNode) -> planner_types::post_asap::ExecutionTiming, + ) -> PostAsapDag { + let mut timed = dag.clone(); + for node in &mut timed.nodes { + node.output_state.timing = timing(node); + } + for edge in &mut timed.edges { + let producer = timed.nodes.iter().find(|node| node.id == edge.producer); + edge.data_state = producer.unwrap().output_state; + } + timed + } + + fn raw_input(dag: &PostAsapDag) -> BTreeMap { + let raw = dag + .nodes + .iter() + .find(|node| matches!(node.payload, Payload::Fallback { .. })) + .unwrap(); + BTreeMap::from([( + u64::from(raw.id.0), + InputContract::bounded(Arc::new(raw.output_schema.clone())), + )]) + } + + /// Cutting one compilation by a retained-state timing and by the all + /// query-time timing (what ContinuouslyMaintained and Ephemeral assign) + /// lowers each Planner node once and matches `compile_candidate`. + #[test] + fn timing_cuts_share_one_lowering() { + use planner_types::post_asap::ExecutionTiming::QueryTime; + let (retained, _, root) = grouped_rate(); + let ephemeral = with_timing(&retained, |_| QueryTime); + let inputs = raw_input(&retained); + let lowered = || crate::physical_planner::LOWERED_NODES.with(|count| count.get()); + let before = lowered(); + let compiled = compile(&ephemeral, inputs.clone(), &[root]).unwrap(); + let once = lowered() - before; + let cuts = [&retained, &ephemeral].map(|timed| { + let frontier = frontier_from_timing(timed).unwrap(); + let cut = cut_candidate(&compiled, &frontier).unwrap(); + (timed, frontier, cut) + }); + assert!(once > 0); + assert_eq!(lowered() - before, once); + assert_eq!(cuts[0].1.len(), 1); + assert!(cuts[1].1.is_empty()); + for (timed, frontier, cut) in cuts { + let expected = compile_candidate(timed, inputs.clone(), &[root], &frontier).unwrap(); + assert_eq!( + serde_json::to_vec(&cut).unwrap(), + serde_json::to_vec(&expected).unwrap() + ); + } + } + + /// The frontier is the ingestion-time nodes read at query time; an + /// ingestion-time root is itself the frontier. + #[test] + fn frontier_from_timing_includes_ingestion_root() { + use planner_types::post_asap::ExecutionTiming::IngestionTime; + let (dag, _, root) = grouped_rate(); + let timed = with_timing(&dag, |_| IngestionTime); + assert_eq!(frontier_from_timing(&timed).unwrap(), [root]); + } + + /// A query-time node feeding an ingestion-time node is rejected. + #[test] + fn frontier_from_timing_rejects_query_time_input_to_ingestion() { + use planner_types::post_asap::ExecutionTiming::{IngestionTime, QueryTime}; + let (dag, _, _) = grouped_rate(); + let timed = with_timing(&dag, |node| { + if node.id == dag.root { + IngestionTime + } else { + QueryTime + } + }); + assert!(frontier_from_timing(&timed).is_err()); + } +} diff --git a/crates/asap-physical-operators/src/physical_planner/compiled.rs b/crates/asap-physical-operators/src/physical_planner/compiled.rs index 70d6a9ff..af0bbe39 100644 --- a/crates/asap-physical-operators/src/physical_planner/compiled.rs +++ b/crates/asap-physical-operators/src/physical_planner/compiled.rs @@ -212,13 +212,8 @@ impl CompiledPhysicalDag { } /// Derive a reachable output contract without opening deployment readers. pub fn output_contract(&self, id: NodeId) -> Result { - let sources = self - .input_contracts() - .map(|(id, contract)| (id, Box::new(contract.clone()) as Source<'_>)) - .collect(); - let graph = self.instantiate(sources)?; - let properties = graph.properties(&self.roots)?; - let properties = *properties + let properties = *self + .output_properties()? .get(&id) .ok_or_else(|| invalid("output is not reachable"))?; let schema = match self @@ -231,6 +226,53 @@ impl CompiledPhysicalDag { }; Ok(InputContract { schema, properties }) } + /// Properties of every reachable node, derived in one contract-only pass. + pub(super) fn output_properties(&self) -> Result, Error> { + let sources = self + .input_contracts() + .map(|(id, contract)| (id, Box::new(contract.clone()) as Source<'_>)) + .collect(); + self.instantiate(sources)?.properties(&self.roots) + } + /// Direct physical dependencies; empty for inputs and unknown IDs. + pub(super) fn dependencies(&self, id: NodeId) -> &[NodeId] { + match self.nodes.get(&id) { + Some(Node::Operator { inputs, .. }) => inputs, + _ => &[], + } + } + pub(super) fn is_operator(&self, id: NodeId) -> bool { + matches!(self.nodes.get(&id), Some(Node::Operator { .. })) + } + /// Keep the already-lowered operators reachable from `roots`, replacing + /// each node in `boundaries` by a typed input. Nothing is lowered again. + pub(super) fn cut( + &self, + boundaries: &BTreeMap, + roots: &[NodeId], + ) -> Result { + let mut result = Self::new(roots.to_vec()); + let mut pending = roots.to_vec(); + while let Some(id) = pending.pop() { + if result.nodes.contains_key(&id) { + continue; + } + let node = match boundaries.get(&id) { + Some(contract) => Node::Input(contract.clone()), + None => self + .nodes + .get(&id) + .cloned() + .ok_or_else(|| invalid(format!("missing physical node {id}")))?, + }; + if let Node::Operator { inputs, .. } = &node { + pending.extend(inputs); + } + result.nodes.insert(id, node); + } + result.validate()?; + Ok(result) + } /// Validate using contract-only sources. No deployment reader is available. pub fn validate(&self) -> Result<(), Error> { let sources = self diff --git a/crates/asap-physical-operators/src/physical_planner/mod.rs b/crates/asap-physical-operators/src/physical_planner/mod.rs index 112e1715..fede3777 100644 --- a/crates/asap-physical-operators/src/physical_planner/mod.rs +++ b/crates/asap-physical-operators/src/physical_planner/mod.rs @@ -36,8 +36,8 @@ pub mod promql_values; mod candidates; pub use candidates::{ - compile_candidate, compile_candidates, enumerate_frontiers, select_candidate, CandidateCost, - CandidateSelection, PhysicalCandidate, + compile_candidate, compile_candidates, cut_candidate, enumerate_frontiers, + frontier_from_timing, select_candidate, CandidateCost, CandidateSelection, PhysicalCandidate, }; mod compiled; @@ -103,6 +103,21 @@ pub fn bind_with_data_sources<'a>( bind(dag, sources, roots) } +#[cfg(test)] +thread_local! { + /// Planner nodes lowered by this thread, for compile-once tests. + static LOWERED_NODES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Helper operators are numbered from their Planner node alone, above the u32 +/// Planner ID range, so every boundary choice yields a subgraph of the same +/// lowering and candidate cuts need not renumber operators. A node lowering to +/// several helpers takes consecutive indices below its base. +fn helper_id(node: NodeId, index: u64) -> NodeId { + debug_assert!(node <= u64::from(u32::MAX) && index < 1 << 16); + u64::MAX - (node << 16) - index +} + fn compile_internal( dag: &PostAsapDag, mut sources: BTreeMap, @@ -159,9 +174,9 @@ fn compile_internal( } } let mut graph = CompiledPhysicalDag::new(roots.to_vec()); - let mut auxiliary = u64::MAX; for id in ordered { let node = nodes[&id]; + let auxiliary = helper_id(id, 0); let output = Arc::new(node.output_schema.clone()); crate::values::validate_schema(&output)?; if let Some(source) = sources.remove(&id) { @@ -170,6 +185,8 @@ fn compile_internal( } graph.add_input(id, source)?; } else { + #[cfg(test)] + LOWERED_NODES.with(|count| count.set(count.get() + 1)); let mut inputs = dependencies.get(&id).cloned().unwrap_or_default(); let mut schemas = inputs .iter() @@ -185,7 +202,6 @@ fn compile_internal( Operator::union(schemas[0].clone(), schemas.len())?, )?; inputs = vec![auxiliary]; - auxiliary -= 1; schemas.truncate(1); } if let Payload::Value { @@ -280,7 +296,6 @@ fn compile_internal( vec![auxiliary], Operator::limit(input, *k as u64, 0, groups)?.with_output_schema(output)?, )?; - auxiliary -= 1; continue; } // A closed row must include either all source labels or the explicit @@ -340,7 +355,6 @@ fn compile_internal( vec![auxiliary], Operator::scope_timestamp(compact, output)?, )?; - auxiliary -= 1; continue; } let mut operator = compile_node(node, &schemas) diff --git a/crates/asap-physical-operators/src/physical_planner/promql_rows.rs b/crates/asap-physical-operators/src/physical_planner/promql_rows.rs index 8b887580..d2c13328 100644 --- a/crates/asap-physical-operators/src/physical_planner/promql_rows.rs +++ b/crates/asap-physical-operators/src/physical_planner/promql_rows.rs @@ -249,16 +249,13 @@ pub fn compile_rate_ranking( Ok((source, program)) } -/// The selected logical placement requires fresh aggregate state per closed window. -/// Compile both physical graphs before deployment chooses storage or scheduling. -/// The input is the complete collection of per-series exact counter states. +/// Compile a lifecycle-timed DAG whose heap or grouped Sum over per-series +/// Rate readouts runs at ingestion time: fresh aggregate state per closed +/// window. The input is the complete collection of per-series counter states. pub fn compile_fixed_window_rate_aggregation( - selected: &Rc, + dag: &planner_types::post_asap::PostAsapDag, ) -> Result { - use planner_types::post_asap::{ - compile_post_asap_dag, ExactKind, ExecutionTiming, SketchAlgorithm, - }; - let dag = compile_post_asap_dag(selected).map_err(|e| invalid(e.to_string()))?; + use planner_types::post_asap::{ExactKind, ExecutionTiming, SketchAlgorithm}; let sources = dag .nodes .iter() @@ -310,7 +307,7 @@ pub fn compile_fixed_window_rate_aggregation( )); } compile_candidate( - &dag, + dag, BTreeMap::from([( u64::from(source.id.0), InputContract::bounded(Arc::new(source.output_schema.clone())), diff --git a/crates/asap-physical-operators/tests/precompute_candidates.rs b/crates/asap-physical-operators/tests/precompute_candidates.rs index c00b388f..f8050118 100644 --- a/crates/asap-physical-operators/tests/precompute_candidates.rs +++ b/crates/asap-physical-operators/tests/precompute_candidates.rs @@ -4,7 +4,8 @@ use asap_physical_operators::{ factory::create_planner_accumulator, operators::Operator, physical_planner::{ - compile_candidates, select_candidate, CandidateCost, CompiledPhysicalDag, InputContract, + compile, compile_candidate, compile_candidates, cut_candidate, enumerate_frontiers, + select_candidate, CandidateCost, CompiledPhysicalDag, InputContract, PhysicalCandidate, Source, }, runtime::{Limits, RunContext, Scope}, @@ -55,7 +56,7 @@ fn grouped_rate() -> PostAsapDag { let space = grouped_rate_space(); let selected = space .global_selection(&DefaultCostModel) - .assemble_selected_dag(&space.roots[0].1) + .assemble_selected_query(&space.roots[0].1) .unwrap() .unwrap(); compile_post_asap_dag(&selected).unwrap() @@ -107,7 +108,10 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { PostAsapOperatorPayload::Value { operation: ValueOperation::FinalizeExactAccumulator } - ) + ) && dag + .edges + .iter() + .any(|edge| edge.producer == state.id && edge.consumer == node.id) }) .unwrap(); let input_schema = Arc::new(state.output_schema.clone()); @@ -546,3 +550,177 @@ fn enumerated_grouped_rate_candidates_execute_numeric_query_outputs() { "must execute both stored and query-time grouped Rate candidates: {executed}" ); } + +/// The per-frontier lowering used before compile-once cuts: each boundary +/// choice lowers the precompute and query DAGs from the logical DAG again. +fn recompiled_candidate( + dag: &PostAsapDag, + inputs: &BTreeMap, + roots: &[u64], + frontier: &[u64], +) -> Result { + use asap_physical_operators::plan::Emission; + if frontier.is_empty() { + return Ok(PhysicalCandidate { + precompute: None, + query: compile(dag, inputs.clone(), roots)?, + materialized_outputs: BTreeMap::new(), + }); + } + let precompute = compile(dag, inputs.clone(), frontier)?; + let mut materialized_outputs = BTreeMap::new(); + for &id in frontier { + let mut output = precompute.output_contract(id)?; + output.properties.emission = Emission::Unknown; + materialized_outputs.insert(id, output); + } + let mut query_inputs = inputs.clone(); + query_inputs.extend(materialized_outputs.clone()); + Ok(PhysicalCandidate { + precompute: Some(precompute), + query: compile(dag, query_inputs, roots)?, + materialized_outputs, + }) +} + +fn assert_cuts_match_recompilation( + dag: &PostAsapDag, + inputs: BTreeMap, + roots: &[u64], + min_frontiers: usize, +) { + let compiled = compile(dag, inputs.clone(), roots).unwrap(); + let frontiers = enumerate_frontiers(dag, &inputs, roots, 4096).unwrap(); + assert!(frontiers.len() >= min_frontiers, "{frontiers:?}"); + for frontier in &frontiers { + let cut = cut_candidate(&compiled, frontier).unwrap(); + let expected = recompiled_candidate(dag, &inputs, roots, frontier).unwrap(); + assert_eq!( + serde_json::to_vec(&cut).unwrap(), + serde_json::to_vec(&expected).unwrap(), + "{frontier:?}" + ); + } +} + +/// Every enumerated grouped Rate→Sum frontier (query-only, stored Rate, +/// stored Sum) cuts to exactly the candidate that per-frontier lowering builds. +#[test] +fn grouped_rate_cuts_equal_per_frontier_compilation() { + let dag = grouped_rate(); + let state = dag + .nodes + .iter() + .find(|node| matches!(node.payload, PostAsapOperatorPayload::SummaryAgg { .. })) + .unwrap(); + let inputs = BTreeMap::from([( + u64::from(state.id.0), + InputContract::bounded(Arc::new(state.output_schema.clone())), + )]); + assert_cuts_match_recompilation(&dag, inputs, &[u64::from(dag.root.0)], 3); +} + +/// Cuts of a DAG whose nodes lower to helper operators (current-series +/// population read by Sort→Limit) keep the same operator IDs as recompilation. +#[test] +fn population_topk_cuts_equal_per_frontier_compilation() { + let workload = PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![BatchEntry { + query: Query("topk by(job)(1, m)".into()), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit(AccuracyTarget::Exact), + ..Default::default() + }, + predictability: Predictability::Unknown, + invocations: 1, + execute_at: None, + time_selection: TimeSelection::default(), + }]), + repeating_queries: None, + }, + data_workload: Some(DataWorkload { + data_ingestion_interval: Evidence { + value: Some(DurationMs(60_000)), + ..Default::default() + }, + ..Default::default() + }), + }; + let original = asap_frontend_promql::lower_promql_workload(&workload, 0) + .unwrap() + .remove(0); + let root = Rc::new( + asap_physical_operators::physical_planner::promql_rows::with_series_identity(&original) + .unwrap(), + ); + let selected = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( + std::slice::from_ref(&root), + ) + .candidate(&root) + .unwrap(); + let dag = compile_post_asap_dag(&selected).unwrap(); + let raw = dag + .nodes + .iter() + .find(|node| matches!(node.payload, PostAsapOperatorPayload::Fallback { .. })) + .unwrap(); + let inputs = BTreeMap::from([( + u64::from(raw.id.0), + InputContract::bounded(Arc::new(raw.output_schema.clone())), + )]); + let roots = [u64::from(dag.root.0)]; + let compiled = compile(&dag, inputs.clone(), &roots).unwrap(); + // The root reads its population through a Sort helper numbered by the root. + let helper = u64::MAX - (roots[0] << 16); + assert_eq!(compiled.operator_name(helper), Some("Sort")); + assert!( + cut_candidate(&compiled, &[helper]).is_err(), + "helper operators are not Planner boundaries" + ); + assert_cuts_match_recompilation(&dag, inputs, &roots, 2); +} + +/// Cuts reject frontiers that recompilation rejects: duplicates, inputs, +/// unknown IDs, and an output shadowed by its descendant. +#[test] +fn cut_candidate_rejects_invalid_frontiers() { + let dag = grouped_rate(); + let state = dag + .nodes + .iter() + .find(|node| matches!(node.payload, PostAsapOperatorPayload::SummaryAgg { .. })) + .unwrap(); + let readout = dag + .nodes + .iter() + .find(|node| { + matches!( + node.payload, + PostAsapOperatorPayload::Value { + operation: ValueOperation::FinalizeExactAccumulator + } + ) + }) + .unwrap(); + let (state_id, rate_id, root) = ( + u64::from(state.id.0), + u64::from(readout.id.0), + u64::from(dag.root.0), + ); + let inputs = BTreeMap::from([( + state_id, + InputContract::bounded(Arc::new(state.output_schema.clone())), + )]); + let compiled = compile(&dag, inputs.clone(), &[root]).unwrap(); + for frontier in [ + vec![rate_id, rate_id], + vec![state_id], + vec![999], + vec![root, rate_id], + ] { + assert!(cut_candidate(&compiled, &frontier).is_err(), "{frontier:?}"); + assert!(compile_candidate(&dag, inputs.clone(), &[root], &frontier).is_err()); + } +} diff --git a/crates/asap-physical-operators/tests/weighted_topk_binding.rs b/crates/asap-physical-operators/tests/weighted_topk_binding.rs index 664ae799..ef086b1b 100644 --- a/crates/asap-physical-operators/tests/weighted_topk_binding.rs +++ b/crates/asap-physical-operators/tests/weighted_topk_binding.rs @@ -756,10 +756,102 @@ fn spatial_topk_exposes_signed_heap_candidate_over_complete_snapshot() { } } -// Placement changes execution ownership only. Every fixed-window candidate -// contains Rate finalization before a fresh heap, with query readout downstream. +/// Deployment-side lifecycle choice: every summary state of `candidate` is +/// continuously maintained, and the chosen lifecycles set execution timing. +fn continuously_maintained_dag(candidate: &Rc) -> PostAsapDag { + use asap_aware_mapping::{ + cost_model::{Cost, CostModel}, + enumerate_summary_maintenance_lifecycles, CostRate, Horizon, + SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCapabilities, + SummaryMaintenanceLifecycleCostInputs, WorkloadDemand, + }; + use planner_types::workload::{ + DataArrival, Rate, RepeatedDemand, RepeatingEntry, RepetitionInterval, + }; + struct Costed; + impl CostModel for Costed { + fn rank_candidates( + &self, + _: &planner_types::pre_asap::agg_intent::AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + fn summary_maintenance_lifecycle_cost_inputs( + &self, + _: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + SummaryMaintenanceLifecycleCostInputs { + build_cost: Some(Cost(10.)), + maintenance_cost_per_update: Some(Cost(1.)), + summary_read_cost: Some(Cost(1.)), + retention_cost_rate: Some(CostRate(0.1)), + retirement_cost: Some(Cost(1.)), + } + } + fn summary_maintenance_capabilities( + &self, + _: &SummaryNode, + ) -> SummaryMaintenanceCapabilities { + SummaryMaintenanceCapabilities { + incremental_update: true, + merge: true, + delete: true, + } + } + } + const NOW_MS: u64 = 1_000_000; + let queries = QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: None, + repeating_queries: Some(vec![RepeatingEntry { + query: Query("topk by(job)(2, rate(m[1m]))".into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(60_000)), + requirements: QueryRequirements::default(), + predictability: Predictability::Predictable { known_at: None }, + time_selection: TimeSelection::default(), + }]), + }; + let data = DataWorkload { + arrival: DataArrival::ContinuouslyIngesting, + ingestion_rate: WorkloadEvidence { + value: Some(Rate(1.)), + source: planner_types::workload::EvidenceSource::Observed, + observed_at_ms: Some(NOW_MS), + valid_for_ms: Some(60_000), + }, + ..Default::default() + }; + let lifecycles = enumerate_summary_maintenance_lifecycles( + Rc::clone(candidate), + WorkloadDemand::new_with_data(&queries, &data, &[0]), + NOW_MS, + Some(Horizon(100.)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &Costed, + ) + .unwrap(); + let choices = lifecycles + .deployments() + .iter() + .map(|deployment| { + ( + deployment.post_asap_node_id, + SummaryMaintenanceLifecycle::ContinuouslyMaintained, + ) + }) + .collect::>(); + lifecycles + .select(&choices) + .unwrap() + .execution_timed_dag() + .unwrap() +} + +// A maintained heap over finalized per-series Rate is the fixed-window +// placement: lifecycle timing, not a separate candidate, puts it in precompute. #[test] -fn planner_exposes_fixed_window_rate_heap_precompute_candidates() { +fn maintained_rate_heap_lifecycle_compiles_fixed_window_precompute() { use asap_physical_operators::physical_planner::{ compile_candidate, promql_rows::with_series_identity, }; @@ -775,13 +867,17 @@ fn planner_exposes_fixed_window_rate_heap_precompute_candidates() { &EqualSplitAllocator, &Evidence, ); - let candidates = strategy.fixed_window_rate_candidates(&root).candidates; + let candidates = strategy + .replacements(&TargetSubDAG::new(&root)) + .into_iter() + .filter_map(|candidate| match candidate.replacement { + Replacement::Summary(root) if candidate.rationale.contains("WithHeap") => Some(root), + _ => None, + }) + .collect::>(); assert_eq!(candidates.len(), 2); - for candidate in candidates { - let Replacement::Summary(root) = candidate.replacement else { - panic!() - }; - let dag = compile_post_asap_dag(&root).unwrap(); + for root in candidates { + let dag = continuously_maintained_dag(&root); let state = dag .nodes .iter() @@ -819,16 +915,11 @@ fn planner_exposes_fixed_window_rate_heap_precompute_candidates() { &[u64::from(heap.id.0)], ) .unwrap(); - let exported = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&root).unwrap(); + let exported = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&dag).unwrap(); assert_eq!( serde_json::to_vec(&exported).unwrap(), serde_json::to_vec(&physical).unwrap() ); - assert!( - asap_physical_operators::physical_planner::promql_rows::compile_rate_ranking(&root) - .is_err(), - "query binding must not move the selected precompute frontier" - ); // Execute the selected split across a state serialization boundary. // Each run builds fresh weights from that window's counters. let execute = |plan: &asap_physical_operators::physical_planner::CompiledPhysicalDag, @@ -959,60 +1050,3 @@ fn planner_exposes_fixed_window_rate_heap_precompute_candidates() { ); } } - -// Grouped Rate has a legal stored Sum candidate as well as query-time reduction. -#[test] -fn grouped_rate_exposes_precomputed_sum_with_query_readout() { - let root = Rc::new( - asap_physical_operators::physical_planner::promql_rows::with_series_identity( - &lower_promql("sum by(job)(rate(m[1m]))", AccuracyTarget::Exact).unwrap(), - ) - .unwrap(), - ); - let strategy = SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence( - &DefaultCostModel, - &DefaultAccuracyModel, - &EqualSplitAllocator, - &Evidence, - ); - let direct = strategy.query_time_rate_aggregation_candidates(&root); - assert!( - direct.candidates.iter().any(|candidate| { - let Replacement::Summary(root) = &candidate.replacement else { - return false; - }; - let Ok((_, program)) = - asap_physical_operators::physical_planner::promql_rows::compile_rate_ranking(root) - else { - return false; - }; - let output = program.output_contract(program.roots()[0]).unwrap(); - output - .schema - .fields - .iter() - .all(|field| matches!(field.dtype, SummaryFamilyType::Plain(_))) - }), - "query-time grouped Rate must finalize Sum inside the physical graph" - ); - let candidates = strategy.fixed_window_rate_candidates(&root).candidates; - assert!( - !candidates.is_empty(), - "Planner must expose Rate -> grouped Sum at ingestion" - ); - for candidate in candidates { - let Replacement::Summary(root) = candidate.replacement else { - panic!() - }; - let physical = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&root).unwrap(); - let precompute = - String::from_utf8(serde_json::to_vec(&physical.precompute.unwrap()).unwrap()).unwrap(); - assert!( - precompute.contains("SummaryBuild") - && precompute.contains("Rate") - && precompute.contains("Sum") - ); - let query = String::from_utf8(serde_json::to_vec(&physical.query).unwrap()).unwrap(); - assert!(query.contains("Readout") && !query.contains("SummaryBuild")); - } -} diff --git a/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs b/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs index d5406f23..08e3e3b1 100644 --- a/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs +++ b/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs @@ -212,6 +212,15 @@ fn selected_plan_with_horizon( .into_iter() .next() .expect("one normalized workload entry"); + selected_plan_for_lowered(workload, lowered, model, horizon) +} + +fn selected_plan_for_lowered( + workload: &PlanningWorkload, + lowered: asap_types::pre_asap::QueryExpr, + model: &dyn CostModel, + horizon: Horizon, +) -> asap_aware_mapping::SummaryMaintenanceLifecyclePlan { let root = Rc::new(lowered); let strategies = asap_aware_mapping::default_strategies_with(model); let space = search_workload_with(vec![("dashboard", Rc::clone(&root))], &strategies); @@ -426,3 +435,530 @@ fn continuous_lifecycle_compiles_and_executes_spatial_kll() { ); } } + +fn quantile_workload(query: &str) -> PlanningWorkload { + let mut workload = dashboard_workload(); + workload.query_workload.query_batch.as_mut().unwrap()[0].query = Query(query.into()); + workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = Query(query.into()); + workload +} + +/// Timed DAG for `query` after binding every summary state to `lifecycle`. +/// Grouped queries carry a physical series identity, as per-entity state needs. +fn lifecycle_timed_dag( + query: &str, + lifecycle: &SummaryMaintenanceLifecycle, +) -> (asap_types::post_asap::PostAsapDag, Vec) { + use asap_aware_mapping::enumerate_summary_maintenance_lifecycles; + let workload = quantile_workload(query); + let mut lowered = lower_promql_workload(&workload, 0).unwrap().remove(0); + if query.contains(" by(") { + lowered = + asap_physical_operators::physical_planner::promql_rows::with_series_identity(&lowered) + .unwrap(); + } + let root = + selected_plan_for_lowered(&workload, lowered, &FullyCostedRuntime, Horizon(100.)).root; + let candidates = enumerate_summary_maintenance_lifecycles( + root, + WorkloadDemand::new_with_data( + &workload.query_workload, + workload.data_workload.as_ref().unwrap(), + &[1], + ), + NOW_MS, + Some(Horizon(100.)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &FullyCostedRuntime, + ) + .unwrap(); + let choices: Vec<_> = candidates + .deployments() + .iter() + .map(|deployment| (deployment.post_asap_node_id, lifecycle.clone())) + .collect(); + let mut states: Vec<_> = choices.iter().map(|(id, _)| u64::from(id.0)).collect(); + states.sort_unstable(); + let dag = candidates + .select(&choices) + .unwrap() + .execution_timed_dag() + .unwrap(); + (dag, states) +} + +/// Compile inputs for a timed DAG: its raw source, available at either phase. +fn raw_inputs( + dag: &asap_types::post_asap::PostAsapDag, +) -> std::collections::BTreeMap { + let raw = dag + .nodes + .iter() + .find(|node| { + matches!( + node.payload, + asap_types::post_asap::PostAsapOperatorPayload::Fallback { .. } + ) + }) + .unwrap(); + std::collections::BTreeMap::from([( + u64::from(raw.id.0), + asap_physical_operators::physical_planner::InputContract::bounded(std::sync::Arc::new( + raw.output_schema.clone(), + )), + )]) +} + +/// For existing PromQL fixtures, Planner's own retained lifecycle selection +/// reproduces the timing that realization strategies assign today. +#[test] +fn planner_lifecycle_selection_reproduces_strategy_timing() { + for query in [ + "quantile_over_time(0.99, latency[5m])", + "quantile(0.99, latency)", + "sum by(job)(rate(m[1m]))", + ] { + let plan = selected_plan(&quantile_workload(query)); + assert!(!plan.selected_raw_recompute, "{query}"); + assert!(plan.deployments.iter().all(|deployment| { + deployment + .summary_maintenance_lifecycle_guarantee + .as_ref() + .is_some_and(|guarantee| { + guarantee.summary_maintenance_lifecycle + != SummaryMaintenanceLifecycle::Ephemeral + }) + })); + let strategy = asap_types::post_asap::compile_post_asap_dag(&plan.root).unwrap(); + assert_eq!(plan.execution_timed_dag().unwrap(), strategy, "{query}"); + } +} + +/// An explicitly chosen lifecycle reaches physical compilation through timing: +/// ContinuouslyMaintained puts the state in precompute, Ephemeral leaves +/// precompute empty and reads the raw source at query time; both answer alike. +#[test] +fn chosen_lifecycle_timing_decides_precompute_contents() { + use asap_physical_operators::{ + physical_planner::{compile_candidate, frontier_from_timing}, + runtime::Scope, + values::{Batch, Value}, + }; + use asap_types::{post_asap::SummaryFamilyType, pre_asap::DataType}; + use std::collections::BTreeMap; + + let mut answers = Vec::new(); + for lifecycle in [ + SummaryMaintenanceLifecycle::ContinuouslyMaintained, + SummaryMaintenanceLifecycle::Ephemeral, + ] { + let (dag, states) = lifecycle_timed_dag("quantile(0.99, latency)", &lifecycle); + let [state] = states[..] else { + panic!("one summary state"); + }; + let inputs = raw_inputs(&dag); + let (&raw_id, contract) = inputs.iter().next().unwrap(); + let schema = contract.schema.clone(); + let frontier = frontier_from_timing(&dag).unwrap(); + let candidate = + compile_candidate(&dag, inputs, &[u64::from(dag.root.0)], &frontier).unwrap(); + let rows = (1..=100) + .map(|value| { + schema + .fields + .iter() + .map(|field| match field.dtype { + SummaryFamilyType::Plain(DataType::Float64) => { + Value::Float64(f64::from(value)) + } + SummaryFamilyType::Plain(DataType::Timestamp) => Value::Timestamp(300_000), + _ => panic!("unexpected field {field:?}"), + }) + .collect() + }) + .collect(); + let raw_batch = Batch::try_new(schema.clone(), rows).unwrap(); + let query_scope = Scope::Query { + evaluation_time_ms: 300_000, + revision: 1, + }; + let result = if lifecycle == SummaryMaintenanceLifecycle::Ephemeral { + assert!(frontier.is_empty()); + assert!(candidate.precompute.is_none()); + physical_common::execute( + &candidate.query, + BTreeMap::from([(raw_id, raw_batch)]), + query_scope, + ) + } else { + assert_eq!(frontier, [state]); + assert_eq!( + candidate + .materialized_outputs + .keys() + .copied() + .collect::>(), + [state] + ); + let stored = physical_common::execute( + candidate.precompute.as_ref().unwrap(), + BTreeMap::from([(raw_id, raw_batch)]), + Scope::Ingestion { + window_start_ms: 0, + window_end_ms: 300_000, + revision: 1, + }, + ); + physical_common::execute( + &candidate.query, + BTreeMap::from([(state, stored[0][0].clone())]), + query_scope, + ) + }; + answers.push( + result[0] + .iter() + .flat_map(|batch| batch.rows()) + .flat_map(|row| row.iter()) + .filter_map(|value| match value { + Value::Float64(value) => Some(*value), + _ => None, + }) + .collect::>(), + ); + } + assert_eq!(answers[0], answers[1]); + assert_eq!(answers[0].len(), 1); +} + +/// One compilation, cut by each lifecycle assignment's timing, yields exactly +/// the candidate `compile_candidate` builds for that timed DAG: the retained +/// state is the frontier under ContinuouslyMaintained, and nothing under +/// Ephemeral. Covers the KLL quantile fixture and grouped Rate→Sum. +#[test] +fn lifecycle_timing_cuts_one_compilation() { + use asap_physical_operators::physical_planner::{ + compile, compile_candidate, cut_candidate, frontier_from_timing, + }; + for query in ["quantile(0.99, latency)", "sum by(job)(rate(m[1m]))"] { + let ephemeral = SummaryMaintenanceLifecycle::Ephemeral; + let (compiled_dag, _) = lifecycle_timed_dag(query, &ephemeral); + let inputs = raw_inputs(&compiled_dag); + let roots = [u64::from(compiled_dag.root.0)]; + let compiled = compile(&compiled_dag, inputs.clone(), &roots).unwrap(); + for lifecycle in [ + SummaryMaintenanceLifecycle::ContinuouslyMaintained, + ephemeral, + ] { + let (dag, states) = lifecycle_timed_dag(query, &lifecycle); + let frontier = frontier_from_timing(&dag).unwrap(); + // Retained states read by a query-time consumer, or the root itself. + let query_time = |id: u64| { + dag.nodes.iter().any(|node| { + u64::from(node.id.0) == id + && node.output_state.timing + == asap_types::post_asap::ExecutionTiming::QueryTime + }) + }; + let expected_frontier = if lifecycle == SummaryMaintenanceLifecycle::Ephemeral { + vec![] + } else { + states + .iter() + .copied() + .filter(|state| { + *state == u64::from(dag.root.0) + || dag.edges.iter().any(|edge| { + u64::from(edge.producer.0) == *state + && query_time(u64::from(edge.consumer.0)) + }) + }) + .collect() + }; + assert_eq!(frontier, expected_frontier, "{query} {lifecycle:?}"); + let cut = cut_candidate(&compiled, &frontier).unwrap(); + let expected = compile_candidate(&dag, inputs.clone(), &roots, &frontier).unwrap(); + assert_eq!( + serde_json::to_vec(&cut).unwrap(), + serde_json::to_vec(&expected).unwrap(), + "{query} {lifecycle:?}" + ); + } + } +} + +/// A maintained current-series population is placed by its lifecycle choice: +/// ContinuouslyMaintained stores the population in precompute, Ephemeral +/// rebuilds it from the raw source at query time; both rank alike. +#[test] +fn chosen_population_lifecycle_decides_precompute_contents() { + use asap_aware_mapping::{ + enumerate_summary_maintenance_lifecycles, + maintained_population::MaintainedPopulationStrategy, + }; + use asap_physical_operators::{ + physical_planner::{ + compile_candidate, + promql_rows::{series_row, with_series_identity}, + InputContract, + }, + runtime::Scope, + values::{Batch, Value}, + }; + use asap_types::post_asap::{ + maintained_population::PopulationInput, PostAsapOperatorPayload, ValueOperation, + }; + use std::{collections::BTreeMap, sync::Arc}; + + let workload = quantile_workload("topk by(job)(1, m)"); + let root = Rc::new( + with_series_identity(&lower_promql_workload(&workload, 0).unwrap().remove(0)).unwrap(), + ); + let root = MaintainedPopulationStrategy::new(std::slice::from_ref(&root)) + .candidate(&root) + .unwrap(); + let mut answers = Vec::new(); + for lifecycle in [ + SummaryMaintenanceLifecycle::ContinuouslyMaintained, + SummaryMaintenanceLifecycle::Ephemeral, + ] { + let candidates = enumerate_summary_maintenance_lifecycles( + Rc::clone(&root), + WorkloadDemand::new_with_data( + &workload.query_workload, + workload.data_workload.as_ref().unwrap(), + &[1], + ), + NOW_MS, + Some(Horizon(100.)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &FullyCostedRuntime, + ) + .unwrap(); + let [deployment] = candidates.deployments() else { + panic!("one population state"); + }; + let id = deployment.post_asap_node_id; + let dag = candidates + .select(&[(id, lifecycle.clone())]) + .unwrap() + .execution_timed_dag() + .unwrap(); + let population = dag.nodes.iter().find(|node| node.id == id).unwrap(); + let PostAsapOperatorPayload::Value { + operation: ValueOperation::MaintainPopulation { population }, + } = &population.payload + else { + panic!("the deployment is the maintained population"); + }; + let PopulationInput::CurrentSeries(spec) = &population.input else { + panic!("current-series population"); + }; + let lookback = i64::try_from(spec.lookback_ms).unwrap(); + let raw = dag + .nodes + .iter() + .find(|node| matches!(node.payload, PostAsapOperatorPayload::Fallback { .. })) + .unwrap(); + let (raw_id, schema) = (u64::from(raw.id.0), Arc::new(raw.output_schema.clone())); + let frontier = + asap_physical_operators::physical_planner::frontier_from_timing(&dag).unwrap(); + let candidate = compile_candidate( + &dag, + BTreeMap::from([(raw_id, InputContract::bounded(schema.clone()))]), + &[u64::from(dag.root.0)], + &frontier, + ) + .unwrap(); + let end = 60_000; + let rows = [("a", end - 1, 100.), ("a", end, 1.), ("b", end, 20.)] + .into_iter() + .map(|(instance, at, value)| { + series_row( + &schema, + &BTreeMap::from([ + ("job".into(), "api".into()), + ("instance".into(), instance.into()), + ]), + at, + value, + ) + .unwrap() + }) + .collect(); + let raw_batch = Batch::try_new(schema.clone(), rows).unwrap(); + let query_scope = Scope::Query { + evaluation_time_ms: end, + revision: 1, + }; + let result = if lifecycle == SummaryMaintenanceLifecycle::Ephemeral { + assert!(frontier.is_empty()); + assert!(candidate.precompute.is_none()); + physical_common::execute( + &candidate.query, + BTreeMap::from([(raw_id, raw_batch)]), + query_scope, + ) + } else { + let state = u64::from(id.0); + assert_eq!(frontier, [state]); + let stored = physical_common::execute( + candidate.precompute.as_ref().unwrap(), + BTreeMap::from([(raw_id, raw_batch)]), + Scope::Ingestion { + window_start_ms: end - lookback, + window_end_ms: end, + revision: 1, + }, + ); + physical_common::execute( + &candidate.query, + BTreeMap::from([(state, stored[0][0].clone())]), + query_scope, + ) + }; + answers.push( + result[0] + .iter() + .flat_map(|batch| batch.rows()) + .flat_map(|row| row.iter()) + .filter_map(|value| match value { + Value::Float64(value) => Some(*value), + _ => None, + }) + .collect::>(), + ); + } + assert_eq!(answers[0], answers[1]); + assert_eq!(answers[0], [20.]); +} + +/// Grouped Rate→Sum is one inventory candidate: retaining the Sum state puts +/// Rate and Sum in precompute, while an `Ephemeral` Sum over a retained Rate +/// state leaves Sum in the query DAG. +#[test] +fn grouped_rate_sum_placement_is_a_lifecycle_choice() { + use asap_aware_mapping::enumerate_summary_maintenance_lifecycles; + use asap_physical_operators::physical_planner::{compile_candidate, InputContract}; + use asap_types::post_asap::{ + ExactKind, PostAsapOperatorPayload, SummaryExpr, SummaryFamilyType, + }; + use std::{collections::BTreeMap, sync::Arc}; + + let workload = quantile_workload("sum by(job)(rate(m[1m]))"); + let root = Rc::new( + asap_physical_operators::physical_planner::promql_rows::with_series_identity( + &lower_promql_workload(&workload, 0).unwrap().remove(0), + ) + .unwrap(), + ); + let is_exact = |node: &SummaryNode, kind: ExactKind| { + matches!(&node.expr, SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(k, _), .. + } if *k == kind) + }; + let inventory = asap_aware_mapping::search_workload(vec![("q", root)]) + .enumerate_candidate_dags(4096) + .unwrap(); + let candidates = inventory + .candidates + .into_iter() + .map(|mut forest| forest.remove(0).1) + .filter(|candidate| { + matches!(&candidate.expr, SummaryExpr::ValueOperation { child, .. } + if is_exact(child, ExactKind::Sum)) + }) + .collect::>(); + let [candidate] = candidates.as_slice() else { + panic!("one grouped Sum candidate, got {}", candidates.len()); + }; + let mut placements = Vec::new(); + for sum_lifecycle in [ + SummaryMaintenanceLifecycle::ContinuouslyMaintained, + SummaryMaintenanceLifecycle::Ephemeral, + ] { + let lifecycles = enumerate_summary_maintenance_lifecycles( + Rc::clone(candidate), + WorkloadDemand::new_with_data( + &workload.query_workload, + workload.data_workload.as_ref().unwrap(), + &[1], + ), + NOW_MS, + Some(Horizon(100.)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &FullyCostedRuntime, + ) + .unwrap(); + let choices = lifecycles + .deployments() + .iter() + .map(|deployment| { + let lifecycle = if is_exact(&deployment.summary, ExactKind::Sum) { + sum_lifecycle.clone() + } else { + SummaryMaintenanceLifecycle::ContinuouslyMaintained + }; + (deployment.post_asap_node_id, lifecycle) + }) + .collect::>(); + assert_eq!(choices.len(), 2, "Rate and Sum states"); + let dag = lifecycles + .select(&choices) + .unwrap() + .execution_timed_dag() + .unwrap(); + let raw = dag + .nodes + .iter() + .find(|node| matches!(node.payload, PostAsapOperatorPayload::Fallback { .. })) + .unwrap(); + let frontier = + asap_physical_operators::physical_planner::frontier_from_timing(&dag).unwrap(); + let [boundary] = frontier.as_slice() else { + panic!("one precompute output, got {frontier:?}"); + }; + let boundary = dag + .nodes + .iter() + .find(|node| u64::from(node.id.0) == *boundary) + .unwrap(); + let physical = compile_candidate( + &dag, + BTreeMap::from([( + u64::from(raw.id.0), + InputContract::bounded(Arc::new(raw.output_schema.clone())), + )]), + &[u64::from(dag.root.0)], + &frontier, + ) + .unwrap(); + let json = |value| String::from_utf8(serde_json::to_vec(value).unwrap()).unwrap(); + placements.push(( + boundary.payload.clone(), + json(physical.precompute.as_ref().unwrap()), + json(&physical.query), + )); + } + let builds = |json: &str, kind: &str| { + json.contains(&format!( + r#"{{"SummaryBuild":{{"family":{{"ExactAggregate":["{kind}","{kind}"]}}"# + )) + }; + let [(retained, retained_pre, retained_query), (ephemeral, ephemeral_pre, ephemeral_query)] = + placements.as_slice() + else { + unreachable!() + }; + let state = |payload: &PostAsapOperatorPayload, kind: ExactKind| { + matches!(payload, PostAsapOperatorPayload::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(k, _), .. + } if *k == kind) + }; + assert!(state(retained, ExactKind::Sum)); + assert!(builds(retained_pre, "Rate") && builds(retained_pre, "Sum")); + assert!(!retained_query.contains("SummaryBuild")); + assert!(state(ephemeral, ExactKind::Rate)); + assert!(builds(ephemeral_pre, "Rate") && !builds(ephemeral_pre, "Sum")); + assert!(builds(ephemeral_query, "Sum")); +} diff --git a/crates/types/src/post_asap/execution_data_state.rs b/crates/types/src/post_asap/execution_data_state.rs index 43f2cd05..a5d87c44 100644 --- a/crates/types/src/post_asap/execution_data_state.rs +++ b/crates/types/src/post_asap/execution_data_state.rs @@ -483,14 +483,17 @@ fn visit( operation, timing, } => { + // Population timing is a lifecycle decision: a retained population + // is maintained at ingestion time, an ephemeral one is rebuilt + // from raw input per query. Its input and readout contracts are + // structural and hold either way. let valid_population = match operation { ValueOperation::MaintainPopulation { population } => { - *timing == ExecutionTiming::IngestionTime - && matches!(&child.expr, SummaryExpr::KeepPreAsap(input) if population.matches_input(input)) + matches!(&child.expr, SummaryExpr::KeepPreAsap(input) if population.matches_input(input)) } ValueOperation::ReadPopulation { readout } => { *timing == ExecutionTiming::QueryTime - && matches!(&child.expr, SummaryExpr::ValueOperation { operation: ValueOperation::MaintainPopulation { population }, timing: ExecutionTiming::IngestionTime, .. } if population.supports(readout)) + && matches!(&child.expr, SummaryExpr::ValueOperation { operation: ValueOperation::MaintainPopulation { population }, .. } if population.supports(readout)) } _ => true, }; @@ -507,13 +510,13 @@ fn visit( && s.primitive == DataPrimitive::SummaryState && (*timing == ExecutionTiming::QueryTime || s.timing == *timing) && is_exact_accumulator_state(&child.schema).is_ok(); + // A query-time readout may read a population retained at ingestion. let population_readout = matches!(operation, ValueOperation::ReadPopulation { .. }) && *timing == ExecutionTiming::QueryTime && matches!( &child.expr, SummaryExpr::ValueOperation { operation: ValueOperation::MaintainPopulation { .. }, - timing: ExecutionTiming::IngestionTime, .. } ); diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index fb9a7f42..32c0b093 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -34,7 +34,9 @@ fields and [frontend dependencies](#frontend-specific-dependencies). DAG assembly](#selection-and-dag-assembly), and [summary-maintenance lifecycle](#summary-maintenance-lifecycle-aware-helper) APIs operate on this `PlanSpace`. These are alternative uses of the candidate space, not mandatory sequential -stages. `PlanSpace` itself has no selected summary-maintenance lifecycle. +stages. `PlanSpace` itself has no selected summary-maintenance lifecycle, and +its candidates do not choose precompute versus query-time placement: a chosen +lifecycle assignment sets each node's execution timing. The candidate DAGs are logical planning artifacts. ASAPPlanner does **not** produce a deployed executable plan; downstream systems bind physical operators, diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index 7e7b3bd0..81afc40b 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -32,10 +32,34 @@ The Logical Post-ASAP DAG is preceded by the Pre-ASAP DAG (`QueryExpr`), the language-independent query semantics before summary selection. Both are logical. Planning builds Post-ASAP `SummaryNode` trees; `compile_post_asap_dag` exports the selected tree as a `PostAsapDag`, which is the Physical Plan -Compiler's input. Its per-node execution phase (ingestion or query time) is an -initial placement: compilation places ingestion-time nodes in the precompute DAG, -while frontier enumeration proposes alternative materialization splits. Which -layer owns placement is an open design question, deferred to a later change. +Compiler's input. Its per-node execution phase (ingestion or query time) is +decided by the selected summary maintenance lifecycle, as the layer contract +below states. + +### Layer contract + +1. **Logical Post-ASAP** (`PlanSpace`) decides what to compute: summary + families, readouts and sharing. It does not decide placement; timing that a + realization strategy writes while building a candidate is provisional. +2. **Summary maintenance lifecycle** (Planner) lists the lifecycle choices for + each unique retained state: every summary state (`SummaryAgg`) and every + maintained population that does not feed a summary state. + A chosen assignment determines every node's + `ExecutionTiming`, plus window framework and retention. + `SummaryMaintenanceLifecyclePlan::execution_timed_dag` applies it: a retained + (non-`Ephemeral`) state and all of its inputs run at ingestion time; + readouts, other consumers, and `Ephemeral` states not consumed by retained + state run at query time. A population that feeds a summary state is one of + that state's inputs and follows its timing. +3. **Physical compile** (Planner) reads timing: ingestion-time nodes form the + precompute DAG and the rest form the query DAG, joined by typed outputs. It + does not see raw ingestion, panes, storage or stored-state readout. +4. **Backend** chooses the lifecycle assignment with its own `CostModel`: + precompute CPU (`maintenance_cost_per_update`), sketch/summary store cost + (`retention_cost_rate`), query reads (`summary_read_cost`) and per-query + builds (`build_cost`, for `Ephemeral`), counting shared state once. + `Ephemeral` requires the deployment to supply the state's raw input as a + query-time source. ### Candidate generation and deployment selection @@ -68,10 +92,12 @@ separate unsupported compilation, deployment infeasibility, missing evidence, and a feasible candidate that loses on cost. Absence is not a cost comparison. For `sum by(job)(rate(m[1m]))`, Rate remains per series before grouped Sum. -When lifecycle requirements permit it, a candidate may finalize Rate and Sum -within a bounded precompute run and persist the grouped value. Another may leave -those operators in the query DAG. Storing a value requires its exact evaluation -window, revision, readiness and serving cadence to match the query contract. +`PlanSpace` offers one such candidate, with a per-series Rate state and a grouped +Sum state. Its lifecycle assignment places it: a retained Sum state finalizes +Rate and builds Sum within a bounded precompute run; an `Ephemeral` Sum over a +retained Rate state leaves the Rate readout and Sum in the query DAG. Storing a +value requires its exact evaluation window, revision, readiness and serving +cadence to match the query contract. For instant-vector TopK, CMS/CountSketch with a candidate heap requires explicit series identity and a supported latest-value input protocol. Appending historical @@ -236,6 +262,29 @@ using workload demand, window/freshness requirements and supported physical implementations. Backend selection uses runtime feasibility and cost after physical compilation. The following example follows one candidate. +Candidate generation and selection are separate steps. For every unique retained +state, enumeration reports each lifecycle (ephemeral, prepared, shared, +continuously maintained) as legal, with a Planner cost or explicitly unknown +cost, or as rejected with a reason. Planner does not remove a legal alternative +because its own estimate prefers another. A deployment prices the legal +alternatives over the whole workload, counting shared state once, and binds one +lifecycle per state. Binding checks that the choice is legal and that states on +one maintenance path share an evaluation schedule. An alternative whose cost is +unknown can be bound only when the deployment's cost model is authoritative for +complete-candidate cost; unknown cost is never treated as zero. It then yields the same +lifecycle guarantee and window framework the physical compiler consumes when +Planner selects. Planner's own cheapest-alternative selection remains available +for callers without deployment pricing. The window framework is decided for the +complete combination, not for one alternative in isolation. + +A maintained population (for example, the current series of `topk by(job)(1, m)`) +is retained state like a summary. Retaining it maintains the latest sample per +series at ingestion and leaves only the readout at query time. Choosing +`Ephemeral` rebuilds that snapshot from raw samples for each query, so the +deployment must supply the raw source at query time. The caller's `CostModel` +prices both through the same lifecycle hooks; a model without population +evidence leaves them unknown, and they are not selected. + For the running example, assume it selects: ```text @@ -348,21 +397,26 @@ DAG. If the required behavior cannot be realized, physical compilation fails. Materialization frontiers are Planner decisions. A candidate records both the precompute Physical DAG and the query Physical DAG, with typed outputs connecting them. The deployment compiler binds those outputs; it does not move operators. - -For `sum by(job)(rate(m[1m]))`, legal physical candidates can include: +Lifecycle timing gives the frontier: ingestion-time nodes read by query-time +nodes. For `sum by(job)(rate(m[1m]))`, the two lifecycle choices of the single +logical candidate give: ```text -Candidate A: - precompute: compatible per-series counter states → per-series Rate - materialized output: per-series rate values for window/evaluation/revision - query: stored per-series rate values → grouped Sum - -Candidate B: - precompute: compatible per-series counter states → per-series Rate → grouped Sum - materialized output: grouped values for window/evaluation/revision - query: stored grouped values → result +Candidate A (Rate state retained, Sum Ephemeral): + precompute: counter samples → per-series Rate state + materialized output: per-series Rate states for window/evaluation/revision + query: stored Rate states → Rate readout → grouped Sum → result + +Candidate B (Rate and Sum states retained): + precompute: counter samples → per-series Rate → grouped Sum state + materialized output: grouped Sum states for window/evaluation/revision + query: stored grouped Sum states → Sum readout → result ``` +Explicit frontiers passed to `compile_candidates` can also persist per-series +rate values; deriving that frontier from timing inside the physical planner is +not yet implemented. + Both preserve reset-aware Rate before Sum. Summing raw counters before Rate is not equivalent. The counter-state build may be another precompute DAG; typed state inputs do not imply that a deployment can construct or bind those states. @@ -376,6 +430,22 @@ feasibility is rejected before pricing. The optimizer supplies candidate frontiers and cost evidence, including updates, retention, recurrence and sharing. `enumerate_frontiers` constructs bounded, reachable antichain frontiers above explicit input boundaries, including query-only and fully precomputed results. It fails explicitly when the candidate budget is exceeded. Maintenance selection must still reject frontiers that violate window, freshness, or reuse requirements; deployment feasibility is checked before pricing. +The lifecycle layer decides timing; physical compilation reads it. Lowering a +node does not depend on the frontier, so each query DAG is lowered once and +different lifecycle assignments are different cuts of that lowering. +`compile(dag, inputs, roots)` yields the complete `CompiledPhysicalDag`. +`frontier_from_timing(&timed_dag)` reads an assignment's timed DAG (from +`execution_timed_dag`) and returns its frontier: ingestion-time nodes read by +query-time nodes, or an ingestion-time root; a query-time node feeding an +ingestion-time node is rejected. `cut_candidate(&compiled, &frontier)` then +partitions the lowered operators: the frontier's ancestors form the precompute +DAG and the rest form the query DAG. Helper operators are numbered by their +Planner node (`u64::MAX - (node_id << 16) - index`), so a cut is byte-identical to +`compile_candidate` for that frontier. One exception: an ingestion-time +`Binary` lowers differently, so its timing must match at compile time. +`compile_candidate(s)` and `enumerate_frontiers` wrap the same path. Temporal +pane candidates remain a separate lowering. + Physical compilation opens no readers. Bounded precompute outputs become typed query inputs. Their source, filters, grouping, build window, evaluation time, readiness and revision contracts must accompany the selected lifecycle and be checked during @@ -454,7 +524,7 @@ The complete example makes the ownership boundary explicit: | --- | --- | | **Logical Post-ASAP DAG** | Use `KLL(k=200)` with shared merge for p50/p99 | | **Summary Maintenance Candidate Generation** | Maintain 1-minute panes and reuse them for aligned five-minute queries | -| **Summary Maintenance Lifecycle** | Record pane/window/freshness/reuse requirements | +| **Summary Maintenance Lifecycle** | Record pane/window/freshness/reuse requirements and each node's execution timing | | **Physical Plan Compiler** | Lower to native KLL build, merge, and readout operators | | **Physical DAG** | Define precompute and query DAGs with typed input/output boundaries | | **Deployment Plan Compiler** | Bind raw input and KLL state slots to concrete sources/materializations | @@ -500,6 +570,11 @@ operator/runtime fixtures: | Test | Contract exercised | | --- | --- | | `summary_maintenance_lifecycle_e2e::continuous_lifecycle_compiles_and_executes_spatial_kll` | PromQL workload → selected continuous lifecycle → logical DAG → compiled precompute/query candidate → results in independent revisions; an unbounded candidate fails before pricing, and a bounded request candidate summarizes the same input samples | +| `summary_maintenance_lifecycle_e2e::chosen_lifecycle_timing_decides_precompute_contents` | PromQL workload → enumerated lifecycles → explicit choice → timed DAG → compiled candidate; ContinuouslyMaintained stores the state in precompute, Ephemeral leaves precompute empty and reads the raw source at query time; both return the same p99 | +| `summary_maintenance_lifecycle_e2e::lifecycle_timing_cuts_one_compilation` | KLL quantile and grouped Rate→Sum: one compilation cut by the ContinuouslyMaintained and Ephemeral timed DAGs equals `compile_candidate` for each; the frontier is the retained state or empty | + +| `summary_maintenance_lifecycle_e2e::chosen_population_lifecycle_decides_precompute_contents` | PromQL `topk by(job)` over a maintained population → explicit choice → timed DAG → compiled candidate; ContinuouslyMaintained stores the population in precompute, Ephemeral rebuilds it from raw samples at query time; both rank alike | +| `summary_maintenance_lifecycle_e2e::planner_lifecycle_selection_reproduces_strategy_timing` | For PromQL summary fixtures, the timed DAG from Planner's retained selection equals the DAG realization strategies produce | | `kll_pane_execution::five_panes_roundtrip_and_shared_merge_runs_once` | Explicit one-minute precompute DAGs → real MessagePack state bytes → five required query inputs → shared native merge → p50/p99; counts every sample once, checks adjacent aligned windows and instruments one merge start per run | | `kll_pane_execution::restored_panes_reject_corruption_parameters_schema_and_missing_binding` | Corrupt bytes, parameter relabelling, incompatible schemas and absent bindings fail explicitly | | `precompute_candidates::grouped_rate_can_be_materialized_before_or_after_grouped_sum` | Cost changes select different legal precompute frontiers; both selected candidates execute with the same reset-sensitive result; uncompilable candidates are not priced | diff --git a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md index 99501e97..06555965 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md +++ b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md @@ -70,7 +70,7 @@ columns and multi-measure aggregates need additional rules. ```text KeepPreAsap(input) - -> MaintainPopulation { input, max_k, quantiles } [maintenance] + -> MaintainPopulation { input, max_k, quantiles } [lifecycle-timed] -> ReadPopulation { Quantile(q1) } [read] -> ReadPopulation { Quantile(q2) } [read] -> ReadPopulation { TopK(k1) } [read] @@ -115,8 +115,9 @@ because their source names or numeric values happen to agree. ## Validation, selection and execution responsibilities -Planner validates the declared input, maintenance/read phases and readout -compatibility. Its intended guarantee is exact membership and exact readout; +Planner validates the declared input, the query-time readout and readout +compatibility; the population's lifecycle decides whether it is maintained at +ingestion or rebuilt per query. Its intended guarantee is exact membership and exact readout; a physical implementation still must preserve the language's numeric and empty-input semantics. In particular, SQL global COUNT over an empty population returns a row with zero, while PromQL COUNT over an empty vector returns an empty vector. diff --git a/docs/design_docs/proposals/operator-sharing.md b/docs/design_docs/proposals/operator-sharing.md index 03b36cf5..3d075105 100644 --- a/docs/design_docs/proposals/operator-sharing.md +++ b/docs/design_docs/proposals/operator-sharing.md @@ -287,7 +287,7 @@ Per node kind: | `SummaryAgg` | from the child; ingestion time under `KeepPreAsap` | **set** by binding, as today's fallback: `IngestionTime`, or `QueryTime` over a query-time child | | `FinalizeExactAccumulator` | a stored field, set by the planner | **set** by the planner: the same position allows either time | | `SummaryEstimate` | query time, fixed by the kind | **derived** from the kind: query time | -| `MaintainPopulation` / `ReadPopulation` | a stored field, always ingestion / query time | **derived** from the kind: ingestion / query time | +| `MaintainPopulation` / `ReadPopulation` | a stored field: population timing set by its lifecycle; readout always query time | population: **set** by its lifecycle; readout: **derived**, query time | | unused variants | `SummaryMerge`: a stored field; `Join` / `Subtract` / `Delete`: ingestion time | unimplemented (§1.3) | Unlike a guarantee, a timing depends on the parents, so `derive_timings` needs the whole diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 92288067..a4119122 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -633,6 +633,8 @@ that prepared or retained shared state is supported. | `plan_summary_maintenance_lifecycles` | Assembled logical DAG root, `WorkloadDemand`, `now_ms`, optional horizon, runtime capabilities, cost model | `Result` for that fixed root; does not revisit all semantic candidates | | `global_selection_with_summary_maintenance_lifecycles` | `PlanSpace`, workload/root-entry associations, time, horizon, capabilities, cost model | Lifecycle-aware compatible selection/error, using eligible cost evidence | | `assemble_selected_dag_with_summary_maintenance_lifecycles` | Selection, target root and lifecycle context | Optional lifecycle plan/error; attaches state deployment decisions | +| `enumerate_summary_maintenance_lifecycles` | Same inputs as `plan_summary_maintenance_lifecycles` | `SummaryMaintenanceLifecycleCandidates`: per unique retained state, every alternative with its cost or rejection; nothing selected. `guarantee(&lifecycle)` gives the mode/schedule that alternative would carry | +| `SummaryMaintenanceLifecycleCandidates::select(choices)` | One `(PostAsapNodeId, SummaryMaintenanceLifecycle)` per state, copied from `deployments()` | The same `SummaryMaintenanceLifecyclePlan` Planner selection would produce for that combination, or `SummaryMaintenanceLifecycleChoiceError` when a choice is unknown, missing, duplicated, rejected, schedule-incompatible, or not completely estimable | Inspect `deployments`, their selected lifecycle/alternatives/rejections, `selected_raw_recompute`, and optional summary/raw costs. Success of a function @@ -644,6 +646,52 @@ lifecycle analysis after structural selection can evaluate the selected root, but does not make the earlier selection lifecycle-optimal. An application may consume ranked candidates and perform this comparison downstream instead. +A deployment that prices lifecycles itself calls +`enumerate_summary_maintenance_lifecycles`, prices the alternatives, and binds +its choice with `select`. A choice is accepted only if Planner could select it: +an alternative with `MissingCostEvidence` is accepted only when the cost model's +complete-candidate hook covers lifecycle costs. Window frameworks and totals come +from that hook, as in Planner selection. + +A lifecycle choice then fixes each physical placement through timing: a +continuously maintained state and its inputs run at ingestion time, while an +ephemeral one stays at query time. Compile each query's `PostAsapDag` once and +cut every chosen assignment from that result: + +```rust +use asap_physical_operators::physical_planner::{ + compile, cut_candidate, frontier_from_timing, +}; + +let compiled = compile(&dag, inputs, &roots)?; // each node lowered once +for plan in lifecycle_plans { + let frontier = frontier_from_timing(&plan.execution_timed_dag()?)?; + // Precompute/query DAGs split at `frontier`; no logical lowering. + let candidate = cut_candidate(&compiled, &frontier)?; + // Check feasibility and price `candidate`; bind the selected one as is. +} +``` + +The frontier is the set of ingestion-time nodes read by query-time nodes (or an +ingestion-time root). `frontier_from_timing` rejects a query-time node feeding +an ingestion-time node. `cut_candidate` returns exactly what +`compile_candidate(&dag, inputs, &roots, &frontier)` returns and rejects the +same invalid frontiers. If the DAG has an ingestion-time `Binary`, compile with +the same timing for that node, because it lowers differently. Temporal pane +candidates are a different lowering and still use +`compile_temporal_pane_candidate`. + +Retained states are `SummaryAgg` nodes and `MaintainPopulation` nodes that do +not feed a `SummaryAgg`; a population that does feed one is part of that +state's input. The lifecycle cost hooks (`summary_maintenance_capabilities`, +`summary_maintenance_lifecycle_cost_inputs_for_horizon`) and the complete-candidate +hook therefore also receive `MaintainPopulation` nodes. A model that does not +recognize one should return unknown costs, which keep its alternatives +unselected; a model that prices every node uniformly now also prices +populations, so population candidates can win lifecycle-aware selection. `SummaryMaintenanceLifecyclePlan::execution_timed_dag` times a +population as it times a summary state: retained at ingestion, `Ephemeral` at +query time from the raw source. + ## Optional whole-plan selection and DAG assembly ### What does global selection mean?