From 631ecb8ee589792cff4c654f105fdde0f039da5f Mon Sep 17 00:00:00 2001 From: Ky Le Date: Mon, 16 Nov 2026 23:52:35 +0700 Subject: [PATCH 1/3] fix(metrics): preserve nested aggregate metric formulas Signed-off-by: Ky Le --- sqlmesh/core/metric/definition.py | 90 ++++++++++++++++++++----- tests/core/metric/test_rewriter.py | 103 ++++++++++++++++++++++++++++- 2 files changed, 176 insertions(+), 17 deletions(-) diff --git a/sqlmesh/core/metric/definition.py b/sqlmesh/core/metric/definition.py index fc523298c8..593ad7aef3 100644 --- a/sqlmesh/core/metric/definition.py +++ b/sqlmesh/core/metric/definition.py @@ -141,25 +141,55 @@ def to_metric( class Metric(MetricMeta, frozen=True): expanded: exp.Expr + def _aggregate_output_names(self) -> t.List[str]: + """Returns the output column name for each aggregate in the expanded metric.""" + nested_aggregate_names: t.Dict[exp.AggFunc, str] = {} + output_names = [] + + for agg in self.expanded.find_all(exp.AggFunc): + if isinstance(agg.parent, exp.Alias): + output_names.append(agg.parent.alias) + continue + + if agg not in nested_aggregate_names: + nested_aggregate_names[agg] = ( + f"__sqlmesh_metric_{self.name.replace('.', '_')}__agg_" + f"{len(nested_aggregate_names)}" + ) + output_names.append(nested_aggregate_names[agg]) + + return output_names + @property def aggs(self) -> t.Dict[exp.AggFunc, MeasureAndDimTables]: """Returns a dictionary of aggregation to referenced tables. This method removes catalog and schema information from columns. """ - return { - t.cast( - exp.AggFunc, - t.cast(exp.Expr, agg.parent).transform( - lambda node: ( - exp.column(node.this, table=remove_namespace(node)) - if isinstance(node, exp.Column) and node.table - else node - ) - ), - ): _get_measure_and_dim_tables(agg) - for agg in self.expanded.find_all(exp.AggFunc) - } + aggs: t.Dict[exp.AggFunc, MeasureAndDimTables] = {} + + for agg, output_name in zip( + self.expanded.find_all(exp.AggFunc), self._aggregate_output_names() + ): + projection = ( + agg.parent + if isinstance(agg.parent, exp.Alias) + else exp.alias_(agg.copy(), output_name) + ) + aggs[ + t.cast( + exp.AggFunc, + projection.transform( + lambda node: ( + exp.column(node.this, table=remove_namespace(node)) + if isinstance(node, exp.Column) and node.table + else node + ) + ), + ) + ] = _get_measure_and_dim_tables(agg) + + return aggs @property def formula(self) -> exp.Expr: @@ -168,10 +198,38 @@ def formula(self) -> exp.Expr: For simple metrics it is just the metric name. For derived metrics, it consists of the operations of the derived metrics without aggregations. """ + formula = ( + self.expanded.this.copy() + if isinstance(self.expanded, exp.Alias) + else self.expanded.copy() + ) + formula_aggregates = list(formula.find_all(exp.AggFunc)) + aggregate_output_names = { + id(agg): output_name + for agg, output_name in zip(formula_aggregates, self._aggregate_output_names()) + } + + def _replace_metric_alias(node: exp.Expr) -> exp.Expr: + if not isinstance(node, exp.Alias): + return node + + if isinstance(node.this, exp.AggFunc): + return exp.Column(this=exp.to_identifier(node.alias)) + if node.this.find(exp.AggFunc): + return node.this + return exp.Column(this=exp.to_identifier(node.alias)) + + formula = formula.transform(_replace_metric_alias, copy=False) + + def _replace_aggregate(node: exp.Expr) -> exp.Expr: + if not isinstance(node, exp.AggFunc): + return node + output_name = aggregate_output_names[id(node)] + return exp.Column(this=exp.to_identifier(output_name)) + + formula = formula.transform(_replace_aggregate, copy=False) return exp.alias_( - self.expanded.transform( - lambda node: exp.column(node.args["alias"]) if isinstance(node, exp.Alias) else node - ), + formula, self.name, copy=False, ) diff --git a/tests/core/metric/test_rewriter.py b/tests/core/metric/test_rewriter.py index d2ce821c2d..c918637f67 100644 --- a/tests/core/metric/test_rewriter.py +++ b/tests/core/metric/test_rewriter.py @@ -1,7 +1,8 @@ -from sqlglot import parse_one +from sqlglot import exp, parse_one from sqlmesh.core import dialect as d from sqlmesh.core.metric import expand_metrics, load_metric_ddl, rewrite +from sqlmesh.core.model import load_sql_based_model from sqlmesh.core.reference import ReferenceGraph from sqlmesh.utils import UniqueKeyDict @@ -248,3 +249,103 @@ def test_rewrite_order_by_metric(sushi_context_pre_scheduling): assert "ARRAY_AGG(sushi__orders.amount ORDER BY sushi__orders.amount" in sql assert "sushi.orders AS sushi__orders" in sql parse_one(sql, dialect=context.config.dialect) + + +def test_rewrite_nested_metric_aggregates(): + dialect = "bigquery" + model = load_sql_based_model( + d.parse( + "MODEL (name sushi.orders, kind FULL); SELECT 1 AS customer_id, 10 AS amount", + default_dialect=dialect, + ), + dialect=dialect, + ) + metric_statements = d.parse( + """ + METRIC (name total_amount, expression SUM(sushi.orders.amount)); + METRIC (name derived_amount, expression total_amount + 1); + METRIC (name median_amount, expression APPROX_QUANTILES(sushi.orders.amount, 2)[OFFSET(1)]); + METRIC (name safe_median_amount, expression COALESCE(CAST(APPROX_QUANTILES(sushi.orders.amount, 2)[OFFSET(1)] AS FLOAT64), 0)); + METRIC (name scaled_median, expression median_amount * 2); + METRIC (name safe_median_plus_one, expression safe_median_amount + 1); + METRIC (name combined_amount, expression SUM(sushi.orders.amount) + MAX(sushi.orders.amount)); + METRIC (name reused_amount, expression SUM(sushi.orders.amount) / NULLIF(SUM(sushi.orders.amount), 0)); + """, + default_dialect=dialect, + ) + metas = UniqueKeyDict( + "metas", + { + meta.name: meta + for meta in (load_metric_ddl(stmt, dialect=dialect) for stmt in metric_statements) + }, + ) + metrics = expand_metrics(metas) + graph = ReferenceGraph([model]) + + query = rewrite( + """ + SELECT + customer_id, + METRIC(median_amount), + METRIC(safe_median_amount), + METRIC(scaled_median), + METRIC(safe_median_plus_one), + METRIC(combined_amount), + METRIC(reused_amount), + METRIC(total_amount), + METRIC(derived_amount) + FROM __semantic.__table + GROUP BY customer_id + """, + graph=graph, + metrics=metrics, + dialect=dialect, + ) + + outer_projections = { + projection.alias_or_name: ( + projection.this if isinstance(projection, exp.Alias) else projection + ) + for projection in query.selects + } + assert isinstance(outer_projections["median_amount"], exp.Bracket) + safe_median = outer_projections["safe_median_amount"] + assert isinstance(safe_median, exp.Coalesce) + assert safe_median.find(exp.Cast) is not None + assert safe_median.find(exp.Bracket) is not None + + scaled_median = outer_projections["scaled_median"] + assert isinstance(scaled_median, exp.Mul) + assert isinstance(scaled_median.this, exp.Bracket) + safe_median_plus_one = outer_projections["safe_median_plus_one"] + assert isinstance(safe_median_plus_one, exp.Add) + assert isinstance(safe_median_plus_one.this, exp.Coalesce) + assert safe_median_plus_one.find(exp.Cast) is not None + + median_aggregate_ref = outer_projections["median_amount"].this + safe_median_aggregate_ref = outer_projections["safe_median_amount"].find(exp.Bracket).this + assert median_aggregate_ref.name != safe_median_aggregate_ref.name + + combined = outer_projections["combined_amount"] + assert isinstance(combined, exp.Add) + assert not list(combined.find_all(exp.AggFunc)) + reused = outer_projections["reused_amount"] + reused_aggregate_refs = list(reused.find_all(exp.Column)) + assert len(reused_aggregate_refs) == 2 + assert reused_aggregate_refs[0].name == reused_aggregate_refs[1].name + + # The aggregate expressions are projected once, with aliases that the outer + # metric formulas can reference after applying their wrappers. + source = query.args["from_"].this + inner = source.this if isinstance(source, exp.Subquery) else None + assert isinstance(inner, exp.Select) + aggregate_projections = [ + projection for projection in inner.selects if projection.find(exp.AggFunc) + ] + assert len(aggregate_projections) >= 3 + assert all(isinstance(projection, exp.Alias) for projection in aggregate_projections) + + # Existing simple and derived metric behavior remains intact. + assert isinstance(outer_projections["total_amount"], exp.Column) + assert isinstance(outer_projections["derived_amount"], exp.Add) From 759fb30da813f133f54a210029d7a1fceda17729 Mon Sep 17 00:00:00 2001 From: Ky Le Date: Tue, 17 Nov 2026 00:12:00 +0700 Subject: [PATCH 2/3] fix(review): avoid metric aggregate alias collisions Signed-off-by: Ky Le --- sqlmesh/core/metric/definition.py | 40 +++++++++++++++++++++++------- sqlmesh/core/metric/rewriter.py | 21 +++++++++++++--- tests/core/metric/test_rewriter.py | 12 ++++++++- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/sqlmesh/core/metric/definition.py b/sqlmesh/core/metric/definition.py index 593ad7aef3..18d8fda35b 100644 --- a/sqlmesh/core/metric/definition.py +++ b/sqlmesh/core/metric/definition.py @@ -141,21 +141,37 @@ def to_metric( class Metric(MetricMeta, frozen=True): expanded: exp.Expr - def _aggregate_output_names(self) -> t.List[str]: + def _aggregate_output_names( + self, + aggregates: t.Optional[t.List[exp.AggFunc]] = None, + used_names: t.Optional[t.Set[str]] = None, + ) -> t.List[str]: """Returns the output column name for each aggregate in the expanded metric.""" nested_aggregate_names: t.Dict[exp.AggFunc, str] = {} output_names = [] + used_names = used_names if used_names is not None else set() - for agg in self.expanded.find_all(exp.AggFunc): + if aggregates is None: + aggregates = list(self.expanded.find_all(exp.AggFunc)) + + for agg in aggregates: if isinstance(agg.parent, exp.Alias): output_names.append(agg.parent.alias) + used_names.add(agg.parent.alias) continue if agg not in nested_aggregate_names: - nested_aggregate_names[agg] = ( + base_name = ( f"__sqlmesh_metric_{self.name.replace('.', '_')}__agg_" f"{len(nested_aggregate_names)}" ) + output_name = base_name + suffix = 1 + while output_name in used_names: + output_name = f"{base_name}_{suffix}" + suffix += 1 + nested_aggregate_names[agg] = output_name + used_names.add(output_name) output_names.append(nested_aggregate_names[agg]) return output_names @@ -166,11 +182,15 @@ def aggs(self) -> t.Dict[exp.AggFunc, MeasureAndDimTables]: This method removes catalog and schema information from columns. """ - aggs: t.Dict[exp.AggFunc, MeasureAndDimTables] = {} + aggregates = list(self.expanded.find_all(exp.AggFunc)) + return self._aggs_with_output_names(self._aggregate_output_names(aggregates)) - for agg, output_name in zip( - self.expanded.find_all(exp.AggFunc), self._aggregate_output_names() - ): + def _aggs_with_output_names( + self, output_names: t.List[str] + ) -> t.Dict[exp.AggFunc, MeasureAndDimTables]: + aggs: t.Dict[exp.AggFunc, MeasureAndDimTables] = {} + aggregates = list(self.expanded.find_all(exp.AggFunc)) + for agg, output_name in zip(aggregates, output_names): projection = ( agg.parent if isinstance(agg.parent, exp.Alias) @@ -198,6 +218,9 @@ def formula(self) -> exp.Expr: For simple metrics it is just the metric name. For derived metrics, it consists of the operations of the derived metrics without aggregations. """ + return self._formula_with_output_names(self._aggregate_output_names()) + + def _formula_with_output_names(self, output_names: t.List[str]) -> exp.Expr: formula = ( self.expanded.this.copy() if isinstance(self.expanded, exp.Alias) @@ -205,8 +228,7 @@ def formula(self) -> exp.Expr: ) formula_aggregates = list(formula.find_all(exp.AggFunc)) aggregate_output_names = { - id(agg): output_name - for agg, output_name in zip(formula_aggregates, self._aggregate_output_names()) + id(agg): output_name for agg, output_name in zip(formula_aggregates, output_names) } def _replace_metric_alias(node: exp.Expr) -> exp.Expr: diff --git a/sqlmesh/core/metric/rewriter.py b/sqlmesh/core/metric/rewriter.py index 6c9ec429a8..b6f131df68 100644 --- a/sqlmesh/core/metric/rewriter.py +++ b/sqlmesh/core/metric/rewriter.py @@ -42,13 +42,28 @@ def rewrite(self, expression: exp.Expr) -> exp.Expr: def _build_sources(self, projections: t.List[exp.Expr]) -> SourceAggsAndJoins: sources: SourceAggsAndJoins = {} + used_names = {metric.name for metric in self.metrics.values()} + used_names.update( + projection.alias for projection in projections if isinstance(projection, exp.Alias) + ) + for metric in self.metrics.values(): + used_names.update( + node.alias for node in metric.expanded.find_all(exp.Alias) if node.alias + ) + + output_names_by_metric: t.Dict[str, t.List[str]] = {} for projection in projections: for ref in find_all_in_scope(projection, d.MetricAgg): metric = self.metrics[ref.this.name] - ref.replace(metric.formula.this) - - for agg, (measure, dims) in metric.aggs.items(): + output_names = output_names_by_metric.get(metric.name) + if output_names is None: + aggregates = list(metric.expanded.find_all(exp.AggFunc)) + output_names = metric._aggregate_output_names(aggregates, used_names) + output_names_by_metric[metric.name] = output_names + ref.replace(metric._formula_with_output_names(output_names).this) + + for agg, (measure, dims) in metric._aggs_with_output_names(output_names).items(): aggs, joins = sources.setdefault(measure, (set(), dict())) aggs.add(agg) for dim in dims: diff --git a/tests/core/metric/test_rewriter.py b/tests/core/metric/test_rewriter.py index c918637f67..be3f3fb5d7 100644 --- a/tests/core/metric/test_rewriter.py +++ b/tests/core/metric/test_rewriter.py @@ -255,7 +255,7 @@ def test_rewrite_nested_metric_aggregates(): dialect = "bigquery" model = load_sql_based_model( d.parse( - "MODEL (name sushi.orders, kind FULL); SELECT 1 AS customer_id, 10 AS amount", + "MODEL (name sushi.orders, kind FULL); SELECT 1 AS customer_id, 10 AS amount, 100 AS other_amount", default_dialect=dialect, ), dialect=dialect, @@ -270,6 +270,7 @@ def test_rewrite_nested_metric_aggregates(): METRIC (name safe_median_plus_one, expression safe_median_amount + 1); METRIC (name combined_amount, expression SUM(sushi.orders.amount) + MAX(sushi.orders.amount)); METRIC (name reused_amount, expression SUM(sushi.orders.amount) / NULLIF(SUM(sushi.orders.amount), 0)); + METRIC (name __sqlmesh_metric_safe_median_amount__agg_0, expression SUM(sushi.orders.other_amount)); """, default_dialect=dialect, ) @@ -293,6 +294,7 @@ def test_rewrite_nested_metric_aggregates(): METRIC(safe_median_plus_one), METRIC(combined_amount), METRIC(reused_amount), + METRIC(__sqlmesh_metric_safe_median_amount__agg_0), METRIC(total_amount), METRIC(derived_amount) FROM __semantic.__table @@ -349,3 +351,11 @@ def test_rewrite_nested_metric_aggregates(): # Existing simple and derived metric behavior remains intact. assert isinstance(outer_projections["total_amount"], exp.Column) assert isinstance(outer_projections["derived_amount"], exp.Add) + + # A public metric name can match the generated alias for an aggregate + # nested under a wrapper. The source query must allocate distinct aliases. + colliding_metric = outer_projections["__sqlmesh_metric_safe_median_amount__agg_0"] + assert isinstance(colliding_metric, exp.Column) + assert colliding_metric.name != safe_median_aggregate_ref.name + projected_aliases = [projection.alias for projection in aggregate_projections] + assert len(projected_aliases) == len(set(projected_aliases)) From 129b4732cdb54d08a9af801d0c63372b46d4da40 Mon Sep 17 00:00:00 2001 From: Ky Le Date: Tue, 17 Nov 2026 00:15:31 +0700 Subject: [PATCH 3/3] fix(review): reserve metric grain aliases Signed-off-by: Ky Le --- sqlmesh/core/metric/rewriter.py | 18 ++++++++----- tests/core/metric/test_rewriter.py | 41 +++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/sqlmesh/core/metric/rewriter.py b/sqlmesh/core/metric/rewriter.py index b6f131df68..1c605dbce7 100644 --- a/sqlmesh/core/metric/rewriter.py +++ b/sqlmesh/core/metric/rewriter.py @@ -40,11 +40,18 @@ def rewrite(self, expression: exp.Expr) -> exp.Expr: return expression - def _build_sources(self, projections: t.List[exp.Expr]) -> SourceAggsAndJoins: + def _build_sources( + self, projections: t.List[exp.Expr], group_by: t.List[exp.Expr] + ) -> SourceAggsAndJoins: sources: SourceAggsAndJoins = {} used_names = {metric.name for metric in self.metrics.values()} used_names.update( - projection.alias for projection in projections if isinstance(projection, exp.Alias) + projection.alias if isinstance(projection, exp.Alias) else projection.output_name + for projection in projections + if projection.alias_or_name + ) + used_names.update( + expression.output_name for expression in group_by if expression.output_name ) for metric in self.metrics.values(): used_names.update( @@ -75,14 +82,13 @@ def _expand(self, select: exp.Select) -> None: base = select.args["from_"].this.find(exp.Table) base_alias = base.alias_or_name base_name = exp.table_name(base) + group = select.args.pop("group", None) + group_by = group.expressions if group else [] sources: SourceAggsAndJoins = ( {} if base_name == self.semantic_name else {base_name: (set(), {})} ) - sources.update(self._build_sources(select.selects)) - - group = select.args.pop("group", None) - group_by = group.expressions if group else [] + sources.update(self._build_sources(select.selects, group_by)) mapping = { remove_namespace(exp.table_name(source.assert_is(exp.Table))): name diff --git a/tests/core/metric/test_rewriter.py b/tests/core/metric/test_rewriter.py index be3f3fb5d7..c8cb7e38c2 100644 --- a/tests/core/metric/test_rewriter.py +++ b/tests/core/metric/test_rewriter.py @@ -255,7 +255,7 @@ def test_rewrite_nested_metric_aggregates(): dialect = "bigquery" model = load_sql_based_model( d.parse( - "MODEL (name sushi.orders, kind FULL); SELECT 1 AS customer_id, 10 AS amount, 100 AS other_amount", + "MODEL (name sushi.orders, kind FULL); SELECT 1 AS customer_id, 10 AS amount, 100 AS other_amount, 1000 AS __sqlmesh_metric_median_amount__agg_0", default_dialect=dialect, ), dialect=dialect, @@ -359,3 +359,42 @@ def test_rewrite_nested_metric_aggregates(): assert colliding_metric.name != safe_median_aggregate_ref.name projected_aliases = [projection.alias for projection in aggregate_projections] assert len(projected_aliases) == len(set(projected_aliases)) + + dimension_metrics = {"median_amount": metrics["median_amount"]} + selected_dimension_query = rewrite( + """ + SELECT __sqlmesh_metric_median_amount__agg_0, METRIC(median_amount) + FROM __semantic.__table + GROUP BY __sqlmesh_metric_median_amount__agg_0 + """, + graph=graph, + metrics=dimension_metrics, + dialect=dialect, + ) + selected_dimension_source = selected_dimension_query.args["from_"].this + selected_dimension_inner = ( + selected_dimension_source.this + if isinstance(selected_dimension_source, exp.Subquery) + else None + ) + assert isinstance(selected_dimension_inner, exp.Select) + selected_dimension_names = [ + projection.output_name for projection in selected_dimension_inner.selects + ] + assert len(selected_dimension_names) == len(set(selected_dimension_names)) + + dimension_collision_query = rewrite( + """ + SELECT METRIC(median_amount) + FROM __semantic.__table + GROUP BY __sqlmesh_metric_median_amount__agg_0 + """, + graph=graph, + metrics=dimension_metrics, + dialect=dialect, + ) + dimension_source = dimension_collision_query.args["from_"].this + dimension_inner = dimension_source.this if isinstance(dimension_source, exp.Subquery) else None + assert isinstance(dimension_inner, exp.Select) + dimension_projection_names = [projection.output_name for projection in dimension_inner.selects] + assert len(dimension_projection_names) == len(set(dimension_projection_names))