sqlglot.optimizer.annotate_types

  1from __future__ import annotations
  2
  3import functools
  4import typing as t
  5
  6from sqlglot import exp
  7from sqlglot.helper import (
  8    ensure_list,
  9    is_date_unit,
 10    is_iso_date,
 11    is_iso_datetime,
 12    seq_get,
 13)
 14from sqlglot.optimizer.scope import Scope, traverse_scope
 15from sqlglot.schema import MappingSchema, Schema, ensure_schema
 16from sqlglot.dialects.dialect import Dialect
 17
 18if t.TYPE_CHECKING:
 19    from sqlglot._typing import B, E
 20
 21    BinaryCoercionFunc = t.Callable[[exp.Expression, exp.Expression], exp.DataType.Type]
 22    BinaryCoercions = t.Dict[
 23        t.Tuple[exp.DataType.Type, exp.DataType.Type],
 24        BinaryCoercionFunc,
 25    ]
 26
 27    from sqlglot.dialects.dialect import DialectType, AnnotatorsType
 28
 29
 30def annotate_types(
 31    expression: E,
 32    schema: t.Optional[t.Dict | Schema] = None,
 33    annotators: t.Optional[AnnotatorsType] = None,
 34    coerces_to: t.Optional[t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]]] = None,
 35    dialect: DialectType = None,
 36) -> E:
 37    """
 38    Infers the types of an expression, annotating its AST accordingly.
 39
 40    Example:
 41        >>> import sqlglot
 42        >>> schema = {"y": {"cola": "SMALLINT"}}
 43        >>> sql = "SELECT x.cola + 2.5 AS cola FROM (SELECT y.cola AS cola FROM y AS y) AS x"
 44        >>> annotated_expr = annotate_types(sqlglot.parse_one(sql), schema=schema)
 45        >>> annotated_expr.expressions[0].type.this  # Get the type of "x.cola + 2.5 AS cola"
 46        <Type.DOUBLE: 'DOUBLE'>
 47
 48    Args:
 49        expression: Expression to annotate.
 50        schema: Database schema.
 51        annotators: Maps expression type to corresponding annotation function.
 52        coerces_to: Maps expression type to set of types that it can be coerced into.
 53
 54    Returns:
 55        The expression annotated with types.
 56    """
 57
 58    schema = ensure_schema(schema, dialect=dialect)
 59
 60    return TypeAnnotator(schema, annotators, coerces_to).annotate(expression)
 61
 62
 63def _coerce_date_literal(l: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.DataType.Type:
 64    date_text = l.name
 65    is_iso_date_ = is_iso_date(date_text)
 66
 67    if is_iso_date_ and is_date_unit(unit):
 68        return exp.DataType.Type.DATE
 69
 70    # An ISO date is also an ISO datetime, but not vice versa
 71    if is_iso_date_ or is_iso_datetime(date_text):
 72        return exp.DataType.Type.DATETIME
 73
 74    return exp.DataType.Type.UNKNOWN
 75
 76
 77def _coerce_date(l: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.DataType.Type:
 78    if not is_date_unit(unit):
 79        return exp.DataType.Type.DATETIME
 80    return l.type.this if l.type else exp.DataType.Type.UNKNOWN
 81
 82
 83def swap_args(func: BinaryCoercionFunc) -> BinaryCoercionFunc:
 84    @functools.wraps(func)
 85    def _swapped(l: exp.Expression, r: exp.Expression) -> exp.DataType.Type:
 86        return func(r, l)
 87
 88    return _swapped
 89
 90
 91def swap_all(coercions: BinaryCoercions) -> BinaryCoercions:
 92    return {**coercions, **{(b, a): swap_args(func) for (a, b), func in coercions.items()}}
 93
 94
 95class _TypeAnnotator(type):
 96    def __new__(cls, clsname, bases, attrs):
 97        klass = super().__new__(cls, clsname, bases, attrs)
 98
 99        # Highest-to-lowest type precedence, as specified in Spark's docs (ANSI):
100        # https://spark.apache.org/docs/3.2.0/sql-ref-ansi-compliance.html
101        text_precedence = (
102            exp.DataType.Type.TEXT,
103            exp.DataType.Type.NVARCHAR,
104            exp.DataType.Type.VARCHAR,
105            exp.DataType.Type.NCHAR,
106            exp.DataType.Type.CHAR,
107        )
108        numeric_precedence = (
109            exp.DataType.Type.DOUBLE,
110            exp.DataType.Type.FLOAT,
111            exp.DataType.Type.DECIMAL,
112            exp.DataType.Type.BIGINT,
113            exp.DataType.Type.INT,
114            exp.DataType.Type.SMALLINT,
115            exp.DataType.Type.TINYINT,
116        )
117        timelike_precedence = (
118            exp.DataType.Type.TIMESTAMPLTZ,
119            exp.DataType.Type.TIMESTAMPTZ,
120            exp.DataType.Type.TIMESTAMP,
121            exp.DataType.Type.DATETIME,
122            exp.DataType.Type.DATE,
123        )
124
125        for type_precedence in (text_precedence, numeric_precedence, timelike_precedence):
126            coerces_to = set()
127            for data_type in type_precedence:
128                klass.COERCES_TO[data_type] = coerces_to.copy()
129                coerces_to |= {data_type}
130
131        # NULL can be coerced to any type, so e.g. NULL + 1 will have type INT
132        klass.COERCES_TO[exp.DataType.Type.NULL] = {
133            *text_precedence,
134            *numeric_precedence,
135            *timelike_precedence,
136        }
137
138        return klass
139
140
141class TypeAnnotator(metaclass=_TypeAnnotator):
142    NESTED_TYPES = {
143        exp.DataType.Type.ARRAY,
144    }
145
146    # Specifies what types a given type can be coerced into (autofilled)
147    COERCES_TO: t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]] = {}
148
149    # Coercion functions for binary operations.
150    # Map of type pairs to a callable that takes both sides of the binary operation and returns the resulting type.
151    BINARY_COERCIONS: BinaryCoercions = {
152        **swap_all(
153            {
154                (t, exp.DataType.Type.INTERVAL): lambda l, r: _coerce_date_literal(
155                    l, r.args.get("unit")
156                )
157                for t in exp.DataType.TEXT_TYPES
158            }
159        ),
160        **swap_all(
161            {
162                # text + numeric will yield the numeric type to match most dialects' semantics
163                (text, numeric): lambda l, r: t.cast(
164                    exp.DataType.Type, l.type if l.type in exp.DataType.NUMERIC_TYPES else r.type
165                )
166                for text in exp.DataType.TEXT_TYPES
167                for numeric in exp.DataType.NUMERIC_TYPES
168            }
169        ),
170        **swap_all(
171            {
172                (exp.DataType.Type.DATE, exp.DataType.Type.INTERVAL): lambda l, r: _coerce_date(
173                    l, r.args.get("unit")
174                ),
175            }
176        ),
177    }
178
179    def __init__(
180        self,
181        schema: Schema,
182        annotators: t.Optional[AnnotatorsType] = None,
183        coerces_to: t.Optional[t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]]] = None,
184        binary_coercions: t.Optional[BinaryCoercions] = None,
185    ) -> None:
186        self.schema = schema
187        self.annotators = annotators or Dialect.get_or_raise(schema.dialect).ANNOTATORS
188        self.coerces_to = (
189            coerces_to or Dialect.get_or_raise(schema.dialect).COERCES_TO or self.COERCES_TO
190        )
191        self.binary_coercions = binary_coercions or self.BINARY_COERCIONS
192
193        # Caches the ids of annotated sub-Expressions, to ensure we only visit them once
194        self._visited: t.Set[int] = set()
195
196        # Maps an exp.SetOperation's id (e.g. UNION) to its projection types. This is computed if the
197        # exp.SetOperation is the expression of a scope source, as selecting from it multiple times
198        # would reprocess the entire subtree to coerce the types of its operands' projections
199        self._setop_column_types: t.Dict[int, t.Dict[str, exp.DataType | exp.DataType.Type]] = {}
200
201    def _set_type(
202        self, expression: exp.Expression, target_type: t.Optional[exp.DataType | exp.DataType.Type]
203    ) -> None:
204        expression.type = target_type or exp.DataType.Type.UNKNOWN  # type: ignore
205        self._visited.add(id(expression))
206
207    def annotate(self, expression: E) -> E:
208        for scope in traverse_scope(expression):
209            self.annotate_scope(scope)
210        return self._maybe_annotate(expression)  # This takes care of non-traversable expressions
211
212    def annotate_scope(self, scope: Scope) -> None:
213        selects = {}
214        for name, source in scope.sources.items():
215            if not isinstance(source, Scope):
216                continue
217
218            expression = source.expression
219            if isinstance(expression, exp.UDTF):
220                values = []
221
222                if isinstance(expression, exp.Lateral):
223                    if isinstance(expression.this, exp.Explode):
224                        values = [expression.this.this]
225                elif isinstance(expression, exp.Unnest):
226                    values = [expression]
227                elif not isinstance(expression, exp.TableFromRows):
228                    values = expression.expressions[0].expressions
229
230                if not values:
231                    continue
232
233                selects[name] = {
234                    alias: column.type
235                    for alias, column in zip(expression.alias_column_names, values)
236                }
237            elif isinstance(expression, exp.SetOperation) and len(expression.left.selects) == len(
238                expression.right.selects
239            ):
240                selects[name] = col_types = self._setop_column_types.setdefault(id(expression), {})
241
242                if not col_types:
243                    # Process a chain / sub-tree of set operations
244                    for set_op in expression.walk(
245                        prune=lambda n: not isinstance(n, (exp.SetOperation, exp.Subquery))
246                    ):
247                        if not isinstance(set_op, exp.SetOperation):
248                            continue
249
250                        if set_op.args.get("by_name"):
251                            r_type_by_select = {
252                                s.alias_or_name: s.type for s in set_op.right.selects
253                            }
254                            setop_cols = {
255                                s.alias_or_name: self._maybe_coerce(
256                                    t.cast(exp.DataType, s.type),
257                                    r_type_by_select.get(s.alias_or_name)
258                                    or exp.DataType.Type.UNKNOWN,
259                                )
260                                for s in set_op.left.selects
261                            }
262                        else:
263                            setop_cols = {
264                                ls.alias_or_name: self._maybe_coerce(
265                                    t.cast(exp.DataType, ls.type), t.cast(exp.DataType, rs.type)
266                                )
267                                for ls, rs in zip(set_op.left.selects, set_op.right.selects)
268                            }
269
270                        # Coerce intermediate results with the previously registered types, if they exist
271                        for col_name, col_type in setop_cols.items():
272                            col_types[col_name] = self._maybe_coerce(
273                                col_type, col_types.get(col_name, exp.DataType.Type.NULL)
274                            )
275
276            else:
277                selects[name] = {s.alias_or_name: s.type for s in expression.selects}
278
279        # First annotate the current scope's column references
280        for col in scope.columns:
281            if not col.table:
282                continue
283
284            source = scope.sources.get(col.table)
285            if isinstance(source, exp.Table):
286                self._set_type(col, self.schema.get_column_type(source, col))
287            elif source:
288                if col.table in selects and col.name in selects[col.table]:
289                    self._set_type(col, selects[col.table][col.name])
290                elif isinstance(source.expression, exp.Unnest):
291                    self._set_type(col, source.expression.type)
292
293        if isinstance(self.schema, MappingSchema):
294            for table_column in scope.table_columns:
295                source = scope.sources.get(table_column.name)
296
297                if isinstance(source, exp.Table):
298                    schema = self.schema.find(
299                        source, raise_on_missing=False, ensure_data_types=True
300                    )
301                    if not isinstance(schema, dict):
302                        continue
303
304                    struct_type = exp.DataType(
305                        this=exp.DataType.Type.STRUCT,
306                        expressions=[
307                            exp.ColumnDef(this=exp.to_identifier(c), kind=kind)
308                            for c, kind in schema.items()
309                        ],
310                        nested=True,
311                    )
312                    self._set_type(table_column, struct_type)
313                elif (
314                    isinstance(source, Scope)
315                    and isinstance(source.expression, exp.Query)
316                    and (
317                        source.expression.meta.get("query_type") or exp.DataType.build("UNKNOWN")
318                    ).is_type(exp.DataType.Type.STRUCT)
319                ):
320                    self._set_type(table_column, source.expression.meta["query_type"])
321
322        # Then (possibly) annotate the remaining expressions in the scope
323        self._maybe_annotate(scope.expression)
324
325        if self.schema.dialect == "bigquery" and isinstance(scope.expression, exp.Query):
326            struct_type = exp.DataType(
327                this=exp.DataType.Type.STRUCT,
328                expressions=[
329                    exp.ColumnDef(
330                        this=exp.to_identifier(select.output_name),
331                        kind=select.type.copy() if select.type else None,
332                    )
333                    for select in scope.expression.selects
334                ],
335                nested=True,
336            )
337
338            if not any(
339                cd.kind.is_type(exp.DataType.Type.UNKNOWN)
340                for cd in struct_type.expressions
341                if cd.kind
342            ):
343                # We don't use `_set_type` on purpose here. If we annotated the query directly, then
344                # using it in other contexts (e.g., ARRAY(<query>)) could result in incorrect type
345                # annotations, i.e., it shouldn't be interpreted as a STRUCT value.
346                scope.expression.meta["query_type"] = struct_type
347
348    def _maybe_annotate(self, expression: E) -> E:
349        if id(expression) in self._visited:
350            return expression  # We've already inferred the expression's type
351
352        annotator = self.annotators.get(expression.__class__)
353
354        return (
355            annotator(self, expression)
356            if annotator
357            else self._annotate_with_type(expression, exp.DataType.Type.UNKNOWN)
358        )
359
360    def _annotate_args(self, expression: E) -> E:
361        for value in expression.iter_expressions():
362            self._maybe_annotate(value)
363
364        return expression
365
366    def _maybe_coerce(
367        self,
368        type1: exp.DataType | exp.DataType.Type,
369        type2: exp.DataType | exp.DataType.Type,
370    ) -> exp.DataType | exp.DataType.Type:
371        """
372        Returns type2 if type1 can be coerced into it, otherwise type1.
373
374        If either type is parameterized (e.g. DECIMAL(18, 2) contains two parameters),
375        we assume type1 does not coerce into type2, so we also return it in this case.
376        """
377        if isinstance(type1, exp.DataType):
378            if type1.expressions:
379                return type1
380            type1_value = type1.this
381        else:
382            type1_value = type1
383
384        if isinstance(type2, exp.DataType):
385            if type2.expressions:
386                return type2
387            type2_value = type2.this
388        else:
389            type2_value = type2
390
391        # We propagate the UNKNOWN type upwards if found
392        if exp.DataType.Type.UNKNOWN in (type1_value, type2_value):
393            return exp.DataType.Type.UNKNOWN
394
395        return type2_value if type2_value in self.coerces_to.get(type1_value, {}) else type1_value
396
397    def _annotate_binary(self, expression: B) -> B:
398        self._annotate_args(expression)
399
400        left, right = expression.left, expression.right
401        left_type, right_type = left.type.this, right.type.this  # type: ignore
402
403        if isinstance(expression, (exp.Connector, exp.Predicate)):
404            self._set_type(expression, exp.DataType.Type.BOOLEAN)
405        elif (left_type, right_type) in self.binary_coercions:
406            self._set_type(expression, self.binary_coercions[(left_type, right_type)](left, right))
407        else:
408            self._set_type(expression, self._maybe_coerce(left_type, right_type))
409
410        return expression
411
412    def _annotate_unary(self, expression: E) -> E:
413        self._annotate_args(expression)
414
415        if isinstance(expression, exp.Not):
416            self._set_type(expression, exp.DataType.Type.BOOLEAN)
417        else:
418            self._set_type(expression, expression.this.type)
419
420        return expression
421
422    def _annotate_literal(self, expression: exp.Literal) -> exp.Literal:
423        if expression.is_string:
424            self._set_type(expression, exp.DataType.Type.VARCHAR)
425        elif expression.is_int:
426            self._set_type(expression, exp.DataType.Type.INT)
427        else:
428            self._set_type(expression, exp.DataType.Type.DOUBLE)
429
430        return expression
431
432    def _annotate_with_type(
433        self, expression: E, target_type: exp.DataType | exp.DataType.Type
434    ) -> E:
435        self._set_type(expression, target_type)
436        return self._annotate_args(expression)
437
438    @t.no_type_check
439    def _annotate_by_args(
440        self,
441        expression: E,
442        *args: str,
443        promote: bool = False,
444        array: bool = False,
445    ) -> E:
446        self._annotate_args(expression)
447
448        expressions: t.List[exp.Expression] = []
449        for arg in args:
450            arg_expr = expression.args.get(arg)
451            expressions.extend(expr for expr in ensure_list(arg_expr) if expr)
452
453        last_datatype = None
454        for expr in expressions:
455            expr_type = expr.type
456
457            # Stop at the first nested data type found - we don't want to _maybe_coerce nested types
458            if expr_type.args.get("nested"):
459                last_datatype = expr_type
460                break
461
462            if not expr_type.is_type(exp.DataType.Type.UNKNOWN):
463                last_datatype = self._maybe_coerce(last_datatype or expr_type, expr_type)
464
465        self._set_type(expression, last_datatype or exp.DataType.Type.UNKNOWN)
466
467        if promote:
468            if expression.type.this in exp.DataType.INTEGER_TYPES:
469                self._set_type(expression, exp.DataType.Type.BIGINT)
470            elif expression.type.this in exp.DataType.FLOAT_TYPES:
471                self._set_type(expression, exp.DataType.Type.DOUBLE)
472
473        if array:
474            self._set_type(
475                expression,
476                exp.DataType(
477                    this=exp.DataType.Type.ARRAY, expressions=[expression.type], nested=True
478                ),
479            )
480
481        return expression
482
483    def _annotate_timeunit(
484        self, expression: exp.TimeUnit | exp.DateTrunc
485    ) -> exp.TimeUnit | exp.DateTrunc:
486        self._annotate_args(expression)
487
488        if expression.this.type.this in exp.DataType.TEXT_TYPES:
489            datatype = _coerce_date_literal(expression.this, expression.unit)
490        elif expression.this.type.this in exp.DataType.TEMPORAL_TYPES:
491            datatype = _coerce_date(expression.this, expression.unit)
492        else:
493            datatype = exp.DataType.Type.UNKNOWN
494
495        self._set_type(expression, datatype)
496        return expression
497
498    def _annotate_bracket(self, expression: exp.Bracket) -> exp.Bracket:
499        self._annotate_args(expression)
500
501        bracket_arg = expression.expressions[0]
502        this = expression.this
503
504        if isinstance(bracket_arg, exp.Slice):
505            self._set_type(expression, this.type)
506        elif this.type.is_type(exp.DataType.Type.ARRAY):
507            self._set_type(expression, seq_get(this.type.expressions, 0))
508        elif isinstance(this, (exp.Map, exp.VarMap)) and bracket_arg in this.keys:
509            index = this.keys.index(bracket_arg)
510            value = seq_get(this.values, index)
511            self._set_type(expression, value.type if value else None)
512        else:
513            self._set_type(expression, exp.DataType.Type.UNKNOWN)
514
515        return expression
516
517    def _annotate_div(self, expression: exp.Div) -> exp.Div:
518        self._annotate_args(expression)
519
520        left_type, right_type = expression.left.type.this, expression.right.type.this  # type: ignore
521
522        if (
523            expression.args.get("typed")
524            and left_type in exp.DataType.INTEGER_TYPES
525            and right_type in exp.DataType.INTEGER_TYPES
526        ):
527            self._set_type(expression, exp.DataType.Type.BIGINT)
528        else:
529            self._set_type(expression, self._maybe_coerce(left_type, right_type))
530            if expression.type and expression.type.this not in exp.DataType.REAL_TYPES:
531                self._set_type(
532                    expression, self._maybe_coerce(expression.type, exp.DataType.Type.DOUBLE)
533                )
534
535        return expression
536
537    def _annotate_dot(self, expression: exp.Dot) -> exp.Dot:
538        self._annotate_args(expression)
539        self._set_type(expression, None)
540        this_type = expression.this.type
541
542        if this_type and this_type.is_type(exp.DataType.Type.STRUCT):
543            for e in this_type.expressions:
544                if e.name == expression.expression.name:
545                    self._set_type(expression, e.kind)
546                    break
547
548        return expression
549
550    def _annotate_explode(self, expression: exp.Explode) -> exp.Explode:
551        self._annotate_args(expression)
552        self._set_type(expression, seq_get(expression.this.type.expressions, 0))
553        return expression
554
555    def _annotate_unnest(self, expression: exp.Unnest) -> exp.Unnest:
556        self._annotate_args(expression)
557        child = seq_get(expression.expressions, 0)
558
559        if child and child.is_type(exp.DataType.Type.ARRAY):
560            expr_type = seq_get(child.type.expressions, 0)
561        else:
562            expr_type = None
563
564        self._set_type(expression, expr_type)
565        return expression
566
567    def _annotate_struct_value(
568        self, expression: exp.Expression
569    ) -> t.Optional[exp.DataType] | exp.ColumnDef:
570        alias = expression.args.get("alias")
571        if alias:
572            return exp.ColumnDef(this=alias.copy(), kind=expression.type)
573
574        # Case: key = value or key := value
575        if expression.expression:
576            return exp.ColumnDef(this=expression.this.copy(), kind=expression.expression.type)
577
578        return expression.type
579
580    def _annotate_struct(self, expression: exp.Struct) -> exp.Struct:
581        self._annotate_args(expression)
582        self._set_type(
583            expression,
584            exp.DataType(
585                this=exp.DataType.Type.STRUCT,
586                expressions=[self._annotate_struct_value(expr) for expr in expression.expressions],
587                nested=True,
588            ),
589        )
590        return expression
591
592    @t.overload
593    def _annotate_map(self, expression: exp.Map) -> exp.Map: ...
594
595    @t.overload
596    def _annotate_map(self, expression: exp.VarMap) -> exp.VarMap: ...
597
598    def _annotate_map(self, expression):
599        self._annotate_args(expression)
600
601        keys = expression.args.get("keys")
602        values = expression.args.get("values")
603
604        map_type = exp.DataType(this=exp.DataType.Type.MAP)
605        if isinstance(keys, exp.Array) and isinstance(values, exp.Array):
606            key_type = seq_get(keys.type.expressions, 0) or exp.DataType.Type.UNKNOWN
607            value_type = seq_get(values.type.expressions, 0) or exp.DataType.Type.UNKNOWN
608
609            if key_type != exp.DataType.Type.UNKNOWN and value_type != exp.DataType.Type.UNKNOWN:
610                map_type.set("expressions", [key_type, value_type])
611                map_type.set("nested", True)
612
613        self._set_type(expression, map_type)
614        return expression
615
616    def _annotate_to_map(self, expression: exp.ToMap) -> exp.ToMap:
617        self._annotate_args(expression)
618
619        map_type = exp.DataType(this=exp.DataType.Type.MAP)
620        arg = expression.this
621        if arg.is_type(exp.DataType.Type.STRUCT):
622            for coldef in arg.type.expressions:
623                kind = coldef.kind
624                if kind != exp.DataType.Type.UNKNOWN:
625                    map_type.set("expressions", [exp.DataType.build("varchar"), kind])
626                    map_type.set("nested", True)
627                    break
628
629        self._set_type(expression, map_type)
630        return expression
631
632    def _annotate_extract(self, expression: exp.Extract) -> exp.Extract:
633        self._annotate_args(expression)
634        part = expression.name
635        if part == "TIME":
636            self._set_type(expression, exp.DataType.Type.TIME)
637        elif part == "DATE":
638            self._set_type(expression, exp.DataType.Type.DATE)
639        else:
640            self._set_type(expression, exp.DataType.Type.INT)
641        return expression
642
643    def _annotate_by_array_element(self, expression: exp.Expression) -> exp.Expression:
644        self._annotate_args(expression)
645
646        array_arg = expression.this
647        if array_arg.type.is_type(exp.DataType.Type.ARRAY):
648            element_type = seq_get(array_arg.type.expressions, 0) or exp.DataType.Type.UNKNOWN
649            self._set_type(expression, element_type)
650        else:
651            self._set_type(expression, exp.DataType.Type.UNKNOWN)
652
653        return expression