sqlglot.optimizer.annotate_types

  1from __future__ import annotations
  2
  3import functools
  4import typing as t
  5
  6from sqlglot import exp
  7from sqlglot.helper import (
  8    ensure_list,
  9    is_date_unit,
 10    is_iso_date,
 11    is_iso_datetime,
 12    seq_get,
 13)
 14from sqlglot.optimizer.scope import Scope, traverse_scope
 15from sqlglot.schema import MappingSchema, Schema, ensure_schema
 16from sqlglot.dialects.dialect import Dialect
 17
 18if t.TYPE_CHECKING:
 19    from sqlglot._typing import B, E
 20
 21    BinaryCoercionFunc = t.Callable[[exp.Expression, exp.Expression], exp.DataType.Type]
 22    BinaryCoercions = t.Dict[
 23        t.Tuple[exp.DataType.Type, exp.DataType.Type],
 24        BinaryCoercionFunc,
 25    ]
 26
 27    from sqlglot.dialects.dialect import DialectType, AnnotatorsType
 28
 29
 30def annotate_types(
 31    expression: E,
 32    schema: t.Optional[t.Dict | Schema] = None,
 33    annotators: t.Optional[AnnotatorsType] = None,
 34    coerces_to: t.Optional[t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]]] = None,
 35    dialect: DialectType = None,
 36) -> E:
 37    """
 38    Infers the types of an expression, annotating its AST accordingly.
 39
 40    Example:
 41        >>> import sqlglot
 42        >>> schema = {"y": {"cola": "SMALLINT"}}
 43        >>> sql = "SELECT x.cola + 2.5 AS cola FROM (SELECT y.cola AS cola FROM y AS y) AS x"
 44        >>> annotated_expr = annotate_types(sqlglot.parse_one(sql), schema=schema)
 45        >>> annotated_expr.expressions[0].type.this  # Get the type of "x.cola + 2.5 AS cola"
 46        <Type.DOUBLE: 'DOUBLE'>
 47
 48    Args:
 49        expression: Expression to annotate.
 50        schema: Database schema.
 51        annotators: Maps expression type to corresponding annotation function.
 52        coerces_to: Maps expression type to set of types that it can be coerced into.
 53
 54    Returns:
 55        The expression annotated with types.
 56    """
 57
 58    schema = ensure_schema(schema, dialect=dialect)
 59
 60    return TypeAnnotator(schema, annotators, coerces_to).annotate(expression)
 61
 62
 63def _coerce_date_literal(l: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.DataType.Type:
 64    date_text = l.name
 65    is_iso_date_ = is_iso_date(date_text)
 66
 67    if is_iso_date_ and is_date_unit(unit):
 68        return exp.DataType.Type.DATE
 69
 70    # An ISO date is also an ISO datetime, but not vice versa
 71    if is_iso_date_ or is_iso_datetime(date_text):
 72        return exp.DataType.Type.DATETIME
 73
 74    return exp.DataType.Type.UNKNOWN
 75
 76
 77def _coerce_date(l: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.DataType.Type:
 78    if not is_date_unit(unit):
 79        return exp.DataType.Type.DATETIME
 80    return l.type.this if l.type else exp.DataType.Type.UNKNOWN
 81
 82
 83def swap_args(func: BinaryCoercionFunc) -> BinaryCoercionFunc:
 84    @functools.wraps(func)
 85    def _swapped(l: exp.Expression, r: exp.Expression) -> exp.DataType.Type:
 86        return func(r, l)
 87
 88    return _swapped
 89
 90
 91def swap_all(coercions: BinaryCoercions) -> BinaryCoercions:
 92    return {**coercions, **{(b, a): swap_args(func) for (a, b), func in coercions.items()}}
 93
 94
 95class _TypeAnnotator(type):
 96    def __new__(cls, clsname, bases, attrs):
 97        klass = super().__new__(cls, clsname, bases, attrs)
 98
 99        # Highest-to-lowest type precedence, as specified in Spark's docs (ANSI):
100        # https://spark.apache.org/docs/3.2.0/sql-ref-ansi-compliance.html
101        text_precedence = (
102            exp.DataType.Type.TEXT,
103            exp.DataType.Type.NVARCHAR,
104            exp.DataType.Type.VARCHAR,
105            exp.DataType.Type.NCHAR,
106            exp.DataType.Type.CHAR,
107        )
108        numeric_precedence = (
109            exp.DataType.Type.DOUBLE,
110            exp.DataType.Type.FLOAT,
111            exp.DataType.Type.DECIMAL,
112            exp.DataType.Type.BIGINT,
113            exp.DataType.Type.INT,
114            exp.DataType.Type.SMALLINT,
115            exp.DataType.Type.TINYINT,
116        )
117        timelike_precedence = (
118            exp.DataType.Type.TIMESTAMPLTZ,
119            exp.DataType.Type.TIMESTAMPTZ,
120            exp.DataType.Type.TIMESTAMP,
121            exp.DataType.Type.DATETIME,
122            exp.DataType.Type.DATE,
123        )
124
125        for type_precedence in (text_precedence, numeric_precedence, timelike_precedence):
126            coerces_to = set()
127            for data_type in type_precedence:
128                klass.COERCES_TO[data_type] = coerces_to.copy()
129                coerces_to |= {data_type}
130
131        # NULL can be coerced to any type, so e.g. NULL + 1 will have type INT
132        klass.COERCES_TO[exp.DataType.Type.NULL] = {
133            *text_precedence,
134            *numeric_precedence,
135            *timelike_precedence,
136        }
137
138        return klass
139
140
141class TypeAnnotator(metaclass=_TypeAnnotator):
142    NESTED_TYPES = {
143        exp.DataType.Type.ARRAY,
144    }
145
146    # Specifies what types a given type can be coerced into (autofilled)
147    COERCES_TO: t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]] = {}
148
149    # Coercion functions for binary operations.
150    # Map of type pairs to a callable that takes both sides of the binary operation and returns the resulting type.
151    BINARY_COERCIONS: BinaryCoercions = {
152        **swap_all(
153            {
154                (t, exp.DataType.Type.INTERVAL): lambda l, r: _coerce_date_literal(
155                    l, r.args.get("unit")
156                )
157                for t in exp.DataType.TEXT_TYPES
158            }
159        ),
160        **swap_all(
161            {
162                # text + numeric will yield the numeric type to match most dialects' semantics
163                (text, numeric): lambda l, r: t.cast(
164                    exp.DataType.Type, l.type if l.type in exp.DataType.NUMERIC_TYPES else r.type
165                )
166                for text in exp.DataType.TEXT_TYPES
167                for numeric in exp.DataType.NUMERIC_TYPES
168            }
169        ),
170        **swap_all(
171            {
172                (exp.DataType.Type.DATE, exp.DataType.Type.INTERVAL): lambda l, r: _coerce_date(
173                    l, r.args.get("unit")
174                ),
175            }
176        ),
177    }
178
179    def __init__(
180        self,
181        schema: Schema,
182        annotators: t.Optional[AnnotatorsType] = None,
183        coerces_to: t.Optional[t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]]] = None,
184        binary_coercions: t.Optional[BinaryCoercions] = None,
185    ) -> None:
186        self.schema = schema
187        self.annotators = annotators or Dialect.get_or_raise(schema.dialect).ANNOTATORS
188        self.coerces_to = (
189            coerces_to or Dialect.get_or_raise(schema.dialect).COERCES_TO or self.COERCES_TO
190        )
191        self.binary_coercions = binary_coercions or self.BINARY_COERCIONS
192
193        # Caches the ids of annotated sub-Expressions, to ensure we only visit them once
194        self._visited: t.Set[int] = set()
195
196        # Maps an exp.SetOperation's id (e.g. UNION) to its projection types. This is computed if the
197        # exp.SetOperation is the expression of a scope source, as selecting from it multiple times
198        # would reprocess the entire subtree to coerce the types of its operands' projections
199        self._setop_column_types: t.Dict[int, t.Dict[str, exp.DataType | exp.DataType.Type]] = {}
200
201    def _set_type(
202        self, expression: exp.Expression, target_type: t.Optional[exp.DataType | exp.DataType.Type]
203    ) -> None:
204        expression.type = target_type or exp.DataType.Type.UNKNOWN  # type: ignore
205        self._visited.add(id(expression))
206
207    def annotate(self, expression: E) -> E:
208        for scope in traverse_scope(expression):
209            self.annotate_scope(scope)
210        return self._maybe_annotate(expression)  # This takes care of non-traversable expressions
211
212    def annotate_scope(self, scope: Scope) -> None:
213        selects = {}
214        for name, source in scope.sources.items():
215            if not isinstance(source, Scope):
216                continue
217
218            expression = source.expression
219            if isinstance(expression, exp.UDTF):
220                values = []
221
222                if isinstance(expression, exp.Lateral):
223                    if isinstance(expression.this, exp.Explode):
224                        values = [expression.this.this]
225                elif isinstance(expression, exp.Unnest):
226                    values = [expression]
227                elif not isinstance(expression, exp.TableFromRows):
228                    values = expression.expressions[0].expressions
229
230                if not values:
231                    continue
232
233                selects[name] = {
234                    alias: column.type
235                    for alias, column in zip(expression.alias_column_names, values)
236                }
237            elif isinstance(expression, exp.SetOperation) and len(expression.left.selects) == len(
238                expression.right.selects
239            ):
240                selects[name] = col_types = self._setop_column_types.setdefault(id(expression), {})
241
242                if not col_types:
243                    # Process a chain / sub-tree of set operations
244                    for set_op in expression.walk(
245                        prune=lambda n: not isinstance(n, (exp.SetOperation, exp.Subquery))
246                    ):
247                        if not isinstance(set_op, exp.SetOperation):
248                            continue
249
250                        if set_op.args.get("by_name"):
251                            r_type_by_select = {
252                                s.alias_or_name: s.type for s in set_op.right.selects
253                            }
254                            setop_cols = {
255                                s.alias_or_name: self._maybe_coerce(
256                                    t.cast(exp.DataType, s.type),
257                                    r_type_by_select.get(s.alias_or_name)
258                                    or exp.DataType.Type.UNKNOWN,
259                                )
260                                for s in set_op.left.selects
261                            }
262                        else:
263                            setop_cols = {
264                                ls.alias_or_name: self._maybe_coerce(
265                                    t.cast(exp.DataType, ls.type), t.cast(exp.DataType, rs.type)
266                                )
267                                for ls, rs in zip(set_op.left.selects, set_op.right.selects)
268                            }
269
270                        # Coerce intermediate results with the previously registered types, if they exist
271                        for col_name, col_type in setop_cols.items():
272                            col_types[col_name] = self._maybe_coerce(
273                                col_type, col_types.get(col_name, exp.DataType.Type.NULL)
274                            )
275
276            else:
277                selects[name] = {s.alias_or_name: s.type for s in expression.selects}
278
279        # First annotate the current scope's column references
280        for col in scope.columns:
281            if not col.table:
282                continue
283
284            source = scope.sources.get(col.table)
285            if isinstance(source, exp.Table):
286                self._set_type(col, self.schema.get_column_type(source, col))
287            elif source:
288                if col.table in selects and col.name in selects[col.table]:
289                    self._set_type(col, selects[col.table][col.name])
290                elif isinstance(source.expression, exp.Unnest):
291                    self._set_type(col, source.expression.type)
292
293        if isinstance(self.schema, MappingSchema):
294            for table_column in scope.table_columns:
295                source = scope.sources.get(table_column.name)
296
297                if isinstance(source, exp.Table):
298                    schema = self.schema.find(
299                        source, raise_on_missing=False, ensure_data_types=True
300                    )
301                    if not isinstance(schema, dict):
302                        continue
303
304                    struct_type = exp.DataType(
305                        this=exp.DataType.Type.STRUCT,
306                        expressions=[
307                            exp.ColumnDef(this=exp.to_identifier(c), kind=kind)
308                            for c, kind in schema.items()
309                        ],
310                        nested=True,
311                    )
312                    self._set_type(table_column, struct_type)
313                elif (
314                    isinstance(source, Scope)
315                    and isinstance(source.expression, exp.Query)
316                    and source.expression.is_type(exp.DataType.Type.STRUCT)
317                ):
318                    self._set_type(table_column, source.expression.type)
319
320        # Then (possibly) annotate the remaining expressions in the scope
321        self._maybe_annotate(scope.expression)
322
323        if self.schema.dialect == "bigquery" and isinstance(scope.expression, exp.Query):
324            struct_type = exp.DataType(
325                this=exp.DataType.Type.STRUCT,
326                expressions=[
327                    exp.ColumnDef(this=exp.to_identifier(select.output_name), kind=select.type)
328                    for select in scope.expression.selects
329                ],
330                nested=True,
331            )
332            if not any(
333                cd.kind.is_type(exp.DataType.Type.UNKNOWN)
334                for cd in struct_type.expressions
335                if cd.kind
336            ):
337                self._set_type(scope.expression, struct_type)
338
339    def _maybe_annotate(self, expression: E) -> E:
340        if id(expression) in self._visited:
341            return expression  # We've already inferred the expression's type
342
343        annotator = self.annotators.get(expression.__class__)
344
345        return (
346            annotator(self, expression)
347            if annotator
348            else self._annotate_with_type(expression, exp.DataType.Type.UNKNOWN)
349        )
350
351    def _annotate_args(self, expression: E) -> E:
352        for value in expression.iter_expressions():
353            self._maybe_annotate(value)
354
355        return expression
356
357    def _maybe_coerce(
358        self,
359        type1: exp.DataType | exp.DataType.Type,
360        type2: exp.DataType | exp.DataType.Type,
361    ) -> exp.DataType | exp.DataType.Type:
362        """
363        Returns type2 if type1 can be coerced into it, otherwise type1.
364
365        If either type is parameterized (e.g. DECIMAL(18, 2) contains two parameters),
366        we assume type1 does not coerce into type2, so we also return it in this case.
367        """
368        if isinstance(type1, exp.DataType):
369            if type1.expressions:
370                return type1
371            type1_value = type1.this
372        else:
373            type1_value = type1
374
375        if isinstance(type2, exp.DataType):
376            if type2.expressions:
377                return type2
378            type2_value = type2.this
379        else:
380            type2_value = type2
381
382        # We propagate the UNKNOWN type upwards if found
383        if exp.DataType.Type.UNKNOWN in (type1_value, type2_value):
384            return exp.DataType.Type.UNKNOWN
385
386        return type2_value if type2_value in self.coerces_to.get(type1_value, {}) else type1_value
387
388    def _annotate_binary(self, expression: B) -> B:
389        self._annotate_args(expression)
390
391        left, right = expression.left, expression.right
392        left_type, right_type = left.type.this, right.type.this  # type: ignore
393
394        if isinstance(expression, (exp.Connector, exp.Predicate)):
395            self._set_type(expression, exp.DataType.Type.BOOLEAN)
396        elif (left_type, right_type) in self.binary_coercions:
397            self._set_type(expression, self.binary_coercions[(left_type, right_type)](left, right))
398        else:
399            self._set_type(expression, self._maybe_coerce(left_type, right_type))
400
401        return expression
402
403    def _annotate_unary(self, expression: E) -> E:
404        self._annotate_args(expression)
405
406        if isinstance(expression, exp.Not):
407            self._set_type(expression, exp.DataType.Type.BOOLEAN)
408        else:
409            self._set_type(expression, expression.this.type)
410
411        return expression
412
413    def _annotate_literal(self, expression: exp.Literal) -> exp.Literal:
414        if expression.is_string:
415            self._set_type(expression, exp.DataType.Type.VARCHAR)
416        elif expression.is_int:
417            self._set_type(expression, exp.DataType.Type.INT)
418        else:
419            self._set_type(expression, exp.DataType.Type.DOUBLE)
420
421        return expression
422
423    def _annotate_with_type(
424        self, expression: E, target_type: exp.DataType | exp.DataType.Type
425    ) -> E:
426        self._set_type(expression, target_type)
427        return self._annotate_args(expression)
428
429    @t.no_type_check
430    def _annotate_by_args(
431        self,
432        expression: E,
433        *args: str,
434        promote: bool = False,
435        array: bool = False,
436    ) -> E:
437        self._annotate_args(expression)
438
439        expressions: t.List[exp.Expression] = []
440        for arg in args:
441            arg_expr = expression.args.get(arg)
442            expressions.extend(expr for expr in ensure_list(arg_expr) if expr)
443
444        last_datatype = None
445        for expr in expressions:
446            expr_type = expr.type
447
448            # Stop at the first nested data type found - we don't want to _maybe_coerce nested types
449            if expr_type.args.get("nested"):
450                last_datatype = expr_type
451                break
452
453            if not expr_type.is_type(exp.DataType.Type.UNKNOWN):
454                last_datatype = self._maybe_coerce(last_datatype or expr_type, expr_type)
455
456        self._set_type(expression, last_datatype or exp.DataType.Type.UNKNOWN)
457
458        if promote:
459            if expression.type.this in exp.DataType.INTEGER_TYPES:
460                self._set_type(expression, exp.DataType.Type.BIGINT)
461            elif expression.type.this in exp.DataType.FLOAT_TYPES:
462                self._set_type(expression, exp.DataType.Type.DOUBLE)
463
464        if array:
465            self._set_type(
466                expression,
467                exp.DataType(
468                    this=exp.DataType.Type.ARRAY, expressions=[expression.type], nested=True
469                ),
470            )
471
472        return expression
473
474    def _annotate_timeunit(
475        self, expression: exp.TimeUnit | exp.DateTrunc
476    ) -> exp.TimeUnit | exp.DateTrunc:
477        self._annotate_args(expression)
478
479        if expression.this.type.this in exp.DataType.TEXT_TYPES:
480            datatype = _coerce_date_literal(expression.this, expression.unit)
481        elif expression.this.type.this in exp.DataType.TEMPORAL_TYPES:
482            datatype = _coerce_date(expression.this, expression.unit)
483        else:
484            datatype = exp.DataType.Type.UNKNOWN
485
486        self._set_type(expression, datatype)
487        return expression
488
489    def _annotate_bracket(self, expression: exp.Bracket) -> exp.Bracket:
490        self._annotate_args(expression)
491
492        bracket_arg = expression.expressions[0]
493        this = expression.this
494
495        if isinstance(bracket_arg, exp.Slice):
496            self._set_type(expression, this.type)
497        elif this.type.is_type(exp.DataType.Type.ARRAY):
498            self._set_type(expression, seq_get(this.type.expressions, 0))
499        elif isinstance(this, (exp.Map, exp.VarMap)) and bracket_arg in this.keys:
500            index = this.keys.index(bracket_arg)
501            value = seq_get(this.values, index)
502            self._set_type(expression, value.type if value else None)
503        else:
504            self._set_type(expression, exp.DataType.Type.UNKNOWN)
505
506        return expression
507
508    def _annotate_div(self, expression: exp.Div) -> exp.Div:
509        self._annotate_args(expression)
510
511        left_type, right_type = expression.left.type.this, expression.right.type.this  # type: ignore
512
513        if (
514            expression.args.get("typed")
515            and left_type in exp.DataType.INTEGER_TYPES
516            and right_type in exp.DataType.INTEGER_TYPES
517        ):
518            self._set_type(expression, exp.DataType.Type.BIGINT)
519        else:
520            self._set_type(expression, self._maybe_coerce(left_type, right_type))
521            if expression.type and expression.type.this not in exp.DataType.REAL_TYPES:
522                self._set_type(
523                    expression, self._maybe_coerce(expression.type, exp.DataType.Type.DOUBLE)
524                )
525
526        return expression
527
528    def _annotate_dot(self, expression: exp.Dot) -> exp.Dot:
529        self._annotate_args(expression)
530        self._set_type(expression, None)
531        this_type = expression.this.type
532
533        if this_type and this_type.is_type(exp.DataType.Type.STRUCT):
534            for e in this_type.expressions:
535                if e.name == expression.expression.name:
536                    self._set_type(expression, e.kind)
537                    break
538
539        return expression
540
541    def _annotate_explode(self, expression: exp.Explode) -> exp.Explode:
542        self._annotate_args(expression)
543        self._set_type(expression, seq_get(expression.this.type.expressions, 0))
544        return expression
545
546    def _annotate_unnest(self, expression: exp.Unnest) -> exp.Unnest:
547        self._annotate_args(expression)
548        child = seq_get(expression.expressions, 0)
549
550        if child and child.is_type(exp.DataType.Type.ARRAY):
551            expr_type = seq_get(child.type.expressions, 0)
552        else:
553            expr_type = None
554
555        self._set_type(expression, expr_type)
556        return expression
557
558    def _annotate_struct_value(
559        self, expression: exp.Expression
560    ) -> t.Optional[exp.DataType] | exp.ColumnDef:
561        alias = expression.args.get("alias")
562        if alias:
563            return exp.ColumnDef(this=alias.copy(), kind=expression.type)
564
565        # Case: key = value or key := value
566        if expression.expression:
567            return exp.ColumnDef(this=expression.this.copy(), kind=expression.expression.type)
568
569        return expression.type
570
571    def _annotate_struct(self, expression: exp.Struct) -> exp.Struct:
572        self._annotate_args(expression)
573        self._set_type(
574            expression,
575            exp.DataType(
576                this=exp.DataType.Type.STRUCT,
577                expressions=[self._annotate_struct_value(expr) for expr in expression.expressions],
578                nested=True,
579            ),
580        )
581        return expression
582
583    @t.overload
584    def _annotate_map(self, expression: exp.Map) -> exp.Map: ...
585
586    @t.overload
587    def _annotate_map(self, expression: exp.VarMap) -> exp.VarMap: ...
588
589    def _annotate_map(self, expression):
590        self._annotate_args(expression)
591
592        keys = expression.args.get("keys")
593        values = expression.args.get("values")
594
595        map_type = exp.DataType(this=exp.DataType.Type.MAP)
596        if isinstance(keys, exp.Array) and isinstance(values, exp.Array):
597            key_type = seq_get(keys.type.expressions, 0) or exp.DataType.Type.UNKNOWN
598            value_type = seq_get(values.type.expressions, 0) or exp.DataType.Type.UNKNOWN
599
600            if key_type != exp.DataType.Type.UNKNOWN and value_type != exp.DataType.Type.UNKNOWN:
601                map_type.set("expressions", [key_type, value_type])
602                map_type.set("nested", True)
603
604        self._set_type(expression, map_type)
605        return expression
606
607    def _annotate_to_map(self, expression: exp.ToMap) -> exp.ToMap:
608        self._annotate_args(expression)
609
610        map_type = exp.DataType(this=exp.DataType.Type.MAP)
611        arg = expression.this
612        if arg.is_type(exp.DataType.Type.STRUCT):
613            for coldef in arg.type.expressions:
614                kind = coldef.kind
615                if kind != exp.DataType.Type.UNKNOWN:
616                    map_type.set("expressions", [exp.DataType.build("varchar"), kind])
617                    map_type.set("nested", True)
618                    break
619
620        self._set_type(expression, map_type)
621        return expression
622
623    def _annotate_extract(self, expression: exp.Extract) -> exp.Extract:
624        self._annotate_args(expression)
625        part = expression.name
626        if part == "TIME":
627            self._set_type(expression, exp.DataType.Type.TIME)
628        elif part == "DATE":
629            self._set_type(expression, exp.DataType.Type.DATE)
630        else:
631            self._set_type(expression, exp.DataType.Type.INT)
632        return expression