sqlglot.optimizer.annotate_types

  1from __future__ import annotations
  2
  3import functools
  4import typing as t
  5
  6from sqlglot import exp
  7from sqlglot.helper import (
  8    ensure_list,
  9    is_date_unit,
 10    is_iso_date,
 11    is_iso_datetime,
 12    seq_get,
 13)
 14from sqlglot.optimizer.scope import Scope, traverse_scope
 15from sqlglot.schema import Schema, ensure_schema
 16from sqlglot.dialects.dialect import Dialect
 17
 18if t.TYPE_CHECKING:
 19    from sqlglot._typing import B, E
 20
 21    BinaryCoercionFunc = t.Callable[[exp.Expression, exp.Expression], exp.DataType.Type]
 22    BinaryCoercions = t.Dict[
 23        t.Tuple[exp.DataType.Type, exp.DataType.Type],
 24        BinaryCoercionFunc,
 25    ]
 26
 27    from sqlglot.dialects.dialect import DialectType, AnnotatorsType
 28
 29
 30def annotate_types(
 31    expression: E,
 32    schema: t.Optional[t.Dict | Schema] = None,
 33    annotators: t.Optional[AnnotatorsType] = None,
 34    coerces_to: t.Optional[t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]]] = None,
 35    dialect: t.Optional[DialectType] = None,
 36) -> E:
 37    """
 38    Infers the types of an expression, annotating its AST accordingly.
 39
 40    Example:
 41        >>> import sqlglot
 42        >>> schema = {"y": {"cola": "SMALLINT"}}
 43        >>> sql = "SELECT x.cola + 2.5 AS cola FROM (SELECT y.cola AS cola FROM y AS y) AS x"
 44        >>> annotated_expr = annotate_types(sqlglot.parse_one(sql), schema=schema)
 45        >>> annotated_expr.expressions[0].type.this  # Get the type of "x.cola + 2.5 AS cola"
 46        <Type.DOUBLE: 'DOUBLE'>
 47
 48    Args:
 49        expression: Expression to annotate.
 50        schema: Database schema.
 51        annotators: Maps expression type to corresponding annotation function.
 52        coerces_to: Maps expression type to set of types that it can be coerced into.
 53
 54    Returns:
 55        The expression annotated with types.
 56    """
 57
 58    schema = ensure_schema(schema)
 59
 60    return TypeAnnotator(schema, annotators, coerces_to, dialect=dialect).annotate(expression)
 61
 62
 63def _coerce_date_literal(l: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.DataType.Type:
 64    date_text = l.name
 65    is_iso_date_ = is_iso_date(date_text)
 66
 67    if is_iso_date_ and is_date_unit(unit):
 68        return exp.DataType.Type.DATE
 69
 70    # An ISO date is also an ISO datetime, but not vice versa
 71    if is_iso_date_ or is_iso_datetime(date_text):
 72        return exp.DataType.Type.DATETIME
 73
 74    return exp.DataType.Type.UNKNOWN
 75
 76
 77def _coerce_date(l: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.DataType.Type:
 78    if not is_date_unit(unit):
 79        return exp.DataType.Type.DATETIME
 80    return l.type.this if l.type else exp.DataType.Type.UNKNOWN
 81
 82
 83def swap_args(func: BinaryCoercionFunc) -> BinaryCoercionFunc:
 84    @functools.wraps(func)
 85    def _swapped(l: exp.Expression, r: exp.Expression) -> exp.DataType.Type:
 86        return func(r, l)
 87
 88    return _swapped
 89
 90
 91def swap_all(coercions: BinaryCoercions) -> BinaryCoercions:
 92    return {**coercions, **{(b, a): swap_args(func) for (a, b), func in coercions.items()}}
 93
 94
 95class _TypeAnnotator(type):
 96    def __new__(cls, clsname, bases, attrs):
 97        klass = super().__new__(cls, clsname, bases, attrs)
 98
 99        # Highest-to-lowest type precedence, as specified in Spark's docs (ANSI):
100        # https://spark.apache.org/docs/3.2.0/sql-ref-ansi-compliance.html
101        text_precedence = (
102            exp.DataType.Type.TEXT,
103            exp.DataType.Type.NVARCHAR,
104            exp.DataType.Type.VARCHAR,
105            exp.DataType.Type.NCHAR,
106            exp.DataType.Type.CHAR,
107        )
108        numeric_precedence = (
109            exp.DataType.Type.DOUBLE,
110            exp.DataType.Type.FLOAT,
111            exp.DataType.Type.DECIMAL,
112            exp.DataType.Type.BIGINT,
113            exp.DataType.Type.INT,
114            exp.DataType.Type.SMALLINT,
115            exp.DataType.Type.TINYINT,
116        )
117        timelike_precedence = (
118            exp.DataType.Type.TIMESTAMPLTZ,
119            exp.DataType.Type.TIMESTAMPTZ,
120            exp.DataType.Type.TIMESTAMP,
121            exp.DataType.Type.DATETIME,
122            exp.DataType.Type.DATE,
123        )
124
125        for type_precedence in (text_precedence, numeric_precedence, timelike_precedence):
126            coerces_to = set()
127            for data_type in type_precedence:
128                klass.COERCES_TO[data_type] = coerces_to.copy()
129                coerces_to |= {data_type}
130
131        # NULL can be coerced to any type, so e.g. NULL + 1 will have type INT
132        klass.COERCES_TO[exp.DataType.Type.NULL] = {
133            *text_precedence,
134            *numeric_precedence,
135            *timelike_precedence,
136        }
137
138        return klass
139
140
141class TypeAnnotator(metaclass=_TypeAnnotator):
142    NESTED_TYPES = {
143        exp.DataType.Type.ARRAY,
144    }
145
146    # Specifies what types a given type can be coerced into (autofilled)
147    COERCES_TO: t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]] = {}
148
149    # Coercion functions for binary operations.
150    # Map of type pairs to a callable that takes both sides of the binary operation and returns the resulting type.
151    BINARY_COERCIONS: BinaryCoercions = {
152        **swap_all(
153            {
154                (t, exp.DataType.Type.INTERVAL): lambda l, r: _coerce_date_literal(
155                    l, r.args.get("unit")
156                )
157                for t in exp.DataType.TEXT_TYPES
158            }
159        ),
160        **swap_all(
161            {
162                # text + numeric will yield the numeric type to match most dialects' semantics
163                (text, numeric): lambda l, r: t.cast(
164                    exp.DataType.Type, l.type if l.type in exp.DataType.NUMERIC_TYPES else r.type
165                )
166                for text in exp.DataType.TEXT_TYPES
167                for numeric in exp.DataType.NUMERIC_TYPES
168            }
169        ),
170        **swap_all(
171            {
172                (exp.DataType.Type.DATE, exp.DataType.Type.INTERVAL): lambda l, r: _coerce_date(
173                    l, r.args.get("unit")
174                ),
175            }
176        ),
177    }
178
179    def __init__(
180        self,
181        schema: Schema,
182        annotators: t.Optional[AnnotatorsType] = None,
183        coerces_to: t.Optional[t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]]] = None,
184        binary_coercions: t.Optional[BinaryCoercions] = None,
185        dialect: t.Optional[DialectType] = None,
186    ) -> None:
187        self.schema = schema
188        self.annotators = annotators or Dialect.get_or_raise(dialect).ANNOTATORS
189        self.coerces_to = coerces_to or self.COERCES_TO
190        self.binary_coercions = binary_coercions or self.BINARY_COERCIONS
191
192        # Caches the ids of annotated sub-Expressions, to ensure we only visit them once
193        self._visited: t.Set[int] = set()
194
195    def _set_type(
196        self, expression: exp.Expression, target_type: t.Optional[exp.DataType | exp.DataType.Type]
197    ) -> None:
198        expression.type = target_type or exp.DataType.Type.UNKNOWN  # type: ignore
199        self._visited.add(id(expression))
200
201    def annotate(self, expression: E) -> E:
202        for scope in traverse_scope(expression):
203            self.annotate_scope(scope)
204        return self._maybe_annotate(expression)  # This takes care of non-traversable expressions
205
206    def annotate_scope(self, scope: Scope) -> None:
207        selects = {}
208        for name, source in scope.sources.items():
209            if not isinstance(source, Scope):
210                continue
211
212            expression = source.expression
213            if isinstance(expression, exp.UDTF):
214                values = []
215
216                if isinstance(expression, exp.Lateral):
217                    if isinstance(expression.this, exp.Explode):
218                        values = [expression.this.this]
219                elif isinstance(expression, exp.Unnest):
220                    values = [expression]
221                else:
222                    values = expression.expressions[0].expressions
223
224                if not values:
225                    continue
226
227                selects[name] = {
228                    alias: column.type
229                    for alias, column in zip(expression.alias_column_names, values)
230                }
231            elif isinstance(expression, exp.SetOperation) and len(expression.left.selects) == len(
232                expression.right.selects
233            ):
234                if expression.args.get("by_name"):
235                    r_type_by_select = {s.alias_or_name: s.type for s in expression.right.selects}
236                    selects[name] = {
237                        s.alias_or_name: self._maybe_coerce(
238                            t.cast(exp.DataType, s.type),
239                            r_type_by_select.get(s.alias_or_name) or exp.DataType.Type.UNKNOWN,
240                        )
241                        for s in expression.left.selects
242                    }
243                else:
244                    selects[name] = {
245                        ls.alias_or_name: self._maybe_coerce(
246                            t.cast(exp.DataType, ls.type), t.cast(exp.DataType, rs.type)
247                        )
248                        for ls, rs in zip(expression.left.selects, expression.right.selects)
249                    }
250            else:
251                selects[name] = {s.alias_or_name: s.type for s in expression.selects}
252
253        # First annotate the current scope's column references
254        for col in scope.columns:
255            if not col.table:
256                continue
257
258            source = scope.sources.get(col.table)
259            if isinstance(source, exp.Table):
260                self._set_type(col, self.schema.get_column_type(source, col))
261            elif source:
262                if col.table in selects and col.name in selects[col.table]:
263                    self._set_type(col, selects[col.table][col.name])
264                elif isinstance(source.expression, exp.Unnest):
265                    self._set_type(col, source.expression.type)
266
267        # Then (possibly) annotate the remaining expressions in the scope
268        self._maybe_annotate(scope.expression)
269
270    def _maybe_annotate(self, expression: E) -> E:
271        if id(expression) in self._visited:
272            return expression  # We've already inferred the expression's type
273
274        annotator = self.annotators.get(expression.__class__)
275
276        return (
277            annotator(self, expression)
278            if annotator
279            else self._annotate_with_type(expression, exp.DataType.Type.UNKNOWN)
280        )
281
282    def _annotate_args(self, expression: E) -> E:
283        for value in expression.iter_expressions():
284            self._maybe_annotate(value)
285
286        return expression
287
288    def _maybe_coerce(
289        self, type1: exp.DataType | exp.DataType.Type, type2: exp.DataType | exp.DataType.Type
290    ) -> exp.DataType.Type:
291        type1_value = type1.this if isinstance(type1, exp.DataType) else type1
292        type2_value = type2.this if isinstance(type2, exp.DataType) else type2
293
294        # We propagate the UNKNOWN type upwards if found
295        if exp.DataType.Type.UNKNOWN in (type1_value, type2_value):
296            return exp.DataType.Type.UNKNOWN
297
298        return t.cast(
299            exp.DataType.Type,
300            type2_value if type2_value in self.coerces_to.get(type1_value, {}) else type1_value,
301        )
302
303    def _annotate_binary(self, expression: B) -> B:
304        self._annotate_args(expression)
305
306        left, right = expression.left, expression.right
307        left_type, right_type = left.type.this, right.type.this  # type: ignore
308
309        if isinstance(expression, (exp.Connector, exp.Predicate)):
310            self._set_type(expression, exp.DataType.Type.BOOLEAN)
311        elif (left_type, right_type) in self.binary_coercions:
312            self._set_type(expression, self.binary_coercions[(left_type, right_type)](left, right))
313        else:
314            self._set_type(expression, self._maybe_coerce(left_type, right_type))
315
316        return expression
317
318    def _annotate_unary(self, expression: E) -> E:
319        self._annotate_args(expression)
320
321        if isinstance(expression, exp.Not):
322            self._set_type(expression, exp.DataType.Type.BOOLEAN)
323        else:
324            self._set_type(expression, expression.this.type)
325
326        return expression
327
328    def _annotate_literal(self, expression: exp.Literal) -> exp.Literal:
329        if expression.is_string:
330            self._set_type(expression, exp.DataType.Type.VARCHAR)
331        elif expression.is_int:
332            self._set_type(expression, exp.DataType.Type.INT)
333        else:
334            self._set_type(expression, exp.DataType.Type.DOUBLE)
335
336        return expression
337
338    def _annotate_with_type(
339        self, expression: E, target_type: exp.DataType | exp.DataType.Type
340    ) -> E:
341        self._set_type(expression, target_type)
342        return self._annotate_args(expression)
343
344    @t.no_type_check
345    def _annotate_by_args(
346        self,
347        expression: E,
348        *args: str,
349        promote: bool = False,
350        array: bool = False,
351    ) -> E:
352        self._annotate_args(expression)
353
354        expressions: t.List[exp.Expression] = []
355        for arg in args:
356            arg_expr = expression.args.get(arg)
357            expressions.extend(expr for expr in ensure_list(arg_expr) if expr)
358
359        last_datatype = None
360        for expr in expressions:
361            expr_type = expr.type
362
363            # Stop at the first nested data type found - we don't want to _maybe_coerce nested types
364            if expr_type.args.get("nested"):
365                last_datatype = expr_type
366                break
367
368            if not expr_type.is_type(exp.DataType.Type.UNKNOWN):
369                last_datatype = self._maybe_coerce(last_datatype or expr_type, expr_type)
370
371        self._set_type(expression, last_datatype or exp.DataType.Type.UNKNOWN)
372
373        if promote:
374            if expression.type.this in exp.DataType.INTEGER_TYPES:
375                self._set_type(expression, exp.DataType.Type.BIGINT)
376            elif expression.type.this in exp.DataType.FLOAT_TYPES:
377                self._set_type(expression, exp.DataType.Type.DOUBLE)
378
379        if array:
380            self._set_type(
381                expression,
382                exp.DataType(
383                    this=exp.DataType.Type.ARRAY, expressions=[expression.type], nested=True
384                ),
385            )
386
387        return expression
388
389    def _annotate_timeunit(
390        self, expression: exp.TimeUnit | exp.DateTrunc
391    ) -> exp.TimeUnit | exp.DateTrunc:
392        self._annotate_args(expression)
393
394        if expression.this.type.this in exp.DataType.TEXT_TYPES:
395            datatype = _coerce_date_literal(expression.this, expression.unit)
396        elif expression.this.type.this in exp.DataType.TEMPORAL_TYPES:
397            datatype = _coerce_date(expression.this, expression.unit)
398        else:
399            datatype = exp.DataType.Type.UNKNOWN
400
401        self._set_type(expression, datatype)
402        return expression
403
404    def _annotate_bracket(self, expression: exp.Bracket) -> exp.Bracket:
405        self._annotate_args(expression)
406
407        bracket_arg = expression.expressions[0]
408        this = expression.this
409
410        if isinstance(bracket_arg, exp.Slice):
411            self._set_type(expression, this.type)
412        elif this.type.is_type(exp.DataType.Type.ARRAY):
413            self._set_type(expression, seq_get(this.type.expressions, 0))
414        elif isinstance(this, (exp.Map, exp.VarMap)) and bracket_arg in this.keys:
415            index = this.keys.index(bracket_arg)
416            value = seq_get(this.values, index)
417            self._set_type(expression, value.type if value else None)
418        else:
419            self._set_type(expression, exp.DataType.Type.UNKNOWN)
420
421        return expression
422
423    def _annotate_div(self, expression: exp.Div) -> exp.Div:
424        self._annotate_args(expression)
425
426        left_type, right_type = expression.left.type.this, expression.right.type.this  # type: ignore
427
428        if (
429            expression.args.get("typed")
430            and left_type in exp.DataType.INTEGER_TYPES
431            and right_type in exp.DataType.INTEGER_TYPES
432        ):
433            self._set_type(expression, exp.DataType.Type.BIGINT)
434        else:
435            self._set_type(expression, self._maybe_coerce(left_type, right_type))
436            if expression.type and expression.type.this not in exp.DataType.REAL_TYPES:
437                self._set_type(
438                    expression, self._maybe_coerce(expression.type, exp.DataType.Type.DOUBLE)
439                )
440
441        return expression
442
443    def _annotate_dot(self, expression: exp.Dot) -> exp.Dot:
444        self._annotate_args(expression)
445        self._set_type(expression, None)
446        this_type = expression.this.type
447
448        if this_type and this_type.is_type(exp.DataType.Type.STRUCT):
449            for e in this_type.expressions:
450                if e.name == expression.expression.name:
451                    self._set_type(expression, e.kind)
452                    break
453
454        return expression
455
456    def _annotate_explode(self, expression: exp.Explode) -> exp.Explode:
457        self._annotate_args(expression)
458        self._set_type(expression, seq_get(expression.this.type.expressions, 0))
459        return expression
460
461    def _annotate_unnest(self, expression: exp.Unnest) -> exp.Unnest:
462        self._annotate_args(expression)
463        child = seq_get(expression.expressions, 0)
464
465        if child and child.is_type(exp.DataType.Type.ARRAY):
466            expr_type = seq_get(child.type.expressions, 0)
467        else:
468            expr_type = None
469
470        self._set_type(expression, expr_type)
471        return expression
472
473    def _annotate_struct_value(
474        self, expression: exp.Expression
475    ) -> t.Optional[exp.DataType] | exp.ColumnDef:
476        alias = expression.args.get("alias")
477        if alias:
478            return exp.ColumnDef(this=alias.copy(), kind=expression.type)
479
480        # Case: key = value or key := value
481        if expression.expression:
482            return exp.ColumnDef(this=expression.this.copy(), kind=expression.expression.type)
483
484        return expression.type
485
486    def _annotate_struct(self, expression: exp.Struct) -> exp.Struct:
487        self._annotate_args(expression)
488        self._set_type(
489            expression,
490            exp.DataType(
491                this=exp.DataType.Type.STRUCT,
492                expressions=[self._annotate_struct_value(expr) for expr in expression.expressions],
493                nested=True,
494            ),
495        )
496        return expression
497
498    @t.overload
499    def _annotate_map(self, expression: exp.Map) -> exp.Map: ...
500
501    @t.overload
502    def _annotate_map(self, expression: exp.VarMap) -> exp.VarMap: ...
503
504    def _annotate_map(self, expression):
505        self._annotate_args(expression)
506
507        keys = expression.args.get("keys")
508        values = expression.args.get("values")
509
510        map_type = exp.DataType(this=exp.DataType.Type.MAP)
511        if isinstance(keys, exp.Array) and isinstance(values, exp.Array):
512            key_type = seq_get(keys.type.expressions, 0) or exp.DataType.Type.UNKNOWN
513            value_type = seq_get(values.type.expressions, 0) or exp.DataType.Type.UNKNOWN
514
515            if key_type != exp.DataType.Type.UNKNOWN and value_type != exp.DataType.Type.UNKNOWN:
516                map_type.set("expressions", [key_type, value_type])
517                map_type.set("nested", True)
518
519        self._set_type(expression, map_type)
520        return expression
521
522    def _annotate_to_map(self, expression: exp.ToMap) -> exp.ToMap:
523        self._annotate_args(expression)
524
525        map_type = exp.DataType(this=exp.DataType.Type.MAP)
526        arg = expression.this
527        if arg.is_type(exp.DataType.Type.STRUCT):
528            for coldef in arg.type.expressions:
529                kind = coldef.kind
530                if kind != exp.DataType.Type.UNKNOWN:
531                    map_type.set("expressions", [exp.DataType.build("varchar"), kind])
532                    map_type.set("nested", True)
533                    break
534
535        self._set_type(expression, map_type)
536        return expression
537
538    def _annotate_extract(self, expression: exp.Extract) -> exp.Extract:
539        self._annotate_args(expression)
540        part = expression.name
541        if part == "TIME":
542            self._set_type(expression, exp.DataType.Type.TIME)
543        elif part == "DATE":
544            self._set_type(expression, exp.DataType.Type.DATE)
545        else:
546            self._set_type(expression, exp.DataType.Type.INT)
547        return expression