sqlglot.optimizer.annotate_types

  1from __future__ import annotations
  2
  3import functools
  4import typing as t
  5
  6from sqlglot import exp
  7from sqlglot.helper import (
  8    ensure_list,
  9    is_date_unit,
 10    is_iso_date,
 11    is_iso_datetime,
 12    seq_get,
 13)
 14from sqlglot.optimizer.scope import Scope, traverse_scope
 15from sqlglot.schema import Schema, ensure_schema
 16from sqlglot.dialects.dialect import Dialect
 17
 18if t.TYPE_CHECKING:
 19    from sqlglot._typing import B, E
 20
 21    BinaryCoercionFunc = t.Callable[[exp.Expression, exp.Expression], exp.DataType.Type]
 22    BinaryCoercions = t.Dict[
 23        t.Tuple[exp.DataType.Type, exp.DataType.Type],
 24        BinaryCoercionFunc,
 25    ]
 26
 27    from sqlglot.dialects.dialect import DialectType, AnnotatorsType
 28
 29
 30def annotate_types(
 31    expression: E,
 32    schema: t.Optional[t.Dict | Schema] = None,
 33    annotators: t.Optional[AnnotatorsType] = None,
 34    coerces_to: t.Optional[t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]]] = None,
 35    dialect: t.Optional[DialectType] = None,
 36) -> E:
 37    """
 38    Infers the types of an expression, annotating its AST accordingly.
 39
 40    Example:
 41        >>> import sqlglot
 42        >>> schema = {"y": {"cola": "SMALLINT"}}
 43        >>> sql = "SELECT x.cola + 2.5 AS cola FROM (SELECT y.cola AS cola FROM y AS y) AS x"
 44        >>> annotated_expr = annotate_types(sqlglot.parse_one(sql), schema=schema)
 45        >>> annotated_expr.expressions[0].type.this  # Get the type of "x.cola + 2.5 AS cola"
 46        <Type.DOUBLE: 'DOUBLE'>
 47
 48    Args:
 49        expression: Expression to annotate.
 50        schema: Database schema.
 51        annotators: Maps expression type to corresponding annotation function.
 52        coerces_to: Maps expression type to set of types that it can be coerced into.
 53
 54    Returns:
 55        The expression annotated with types.
 56    """
 57
 58    schema = ensure_schema(schema)
 59
 60    return TypeAnnotator(schema, annotators, coerces_to, dialect=dialect).annotate(expression)
 61
 62
 63def _coerce_date_literal(l: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.DataType.Type:
 64    date_text = l.name
 65    is_iso_date_ = is_iso_date(date_text)
 66
 67    if is_iso_date_ and is_date_unit(unit):
 68        return exp.DataType.Type.DATE
 69
 70    # An ISO date is also an ISO datetime, but not vice versa
 71    if is_iso_date_ or is_iso_datetime(date_text):
 72        return exp.DataType.Type.DATETIME
 73
 74    return exp.DataType.Type.UNKNOWN
 75
 76
 77def _coerce_date(l: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.DataType.Type:
 78    if not is_date_unit(unit):
 79        return exp.DataType.Type.DATETIME
 80    return l.type.this if l.type else exp.DataType.Type.UNKNOWN
 81
 82
 83def swap_args(func: BinaryCoercionFunc) -> BinaryCoercionFunc:
 84    @functools.wraps(func)
 85    def _swapped(l: exp.Expression, r: exp.Expression) -> exp.DataType.Type:
 86        return func(r, l)
 87
 88    return _swapped
 89
 90
 91def swap_all(coercions: BinaryCoercions) -> BinaryCoercions:
 92    return {**coercions, **{(b, a): swap_args(func) for (a, b), func in coercions.items()}}
 93
 94
 95class _TypeAnnotator(type):
 96    def __new__(cls, clsname, bases, attrs):
 97        klass = super().__new__(cls, clsname, bases, attrs)
 98
 99        # Highest-to-lowest type precedence, as specified in Spark's docs (ANSI):
100        # https://spark.apache.org/docs/3.2.0/sql-ref-ansi-compliance.html
101        text_precedence = (
102            exp.DataType.Type.TEXT,
103            exp.DataType.Type.NVARCHAR,
104            exp.DataType.Type.VARCHAR,
105            exp.DataType.Type.NCHAR,
106            exp.DataType.Type.CHAR,
107        )
108        numeric_precedence = (
109            exp.DataType.Type.DOUBLE,
110            exp.DataType.Type.FLOAT,
111            exp.DataType.Type.DECIMAL,
112            exp.DataType.Type.BIGINT,
113            exp.DataType.Type.INT,
114            exp.DataType.Type.SMALLINT,
115            exp.DataType.Type.TINYINT,
116        )
117        timelike_precedence = (
118            exp.DataType.Type.TIMESTAMPLTZ,
119            exp.DataType.Type.TIMESTAMPTZ,
120            exp.DataType.Type.TIMESTAMP,
121            exp.DataType.Type.DATETIME,
122            exp.DataType.Type.DATE,
123        )
124
125        for type_precedence in (text_precedence, numeric_precedence, timelike_precedence):
126            coerces_to = set()
127            for data_type in type_precedence:
128                klass.COERCES_TO[data_type] = coerces_to.copy()
129                coerces_to |= {data_type}
130
131        return klass
132
133
134class TypeAnnotator(metaclass=_TypeAnnotator):
135    NESTED_TYPES = {
136        exp.DataType.Type.ARRAY,
137    }
138
139    # Specifies what types a given type can be coerced into (autofilled)
140    COERCES_TO: t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]] = {}
141
142    # Coercion functions for binary operations.
143    # Map of type pairs to a callable that takes both sides of the binary operation and returns the resulting type.
144    BINARY_COERCIONS: BinaryCoercions = {
145        **swap_all(
146            {
147                (t, exp.DataType.Type.INTERVAL): lambda l, r: _coerce_date_literal(
148                    l, r.args.get("unit")
149                )
150                for t in exp.DataType.TEXT_TYPES
151            }
152        ),
153        **swap_all(
154            {
155                # text + numeric will yield the numeric type to match most dialects' semantics
156                (text, numeric): lambda l, r: t.cast(
157                    exp.DataType.Type, l.type if l.type in exp.DataType.NUMERIC_TYPES else r.type
158                )
159                for text in exp.DataType.TEXT_TYPES
160                for numeric in exp.DataType.NUMERIC_TYPES
161            }
162        ),
163        **swap_all(
164            {
165                (exp.DataType.Type.DATE, exp.DataType.Type.INTERVAL): lambda l, r: _coerce_date(
166                    l, r.args.get("unit")
167                ),
168            }
169        ),
170    }
171
172    def __init__(
173        self,
174        schema: Schema,
175        annotators: t.Optional[AnnotatorsType] = None,
176        coerces_to: t.Optional[t.Dict[exp.DataType.Type, t.Set[exp.DataType.Type]]] = None,
177        binary_coercions: t.Optional[BinaryCoercions] = None,
178        dialect: t.Optional[DialectType] = None,
179    ) -> None:
180        self.schema = schema
181        self.annotators = annotators or Dialect.get_or_raise(dialect).ANNOTATORS
182        self.coerces_to = coerces_to or self.COERCES_TO
183        self.binary_coercions = binary_coercions or self.BINARY_COERCIONS
184
185        # Caches the ids of annotated sub-Expressions, to ensure we only visit them once
186        self._visited: t.Set[int] = set()
187
188    def _set_type(
189        self, expression: exp.Expression, target_type: t.Optional[exp.DataType | exp.DataType.Type]
190    ) -> None:
191        expression.type = target_type or exp.DataType.Type.UNKNOWN  # type: ignore
192        self._visited.add(id(expression))
193
194    def annotate(self, expression: E) -> E:
195        for scope in traverse_scope(expression):
196            self.annotate_scope(scope)
197        return self._maybe_annotate(expression)  # This takes care of non-traversable expressions
198
199    def annotate_scope(self, scope: Scope) -> None:
200        selects = {}
201        for name, source in scope.sources.items():
202            if not isinstance(source, Scope):
203                continue
204            if isinstance(source.expression, exp.UDTF):
205                values = []
206
207                if isinstance(source.expression, exp.Lateral):
208                    if isinstance(source.expression.this, exp.Explode):
209                        values = [source.expression.this.this]
210                elif isinstance(source.expression, exp.Unnest):
211                    values = [source.expression]
212                else:
213                    values = source.expression.expressions[0].expressions
214
215                if not values:
216                    continue
217
218                selects[name] = {
219                    alias: column
220                    for alias, column in zip(
221                        source.expression.alias_column_names,
222                        values,
223                    )
224                }
225            else:
226                selects[name] = {
227                    select.alias_or_name: select for select in source.expression.selects
228                }
229
230        # First annotate the current scope's column references
231        for col in scope.columns:
232            if not col.table:
233                continue
234
235            source = scope.sources.get(col.table)
236            if isinstance(source, exp.Table):
237                self._set_type(col, self.schema.get_column_type(source, col))
238            elif source:
239                if col.table in selects and col.name in selects[col.table]:
240                    self._set_type(col, selects[col.table][col.name].type)
241                elif isinstance(source.expression, exp.Unnest):
242                    self._set_type(col, source.expression.type)
243
244        # Then (possibly) annotate the remaining expressions in the scope
245        self._maybe_annotate(scope.expression)
246
247    def _maybe_annotate(self, expression: E) -> E:
248        if id(expression) in self._visited:
249            return expression  # We've already inferred the expression's type
250
251        annotator = self.annotators.get(expression.__class__)
252
253        return (
254            annotator(self, expression)
255            if annotator
256            else self._annotate_with_type(expression, exp.DataType.Type.UNKNOWN)
257        )
258
259    def _annotate_args(self, expression: E) -> E:
260        for value in expression.iter_expressions():
261            self._maybe_annotate(value)
262
263        return expression
264
265    def _maybe_coerce(
266        self, type1: exp.DataType | exp.DataType.Type, type2: exp.DataType | exp.DataType.Type
267    ) -> exp.DataType | exp.DataType.Type:
268        type1_value = type1.this if isinstance(type1, exp.DataType) else type1
269        type2_value = type2.this if isinstance(type2, exp.DataType) else type2
270
271        # We propagate the NULL / UNKNOWN types upwards if found
272        if exp.DataType.Type.NULL in (type1_value, type2_value):
273            return exp.DataType.Type.NULL
274        if exp.DataType.Type.UNKNOWN in (type1_value, type2_value):
275            return exp.DataType.Type.UNKNOWN
276
277        return type2_value if type2_value in self.coerces_to.get(type1_value, {}) else type1_value
278
279    def _annotate_binary(self, expression: B) -> B:
280        self._annotate_args(expression)
281
282        left, right = expression.left, expression.right
283        left_type, right_type = left.type.this, right.type.this  # type: ignore
284
285        if isinstance(expression, exp.Connector):
286            if left_type == exp.DataType.Type.NULL and right_type == exp.DataType.Type.NULL:
287                self._set_type(expression, exp.DataType.Type.NULL)
288            elif exp.DataType.Type.NULL in (left_type, right_type):
289                self._set_type(
290                    expression,
291                    exp.DataType.build("NULLABLE", expressions=exp.DataType.build("BOOLEAN")),
292                )
293            else:
294                self._set_type(expression, exp.DataType.Type.BOOLEAN)
295        elif isinstance(expression, exp.Predicate):
296            self._set_type(expression, exp.DataType.Type.BOOLEAN)
297        elif (left_type, right_type) in self.binary_coercions:
298            self._set_type(expression, self.binary_coercions[(left_type, right_type)](left, right))
299        else:
300            self._set_type(expression, self._maybe_coerce(left_type, right_type))
301
302        return expression
303
304    def _annotate_unary(self, expression: E) -> E:
305        self._annotate_args(expression)
306
307        if isinstance(expression, exp.Condition) and not isinstance(expression, exp.Paren):
308            self._set_type(expression, exp.DataType.Type.BOOLEAN)
309        else:
310            self._set_type(expression, expression.this.type)
311
312        return expression
313
314    def _annotate_literal(self, expression: exp.Literal) -> exp.Literal:
315        if expression.is_string:
316            self._set_type(expression, exp.DataType.Type.VARCHAR)
317        elif expression.is_int:
318            self._set_type(expression, exp.DataType.Type.INT)
319        else:
320            self._set_type(expression, exp.DataType.Type.DOUBLE)
321
322        return expression
323
324    def _annotate_with_type(
325        self, expression: E, target_type: exp.DataType | exp.DataType.Type
326    ) -> E:
327        self._set_type(expression, target_type)
328        return self._annotate_args(expression)
329
330    @t.no_type_check
331    def _annotate_by_args(
332        self,
333        expression: E,
334        *args: str,
335        promote: bool = False,
336        array: bool = False,
337    ) -> E:
338        self._annotate_args(expression)
339
340        expressions: t.List[exp.Expression] = []
341        for arg in args:
342            arg_expr = expression.args.get(arg)
343            expressions.extend(expr for expr in ensure_list(arg_expr) if expr)
344
345        last_datatype = None
346        for expr in expressions:
347            expr_type = expr.type
348
349            # Stop at the first nested data type found - we don't want to _maybe_coerce nested types
350            if expr_type.args.get("nested"):
351                last_datatype = expr_type
352                break
353
354            if not expr_type.is_type(exp.DataType.Type.NULL, exp.DataType.Type.UNKNOWN):
355                last_datatype = self._maybe_coerce(last_datatype or expr_type, expr_type)
356
357        self._set_type(expression, last_datatype or exp.DataType.Type.UNKNOWN)
358
359        if promote:
360            if expression.type.this in exp.DataType.INTEGER_TYPES:
361                self._set_type(expression, exp.DataType.Type.BIGINT)
362            elif expression.type.this in exp.DataType.FLOAT_TYPES:
363                self._set_type(expression, exp.DataType.Type.DOUBLE)
364
365        if array:
366            self._set_type(
367                expression,
368                exp.DataType(
369                    this=exp.DataType.Type.ARRAY, expressions=[expression.type], nested=True
370                ),
371            )
372
373        return expression
374
375    def _annotate_timeunit(
376        self, expression: exp.TimeUnit | exp.DateTrunc
377    ) -> exp.TimeUnit | exp.DateTrunc:
378        self._annotate_args(expression)
379
380        if expression.this.type.this in exp.DataType.TEXT_TYPES:
381            datatype = _coerce_date_literal(expression.this, expression.unit)
382        elif expression.this.type.this in exp.DataType.TEMPORAL_TYPES:
383            datatype = _coerce_date(expression.this, expression.unit)
384        else:
385            datatype = exp.DataType.Type.UNKNOWN
386
387        self._set_type(expression, datatype)
388        return expression
389
390    def _annotate_bracket(self, expression: exp.Bracket) -> exp.Bracket:
391        self._annotate_args(expression)
392
393        bracket_arg = expression.expressions[0]
394        this = expression.this
395
396        if isinstance(bracket_arg, exp.Slice):
397            self._set_type(expression, this.type)
398        elif this.type.is_type(exp.DataType.Type.ARRAY):
399            self._set_type(expression, seq_get(this.type.expressions, 0))
400        elif isinstance(this, (exp.Map, exp.VarMap)) and bracket_arg in this.keys:
401            index = this.keys.index(bracket_arg)
402            value = seq_get(this.values, index)
403            self._set_type(expression, value.type if value else None)
404        else:
405            self._set_type(expression, exp.DataType.Type.UNKNOWN)
406
407        return expression
408
409    def _annotate_div(self, expression: exp.Div) -> exp.Div:
410        self._annotate_args(expression)
411
412        left_type, right_type = expression.left.type.this, expression.right.type.this  # type: ignore
413
414        if (
415            expression.args.get("typed")
416            and left_type in exp.DataType.INTEGER_TYPES
417            and right_type in exp.DataType.INTEGER_TYPES
418        ):
419            self._set_type(expression, exp.DataType.Type.BIGINT)
420        else:
421            self._set_type(expression, self._maybe_coerce(left_type, right_type))
422            if expression.type and expression.type.this not in exp.DataType.REAL_TYPES:
423                self._set_type(
424                    expression, self._maybe_coerce(expression.type, exp.DataType.Type.DOUBLE)
425                )
426
427        return expression
428
429    def _annotate_dot(self, expression: exp.Dot) -> exp.Dot:
430        self._annotate_args(expression)
431        self._set_type(expression, None)
432        this_type = expression.this.type
433
434        if this_type and this_type.is_type(exp.DataType.Type.STRUCT):
435            for e in this_type.expressions:
436                if e.name == expression.expression.name:
437                    self._set_type(expression, e.kind)
438                    break
439
440        return expression
441
442    def _annotate_explode(self, expression: exp.Explode) -> exp.Explode:
443        self._annotate_args(expression)
444        self._set_type(expression, seq_get(expression.this.type.expressions, 0))
445        return expression
446
447    def _annotate_unnest(self, expression: exp.Unnest) -> exp.Unnest:
448        self._annotate_args(expression)
449        child = seq_get(expression.expressions, 0)
450
451        if child and child.is_type(exp.DataType.Type.ARRAY):
452            expr_type = seq_get(child.type.expressions, 0)
453        else:
454            expr_type = None
455
456        self._set_type(expression, expr_type)
457        return expression
458
459    def _annotate_struct_value(
460        self, expression: exp.Expression
461    ) -> t.Optional[exp.DataType] | exp.ColumnDef:
462        alias = expression.args.get("alias")
463        if alias:
464            return exp.ColumnDef(this=alias.copy(), kind=expression.type)
465
466        # Case: key = value or key := value
467        if expression.expression:
468            return exp.ColumnDef(this=expression.this.copy(), kind=expression.expression.type)
469
470        return expression.type
471
472    def _annotate_struct(self, expression: exp.Struct) -> exp.Struct:
473        self._annotate_args(expression)
474        self._set_type(
475            expression,
476            exp.DataType(
477                this=exp.DataType.Type.STRUCT,
478                expressions=[self._annotate_struct_value(expr) for expr in expression.expressions],
479                nested=True,
480            ),
481        )
482        return expression
483
484    @t.overload
485    def _annotate_map(self, expression: exp.Map) -> exp.Map: ...
486
487    @t.overload
488    def _annotate_map(self, expression: exp.VarMap) -> exp.VarMap: ...
489
490    def _annotate_map(self, expression):
491        self._annotate_args(expression)
492
493        keys = expression.args.get("keys")
494        values = expression.args.get("values")
495
496        map_type = exp.DataType(this=exp.DataType.Type.MAP)
497        if isinstance(keys, exp.Array) and isinstance(values, exp.Array):
498            key_type = seq_get(keys.type.expressions, 0) or exp.DataType.Type.UNKNOWN
499            value_type = seq_get(values.type.expressions, 0) or exp.DataType.Type.UNKNOWN
500
501            if key_type != exp.DataType.Type.UNKNOWN and value_type != exp.DataType.Type.UNKNOWN:
502                map_type.set("expressions", [key_type, value_type])
503                map_type.set("nested", True)
504
505        self._set_type(expression, map_type)
506        return expression
507
508    def _annotate_to_map(self, expression: exp.ToMap) -> exp.ToMap:
509        self._annotate_args(expression)
510
511        map_type = exp.DataType(this=exp.DataType.Type.MAP)
512        arg = expression.this
513        if arg.is_type(exp.DataType.Type.STRUCT):
514            for coldef in arg.type.expressions:
515                kind = coldef.kind
516                if kind != exp.DataType.Type.UNKNOWN:
517                    map_type.set("expressions", [exp.DataType.build("varchar"), kind])
518                    map_type.set("nested", True)
519                    break
520
521        self._set_type(expression, map_type)
522        return expression
523
524    def _annotate_extract(self, expression: exp.Extract) -> exp.Extract:
525        self._annotate_args(expression)
526        part = expression.name
527        if part == "TIME":
528            self._set_type(expression, exp.DataType.Type.TIME)
529        elif part == "DATE":
530            self._set_type(expression, exp.DataType.Type.DATE)
531        else:
532            self._set_type(expression, exp.DataType.Type.INT)
533        return expression