sqlglot.parsers.hive

  1from __future__ import annotations
  2
  3import typing as t
  4from decimal import Decimal
  5
  6from sqlglot import exp, parser
  7from sqlglot.dialects.dialect import build_formatted_time, build_regexp_extract
  8from sqlglot.helper import seq_get
  9from sqlglot.parser import IsPredicatePrecedence
 10from sqlglot.tokens import TokenType
 11
 12if t.TYPE_CHECKING:
 13    from sqlglot.dialects.dialect import Dialect
 14
 15
 16def build_with_ignore_nulls(
 17    exp_class: type[exp.Expr],
 18) -> t.Callable[[list[exp.Expr]], exp.Expr]:
 19    def _parse(args: list[exp.Expr]) -> exp.Expr:
 20        this = exp_class(this=seq_get(args, 0))
 21        if seq_get(args, 1) == exp.true():
 22            return exp.IgnoreNulls(this=this)
 23        return this
 24
 25    return _parse
 26
 27
 28def _build_to_date(args: list, dialect: Dialect) -> exp.TsOrDsToDate:
 29    expr = build_formatted_time(exp.TsOrDsToDate)(args, dialect)
 30    expr.set("safe", True)
 31    return expr
 32
 33
 34def _build_named_struct(args: list) -> exp.Struct:
 35    """Map named_struct('k', v, ...) to exp.Struct so _annotate_struct sees it."""
 36    expressions: list[exp.Expression] = []
 37    for i in range(0, len(args) - 1, 2):
 38        key, value = args[i], args[i + 1]
 39        name = key.name
 40        expressions.append(exp.PropertyEQ(this=exp.to_identifier(name), expression=value))
 41    return exp.Struct(expressions=expressions)
 42
 43
 44def _build_date_add(args: list) -> exp.TsOrDsAdd:
 45    expression = seq_get(args, 1)
 46    if expression:
 47        expression = expression * -1
 48
 49    return exp.TsOrDsAdd(
 50        this=seq_get(args, 0), expression=expression, unit=exp.Literal.string("DAY")
 51    )
 52
 53
 54class HiveParser(parser.Parser):
 55    IS_PREDICATE_PRECEDENCE = IsPredicatePrecedence.EQUALITY
 56
 57    LOG_DEFAULTS_TO_LN = True
 58    STRICT_CAST = False
 59    VALUES_FOLLOWED_BY_PAREN = False
 60    JOINS_HAVE_EQUAL_PRECEDENCE = True
 61    ADD_JOIN_ON_TRUE = True
 62    ALTER_TABLE_PARTITIONS = True
 63
 64    # Strip trailing fractional zeros and normalize zero when inferring BD literal types.
 65    NORMALIZE_DECIMAL_LITERALS = True
 66
 67    # Allow negative scales for BD exponent literals, e.g. 1E3BD as DECIMAL(1, -3).
 68    DECIMAL_LITERALS_ALLOW_NEGATIVE_SCALE = False
 69
 70    # Whether the dialect supports using ALTER COLUMN syntax with CHANGE COLUMN.
 71    CHANGE_COLUMN_ALTER_SYNTAX = False
 72
 73    FUNCTION_PARSERS = {
 74        **parser.Parser.FUNCTION_PARSERS,
 75        "PERCENTILE": lambda self: self._parse_distinct_arg_function(exp.Quantile),
 76        "PERCENTILE_APPROX": lambda self: self._parse_distinct_arg_function(exp.ApproxQuantile),
 77    }
 78
 79    FUNCTIONS = {
 80        **parser.Parser.FUNCTIONS,
 81        "BASE64": exp.ToBase64.from_arg_list,
 82        "COLLECT_LIST": lambda args: exp.ArrayAgg(this=seq_get(args, 0), nulls_excluded=True),
 83        "COLLECT_SET": exp.ArrayUniqueAgg.from_arg_list,
 84        "CONCAT_WS": lambda args, dialect: exp.ConcatWs(
 85            expressions=args,
 86            safe=not dialect.STRICT_STRING_CONCAT,
 87            coalesce=dialect.CONCAT_WS_COALESCE,
 88            flatten=True,
 89        ),
 90        "DATE_ADD": lambda args: exp.TsOrDsAdd(
 91            this=seq_get(args, 0), expression=seq_get(args, 1), unit=exp.Literal.string("DAY")
 92        ),
 93        "DATE_FORMAT": lambda args, dialect: build_formatted_time(exp.TimeToStr)(
 94            [
 95                exp.TimeStrToTime(this=seq_get(args, 0)),
 96                seq_get(args, 1),
 97            ],
 98            dialect,
 99        ),
100        "DATE_SUB": _build_date_add,
101        "DATEDIFF": lambda args: exp.DateDiff(
102            this=exp.TsOrDsToDate(this=seq_get(args, 0)),
103            expression=exp.TsOrDsToDate(this=seq_get(args, 1)),
104        ),
105        "DAY": lambda args: exp.Day(this=exp.TsOrDsToDate(this=seq_get(args, 0))),
106        "FIRST": build_with_ignore_nulls(exp.First),
107        "FIRST_VALUE": build_with_ignore_nulls(exp.FirstValue),
108        "FROM_UNIXTIME": build_formatted_time(exp.UnixToStr, default=True),
109        "GET_JSON_OBJECT": lambda args, dialect: exp.JSONExtractScalar(
110            this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1))
111        ),
112        "LAST": build_with_ignore_nulls(exp.Last),
113        "LAST_VALUE": build_with_ignore_nulls(exp.LastValue),
114        "MAP": parser.build_var_map,
115        "MONTH": lambda args: exp.Month(this=exp.TsOrDsToDate.from_arg_list(args)),
116        "NAMED_STRUCT": _build_named_struct,
117        "REGEXP_EXTRACT": build_regexp_extract(exp.RegexpExtract),
118        "REGEXP_EXTRACT_ALL": build_regexp_extract(exp.RegexpExtractAll),
119        "SEQUENCE": exp.GenerateSeries.from_arg_list,
120        "SIZE": exp.ArraySize.from_arg_list,
121        "SPLIT": exp.RegexpSplit.from_arg_list,
122        "STR_TO_MAP": lambda args: exp.StrToMap(
123            this=seq_get(args, 0),
124            pair_delim=seq_get(args, 1) or exp.Literal.string(","),
125            key_value_delim=seq_get(args, 2) or exp.Literal.string(":"),
126        ),
127        "TO_DATE": _build_to_date,
128        "TO_JSON": exp.JSONFormat.from_arg_list,
129        "TRUNC": exp.TimestampTrunc.from_arg_list,
130        "UNBASE64": exp.FromBase64.from_arg_list,
131        "UNIX_TIMESTAMP": lambda args, dialect: build_formatted_time(exp.StrToUnix, default=True)(
132            args or [exp.CurrentTimestamp()], dialect
133        ),
134        "YEAR": lambda args: exp.Year(this=exp.TsOrDsToDate.from_arg_list(args)),
135    }
136
137    NO_PAREN_FUNCTION_PARSERS = {
138        **parser.Parser.NO_PAREN_FUNCTION_PARSERS,
139        "TRANSFORM": lambda self: self._parse_transform(),
140    }
141
142    NO_PAREN_FUNCTIONS = {
143        k: v for k, v in parser.Parser.NO_PAREN_FUNCTIONS.items() if k != TokenType.CURRENT_TIME
144    }
145
146    PROPERTY_PARSERS = {
147        **parser.Parser.PROPERTY_PARSERS,
148        "SERDEPROPERTIES": lambda self: exp.SerdeProperties(
149            expressions=self._parse_wrapped_csv(self._parse_property)
150        ),
151        "USING": lambda self: self._parse_using_property(),
152    }
153
154    ALTER_PARSERS = {
155        **parser.Parser.ALTER_PARSERS,
156        "CHANGE": lambda self: self._parse_alter_table_change(),
157        "REPLACE": lambda self: self._parse_alter_table_replace_columns(),
158    }
159
160    def build_cast(self, strict: bool, **kwargs) -> exp.Expr:
161        if self._prev.text.upper() == "BD":
162            this = kwargs["this"]
163            _, digits, exponent = Decimal(this.name).as_tuple()
164            precision, scale = len(digits), -int(exponent)
165
166            if self.NORMALIZE_DECIMAL_LITERALS:
167                if not any(digits):
168                    precision, scale = 1, 0
169                else:
170                    while scale > 0 and digits[precision - 1] == 0:
171                        precision -= 1
172                        scale -= 1
173
174            if scale < 0 and not self.DECIMAL_LITERALS_ALLOW_NEGATIVE_SCALE:
175                precision -= scale
176                scale = 0
177
178            # Unsuffixed exponent literals can be DOUBLE, so cast from text to avoid rounding.
179            kwargs["this"] = this if scale < 0 else exp.Literal.string(this.name)
180            kwargs["to"] = exp.DataType(
181                this=exp.DType.DECIMAL,
182                expressions=[
183                    exp.DataTypeParam(this=exp.Literal.number(value))
184                    for value in (max(precision, scale), scale)
185                ],
186            )
187        return super().build_cast(strict, **kwargs)
188
189    def _parse_transform(self) -> exp.Transform | exp.QueryTransform | None:
190        if not self._match(TokenType.L_PAREN, advance=False):
191            self._retreat(self._index - 1)
192            return None
193
194        args = self._parse_wrapped_csv(self._parse_lambda)
195        row_format_before = self._parse_row_format(match_row=True)
196
197        record_writer = None
198        if self._match_text_seq("RECORDWRITER"):
199            record_writer = self._parse_string()
200
201        if not self._match(TokenType.USING):
202            return exp.Transform.from_arg_list(args)
203
204        command_script = self._parse_string()
205
206        self._match(TokenType.ALIAS)
207        schema = self._parse_schema()
208
209        row_format_after = self._parse_row_format(match_row=True)
210        record_reader = None
211        if self._match_text_seq("RECORDREADER"):
212            record_reader = self._parse_string()
213
214        return self.expression(
215            exp.QueryTransform(
216                expressions=args,
217                command_script=command_script,
218                schema=schema,
219                row_format_before=row_format_before,
220                record_writer=record_writer,
221                row_format_after=row_format_after,
222                record_reader=record_reader,
223            )
224        )
225
226    def _parse_types(
227        self,
228        check_func: bool = False,
229        schema: bool = False,
230        allow_identifiers: bool = True,
231        with_collation: bool = False,
232    ) -> exp.Expr | None:
233        """
234        Spark (and most likely Hive) treats casts to CHAR(length) and VARCHAR(length) as casts to
235        STRING in all contexts except for schema definitions. For example, this is in Spark v3.4.0:
236
237            spark-sql (default)> select cast(1234 as varchar(2));
238            23/06/06 15:51:18 WARN CharVarcharUtils: The Spark cast operator does not support
239            char/varchar type and simply treats them as string type. Please use string type
240            directly to avoid confusion. Otherwise, you can set spark.sql.legacy.charVarcharAsString
241            to true, so that Spark treat them as string type as same as Spark 3.0 and earlier
242
243            1234
244            Time taken: 4.265 seconds, Fetched 1 row(s)
245
246        This shows that Spark doesn't truncate the value into '12', which is inconsistent with
247        what other dialects (e.g. postgres) do, so we need to drop the length to transpile correctly.
248
249        Reference: https://spark.apache.org/docs/latest/sql-ref-datatypes.html
250        """
251        this = super()._parse_types(
252            check_func=check_func,
253            schema=schema,
254            allow_identifiers=allow_identifiers,
255            with_collation=with_collation,
256        )
257
258        if this and not schema:
259
260            def _to_text(node: exp.Expr) -> exp.Expr:
261                if isinstance(node, exp.DataType) and node.is_type("char", "varchar"):
262                    node.set("this", exp.DType.TEXT)
263                    node.set("expressions", None)
264                return node
265
266            return this.transform(_to_text, copy=False)
267
268        return this
269
270    def _parse_alter_table_change(self) -> exp.Expr | None:
271        self._match(TokenType.COLUMN)
272        this = self._parse_field(any_token=True)
273
274        if self.CHANGE_COLUMN_ALTER_SYNTAX and self._match_text_seq("TYPE"):
275            return self.expression(exp.AlterColumn(this=this, dtype=self._parse_types(schema=True)))
276
277        column_new = self._parse_field(any_token=True)
278        dtype = self._parse_types(schema=True)
279
280        comment = self._match(TokenType.COMMENT) and self._parse_string()
281
282        if not this or not column_new or not dtype:
283            self.raise_error(
284                "Expected 'CHANGE COLUMN' to be followed by 'column_name' 'column_name' 'data_type'"
285            )
286
287        return self.expression(
288            exp.AlterColumn(this=this, rename_to=column_new, dtype=dtype, comment=comment)
289        )
290
291    def _parse_alter_table_replace_columns(self) -> exp.ReplaceColumns | None:
292        if not self._match_text_seq("COLUMNS"):
293            return None
294
295        expressions = self._try_parse(
296            lambda: self._parse_wrapped_csv(self._parse_field_def, optional=True)
297        )
298        return self.expression(exp.ReplaceColumns(expressions=expressions)) if expressions else None
299
300    def _parse_using_property(self) -> exp.Property:
301        if self._match_texts(("JAR", "FILE", "ARCHIVE")):
302            kind = self._prev.text.upper()
303            return exp.UsingProperty(this=self._parse_string(), kind=kind)
304
305        return self._parse_property_assignment(exp.FileFormatProperty)
306
307    def _parse_partition_and_order(
308        self,
309    ) -> tuple[list[exp.Expr], exp.Expr | None]:
310        return (
311            (
312                self._parse_csv(self._parse_assignment)
313                if self._match_set({TokenType.PARTITION_BY, TokenType.DISTRIBUTE_BY})
314                else []
315            ),
316            super()._parse_order(skip_order_token=self._match(TokenType.SORT_BY)),
317        )
318
319    def _parse_parameter(self) -> exp.Parameter:
320        self._match(TokenType.L_BRACE)
321        this = self._parse_identifier() or self._parse_primary_or_var()
322        expression = self._match(TokenType.COLON) and (
323            self._parse_identifier() or self._parse_primary_or_var()
324        )
325        self._match(TokenType.R_BRACE)
326        return self.expression(exp.Parameter(this=this, expression=expression))
327
328    def _to_prop_eq(self, expression: exp.Expr, index: int) -> exp.Expr:
329        if expression.is_star:
330            return expression
331
332        if isinstance(expression, exp.Column):
333            key = expression.this
334        else:
335            key = exp.to_identifier(f"col{index + 1}")
336
337        return self.expression(exp.PropertyEQ(this=key, expression=expression))