sqlglot.parsers.hive

  1from __future__ import annotations
  2
  3import typing as t
  4from decimal import Decimal
  5
  6from sqlglot import exp, parser
  7from sqlglot.dialects.dialect import build_formatted_time, build_regexp_extract
  8from sqlglot.helper import seq_get
  9from sqlglot.tokens import TokenType
 10
 11if t.TYPE_CHECKING:
 12    from sqlglot.dialects.dialect import Dialect
 13
 14
 15def build_with_ignore_nulls(
 16    exp_class: type[exp.Expr],
 17) -> t.Callable[[list[exp.Expr]], exp.Expr]:
 18    def _parse(args: list[exp.Expr]) -> exp.Expr:
 19        this = exp_class(this=seq_get(args, 0))
 20        if seq_get(args, 1) == exp.true():
 21            return exp.IgnoreNulls(this=this)
 22        return this
 23
 24    return _parse
 25
 26
 27def _build_to_date(args: list, dialect: Dialect) -> exp.TsOrDsToDate:
 28    expr = build_formatted_time(exp.TsOrDsToDate)(args, dialect)
 29    expr.set("safe", True)
 30    return expr
 31
 32
 33def _build_named_struct(args: list) -> exp.Struct:
 34    """Map named_struct('k', v, ...) to exp.Struct so _annotate_struct sees it."""
 35    expressions: list[exp.Expression] = []
 36    for i in range(0, len(args) - 1, 2):
 37        key, value = args[i], args[i + 1]
 38        name = key.name
 39        expressions.append(exp.PropertyEQ(this=exp.to_identifier(name), expression=value))
 40    return exp.Struct(expressions=expressions)
 41
 42
 43def _build_date_add(args: list) -> exp.TsOrDsAdd:
 44    expression = seq_get(args, 1)
 45    if expression:
 46        expression = expression * -1
 47
 48    return exp.TsOrDsAdd(
 49        this=seq_get(args, 0), expression=expression, unit=exp.Literal.string("DAY")
 50    )
 51
 52
 53class HiveParser(parser.Parser):
 54    LOG_DEFAULTS_TO_LN = True
 55    STRICT_CAST = False
 56    VALUES_FOLLOWED_BY_PAREN = False
 57    JOINS_HAVE_EQUAL_PRECEDENCE = True
 58    ADD_JOIN_ON_TRUE = True
 59    ALTER_TABLE_PARTITIONS = True
 60
 61    # Strip trailing fractional zeros and normalize zero when inferring BD literal types.
 62    NORMALIZE_DECIMAL_LITERALS = True
 63
 64    # Allow negative scales for BD exponent literals, e.g. 1E3BD as DECIMAL(1, -3).
 65    DECIMAL_LITERALS_ALLOW_NEGATIVE_SCALE = False
 66
 67    # Whether the dialect supports using ALTER COLUMN syntax with CHANGE COLUMN.
 68    CHANGE_COLUMN_ALTER_SYNTAX = False
 69
 70    FUNCTION_PARSERS = {
 71        **parser.Parser.FUNCTION_PARSERS,
 72        "PERCENTILE": lambda self: self._parse_distinct_arg_function(exp.Quantile),
 73        "PERCENTILE_APPROX": lambda self: self._parse_distinct_arg_function(exp.ApproxQuantile),
 74    }
 75
 76    FUNCTIONS = {
 77        **parser.Parser.FUNCTIONS,
 78        "BASE64": exp.ToBase64.from_arg_list,
 79        "COLLECT_LIST": lambda args: exp.ArrayAgg(this=seq_get(args, 0), nulls_excluded=True),
 80        "COLLECT_SET": exp.ArrayUniqueAgg.from_arg_list,
 81        "CONCAT_WS": lambda args, dialect: exp.ConcatWs(
 82            expressions=args,
 83            safe=not dialect.STRICT_STRING_CONCAT,
 84            coalesce=dialect.CONCAT_WS_COALESCE,
 85            flatten=True,
 86        ),
 87        "DATE_ADD": lambda args: exp.TsOrDsAdd(
 88            this=seq_get(args, 0), expression=seq_get(args, 1), unit=exp.Literal.string("DAY")
 89        ),
 90        "DATE_FORMAT": lambda args, dialect: build_formatted_time(exp.TimeToStr)(
 91            [
 92                exp.TimeStrToTime(this=seq_get(args, 0)),
 93                seq_get(args, 1),
 94            ],
 95            dialect,
 96        ),
 97        "DATE_SUB": _build_date_add,
 98        "DATEDIFF": lambda args: exp.DateDiff(
 99            this=exp.TsOrDsToDate(this=seq_get(args, 0)),
100            expression=exp.TsOrDsToDate(this=seq_get(args, 1)),
101        ),
102        "DAY": lambda args: exp.Day(this=exp.TsOrDsToDate(this=seq_get(args, 0))),
103        "FIRST": build_with_ignore_nulls(exp.First),
104        "FIRST_VALUE": build_with_ignore_nulls(exp.FirstValue),
105        "FROM_UNIXTIME": build_formatted_time(exp.UnixToStr, default=True),
106        "GET_JSON_OBJECT": lambda args, dialect: exp.JSONExtractScalar(
107            this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1))
108        ),
109        "LAST": build_with_ignore_nulls(exp.Last),
110        "LAST_VALUE": build_with_ignore_nulls(exp.LastValue),
111        "MAP": parser.build_var_map,
112        "MONTH": lambda args: exp.Month(this=exp.TsOrDsToDate.from_arg_list(args)),
113        "NAMED_STRUCT": _build_named_struct,
114        "REGEXP_EXTRACT": build_regexp_extract(exp.RegexpExtract),
115        "REGEXP_EXTRACT_ALL": build_regexp_extract(exp.RegexpExtractAll),
116        "SEQUENCE": exp.GenerateSeries.from_arg_list,
117        "SIZE": exp.ArraySize.from_arg_list,
118        "SPLIT": exp.RegexpSplit.from_arg_list,
119        "STR_TO_MAP": lambda args: exp.StrToMap(
120            this=seq_get(args, 0),
121            pair_delim=seq_get(args, 1) or exp.Literal.string(","),
122            key_value_delim=seq_get(args, 2) or exp.Literal.string(":"),
123        ),
124        "TO_DATE": _build_to_date,
125        "TO_JSON": exp.JSONFormat.from_arg_list,
126        "TRUNC": exp.TimestampTrunc.from_arg_list,
127        "UNBASE64": exp.FromBase64.from_arg_list,
128        "UNIX_TIMESTAMP": lambda args, dialect: build_formatted_time(exp.StrToUnix, default=True)(
129            args or [exp.CurrentTimestamp()], dialect
130        ),
131        "YEAR": lambda args: exp.Year(this=exp.TsOrDsToDate.from_arg_list(args)),
132    }
133
134    NO_PAREN_FUNCTION_PARSERS = {
135        **parser.Parser.NO_PAREN_FUNCTION_PARSERS,
136        "TRANSFORM": lambda self: self._parse_transform(),
137    }
138
139    NO_PAREN_FUNCTIONS = {
140        k: v for k, v in parser.Parser.NO_PAREN_FUNCTIONS.items() if k != TokenType.CURRENT_TIME
141    }
142
143    PROPERTY_PARSERS = {
144        **parser.Parser.PROPERTY_PARSERS,
145        "SERDEPROPERTIES": lambda self: exp.SerdeProperties(
146            expressions=self._parse_wrapped_csv(self._parse_property)
147        ),
148        "USING": lambda self: self._parse_using_property(),
149    }
150
151    ALTER_PARSERS = {
152        **parser.Parser.ALTER_PARSERS,
153        "CHANGE": lambda self: self._parse_alter_table_change(),
154    }
155
156    def build_cast(self, strict: bool, **kwargs) -> exp.Expr:
157        if self._prev.text.upper() == "BD":
158            this = kwargs["this"]
159            _, digits, exponent = Decimal(this.name).as_tuple()
160            precision, scale = len(digits), -int(exponent)
161
162            if self.NORMALIZE_DECIMAL_LITERALS:
163                if not any(digits):
164                    precision, scale = 1, 0
165                else:
166                    while scale > 0 and digits[precision - 1] == 0:
167                        precision -= 1
168                        scale -= 1
169
170            if scale < 0 and not self.DECIMAL_LITERALS_ALLOW_NEGATIVE_SCALE:
171                precision -= scale
172                scale = 0
173
174            # Unsuffixed exponent literals can be DOUBLE, so cast from text to avoid rounding.
175            kwargs["this"] = this if scale < 0 else exp.Literal.string(this.name)
176            kwargs["to"] = exp.DataType(
177                this=exp.DType.DECIMAL,
178                expressions=[
179                    exp.DataTypeParam(this=exp.Literal.number(value))
180                    for value in (max(precision, scale), scale)
181                ],
182            )
183        return super().build_cast(strict, **kwargs)
184
185    def _parse_transform(self) -> exp.Transform | exp.QueryTransform | None:
186        if not self._match(TokenType.L_PAREN, advance=False):
187            self._retreat(self._index - 1)
188            return None
189
190        args = self._parse_wrapped_csv(self._parse_lambda)
191        row_format_before = self._parse_row_format(match_row=True)
192
193        record_writer = None
194        if self._match_text_seq("RECORDWRITER"):
195            record_writer = self._parse_string()
196
197        if not self._match(TokenType.USING):
198            return exp.Transform.from_arg_list(args)
199
200        command_script = self._parse_string()
201
202        self._match(TokenType.ALIAS)
203        schema = self._parse_schema()
204
205        row_format_after = self._parse_row_format(match_row=True)
206        record_reader = None
207        if self._match_text_seq("RECORDREADER"):
208            record_reader = self._parse_string()
209
210        return self.expression(
211            exp.QueryTransform(
212                expressions=args,
213                command_script=command_script,
214                schema=schema,
215                row_format_before=row_format_before,
216                record_writer=record_writer,
217                row_format_after=row_format_after,
218                record_reader=record_reader,
219            )
220        )
221
222    def _parse_types(
223        self,
224        check_func: bool = False,
225        schema: bool = False,
226        allow_identifiers: bool = True,
227        with_collation: bool = False,
228    ) -> exp.Expr | None:
229        """
230        Spark (and most likely Hive) treats casts to CHAR(length) and VARCHAR(length) as casts to
231        STRING in all contexts except for schema definitions. For example, this is in Spark v3.4.0:
232
233            spark-sql (default)> select cast(1234 as varchar(2));
234            23/06/06 15:51:18 WARN CharVarcharUtils: The Spark cast operator does not support
235            char/varchar type and simply treats them as string type. Please use string type
236            directly to avoid confusion. Otherwise, you can set spark.sql.legacy.charVarcharAsString
237            to true, so that Spark treat them as string type as same as Spark 3.0 and earlier
238
239            1234
240            Time taken: 4.265 seconds, Fetched 1 row(s)
241
242        This shows that Spark doesn't truncate the value into '12', which is inconsistent with
243        what other dialects (e.g. postgres) do, so we need to drop the length to transpile correctly.
244
245        Reference: https://spark.apache.org/docs/latest/sql-ref-datatypes.html
246        """
247        this = super()._parse_types(
248            check_func=check_func,
249            schema=schema,
250            allow_identifiers=allow_identifiers,
251            with_collation=with_collation,
252        )
253
254        if this and not schema:
255
256            def _to_text(node: exp.Expr) -> exp.Expr:
257                if isinstance(node, exp.DataType) and node.is_type("char", "varchar"):
258                    node.set("this", exp.DType.TEXT)
259                    node.set("expressions", None)
260                return node
261
262            return this.transform(_to_text, copy=False)
263
264        return this
265
266    def _parse_alter_table_change(self) -> exp.Expr | None:
267        self._match(TokenType.COLUMN)
268        this = self._parse_field(any_token=True)
269
270        if self.CHANGE_COLUMN_ALTER_SYNTAX and self._match_text_seq("TYPE"):
271            return self.expression(exp.AlterColumn(this=this, dtype=self._parse_types(schema=True)))
272
273        column_new = self._parse_field(any_token=True)
274        dtype = self._parse_types(schema=True)
275
276        comment = self._match(TokenType.COMMENT) and self._parse_string()
277
278        if not this or not column_new or not dtype:
279            self.raise_error(
280                "Expected 'CHANGE COLUMN' to be followed by 'column_name' 'column_name' 'data_type'"
281            )
282
283        return self.expression(
284            exp.AlterColumn(this=this, rename_to=column_new, dtype=dtype, comment=comment)
285        )
286
287    def _parse_using_property(self) -> exp.Property:
288        if self._match_texts(("JAR", "FILE", "ARCHIVE")):
289            kind = self._prev.text.upper()
290            return exp.UsingProperty(this=self._parse_string(), kind=kind)
291
292        return self._parse_property_assignment(exp.FileFormatProperty)
293
294    def _parse_partition_and_order(
295        self,
296    ) -> tuple[list[exp.Expr], exp.Expr | None]:
297        return (
298            (
299                self._parse_csv(self._parse_assignment)
300                if self._match_set({TokenType.PARTITION_BY, TokenType.DISTRIBUTE_BY})
301                else []
302            ),
303            super()._parse_order(skip_order_token=self._match(TokenType.SORT_BY)),
304        )
305
306    def _parse_parameter(self) -> exp.Parameter:
307        self._match(TokenType.L_BRACE)
308        this = self._parse_identifier() or self._parse_primary_or_var()
309        expression = self._match(TokenType.COLON) and (
310            self._parse_identifier() or self._parse_primary_or_var()
311        )
312        self._match(TokenType.R_BRACE)
313        return self.expression(exp.Parameter(this=this, expression=expression))
314
315    def _to_prop_eq(self, expression: exp.Expr, index: int) -> exp.Expr:
316        if expression.is_star:
317            return expression
318
319        if isinstance(expression, exp.Column):
320            key = expression.this
321        else:
322            key = exp.to_identifier(f"col{index + 1}")
323
324        return self.expression(exp.PropertyEQ(this=key, expression=expression))