sqlglot.parsers.hive

  1from __future__ import annotations
  2
  3import typing as t
  4
  5from sqlglot import exp, parser
  6from sqlglot.dialects.dialect import build_formatted_time, build_regexp_extract
  7from sqlglot.helper import seq_get
  8from sqlglot.tokens import TokenType
  9
 10if t.TYPE_CHECKING:
 11    from sqlglot._typing import F
 12    from sqlglot.dialects.dialect import Dialect
 13
 14
 15def build_with_ignore_nulls(
 16    exp_class: type[exp.Expr],
 17) -> t.Callable[[list[exp.Expr]], exp.Expr]:
 18    def _parse(args: list[exp.Expr]) -> exp.Expr:
 19        this = exp_class(this=seq_get(args, 0))
 20        if seq_get(args, 1) == exp.true():
 21            return exp.IgnoreNulls(this=this)
 22        return this
 23
 24    return _parse
 25
 26
 27def _build_to_date(args: list, dialect: Dialect) -> exp.TsOrDsToDate:
 28    expr = build_formatted_time(exp.TsOrDsToDate)(args, dialect)
 29    expr.set("safe", True)
 30    return expr
 31
 32
 33def _build_named_struct(args: list) -> exp.Struct:
 34    """Map named_struct('k', v, ...) to exp.Struct so _annotate_struct sees it."""
 35    expressions: list[exp.Expression] = []
 36    for i in range(0, len(args) - 1, 2):
 37        key, value = args[i], args[i + 1]
 38        name = key.name
 39        expressions.append(exp.PropertyEQ(this=exp.to_identifier(name), expression=value))
 40    return exp.Struct(expressions=expressions)
 41
 42
 43def _build_date_add(args: list) -> exp.TsOrDsAdd:
 44    expression = seq_get(args, 1)
 45    if expression:
 46        expression = expression * -1
 47
 48    return exp.TsOrDsAdd(
 49        this=seq_get(args, 0), expression=expression, unit=exp.Literal.string("DAY")
 50    )
 51
 52
 53class HiveParser(parser.Parser):
 54    LOG_DEFAULTS_TO_LN = True
 55    STRICT_CAST = False
 56    VALUES_FOLLOWED_BY_PAREN = False
 57    JOINS_HAVE_EQUAL_PRECEDENCE = True
 58    ADD_JOIN_ON_TRUE = True
 59    ALTER_TABLE_PARTITIONS = True
 60
 61    CHANGE_COLUMN_ALTER_SYNTAX = False
 62    # Whether the dialect supports using ALTER COLUMN syntax with CHANGE COLUMN.
 63
 64    FUNCTION_PARSERS = {
 65        **parser.Parser.FUNCTION_PARSERS,
 66        "PERCENTILE": lambda self: self._parse_quantile_function(exp.Quantile),
 67        "PERCENTILE_APPROX": lambda self: self._parse_quantile_function(exp.ApproxQuantile),
 68    }
 69
 70    FUNCTIONS = {
 71        **parser.Parser.FUNCTIONS,
 72        "BASE64": exp.ToBase64.from_arg_list,
 73        "COLLECT_LIST": lambda args: exp.ArrayAgg(this=seq_get(args, 0), nulls_excluded=True),
 74        "COLLECT_SET": exp.ArrayUniqueAgg.from_arg_list,
 75        "DATE_ADD": lambda args: exp.TsOrDsAdd(
 76            this=seq_get(args, 0), expression=seq_get(args, 1), unit=exp.Literal.string("DAY")
 77        ),
 78        "DATE_FORMAT": lambda args, dialect: build_formatted_time(exp.TimeToStr)(
 79            [
 80                exp.TimeStrToTime(this=seq_get(args, 0)),
 81                seq_get(args, 1),
 82            ],
 83            dialect,
 84        ),
 85        "DATE_SUB": _build_date_add,
 86        "DATEDIFF": lambda args: exp.DateDiff(
 87            this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 88            expression=exp.TsOrDsToDate(this=seq_get(args, 1)),
 89        ),
 90        "DAY": lambda args: exp.Day(this=exp.TsOrDsToDate(this=seq_get(args, 0))),
 91        "FIRST": build_with_ignore_nulls(exp.First),
 92        "FIRST_VALUE": build_with_ignore_nulls(exp.FirstValue),
 93        "FROM_UNIXTIME": build_formatted_time(exp.UnixToStr, default=True),
 94        "GET_JSON_OBJECT": lambda args, dialect: exp.JSONExtractScalar(
 95            this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1))
 96        ),
 97        "LAST": build_with_ignore_nulls(exp.Last),
 98        "LAST_VALUE": build_with_ignore_nulls(exp.LastValue),
 99        "MAP": parser.build_var_map,
100        "MONTH": lambda args: exp.Month(this=exp.TsOrDsToDate.from_arg_list(args)),
101        "NAMED_STRUCT": _build_named_struct,
102        "REGEXP_EXTRACT": build_regexp_extract(exp.RegexpExtract),
103        "REGEXP_EXTRACT_ALL": build_regexp_extract(exp.RegexpExtractAll),
104        "SEQUENCE": exp.GenerateSeries.from_arg_list,
105        "SIZE": exp.ArraySize.from_arg_list,
106        "SPLIT": exp.RegexpSplit.from_arg_list,
107        "STR_TO_MAP": lambda args: exp.StrToMap(
108            this=seq_get(args, 0),
109            pair_delim=seq_get(args, 1) or exp.Literal.string(","),
110            key_value_delim=seq_get(args, 2) or exp.Literal.string(":"),
111        ),
112        "TO_DATE": _build_to_date,
113        "TO_JSON": exp.JSONFormat.from_arg_list,
114        "TRUNC": exp.TimestampTrunc.from_arg_list,
115        "UNBASE64": exp.FromBase64.from_arg_list,
116        "UNIX_TIMESTAMP": lambda args, dialect: build_formatted_time(exp.StrToUnix, default=True)(
117            args or [exp.CurrentTimestamp()], dialect
118        ),
119        "YEAR": lambda args: exp.Year(this=exp.TsOrDsToDate.from_arg_list(args)),
120    }
121
122    NO_PAREN_FUNCTION_PARSERS = {
123        **parser.Parser.NO_PAREN_FUNCTION_PARSERS,
124        "TRANSFORM": lambda self: self._parse_transform(),
125    }
126
127    NO_PAREN_FUNCTIONS = {
128        k: v for k, v in parser.Parser.NO_PAREN_FUNCTIONS.items() if k != TokenType.CURRENT_TIME
129    }
130
131    PROPERTY_PARSERS = {
132        **parser.Parser.PROPERTY_PARSERS,
133        "SERDEPROPERTIES": lambda self: exp.SerdeProperties(
134            expressions=self._parse_wrapped_csv(self._parse_property)
135        ),
136        "USING": lambda self: self._parse_using_property(),
137    }
138
139    ALTER_PARSERS = {
140        **parser.Parser.ALTER_PARSERS,
141        "CHANGE": lambda self: self._parse_alter_table_change(),
142    }
143
144    def _parse_transform(self) -> exp.Transform | exp.QueryTransform | None:
145        if not self._match(TokenType.L_PAREN, advance=False):
146            self._retreat(self._index - 1)
147            return None
148
149        args = self._parse_wrapped_csv(self._parse_lambda)
150        row_format_before = self._parse_row_format(match_row=True)
151
152        record_writer = None
153        if self._match_text_seq("RECORDWRITER"):
154            record_writer = self._parse_string()
155
156        if not self._match(TokenType.USING):
157            return exp.Transform.from_arg_list(args)
158
159        command_script = self._parse_string()
160
161        self._match(TokenType.ALIAS)
162        schema = self._parse_schema()
163
164        row_format_after = self._parse_row_format(match_row=True)
165        record_reader = None
166        if self._match_text_seq("RECORDREADER"):
167            record_reader = self._parse_string()
168
169        return self.expression(
170            exp.QueryTransform(
171                expressions=args,
172                command_script=command_script,
173                schema=schema,
174                row_format_before=row_format_before,
175                record_writer=record_writer,
176                row_format_after=row_format_after,
177                record_reader=record_reader,
178            )
179        )
180
181    def _parse_quantile_function(self, func: type[F]) -> F:
182        if self._match(TokenType.DISTINCT):
183            first_arg: exp.Expr | None = self.expression(
184                exp.Distinct(expressions=[self._parse_lambda()])
185            )
186        else:
187            self._match(TokenType.ALL)
188            first_arg = self._parse_lambda()
189
190        args = [first_arg]
191        if self._match(TokenType.COMMA):
192            args.extend(self._parse_function_args())
193
194        return func.from_arg_list(args)
195
196    def _parse_types(
197        self,
198        check_func: bool = False,
199        schema: bool = False,
200        allow_identifiers: bool = True,
201        with_collation: bool = False,
202    ) -> exp.Expr | None:
203        """
204        Spark (and most likely Hive) treats casts to CHAR(length) and VARCHAR(length) as casts to
205        STRING in all contexts except for schema definitions. For example, this is in Spark v3.4.0:
206
207            spark-sql (default)> select cast(1234 as varchar(2));
208            23/06/06 15:51:18 WARN CharVarcharUtils: The Spark cast operator does not support
209            char/varchar type and simply treats them as string type. Please use string type
210            directly to avoid confusion. Otherwise, you can set spark.sql.legacy.charVarcharAsString
211            to true, so that Spark treat them as string type as same as Spark 3.0 and earlier
212
213            1234
214            Time taken: 4.265 seconds, Fetched 1 row(s)
215
216        This shows that Spark doesn't truncate the value into '12', which is inconsistent with
217        what other dialects (e.g. postgres) do, so we need to drop the length to transpile correctly.
218
219        Reference: https://spark.apache.org/docs/latest/sql-ref-datatypes.html
220        """
221        this = super()._parse_types(
222            check_func=check_func,
223            schema=schema,
224            allow_identifiers=allow_identifiers,
225            with_collation=with_collation,
226        )
227
228        if this and not schema:
229
230            def _to_text(node: exp.Expr) -> exp.Expr:
231                if isinstance(node, exp.DataType) and node.is_type("char", "varchar"):
232                    node.set("this", exp.DType.TEXT)
233                    node.set("expressions", None)
234                return node
235
236            return this.transform(_to_text, copy=False)
237
238        return this
239
240    def _parse_alter_table_change(self) -> exp.Expr | None:
241        self._match(TokenType.COLUMN)
242        this = self._parse_field(any_token=True)
243
244        if self.CHANGE_COLUMN_ALTER_SYNTAX and self._match_text_seq("TYPE"):
245            return self.expression(exp.AlterColumn(this=this, dtype=self._parse_types(schema=True)))
246
247        column_new = self._parse_field(any_token=True)
248        dtype = self._parse_types(schema=True)
249
250        comment = self._match(TokenType.COMMENT) and self._parse_string()
251
252        if not this or not column_new or not dtype:
253            self.raise_error(
254                "Expected 'CHANGE COLUMN' to be followed by 'column_name' 'column_name' 'data_type'"
255            )
256
257        return self.expression(
258            exp.AlterColumn(this=this, rename_to=column_new, dtype=dtype, comment=comment)
259        )
260
261    def _parse_using_property(self) -> exp.Property:
262        if self._match_texts(("JAR", "FILE", "ARCHIVE")):
263            kind = self._prev.text.upper()
264            return exp.UsingProperty(this=self._parse_string(), kind=kind)
265
266        return self._parse_property_assignment(exp.FileFormatProperty)
267
268    def _parse_partition_and_order(
269        self,
270    ) -> tuple[list[exp.Expr], exp.Expr | None]:
271        return (
272            (
273                self._parse_csv(self._parse_assignment)
274                if self._match_set({TokenType.PARTITION_BY, TokenType.DISTRIBUTE_BY})
275                else []
276            ),
277            super()._parse_order(skip_order_token=self._match(TokenType.SORT_BY)),
278        )
279
280    def _parse_parameter(self) -> exp.Parameter:
281        self._match(TokenType.L_BRACE)
282        this = self._parse_identifier() or self._parse_primary_or_var()
283        expression = self._match(TokenType.COLON) and (
284            self._parse_identifier() or self._parse_primary_or_var()
285        )
286        self._match(TokenType.R_BRACE)
287        return self.expression(exp.Parameter(this=this, expression=expression))
288
289    def _to_prop_eq(self, expression: exp.Expr, index: int) -> exp.Expr:
290        if expression.is_star:
291            return expression
292
293        if isinstance(expression, exp.Column):
294            key = expression.this
295        else:
296            key = exp.to_identifier(f"col{index + 1}")
297
298        return self.expression(exp.PropertyEQ(this=key, expression=expression))