sqlglot.parsers.hive

  1from __future__ import annotations
  2
  3import typing as t
  4
  5from sqlglot import exp, parser
  6from sqlglot.dialects.dialect import build_formatted_time, build_regexp_extract
  7from sqlglot.helper import seq_get
  8from sqlglot.tokens import TokenType
  9
 10if t.TYPE_CHECKING:
 11    from sqlglot._typing import F
 12
 13
 14def build_with_ignore_nulls(
 15    exp_class: type[exp.Expr],
 16) -> t.Callable[[list[exp.Expr]], exp.Expr]:
 17    def _parse(args: list[exp.Expr]) -> exp.Expr:
 18        this = exp_class(this=seq_get(args, 0))
 19        if seq_get(args, 1) == exp.true():
 20            return exp.IgnoreNulls(this=this)
 21        return this
 22
 23    return _parse
 24
 25
 26def _build_to_date(args: list) -> exp.TsOrDsToDate:
 27    expr = build_formatted_time(exp.TsOrDsToDate, "hive")(args)
 28    expr.set("safe", True)
 29    return expr
 30
 31
 32def _build_date_add(args: list) -> exp.TsOrDsAdd:
 33    expression = seq_get(args, 1)
 34    if expression:
 35        expression = expression * -1
 36
 37    return exp.TsOrDsAdd(
 38        this=seq_get(args, 0), expression=expression, unit=exp.Literal.string("DAY")
 39    )
 40
 41
 42class HiveParser(parser.Parser):
 43    LOG_DEFAULTS_TO_LN = True
 44    STRICT_CAST = False
 45    VALUES_FOLLOWED_BY_PAREN = False
 46    JOINS_HAVE_EQUAL_PRECEDENCE = True
 47    ADD_JOIN_ON_TRUE = True
 48    ALTER_TABLE_PARTITIONS = True
 49
 50    CHANGE_COLUMN_ALTER_SYNTAX = False
 51    # Whether the dialect supports using ALTER COLUMN syntax with CHANGE COLUMN.
 52
 53    FUNCTION_PARSERS = {
 54        **parser.Parser.FUNCTION_PARSERS,
 55        "PERCENTILE": lambda self: self._parse_quantile_function(exp.Quantile),
 56        "PERCENTILE_APPROX": lambda self: self._parse_quantile_function(exp.ApproxQuantile),
 57    }
 58
 59    FUNCTIONS = {
 60        **parser.Parser.FUNCTIONS,
 61        "BASE64": exp.ToBase64.from_arg_list,
 62        "COLLECT_LIST": lambda args: exp.ArrayAgg(this=seq_get(args, 0), nulls_excluded=True),
 63        "COLLECT_SET": exp.ArrayUniqueAgg.from_arg_list,
 64        "DATE_ADD": lambda args: exp.TsOrDsAdd(
 65            this=seq_get(args, 0), expression=seq_get(args, 1), unit=exp.Literal.string("DAY")
 66        ),
 67        "DATE_FORMAT": lambda args: build_formatted_time(exp.TimeToStr, "hive")(
 68            [
 69                exp.TimeStrToTime(this=seq_get(args, 0)),
 70                seq_get(args, 1),
 71            ]
 72        ),
 73        "DATE_SUB": _build_date_add,
 74        "DATEDIFF": lambda args: exp.DateDiff(
 75            this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 76            expression=exp.TsOrDsToDate(this=seq_get(args, 1)),
 77        ),
 78        "DAY": lambda args: exp.Day(this=exp.TsOrDsToDate(this=seq_get(args, 0))),
 79        "FIRST": build_with_ignore_nulls(exp.First),
 80        "FIRST_VALUE": build_with_ignore_nulls(exp.FirstValue),
 81        "FROM_UNIXTIME": build_formatted_time(exp.UnixToStr, "hive", True),
 82        "GET_JSON_OBJECT": lambda args, dialect: exp.JSONExtractScalar(
 83            this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1))
 84        ),
 85        "LAST": build_with_ignore_nulls(exp.Last),
 86        "LAST_VALUE": build_with_ignore_nulls(exp.LastValue),
 87        "MAP": parser.build_var_map,
 88        "MONTH": lambda args: exp.Month(this=exp.TsOrDsToDate.from_arg_list(args)),
 89        "REGEXP_EXTRACT": build_regexp_extract(exp.RegexpExtract),
 90        "REGEXP_EXTRACT_ALL": build_regexp_extract(exp.RegexpExtractAll),
 91        "SEQUENCE": exp.GenerateSeries.from_arg_list,
 92        "SIZE": exp.ArraySize.from_arg_list,
 93        "SPLIT": exp.RegexpSplit.from_arg_list,
 94        "STR_TO_MAP": lambda args: exp.StrToMap(
 95            this=seq_get(args, 0),
 96            pair_delim=seq_get(args, 1) or exp.Literal.string(","),
 97            key_value_delim=seq_get(args, 2) or exp.Literal.string(":"),
 98        ),
 99        "TO_DATE": _build_to_date,
100        "TO_JSON": exp.JSONFormat.from_arg_list,
101        "TRUNC": exp.TimestampTrunc.from_arg_list,
102        "UNBASE64": exp.FromBase64.from_arg_list,
103        "UNIX_TIMESTAMP": lambda args: build_formatted_time(exp.StrToUnix, "hive", True)(
104            args or [exp.CurrentTimestamp()]
105        ),
106        "YEAR": lambda args: exp.Year(this=exp.TsOrDsToDate.from_arg_list(args)),
107    }
108
109    NO_PAREN_FUNCTION_PARSERS = {
110        **parser.Parser.NO_PAREN_FUNCTION_PARSERS,
111        "TRANSFORM": lambda self: self._parse_transform(),
112    }
113
114    NO_PAREN_FUNCTIONS = {
115        k: v for k, v in parser.Parser.NO_PAREN_FUNCTIONS.items() if k != TokenType.CURRENT_TIME
116    }
117
118    PROPERTY_PARSERS = {
119        **parser.Parser.PROPERTY_PARSERS,
120        "SERDEPROPERTIES": lambda self: exp.SerdeProperties(
121            expressions=self._parse_wrapped_csv(self._parse_property)
122        ),
123        "USING": lambda self: self._parse_using_property(),
124    }
125
126    ALTER_PARSERS = {
127        **parser.Parser.ALTER_PARSERS,
128        "CHANGE": lambda self: self._parse_alter_table_change(),
129    }
130
131    def _parse_transform(self) -> exp.Transform | exp.QueryTransform | None:
132        if not self._match(TokenType.L_PAREN, advance=False):
133            self._retreat(self._index - 1)
134            return None
135
136        args = self._parse_wrapped_csv(self._parse_lambda)
137        row_format_before = self._parse_row_format(match_row=True)
138
139        record_writer = None
140        if self._match_text_seq("RECORDWRITER"):
141            record_writer = self._parse_string()
142
143        if not self._match(TokenType.USING):
144            return exp.Transform.from_arg_list(args)
145
146        command_script = self._parse_string()
147
148        self._match(TokenType.ALIAS)
149        schema = self._parse_schema()
150
151        row_format_after = self._parse_row_format(match_row=True)
152        record_reader = None
153        if self._match_text_seq("RECORDREADER"):
154            record_reader = self._parse_string()
155
156        return self.expression(
157            exp.QueryTransform(
158                expressions=args,
159                command_script=command_script,
160                schema=schema,
161                row_format_before=row_format_before,
162                record_writer=record_writer,
163                row_format_after=row_format_after,
164                record_reader=record_reader,
165            )
166        )
167
168    def _parse_quantile_function(self, func: type[F]) -> F:
169        if self._match(TokenType.DISTINCT):
170            first_arg: exp.Expr | None = self.expression(
171                exp.Distinct(expressions=[self._parse_lambda()])
172            )
173        else:
174            self._match(TokenType.ALL)
175            first_arg = self._parse_lambda()
176
177        args = [first_arg]
178        if self._match(TokenType.COMMA):
179            args.extend(self._parse_function_args())
180
181        return func.from_arg_list(args)
182
183    def _parse_types(
184        self, check_func: bool = False, schema: bool = False, allow_identifiers: bool = True
185    ) -> exp.Expr | None:
186        """
187        Spark (and most likely Hive) treats casts to CHAR(length) and VARCHAR(length) as casts to
188        STRING in all contexts except for schema definitions. For example, this is in Spark v3.4.0:
189
190            spark-sql (default)> select cast(1234 as varchar(2));
191            23/06/06 15:51:18 WARN CharVarcharUtils: The Spark cast operator does not support
192            char/varchar type and simply treats them as string type. Please use string type
193            directly to avoid confusion. Otherwise, you can set spark.sql.legacy.charVarcharAsString
194            to true, so that Spark treat them as string type as same as Spark 3.0 and earlier
195
196            1234
197            Time taken: 4.265 seconds, Fetched 1 row(s)
198
199        This shows that Spark doesn't truncate the value into '12', which is inconsistent with
200        what other dialects (e.g. postgres) do, so we need to drop the length to transpile correctly.
201
202        Reference: https://spark.apache.org/docs/latest/sql-ref-datatypes.html
203        """
204        this = super()._parse_types(
205            check_func=check_func, schema=schema, allow_identifiers=allow_identifiers
206        )
207
208        if this and not schema:
209            return this.transform(
210                lambda node: (
211                    node.replace(exp.DType.TEXT.into_expr())
212                    if isinstance(node, exp.DataType) and node.is_type("char", "varchar")
213                    else node
214                ),
215                copy=False,
216            )
217
218        return this
219
220    def _parse_alter_table_change(self) -> exp.Expr | None:
221        self._match(TokenType.COLUMN)
222        this = self._parse_field(any_token=True)
223
224        if self.CHANGE_COLUMN_ALTER_SYNTAX and self._match_text_seq("TYPE"):
225            return self.expression(exp.AlterColumn(this=this, dtype=self._parse_types(schema=True)))
226
227        column_new = self._parse_field(any_token=True)
228        dtype = self._parse_types(schema=True)
229
230        comment = self._match(TokenType.COMMENT) and self._parse_string()
231
232        if not this or not column_new or not dtype:
233            self.raise_error(
234                "Expected 'CHANGE COLUMN' to be followed by 'column_name' 'column_name' 'data_type'"
235            )
236
237        return self.expression(
238            exp.AlterColumn(this=this, rename_to=column_new, dtype=dtype, comment=comment)
239        )
240
241    def _parse_using_property(self) -> exp.Property:
242        if self._match_texts(("JAR", "FILE", "ARCHIVE")):
243            kind = self._prev.text.upper()
244            return exp.UsingProperty(this=self._parse_string(), kind=kind)
245
246        return self._parse_property_assignment(exp.FileFormatProperty)
247
248    def _parse_partition_and_order(
249        self,
250    ) -> tuple[list[exp.Expr], exp.Expr | None]:
251        return (
252            (
253                self._parse_csv(self._parse_assignment)
254                if self._match_set({TokenType.PARTITION_BY, TokenType.DISTRIBUTE_BY})
255                else []
256            ),
257            super()._parse_order(skip_order_token=self._match(TokenType.SORT_BY)),
258        )
259
260    def _parse_parameter(self) -> exp.Parameter:
261        self._match(TokenType.L_BRACE)
262        this = self._parse_identifier() or self._parse_primary_or_var()
263        expression = self._match(TokenType.COLON) and (
264            self._parse_identifier() or self._parse_primary_or_var()
265        )
266        self._match(TokenType.R_BRACE)
267        return self.expression(exp.Parameter(this=this, expression=expression))
268
269    def _to_prop_eq(self, expression: exp.Expr, index: int) -> exp.Expr:
270        if expression.is_star:
271            return expression
272
273        if isinstance(expression, exp.Column):
274            key = expression.this
275        else:
276            key = exp.to_identifier(f"col{index + 1}")
277
278        return self.expression(exp.PropertyEQ(this=key, expression=expression))