sqlglot.parsers.hive

  1from __future__ import annotations
  2
  3import typing as t
  4
  5from sqlglot import exp, parser
  6from sqlglot.dialects.dialect import build_formatted_time, build_regexp_extract
  7from sqlglot.helper import seq_get
  8from sqlglot.tokens import TokenType
  9
 10if t.TYPE_CHECKING:
 11    from sqlglot.dialects.dialect import Dialect
 12
 13
 14def build_with_ignore_nulls(
 15    exp_class: type[exp.Expr],
 16) -> t.Callable[[list[exp.Expr]], exp.Expr]:
 17    def _parse(args: list[exp.Expr]) -> exp.Expr:
 18        this = exp_class(this=seq_get(args, 0))
 19        if seq_get(args, 1) == exp.true():
 20            return exp.IgnoreNulls(this=this)
 21        return this
 22
 23    return _parse
 24
 25
 26def _build_to_date(args: list, dialect: Dialect) -> exp.TsOrDsToDate:
 27    expr = build_formatted_time(exp.TsOrDsToDate)(args, dialect)
 28    expr.set("safe", True)
 29    return expr
 30
 31
 32def _build_named_struct(args: list) -> exp.Struct:
 33    """Map named_struct('k', v, ...) to exp.Struct so _annotate_struct sees it."""
 34    expressions: list[exp.Expression] = []
 35    for i in range(0, len(args) - 1, 2):
 36        key, value = args[i], args[i + 1]
 37        name = key.name
 38        expressions.append(exp.PropertyEQ(this=exp.to_identifier(name), expression=value))
 39    return exp.Struct(expressions=expressions)
 40
 41
 42def _build_date_add(args: list) -> exp.TsOrDsAdd:
 43    expression = seq_get(args, 1)
 44    if expression:
 45        expression = expression * -1
 46
 47    return exp.TsOrDsAdd(
 48        this=seq_get(args, 0), expression=expression, unit=exp.Literal.string("DAY")
 49    )
 50
 51
 52class HiveParser(parser.Parser):
 53    LOG_DEFAULTS_TO_LN = True
 54    STRICT_CAST = False
 55    VALUES_FOLLOWED_BY_PAREN = False
 56    JOINS_HAVE_EQUAL_PRECEDENCE = True
 57    ADD_JOIN_ON_TRUE = True
 58    ALTER_TABLE_PARTITIONS = True
 59
 60    CHANGE_COLUMN_ALTER_SYNTAX = False
 61    # Whether the dialect supports using ALTER COLUMN syntax with CHANGE COLUMN.
 62
 63    FUNCTION_PARSERS = {
 64        **parser.Parser.FUNCTION_PARSERS,
 65        "PERCENTILE": lambda self: self._parse_distinct_arg_function(exp.Quantile),
 66        "PERCENTILE_APPROX": lambda self: self._parse_distinct_arg_function(exp.ApproxQuantile),
 67    }
 68
 69    FUNCTIONS = {
 70        **parser.Parser.FUNCTIONS,
 71        "BASE64": exp.ToBase64.from_arg_list,
 72        "COLLECT_LIST": lambda args: exp.ArrayAgg(this=seq_get(args, 0), nulls_excluded=True),
 73        "COLLECT_SET": exp.ArrayUniqueAgg.from_arg_list,
 74        "DATE_ADD": lambda args: exp.TsOrDsAdd(
 75            this=seq_get(args, 0), expression=seq_get(args, 1), unit=exp.Literal.string("DAY")
 76        ),
 77        "DATE_FORMAT": lambda args, dialect: build_formatted_time(exp.TimeToStr)(
 78            [
 79                exp.TimeStrToTime(this=seq_get(args, 0)),
 80                seq_get(args, 1),
 81            ],
 82            dialect,
 83        ),
 84        "DATE_SUB": _build_date_add,
 85        "DATEDIFF": lambda args: exp.DateDiff(
 86            this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 87            expression=exp.TsOrDsToDate(this=seq_get(args, 1)),
 88        ),
 89        "DAY": lambda args: exp.Day(this=exp.TsOrDsToDate(this=seq_get(args, 0))),
 90        "FIRST": build_with_ignore_nulls(exp.First),
 91        "FIRST_VALUE": build_with_ignore_nulls(exp.FirstValue),
 92        "FROM_UNIXTIME": build_formatted_time(exp.UnixToStr, default=True),
 93        "GET_JSON_OBJECT": lambda args, dialect: exp.JSONExtractScalar(
 94            this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1))
 95        ),
 96        "LAST": build_with_ignore_nulls(exp.Last),
 97        "LAST_VALUE": build_with_ignore_nulls(exp.LastValue),
 98        "MAP": parser.build_var_map,
 99        "MONTH": lambda args: exp.Month(this=exp.TsOrDsToDate.from_arg_list(args)),
100        "NAMED_STRUCT": _build_named_struct,
101        "REGEXP_EXTRACT": build_regexp_extract(exp.RegexpExtract),
102        "REGEXP_EXTRACT_ALL": build_regexp_extract(exp.RegexpExtractAll),
103        "SEQUENCE": exp.GenerateSeries.from_arg_list,
104        "SIZE": exp.ArraySize.from_arg_list,
105        "SPLIT": exp.RegexpSplit.from_arg_list,
106        "STR_TO_MAP": lambda args: exp.StrToMap(
107            this=seq_get(args, 0),
108            pair_delim=seq_get(args, 1) or exp.Literal.string(","),
109            key_value_delim=seq_get(args, 2) or exp.Literal.string(":"),
110        ),
111        "TO_DATE": _build_to_date,
112        "TO_JSON": exp.JSONFormat.from_arg_list,
113        "TRUNC": exp.TimestampTrunc.from_arg_list,
114        "UNBASE64": exp.FromBase64.from_arg_list,
115        "UNIX_TIMESTAMP": lambda args, dialect: build_formatted_time(exp.StrToUnix, default=True)(
116            args or [exp.CurrentTimestamp()], dialect
117        ),
118        "YEAR": lambda args: exp.Year(this=exp.TsOrDsToDate.from_arg_list(args)),
119    }
120
121    NO_PAREN_FUNCTION_PARSERS = {
122        **parser.Parser.NO_PAREN_FUNCTION_PARSERS,
123        "TRANSFORM": lambda self: self._parse_transform(),
124    }
125
126    NO_PAREN_FUNCTIONS = {
127        k: v for k, v in parser.Parser.NO_PAREN_FUNCTIONS.items() if k != TokenType.CURRENT_TIME
128    }
129
130    PROPERTY_PARSERS = {
131        **parser.Parser.PROPERTY_PARSERS,
132        "SERDEPROPERTIES": lambda self: exp.SerdeProperties(
133            expressions=self._parse_wrapped_csv(self._parse_property)
134        ),
135        "USING": lambda self: self._parse_using_property(),
136    }
137
138    ALTER_PARSERS = {
139        **parser.Parser.ALTER_PARSERS,
140        "CHANGE": lambda self: self._parse_alter_table_change(),
141    }
142
143    def _parse_transform(self) -> exp.Transform | exp.QueryTransform | None:
144        if not self._match(TokenType.L_PAREN, advance=False):
145            self._retreat(self._index - 1)
146            return None
147
148        args = self._parse_wrapped_csv(self._parse_lambda)
149        row_format_before = self._parse_row_format(match_row=True)
150
151        record_writer = None
152        if self._match_text_seq("RECORDWRITER"):
153            record_writer = self._parse_string()
154
155        if not self._match(TokenType.USING):
156            return exp.Transform.from_arg_list(args)
157
158        command_script = self._parse_string()
159
160        self._match(TokenType.ALIAS)
161        schema = self._parse_schema()
162
163        row_format_after = self._parse_row_format(match_row=True)
164        record_reader = None
165        if self._match_text_seq("RECORDREADER"):
166            record_reader = self._parse_string()
167
168        return self.expression(
169            exp.QueryTransform(
170                expressions=args,
171                command_script=command_script,
172                schema=schema,
173                row_format_before=row_format_before,
174                record_writer=record_writer,
175                row_format_after=row_format_after,
176                record_reader=record_reader,
177            )
178        )
179
180    def _parse_types(
181        self,
182        check_func: bool = False,
183        schema: bool = False,
184        allow_identifiers: bool = True,
185        with_collation: bool = False,
186    ) -> exp.Expr | None:
187        """
188        Spark (and most likely Hive) treats casts to CHAR(length) and VARCHAR(length) as casts to
189        STRING in all contexts except for schema definitions. For example, this is in Spark v3.4.0:
190
191            spark-sql (default)> select cast(1234 as varchar(2));
192            23/06/06 15:51:18 WARN CharVarcharUtils: The Spark cast operator does not support
193            char/varchar type and simply treats them as string type. Please use string type
194            directly to avoid confusion. Otherwise, you can set spark.sql.legacy.charVarcharAsString
195            to true, so that Spark treat them as string type as same as Spark 3.0 and earlier
196
197            1234
198            Time taken: 4.265 seconds, Fetched 1 row(s)
199
200        This shows that Spark doesn't truncate the value into '12', which is inconsistent with
201        what other dialects (e.g. postgres) do, so we need to drop the length to transpile correctly.
202
203        Reference: https://spark.apache.org/docs/latest/sql-ref-datatypes.html
204        """
205        this = super()._parse_types(
206            check_func=check_func,
207            schema=schema,
208            allow_identifiers=allow_identifiers,
209            with_collation=with_collation,
210        )
211
212        if this and not schema:
213
214            def _to_text(node: exp.Expr) -> exp.Expr:
215                if isinstance(node, exp.DataType) and node.is_type("char", "varchar"):
216                    node.set("this", exp.DType.TEXT)
217                    node.set("expressions", None)
218                return node
219
220            return this.transform(_to_text, copy=False)
221
222        return this
223
224    def _parse_alter_table_change(self) -> exp.Expr | None:
225        self._match(TokenType.COLUMN)
226        this = self._parse_field(any_token=True)
227
228        if self.CHANGE_COLUMN_ALTER_SYNTAX and self._match_text_seq("TYPE"):
229            return self.expression(exp.AlterColumn(this=this, dtype=self._parse_types(schema=True)))
230
231        column_new = self._parse_field(any_token=True)
232        dtype = self._parse_types(schema=True)
233
234        comment = self._match(TokenType.COMMENT) and self._parse_string()
235
236        if not this or not column_new or not dtype:
237            self.raise_error(
238                "Expected 'CHANGE COLUMN' to be followed by 'column_name' 'column_name' 'data_type'"
239            )
240
241        return self.expression(
242            exp.AlterColumn(this=this, rename_to=column_new, dtype=dtype, comment=comment)
243        )
244
245    def _parse_using_property(self) -> exp.Property:
246        if self._match_texts(("JAR", "FILE", "ARCHIVE")):
247            kind = self._prev.text.upper()
248            return exp.UsingProperty(this=self._parse_string(), kind=kind)
249
250        return self._parse_property_assignment(exp.FileFormatProperty)
251
252    def _parse_partition_and_order(
253        self,
254    ) -> tuple[list[exp.Expr], exp.Expr | None]:
255        return (
256            (
257                self._parse_csv(self._parse_assignment)
258                if self._match_set({TokenType.PARTITION_BY, TokenType.DISTRIBUTE_BY})
259                else []
260            ),
261            super()._parse_order(skip_order_token=self._match(TokenType.SORT_BY)),
262        )
263
264    def _parse_parameter(self) -> exp.Parameter:
265        self._match(TokenType.L_BRACE)
266        this = self._parse_identifier() or self._parse_primary_or_var()
267        expression = self._match(TokenType.COLON) and (
268            self._parse_identifier() or self._parse_primary_or_var()
269        )
270        self._match(TokenType.R_BRACE)
271        return self.expression(exp.Parameter(this=this, expression=expression))
272
273    def _to_prop_eq(self, expression: exp.Expr, index: int) -> exp.Expr:
274        if expression.is_star:
275            return expression
276
277        if isinstance(expression, exp.Column):
278            key = expression.this
279        else:
280            key = exp.to_identifier(f"col{index + 1}")
281
282        return self.expression(exp.PropertyEQ(this=key, expression=expression))