sqlglot.parsers.duckdb

  1from __future__ import annotations
  2
  3import typing as t
  4
  5from sqlglot import exp, parser
  6from sqlglot.trie import new_trie
  7from sqlglot.dialects.dialect import (
  8    binary_from_function,
  9    build_default_decimal_type,
 10    build_formatted_time,
 11    build_regexp_extract,
 12    date_trunc_to_time,
 13    pivot_column_names,
 14)
 15from sqlglot.helper import seq_get
 16from sqlglot.parser import binary_range_parser
 17from sqlglot.tokens import TokenType
 18from collections.abc import Collection
 19
 20
 21def _build_sort_array_desc(args: list) -> exp.Expr:
 22    return exp.SortArray(this=seq_get(args, 0), asc=exp.false())
 23
 24
 25def _build_array_prepend(args: list) -> exp.Expr:
 26    return exp.ArrayPrepend(this=seq_get(args, 1), expression=seq_get(args, 0))
 27
 28
 29def _build_date_diff(args: list) -> exp.Expr:
 30    return exp.DateDiff(this=seq_get(args, 2), expression=seq_get(args, 1), unit=seq_get(args, 0))
 31
 32
 33def _build_generate_series(end_exclusive: bool = False) -> t.Callable[[list], exp.GenerateSeries]:
 34    def _builder(args: list) -> exp.GenerateSeries:
 35        # Check https://duckdb.org/docs/sql/functions/nested.html#range-functions
 36        if len(args) == 1:
 37            # DuckDB uses 0 as a default for the series' start when it's omitted
 38            args.insert(0, exp.Literal.number("0"))
 39
 40        gen_series = exp.GenerateSeries.from_arg_list(args)
 41        gen_series.set("is_end_exclusive", end_exclusive)
 42
 43        return gen_series
 44
 45    return _builder
 46
 47
 48def _build_make_timestamp(args: list) -> exp.Expr:
 49    if len(args) == 1:
 50        return exp.UnixToTime(this=seq_get(args, 0), scale=exp.UnixToTime.MICROS)
 51
 52    return exp.TimestampFromParts(
 53        year=seq_get(args, 0),
 54        month=seq_get(args, 1),
 55        day=seq_get(args, 2),
 56        hour=seq_get(args, 3),
 57        min=seq_get(args, 4),
 58        sec=seq_get(args, 5),
 59    )
 60
 61
 62def _show_parser(*args: t.Any, **kwargs: t.Any) -> t.Callable[[DuckDBParser], exp.Show]:
 63    def _parse(self: DuckDBParser) -> exp.Show:
 64        return self._parse_show_duckdb(*args, **kwargs)
 65
 66    return _parse
 67
 68
 69def _convert_text_type(dtype: exp.DataType) -> exp.DataType:
 70    dtype.set("expressions", None)
 71    return dtype
 72
 73
 74class DuckDBParser(parser.Parser):
 75    MAP_KEYS_ARE_ARBITRARY_EXPRESSIONS = True
 76
 77    NO_PAREN_FUNCTIONS = {
 78        **parser.Parser.NO_PAREN_FUNCTIONS,
 79        TokenType.LOCALTIME: exp.Localtime,
 80        TokenType.LOCALTIMESTAMP: exp.Localtimestamp,
 81        TokenType.CURRENT_CATALOG: exp.CurrentCatalog,
 82        TokenType.SESSION_USER: exp.SessionUser,
 83    }
 84
 85    BITWISE = {k: v for k, v in parser.Parser.BITWISE.items() if k != TokenType.CARET}
 86
 87    RANGE_PARSERS = {
 88        **parser.Parser.RANGE_PARSERS,
 89        TokenType.DAMP: binary_range_parser(exp.ArrayOverlaps),
 90        TokenType.CARET_AT: binary_range_parser(exp.StartsWith),
 91        TokenType.TILDE: binary_range_parser(exp.RegexpFullMatch),
 92    }
 93
 94    EXPONENT = {
 95        **parser.Parser.EXPONENT,
 96        TokenType.CARET: exp.Pow,
 97        TokenType.DSTAR: exp.Pow,
 98    }
 99
100    FUNCTIONS_WITH_ALIASED_ARGS = {*parser.Parser.FUNCTIONS_WITH_ALIASED_ARGS, "STRUCT_PACK"}
101
102    SHOW_PARSERS = {
103        "TABLES": _show_parser("TABLES"),
104        "ALL TABLES": _show_parser("ALL TABLES"),
105    }
106
107    FUNCTIONS = {
108        **{k: v for k, v in parser.Parser.FUNCTIONS.items() if k not in ("DATE_SUB", "GLOB")},
109        "ANY_VALUE": lambda args: exp.IgnoreNulls(this=exp.AnyValue.from_arg_list(args)),
110        "ARRAY_PREPEND": _build_array_prepend,
111        "ARRAY_REVERSE_SORT": _build_sort_array_desc,
112        "ARRAY_INTERSECT": lambda args: exp.ArrayIntersect(expressions=args),
113        "ARRAY_SORT": exp.SortArray.from_arg_list,
114        "BIT_AND": exp.BitwiseAndAgg.from_arg_list,
115        "BIT_OR": exp.BitwiseOrAgg.from_arg_list,
116        "BIT_XOR": exp.BitwiseXorAgg.from_arg_list,
117        "CURRENT_LOCALTIMESTAMP": exp.Localtimestamp.from_arg_list,
118        "DATEDIFF": _build_date_diff,
119        "DATE_DIFF": _build_date_diff,
120        "DATE_TRUNC": date_trunc_to_time,
121        "DATETRUNC": date_trunc_to_time,
122        "DECODE": lambda args: exp.Decode(
123            this=seq_get(args, 0), charset=exp.Literal.string("utf-8")
124        ),
125        "EDITDIST3": exp.Levenshtein.from_arg_list,
126        "ENCODE": lambda args: exp.Encode(
127            this=seq_get(args, 0), charset=exp.Literal.string("utf-8")
128        ),
129        "EPOCH": exp.TimeToUnix.from_arg_list,
130        "EPOCH_MS": lambda args: exp.UnixToTime(this=seq_get(args, 0), scale=exp.UnixToTime.MILLIS),
131        "GENERATE_SERIES": _build_generate_series(),
132        "GET_CURRENT_TIME": exp.CurrentTime.from_arg_list,
133        "GET_BIT": lambda args: exp.Getbit(
134            this=seq_get(args, 0), expression=seq_get(args, 1), zero_is_msb=True
135        ),
136        "JARO_WINKLER_SIMILARITY": exp.JarowinklerSimilarity.from_arg_list,
137        "JSON": exp.ParseJSON.from_arg_list,
138        "JSON_ARRAY": lambda args: exp.JSONArray(expressions=args),
139        "JSON_EXTRACT_PATH": parser.build_extract_json_with_path(exp.JSONExtract),
140        "JSON_EXTRACT_STRING": parser.build_extract_json_with_path(exp.JSONExtractScalar),
141        "LIST": exp.ArrayAgg.from_arg_list,
142        "LIST_DISTINCT": exp.ArrayDistinct.from_arg_list,
143        "LIST_APPEND": exp.ArrayAppend.from_arg_list,
144        "LIST_CONCAT": parser.build_array_concat,
145        "LIST_CONTAINS": exp.ArrayContains.from_arg_list,
146        "LIST_COSINE_DISTANCE": exp.CosineDistance.from_arg_list,
147        "LIST_DISTANCE": exp.EuclideanDistance.from_arg_list,
148        "LIST_FILTER": exp.ArrayFilter.from_arg_list,
149        "LIST_HAS": exp.ArrayContains.from_arg_list,
150        "LIST_HAS_ANY": exp.ArrayOverlaps.from_arg_list,
151        "LIST_MAX": exp.ArrayMax.from_arg_list,
152        "LIST_MIN": exp.ArrayMin.from_arg_list,
153        "LIST_PREPEND": _build_array_prepend,
154        "LIST_REVERSE_SORT": _build_sort_array_desc,
155        "LIST_SORT": exp.SortArray.from_arg_list,
156        "LIST_TRANSFORM": exp.Transform.from_arg_list,
157        "LIST_VALUE": lambda args: exp.Array(expressions=args),
158        "MAKE_DATE": exp.DateFromParts.from_arg_list,
159        "MAKE_TIME": exp.TimeFromParts.from_arg_list,
160        "MAKE_TIMESTAMP": _build_make_timestamp,
161        "QUANTILE_CONT": exp.PercentileCont.from_arg_list,
162        "QUANTILE_DISC": exp.PercentileDisc.from_arg_list,
163        "RANGE": _build_generate_series(end_exclusive=True),
164        "REGEXP_EXTRACT": build_regexp_extract(exp.RegexpExtract),
165        "REGEXP_EXTRACT_ALL": build_regexp_extract(exp.RegexpExtractAll),
166        "REGEXP_MATCHES": exp.RegexpLike.from_arg_list,
167        "REGEXP_REPLACE": lambda args: exp.RegexpReplace(
168            this=seq_get(args, 0),
169            expression=seq_get(args, 1),
170            replacement=seq_get(args, 2),
171            modifiers=seq_get(args, 3),
172            single_replace=True,
173        ),
174        "SHA256": lambda args: exp.SHA2(this=seq_get(args, 0), length=exp.Literal.number(256)),
175        "STRFTIME": build_formatted_time(exp.TimeToStr),
176        "STRING_SPLIT": exp.Split.from_arg_list,
177        "STRING_SPLIT_REGEX": exp.RegexpSplit.from_arg_list,
178        "STRING_TO_ARRAY": exp.Split.from_arg_list,
179        "STRPTIME": build_formatted_time(exp.StrToTime),
180        "STRUCT_PACK": exp.Struct.from_arg_list,
181        "STR_SPLIT": exp.Split.from_arg_list,
182        "STR_SPLIT_REGEX": exp.RegexpSplit.from_arg_list,
183        "TODAY": exp.CurrentDate.from_arg_list,
184        "TIME_BUCKET": exp.DateBin.from_arg_list,
185        "TO_TIMESTAMP": exp.UnixToTime.from_arg_list,
186        "UNNEST": exp.Explode.from_arg_list,
187        "VERSION": exp.CurrentVersion.from_arg_list,
188        "XOR": binary_from_function(exp.BitwiseXor),
189    }
190
191    FUNCTION_PARSERS = {
192        **{k: v for k, v in parser.Parser.FUNCTION_PARSERS.items() if k != "DECODE"},
193        **dict.fromkeys(
194            ("GROUP_CONCAT", "LISTAGG", "STRINGAGG"), lambda self: self._parse_string_agg()
195        ),
196    }
197
198    NO_PAREN_FUNCTION_PARSERS = {
199        **parser.Parser.NO_PAREN_FUNCTION_PARSERS,
200        "MAP": lambda self: self._parse_map(),
201        "@": lambda self: exp.Abs(this=self._parse_bitwise()),
202    }
203
204    PLACEHOLDER_PARSERS = {
205        **parser.Parser.PLACEHOLDER_PARSERS,
206        TokenType.PARAMETER: lambda self: (
207            self.expression(exp.Placeholder(this=self._prev.text))
208            if self._match(TokenType.NUMBER) or self._match_set(self.ID_VAR_TOKENS)
209            else None
210        ),
211    }
212
213    TYPE_CONVERTERS = {
214        # https://duckdb.org/docs/sql/data_types/numeric
215        exp.DType.DECIMAL: build_default_decimal_type(precision=18, scale=3),
216        # https://duckdb.org/docs/sql/data_types/text
217        exp.DType.TEXT: _convert_text_type,
218    }
219
220    STATEMENT_PARSERS = {
221        **parser.Parser.STATEMENT_PARSERS,
222        TokenType.ATTACH: lambda self: self._parse_attach_detach(),
223        TokenType.DETACH: lambda self: self._parse_attach_detach(is_attach=False),
224        TokenType.FORCE: lambda self: self._parse_force(),
225        TokenType.INSTALL: lambda self: self._parse_install(),
226        TokenType.SHOW: lambda self: self._parse_show(),
227    }
228
229    SET_PARSERS = {
230        **parser.Parser.SET_PARSERS,
231        "VARIABLE": lambda self: self._parse_set_item_assignment("VARIABLE"),
232    }
233
234    SHOW_TRIE = new_trie(key.split(" ") for key in SHOW_PARSERS)
235    SET_TRIE = new_trie(key.split(" ") for key in SET_PARSERS)
236
237    def _parse_function_properties(self) -> exp.Properties | None:
238        if self._match(TokenType.TABLE):
239            return exp.Properties(
240                expressions=[
241                    exp.ReturnsProperty(
242                        this=exp.Schema(this=exp.var("TABLE")),
243                        is_table=True,
244                    ),
245                ]
246            )
247        return super()._parse_function_properties()
248
249    def _parse_lambda(self, alias: bool = False) -> exp.Expr | None:
250        index = self._index
251        if not self._match_text_seq("LAMBDA"):
252            return super()._parse_lambda(alias=alias)
253
254        expressions = self._parse_csv(self._parse_lambda_arg)
255        if not self._match(TokenType.COLON):
256            self._retreat(index)
257            return None
258
259        this = self._replace_lambda(self._parse_assignment(), expressions)
260        return self.expression(exp.Lambda(this=this, expressions=expressions, colon=True))
261
262    def _parse_expression(self) -> exp.Expr | None:
263        # DuckDB supports prefix aliases, e.g. foo: 1
264        if self._next.token_type == TokenType.COLON:
265            alias = self._parse_id_var(tokens=self.ALIAS_TOKENS)
266            self._match(TokenType.COLON)
267            comments = self._prev_comments
268
269            this = self._parse_assignment()
270            if isinstance(this, exp.Expr):
271                # Moves the comment next to the alias in `alias: expr /* comment */`
272                comments += this.pop_comments() or []
273
274            return self.expression(exp.Alias(this=this, alias=alias), comments=comments)
275
276        return super()._parse_expression()
277
278    def _parse_table(
279        self,
280        schema: bool = False,
281        joins: bool = False,
282        alias_tokens: Collection[TokenType] | None = None,
283        parse_bracket: bool = False,
284        is_db_reference: bool = False,
285        parse_partition: bool = False,
286        consume_pipe: bool = False,
287    ) -> exp.Expr | None:
288        # DuckDB supports prefix aliases, e.g. FROM foo: bar
289        if self._next.token_type == TokenType.COLON:
290            alias = self._parse_table_alias(alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS)
291            self._match(TokenType.COLON)
292            comments = self._prev_comments
293        else:
294            alias = None
295            comments = []
296
297        table = super()._parse_table(
298            schema=schema,
299            joins=joins,
300            alias_tokens=alias_tokens,
301            parse_bracket=parse_bracket,
302            is_db_reference=is_db_reference,
303            parse_partition=parse_partition,
304        )
305        if isinstance(table, exp.Expr) and isinstance(alias, exp.TableAlias):
306            # Moves the comment next to the alias in `alias: table /* comment */`
307            comments += table.pop_comments() or []
308            alias.comments = alias.pop_comments() + comments
309            table.set("alias", alias)
310
311        return table
312
313    def _parse_table_sample(self, as_modifier: bool = False) -> exp.TableSample | None:
314        # https://duckdb.org/docs/sql/samples.html
315        sample = super()._parse_table_sample(as_modifier=as_modifier)
316        if sample and not sample.args.get("method"):
317            if sample.args.get("size"):
318                sample.set("method", exp.var("RESERVOIR"))
319            else:
320                sample.set("method", exp.var("SYSTEM"))
321
322        return sample
323
324    def _parse_bracket(self, this: exp.Expr | None = None) -> exp.Expr | None:
325        bracket = super()._parse_bracket(this)
326
327        if self.dialect.version < (1, 2) and isinstance(bracket, exp.Bracket):
328            # https://duckdb.org/2025/02/05/announcing-duckdb-120.html#breaking-changes
329            bracket.set("returns_list_for_maps", True)
330
331        return bracket
332
333    def _parse_map(self) -> exp.ToMap | exp.Map:
334        if self._match(TokenType.L_BRACE, advance=False):
335            return self.expression(exp.ToMap(this=self._parse_bracket()))
336
337        args = self._parse_wrapped_csv(self._parse_assignment)
338        return self.expression(exp.Map(keys=seq_get(args, 0), values=seq_get(args, 1)))
339
340    def _parse_struct_types(self, type_required: bool = False) -> exp.Expr | None:
341        return self._parse_field_def()
342
343    def _pivot_column_names(self, aggregations: list[exp.Expr]) -> list[str]:
344        if len(aggregations) == 1:
345            return super()._pivot_column_names(aggregations)
346        return pivot_column_names(aggregations, dialect="duckdb")
347
348    def _parse_attach_detach(self, is_attach: bool = True) -> exp.Attach | exp.Detach:
349        def _parse_attach_option() -> exp.AttachOption:
350            return self.expression(
351                exp.AttachOption(
352                    this=self._parse_var(any_token=True),
353                    expression=self._parse_field(any_token=True),
354                )
355            )
356
357        self._match(TokenType.DATABASE)
358        exists = self._parse_exists(not_=is_attach)
359        this = self._parse_alias(self._parse_primary_or_var(), explicit=True)
360
361        if self._match(TokenType.L_PAREN, advance=False):
362            expressions = self._parse_wrapped_csv(_parse_attach_option)
363        else:
364            expressions = None
365
366        return (
367            self.expression(exp.Attach(this=this, exists=exists, expressions=expressions))
368            if is_attach
369            else self.expression(exp.Detach(this=this, exists=exists))
370        )
371
372    def _parse_show_duckdb(self, this: str) -> exp.Show:
373        from_ = self._parse_table(schema=True) if self._match(TokenType.FROM) else None
374        return self.expression(exp.Show(this=this, from_=from_))
375
376    def _parse_force(self) -> exp.Install | exp.Command:
377        # FORCE can only be followed by INSTALL or CHECKPOINT
378        # In the case of CHECKPOINT, we fallback
379        if not self._match(TokenType.INSTALL):
380            return self._parse_as_command(self._prev)
381
382        return self._parse_install(force=True)
383
384    def _parse_install(self, force: bool = False) -> exp.Install:
385        return self.expression(
386            exp.Install(
387                this=self._parse_id_var(),
388                from_=self._parse_var_or_string() if self._match(TokenType.FROM) else None,
389                force=force,
390            )
391        )
392
393    def _parse_primary(self) -> exp.Expr | None:
394        if self._match_pair(TokenType.HASH, TokenType.NUMBER):
395            return exp.PositionalColumn(this=exp.Literal.number(self._prev.text))
396
397        return super()._parse_primary()