sqlglot.parsers.duckdb

  1from __future__ import annotations
  2
  3import typing as t
  4
  5from sqlglot import exp, parser
  6from sqlglot.trie import new_trie
  7from sqlglot.dialects.dialect import (
  8    binary_from_function,
  9    build_default_decimal_type,
 10    build_formatted_time,
 11    build_regexp_extract,
 12    date_trunc_to_time,
 13    pivot_column_names,
 14)
 15from sqlglot.helper import seq_get
 16from sqlglot.parser import binary_range_parser
 17from sqlglot.tokens import TokenType
 18from collections.abc import Collection
 19
 20
 21def _build_sort_array_desc(args: list) -> exp.Expr:
 22    return exp.SortArray(this=seq_get(args, 0), asc=exp.false())
 23
 24
 25def _build_array_prepend(args: list) -> exp.Expr:
 26    return exp.ArrayPrepend(this=seq_get(args, 1), expression=seq_get(args, 0))
 27
 28
 29def _build_date_diff(args: list) -> exp.Expr:
 30    return exp.DateDiff(this=seq_get(args, 2), expression=seq_get(args, 1), unit=seq_get(args, 0))
 31
 32
 33def _build_generate_series(end_exclusive: bool = False) -> t.Callable[[list], exp.GenerateSeries]:
 34    def _builder(args: list) -> exp.GenerateSeries:
 35        # Check https://duckdb.org/docs/sql/functions/nested.html#range-functions
 36        if len(args) == 1:
 37            # DuckDB uses 0 as a default for the series' start when it's omitted
 38            args.insert(0, exp.Literal.number("0"))
 39
 40        gen_series = exp.GenerateSeries.from_arg_list(args)
 41        gen_series.set("is_end_exclusive", end_exclusive)
 42
 43        return gen_series
 44
 45    return _builder
 46
 47
 48def _build_make_timestamp(args: list) -> exp.Expr:
 49    if len(args) == 1:
 50        return exp.UnixToTime(this=seq_get(args, 0), scale=exp.UnixToTime.MICROS)
 51
 52    return exp.TimestampFromParts(
 53        year=seq_get(args, 0),
 54        month=seq_get(args, 1),
 55        day=seq_get(args, 2),
 56        hour=seq_get(args, 3),
 57        min=seq_get(args, 4),
 58        sec=seq_get(args, 5),
 59    )
 60
 61
 62def _show_parser(*args: t.Any, **kwargs: t.Any) -> t.Callable[[DuckDBParser], exp.Show]:
 63    def _parse(self: DuckDBParser) -> exp.Show:
 64        return self._parse_show_duckdb(*args, **kwargs)
 65
 66    return _parse
 67
 68
 69def _convert_text_type(dtype: exp.DataType) -> exp.DataType:
 70    dtype.set("expressions", None)
 71    return dtype
 72
 73
 74class DuckDBParser(parser.Parser):
 75    MAP_KEYS_ARE_ARBITRARY_EXPRESSIONS = True
 76    PIVOT_COLUMN_NAMING = "agg_name_if_aliased_or_multiple"
 77
 78    NO_PAREN_FUNCTIONS = {
 79        **parser.Parser.NO_PAREN_FUNCTIONS,
 80        TokenType.LOCALTIME: exp.Localtime,
 81        TokenType.LOCALTIMESTAMP: exp.Localtimestamp,
 82        TokenType.CURRENT_CATALOG: exp.CurrentCatalog,
 83        TokenType.SESSION_USER: exp.SessionUser,
 84    }
 85
 86    BITWISE = {k: v for k, v in parser.Parser.BITWISE.items() if k != TokenType.CARET}
 87
 88    RANGE_PARSERS = {
 89        **parser.Parser.RANGE_PARSERS,
 90        TokenType.DAMP: binary_range_parser(exp.ArrayOverlaps),
 91        TokenType.CARET_AT: binary_range_parser(exp.StartsWith),
 92        TokenType.TILDE: binary_range_parser(exp.RegexpFullMatch),
 93    }
 94
 95    EXPONENT = {
 96        **parser.Parser.EXPONENT,
 97        TokenType.CARET: exp.Pow,
 98        TokenType.DSTAR: exp.Pow,
 99    }
100
101    FUNCTIONS_WITH_ALIASED_ARGS = {*parser.Parser.FUNCTIONS_WITH_ALIASED_ARGS, "STRUCT_PACK"}
102
103    SHOW_PARSERS = {
104        "TABLES": _show_parser("TABLES"),
105        "ALL TABLES": _show_parser("ALL TABLES"),
106    }
107
108    FUNCTIONS = {
109        **{k: v for k, v in parser.Parser.FUNCTIONS.items() if k not in ("DATE_SUB", "GLOB")},
110        "ANY_VALUE": lambda args: exp.IgnoreNulls(this=exp.AnyValue.from_arg_list(args)),
111        "ARRAY_PREPEND": _build_array_prepend,
112        "ARRAY_REVERSE_SORT": _build_sort_array_desc,
113        "ARRAY_INTERSECT": lambda args: exp.ArrayIntersect(expressions=args),
114        "ARRAY_SORT": exp.SortArray.from_arg_list,
115        "BIT_AND": exp.BitwiseAndAgg.from_arg_list,
116        "BIT_OR": exp.BitwiseOrAgg.from_arg_list,
117        "BIT_XOR": exp.BitwiseXorAgg.from_arg_list,
118        "CURRENT_LOCALTIMESTAMP": exp.Localtimestamp.from_arg_list,
119        "DATEDIFF": _build_date_diff,
120        "DATE_DIFF": _build_date_diff,
121        "DATE_TRUNC": date_trunc_to_time,
122        "DATETRUNC": date_trunc_to_time,
123        "DECODE": lambda args: exp.Decode(
124            this=seq_get(args, 0), charset=exp.Literal.string("utf-8")
125        ),
126        "EDITDIST3": exp.Levenshtein.from_arg_list,
127        "ENCODE": lambda args: exp.Encode(
128            this=seq_get(args, 0), charset=exp.Literal.string("utf-8")
129        ),
130        "EPOCH": exp.TimeToUnix.from_arg_list,
131        "EPOCH_MS": lambda args: exp.UnixToTime(this=seq_get(args, 0), scale=exp.UnixToTime.MILLIS),
132        "FROM_HEX": exp.Unhex.from_arg_list,
133        "GENERATE_SERIES": _build_generate_series(),
134        "GET_CURRENT_TIME": exp.CurrentTime.from_arg_list,
135        "GET_BIT": lambda args: exp.Getbit(
136            this=seq_get(args, 0), expression=seq_get(args, 1), zero_is_msb=True
137        ),
138        "JARO_WINKLER_SIMILARITY": exp.JarowinklerSimilarity.from_arg_list,
139        "JSON": exp.ParseJSON.from_arg_list,
140        "JSON_ARRAY": lambda args: exp.JSONArray(expressions=args),
141        "JSON_EXTRACT_PATH": parser.build_extract_json_with_path(exp.JSONExtract),
142        "JSON_EXTRACT_STRING": parser.build_extract_json_with_path(exp.JSONExtractScalar),
143        "LIST": exp.ArrayAgg.from_arg_list,
144        "LIST_DISTINCT": exp.ArrayDistinct.from_arg_list,
145        "LIST_APPEND": exp.ArrayAppend.from_arg_list,
146        "LIST_CONCAT": parser.build_array_concat,
147        "LIST_CONTAINS": exp.ArrayContains.from_arg_list,
148        "LIST_COSINE_DISTANCE": exp.CosineDistance.from_arg_list,
149        "LIST_DISTANCE": exp.EuclideanDistance.from_arg_list,
150        "LIST_FILTER": exp.ArrayFilter.from_arg_list,
151        "LIST_HAS": exp.ArrayContains.from_arg_list,
152        "LIST_HAS_ANY": exp.ArrayOverlaps.from_arg_list,
153        "LIST_MAX": exp.ArrayMax.from_arg_list,
154        "LIST_MIN": exp.ArrayMin.from_arg_list,
155        "LIST_PREPEND": _build_array_prepend,
156        "LIST_REVERSE_SORT": _build_sort_array_desc,
157        "LIST_SORT": exp.SortArray.from_arg_list,
158        "LIST_TRANSFORM": exp.Transform.from_arg_list,
159        "LIST_VALUE": lambda args: exp.Array(expressions=args),
160        "MAKE_DATE": exp.DateFromParts.from_arg_list,
161        "MAKE_TIME": exp.TimeFromParts.from_arg_list,
162        "MAKE_TIMESTAMP": _build_make_timestamp,
163        "QUANTILE_CONT": exp.PercentileCont.from_arg_list,
164        "QUANTILE_DISC": exp.PercentileDisc.from_arg_list,
165        "RANGE": _build_generate_series(end_exclusive=True),
166        "REGEXP_EXTRACT": build_regexp_extract(exp.RegexpExtract),
167        "REGEXP_EXTRACT_ALL": build_regexp_extract(exp.RegexpExtractAll),
168        "REGEXP_MATCHES": exp.RegexpLike.from_arg_list,
169        "REGEXP_REPLACE": lambda args: exp.RegexpReplace(
170            this=seq_get(args, 0),
171            expression=seq_get(args, 1),
172            replacement=seq_get(args, 2),
173            modifiers=seq_get(args, 3),
174            single_replace=True,
175        ),
176        "SHA256": lambda args: exp.SHA2(this=seq_get(args, 0), length=exp.Literal.number(256)),
177        "STRFTIME": build_formatted_time(exp.TimeToStr),
178        "STRING_SPLIT": exp.Split.from_arg_list,
179        "STRING_SPLIT_REGEX": exp.RegexpSplit.from_arg_list,
180        "STRING_TO_ARRAY": exp.Split.from_arg_list,
181        "STRPTIME": build_formatted_time(exp.StrToTime),
182        "STRUCT_PACK": exp.Struct.from_arg_list,
183        "STR_SPLIT": exp.Split.from_arg_list,
184        "STR_SPLIT_REGEX": exp.RegexpSplit.from_arg_list,
185        "TODAY": exp.CurrentDate.from_arg_list,
186        "TIME_BUCKET": exp.DateBin.from_arg_list,
187        "TO_TIMESTAMP": exp.UnixToTime.from_arg_list,
188        "UNNEST": exp.Explode.from_arg_list,
189        "VERSION": exp.CurrentVersion.from_arg_list,
190        "XOR": binary_from_function(exp.BitwiseXor),
191    }
192
193    FUNCTION_PARSERS = {
194        **{k: v for k, v in parser.Parser.FUNCTION_PARSERS.items() if k != "DECODE"},
195        **dict.fromkeys(
196            ("GROUP_CONCAT", "LISTAGG", "STRINGAGG"), lambda self: self._parse_string_agg()
197        ),
198        "APPROX_QUANTILE": lambda self: self._parse_distinct_arg_function(exp.ApproxQuantile),
199        "QUANTILE": lambda self: self._parse_distinct_arg_function(exp.Quantile),
200        "QUANTILE_CONT": lambda self: self._parse_distinct_arg_function(exp.PercentileCont),
201        "QUANTILE_DISC": lambda self: self._parse_distinct_arg_function(exp.PercentileDisc),
202    }
203
204    NO_PAREN_FUNCTION_PARSERS = {
205        **parser.Parser.NO_PAREN_FUNCTION_PARSERS,
206        "MAP": lambda self: self._parse_map(),
207        "@": lambda self: exp.Abs(this=self._parse_bitwise()),
208    }
209
210    PLACEHOLDER_PARSERS = {
211        **parser.Parser.PLACEHOLDER_PARSERS,
212        TokenType.PARAMETER: lambda self: (
213            self.expression(exp.Placeholder(this=self._prev.text))
214            if self._match(TokenType.NUMBER) or self._match_set(self.ID_VAR_TOKENS)
215            else None
216        ),
217    }
218
219    TYPE_CONVERTERS = {
220        # https://duckdb.org/docs/sql/data_types/numeric
221        exp.DType.DECIMAL: build_default_decimal_type(precision=18, scale=3),
222        # https://duckdb.org/docs/sql/data_types/text
223        exp.DType.TEXT: _convert_text_type,
224    }
225
226    STATEMENT_PARSERS = {
227        **parser.Parser.STATEMENT_PARSERS,
228        TokenType.ATTACH: lambda self: self._parse_attach_detach(),
229        TokenType.DETACH: lambda self: self._parse_attach_detach(is_attach=False),
230        TokenType.FORCE: lambda self: self._parse_force(),
231        TokenType.INSTALL: lambda self: self._parse_install(),
232        TokenType.SHOW: lambda self: self._parse_show(),
233    }
234
235    SET_PARSERS = {
236        **parser.Parser.SET_PARSERS,
237        "VARIABLE": lambda self: self._parse_set_item_assignment("VARIABLE"),
238    }
239
240    SHOW_TRIE = new_trie(key.split(" ") for key in SHOW_PARSERS)
241    SET_TRIE = new_trie(key.split(" ") for key in SET_PARSERS)
242
243    def _parse_function_properties(self) -> exp.Properties | None:
244        if self._match(TokenType.TABLE):
245            return exp.Properties(
246                expressions=[
247                    exp.ReturnsProperty(
248                        this=exp.Schema(this=exp.var("TABLE")),
249                        is_table=True,
250                    ),
251                ]
252            )
253        return super()._parse_function_properties()
254
255    def _parse_lambda(self, alias: bool = False) -> exp.Expr | None:
256        index = self._index
257        if not self._match_text_seq("LAMBDA"):
258            return super()._parse_lambda(alias=alias)
259
260        expressions = self._parse_csv(self._parse_lambda_arg)
261        if not self._match(TokenType.COLON):
262            self._retreat(index)
263            return None
264
265        this = self._replace_lambda(self._parse_assignment(), expressions)
266        return self.expression(exp.Lambda(this=this, expressions=expressions, colon=True))
267
268    def _parse_expression(self) -> exp.Expr | None:
269        # DuckDB supports prefix aliases, e.g. foo: 1
270        if self._next.token_type == TokenType.COLON:
271            alias = self._parse_id_var(tokens=self.ALIAS_TOKENS)
272            self._match(TokenType.COLON)
273            comments = self._prev_comments
274
275            this = self._parse_assignment()
276            if isinstance(this, exp.Expr):
277                # Moves the comment next to the alias in `alias: expr /* comment */`
278                comments += this.pop_comments() or []
279
280            return self.expression(exp.Alias(this=this, alias=alias), comments=comments)
281
282        return super()._parse_expression()
283
284    def _parse_table(
285        self,
286        schema: bool = False,
287        joins: bool = False,
288        alias_tokens: Collection[TokenType] | None = None,
289        parse_bracket: bool = False,
290        is_db_reference: bool = False,
291        parse_partition: bool = False,
292        consume_pipe: bool = False,
293    ) -> exp.Expr | None:
294        # DuckDB supports prefix aliases, e.g. FROM foo: bar
295        if self._next.token_type == TokenType.COLON:
296            alias = self._parse_table_alias(alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS)
297            self._match(TokenType.COLON)
298            comments = self._prev_comments
299        else:
300            alias = None
301            comments = []
302
303        table = super()._parse_table(
304            schema=schema,
305            joins=joins,
306            alias_tokens=alias_tokens,
307            parse_bracket=parse_bracket,
308            is_db_reference=is_db_reference,
309            parse_partition=parse_partition,
310        )
311        if isinstance(table, exp.Expr) and isinstance(alias, exp.TableAlias):
312            # Moves the comment next to the alias in `alias: table /* comment */`
313            comments += table.pop_comments() or []
314            alias.comments = alias.pop_comments() + comments
315            table.set("alias", alias)
316
317        return table
318
319    def _parse_table_sample(self, as_modifier: bool = False) -> exp.TableSample | None:
320        # https://duckdb.org/docs/sql/samples.html
321        sample = super()._parse_table_sample(as_modifier=as_modifier)
322        if sample and not sample.args.get("method"):
323            if sample.args.get("size"):
324                sample.set("method", exp.var("RESERVOIR"))
325            else:
326                sample.set("method", exp.var("SYSTEM"))
327
328        return sample
329
330    def _parse_bracket(self, this: exp.Expr | None = None) -> exp.Expr | None:
331        bracket = super()._parse_bracket(this)
332
333        if self.dialect.version < (1, 2) and isinstance(bracket, exp.Bracket):
334            # https://duckdb.org/2025/02/05/announcing-duckdb-120.html#breaking-changes
335            bracket.set("returns_list_for_maps", True)
336
337        return bracket
338
339    def _parse_map(self) -> exp.ToMap | exp.Map:
340        if self._match(TokenType.L_BRACE, advance=False):
341            return self.expression(exp.ToMap(this=self._parse_bracket()))
342
343        args = self._parse_wrapped_csv(self._parse_assignment)
344        return self.expression(exp.Map(keys=seq_get(args, 0), values=seq_get(args, 1)))
345
346    def _parse_struct_types(self, type_required: bool = False) -> exp.Expr | None:
347        return self._parse_field_def()
348
349    def _pivot_column_names(self, aggregations: list[exp.Expr]) -> list[str]:
350        if len(aggregations) == 1:
351            return super()._pivot_column_names(aggregations)
352        return pivot_column_names(aggregations, dialect="duckdb")
353
354    def _parse_attach_detach(self, is_attach: bool = True) -> exp.Attach | exp.Detach:
355        def _parse_attach_option() -> exp.AttachOption:
356            return self.expression(
357                exp.AttachOption(
358                    this=self._parse_var(any_token=True),
359                    expression=self._parse_field(any_token=True),
360                )
361            )
362
363        self._match(TokenType.DATABASE)
364        exists = self._parse_exists(not_=is_attach)
365        this = self._parse_alias(self._parse_primary_or_var(), explicit=True)
366
367        if self._match(TokenType.L_PAREN, advance=False):
368            expressions = self._parse_wrapped_csv(_parse_attach_option)
369        else:
370            expressions = None
371
372        return (
373            self.expression(exp.Attach(this=this, exists=exists, expressions=expressions))
374            if is_attach
375            else self.expression(exp.Detach(this=this, exists=exists))
376        )
377
378    def _parse_show_duckdb(self, this: str) -> exp.Show:
379        from_ = self._parse_table(schema=True) if self._match(TokenType.FROM) else None
380        return self.expression(exp.Show(this=this, from_=from_))
381
382    def _parse_force(self) -> exp.Install | exp.Command:
383        # FORCE can only be followed by INSTALL or CHECKPOINT
384        # In the case of CHECKPOINT, we fallback
385        if not self._match(TokenType.INSTALL):
386            return self._parse_as_command(self._prev)
387
388        return self._parse_install(force=True)
389
390    def _parse_install(self, force: bool = False) -> exp.Install:
391        return self.expression(
392            exp.Install(
393                this=self._parse_id_var(),
394                from_=self._parse_var_or_string() if self._match(TokenType.FROM) else None,
395                force=force,
396            )
397        )
398
399    def _parse_primary(self) -> exp.Expr | None:
400        if self._match_pair(TokenType.HASH, TokenType.NUMBER):
401            return exp.PositionalColumn(this=exp.Literal.number(self._prev.text))
402
403        return super()._parse_primary()