sqlglot.parser
1from __future__ import annotations 2 3import itertools 4import logging 5import re 6import typing as t 7from builtins import type as Type 8from collections import defaultdict 9from collections.abc import Sequence 10 11from sqlglot import exp 12from sqlglot.errors import ( 13 ErrorLevel, 14 ParseError, 15 TokenError, 16 concat_messages, 17 highlight_sql, 18 merge_errors, 19) 20from sqlglot.expressions import apply_index_offset 21from sqlglot.helper import ensure_list, i64, seq_get 22from sqlglot.time import format_time 23from sqlglot.tokens import Token, Tokenizer, TokenType 24from sqlglot.trie import TrieResult, in_trie, new_trie 25 26if t.TYPE_CHECKING: 27 from re import Pattern 28 29 from sqlglot._typing import BuilderArgs, E 30 from sqlglot.dialects.dialect import Dialect, DialectType 31 from sqlglot.expressions import ExpOrStr 32 33 T = t.TypeVar("T") 34 TCeilFloor = t.TypeVar("TCeilFloor", exp.Ceil, exp.Floor) 35 36logger = logging.getLogger("sqlglot") 37 38OPTIONS_TYPE = dict[str, Sequence[t.Union[Sequence[str], str]]] 39 40# Used to detect alphabetical characters and +/- in timestamp literals 41TIME_ZONE_RE: Pattern[str] = re.compile(r":.*?[a-zA-Z\+\-]") 42 43 44def build_var_map(args: BuilderArgs) -> exp.StarMap | exp.VarMap: 45 if len(args) == 1 and args[0].is_star: 46 return exp.StarMap(this=args[0]) 47 48 keys: list[ExpOrStr] = [] 49 values: list[ExpOrStr] = [] 50 for i in range(0, len(args), 2): 51 keys.append(args[i]) 52 values.append(args[i + 1]) 53 54 return exp.VarMap(keys=exp.array(*keys, copy=False), values=exp.array(*values, copy=False)) 55 56 57def build_like(args: BuilderArgs) -> exp.Escape | exp.Like: 58 like = exp.Like(this=seq_get(args, 1), expression=seq_get(args, 0)) 59 return exp.Escape(this=like, expression=seq_get(args, 2)) if len(args) > 2 else like 60 61 62def binary_range_parser( 63 expr_type: Type[exp.Expr], reverse_args: bool = False 64) -> t.Callable[[Parser, exp.Expr | None], exp.Expr | None]: 65 def _parse_binary_range(self: Parser, this: exp.Expr | None) -> exp.Expr | None: 66 expression = self._parse_bitwise() 67 if reverse_args: 68 this, expression = expression, this 69 return self._parse_escape(self.expression(expr_type(this=this, expression=expression))) 70 71 return _parse_binary_range 72 73 74def build_logarithm(args: BuilderArgs, dialect: Dialect) -> exp.Func: 75 # Default argument order is base, expression 76 this = seq_get(args, 0) 77 expression = seq_get(args, 1) 78 79 if expression: 80 if not dialect.LOG_BASE_FIRST: 81 this, expression = expression, this 82 return exp.Log(this=this, expression=expression) 83 84 return (exp.Ln if dialect.parser_class.LOG_DEFAULTS_TO_LN else exp.Log)(this=this) 85 86 87def build_hex(args: BuilderArgs, dialect: Dialect) -> exp.Hex | exp.LowerHex: 88 arg = seq_get(args, 0) 89 return exp.LowerHex(this=arg) if dialect.HEX_LOWERCASE else exp.Hex(this=arg) 90 91 92def build_lower(args: BuilderArgs) -> exp.Lower | exp.Hex: 93 # LOWER(HEX(..)) can be simplified to LowerHex to simplify its transpilation 94 arg = seq_get(args, 0) 95 return exp.LowerHex(this=arg.this) if isinstance(arg, exp.Hex) else exp.Lower(this=arg) 96 97 98def build_upper(args: BuilderArgs) -> exp.Upper | exp.Hex: 99 # UPPER(HEX(..)) can be simplified to Hex to simplify its transpilation 100 arg = seq_get(args, 0) 101 return exp.Hex(this=arg.this) if isinstance(arg, exp.Hex) else exp.Upper(this=arg) 102 103 104def build_extract_json_with_path( 105 expr_type: Type[E], 106) -> t.Callable[[BuilderArgs, Dialect], E]: 107 def _builder(args: BuilderArgs, dialect: Dialect) -> E: 108 expression = expr_type( 109 this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1)) 110 ) 111 if len(args) > 2 and expr_type is exp.JSONExtract: 112 expression.set("expressions", args[2:]) 113 if expr_type is exp.JSONExtractScalar: 114 expression.set("scalar_only", dialect.JSON_EXTRACT_SCALAR_SCALAR_ONLY) 115 116 return expression 117 118 return _builder 119 120 121def build_mod(args: BuilderArgs) -> exp.Mod: 122 this = seq_get(args, 0) 123 expression = seq_get(args, 1) 124 125 # Wrap the operands if they are binary nodes, e.g. MOD(a + 1, 7) -> (a + 1) % 7 126 this = exp.Paren(this=this) if isinstance(this, exp.Binary) else this 127 expression = exp.Paren(this=expression) if isinstance(expression, exp.Binary) else expression 128 129 return exp.Mod(this=this, expression=expression) 130 131 132def build_pad(args: BuilderArgs, is_left: bool = True): 133 return exp.Pad( 134 this=seq_get(args, 0), 135 expression=seq_get(args, 1), 136 fill_pattern=seq_get(args, 2), 137 is_left=is_left, 138 ) 139 140 141def build_array_constructor( 142 exp_class: Type[E], args: list[t.Any], bracket_kind: TokenType, dialect: Dialect 143) -> exp.Expr: 144 array_exp = exp_class(expressions=args) 145 146 if exp_class == exp.Array and dialect.HAS_DISTINCT_ARRAY_CONSTRUCTORS: 147 array_exp.set("bracket_notation", bracket_kind == TokenType.L_BRACKET) 148 149 return array_exp 150 151 152def build_convert_timezone( 153 args: BuilderArgs, default_source_tz: str | None = None 154) -> exp.ConvertTimezone | exp.Anonymous: 155 if len(args) == 2: 156 source_tz = exp.Literal.string(default_source_tz) if default_source_tz else None 157 return exp.ConvertTimezone( 158 source_tz=source_tz, target_tz=seq_get(args, 0), timestamp=seq_get(args, 1) 159 ) 160 161 return exp.ConvertTimezone.from_arg_list(args) 162 163 164def build_trim(args: BuilderArgs, is_left: bool = True, reverse_args: bool = False) -> exp.Trim: 165 this, expression = seq_get(args, 0), seq_get(args, 1) 166 167 if expression and reverse_args: 168 this, expression = expression, this 169 170 return exp.Trim(this=this, expression=expression, position="LEADING" if is_left else "TRAILING") 171 172 173def build_coalesce( 174 args: BuilderArgs, is_nvl: bool | None = None, is_null: bool | None = None 175) -> exp.Coalesce: 176 return exp.Coalesce(this=seq_get(args, 0), expressions=args[1:], is_nvl=is_nvl, is_null=is_null) 177 178 179def build_locate_strposition(args: BuilderArgs) -> exp.StrPosition: 180 return exp.StrPosition( 181 this=seq_get(args, 1), 182 substr=seq_get(args, 0), 183 position=seq_get(args, 2), 184 ) 185 186 187def build_array_append(args: BuilderArgs, dialect: Dialect) -> exp.ArrayAppend: 188 """ 189 Builds ArrayAppend with NULL propagation semantics based on the dialect configuration. 190 191 Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. 192 Others (DuckDB, PostgreSQL) create a new single-element array instead. 193 194 Args: 195 args: Function arguments [array, element] 196 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 197 198 Returns: 199 ArrayAppend expression with appropriate null_propagation flag 200 """ 201 return exp.ArrayAppend( 202 this=seq_get(args, 0), 203 expression=seq_get(args, 1), 204 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 205 ) 206 207 208def build_array_prepend(args: BuilderArgs, dialect: Dialect) -> exp.ArrayPrepend: 209 """ 210 Builds ArrayPrepend with NULL propagation semantics based on the dialect configuration. 211 212 Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. 213 Others (DuckDB, PostgreSQL) create a new single-element array instead. 214 215 Args: 216 args: Function arguments [array, element] 217 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 218 219 Returns: 220 ArrayPrepend expression with appropriate null_propagation flag 221 """ 222 return exp.ArrayPrepend( 223 this=seq_get(args, 0), 224 expression=seq_get(args, 1), 225 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 226 ) 227 228 229def build_array_concat(args: BuilderArgs, dialect: Dialect) -> exp.ArrayConcat: 230 """ 231 Builds ArrayConcat with NULL propagation semantics based on the dialect configuration. 232 233 Some dialects (Redshift, Snowflake) return NULL when any input array is NULL. 234 Others (DuckDB, PostgreSQL) skip NULL arrays and continue concatenation. 235 236 Args: 237 args: Function arguments [array1, array2, ...] (variadic) 238 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 239 240 Returns: 241 ArrayConcat expression with appropriate null_propagation flag 242 """ 243 return exp.ArrayConcat( 244 this=seq_get(args, 0), 245 expressions=args[1:], 246 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 247 ) 248 249 250def build_array_remove(args: BuilderArgs, dialect: Dialect) -> exp.ArrayRemove: 251 """ 252 Builds ArrayRemove with NULL propagation semantics based on the dialect configuration. 253 254 Some dialects (Snowflake) return NULL when the removal value is NULL. 255 Others (DuckDB) may return empty array due to NULL comparison semantics. 256 257 Args: 258 args: Function arguments [array, value_to_remove] 259 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 260 261 Returns: 262 ArrayRemove expression with appropriate null_propagation flag 263 """ 264 return exp.ArrayRemove( 265 this=seq_get(args, 0), 266 expression=seq_get(args, 1), 267 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 268 ) 269 270 271def _resolve_dialect(dialect: DialectType) -> Dialect: 272 from sqlglot.dialects.dialect import Dialect 273 274 return Dialect.get_or_raise(dialect) 275 276 277def _unpivot_target(expr: exp.Expr) -> exp.Expr: 278 # UNPIVOT's pre-FOR values and FOR field are new output names, not column references. 279 if isinstance(expr, exp.Column) and not expr.table: 280 return expr.this 281 if isinstance(expr, exp.Tuple): 282 expr.set("expressions", [_unpivot_target(e) for e in expr.expressions]) 283 return expr 284 285 286SENTINEL_NONE: Token = Token(TokenType.SENTINEL, "SENTINEL") 287 288 289class Parser: 290 """ 291 Parser consumes a list of tokens produced by the Tokenizer and produces a parsed syntax tree. 292 293 Args: 294 error_level: The desired error level. 295 Default: ErrorLevel.IMMEDIATE 296 error_message_context: The amount of context to capture from a query string when displaying 297 the error message (in number of characters). 298 Default: 100 299 max_errors: Maximum number of error messages to include in a raised ParseError. 300 This is only relevant if error_level is ErrorLevel.RAISE. 301 Default: 3 302 max_nodes: Maximum number of AST nodes to prevent memory exhaustion. 303 Set to -1 (default) to disable the check. 304 """ 305 306 __slots__ = ( 307 "error_level", 308 "error_message_context", 309 "max_errors", 310 "max_nodes", 311 "dialect", 312 "sql", 313 "errors", 314 "_tokens", 315 "_index", 316 "_curr", 317 "_next", 318 "_prev", 319 "_prev_comments", 320 "_pipe_cte_counter", 321 "_chunks", 322 "_chunk_index", 323 "_tokens_size", 324 "_node_count", 325 ) 326 327 FUNCTIONS: t.ClassVar[dict[str, t.Callable]] = { 328 **{name: func.from_arg_list for name, func in exp.FUNCTION_BY_NAME.items()}, 329 **dict.fromkeys(("COALESCE", "IFNULL", "NVL"), build_coalesce), 330 "ARRAY": lambda args, dialect: exp.Array(expressions=args), 331 "ARRAYAGG": lambda args, dialect: exp.ArrayAgg( 332 this=seq_get(args, 0), nulls_excluded=dialect.ARRAY_AGG_INCLUDES_NULLS is None or None 333 ), 334 "ARRAY_AGG": lambda args, dialect: exp.ArrayAgg( 335 this=seq_get(args, 0), nulls_excluded=dialect.ARRAY_AGG_INCLUDES_NULLS is None or None 336 ), 337 "ARRAY_APPEND": build_array_append, 338 "ARRAY_CAT": build_array_concat, 339 "ARRAY_CONCAT": build_array_concat, 340 "ARRAY_INTERSECT": lambda args: exp.ArrayIntersect(expressions=args), 341 "ARRAY_INTERSECTION": lambda args: exp.ArrayIntersect(expressions=args), 342 "ARRAY_PREPEND": build_array_prepend, 343 "ARRAY_REMOVE": build_array_remove, 344 "COUNT": lambda args: exp.Count(this=seq_get(args, 0), expressions=args[1:], big_int=True), 345 "CONCAT": lambda args, dialect: exp.Concat( 346 expressions=args, 347 safe=not dialect.STRICT_STRING_CONCAT, 348 coalesce=dialect.CONCAT_COALESCE, 349 ), 350 "CONCAT_WS": lambda args, dialect: exp.ConcatWs( 351 expressions=args, 352 safe=not dialect.STRICT_STRING_CONCAT, 353 coalesce=dialect.CONCAT_WS_COALESCE, 354 ), 355 "CONVERT_TIMEZONE": build_convert_timezone, 356 "DATE_TO_DATE_STR": lambda args: exp.Cast( 357 this=seq_get(args, 0), 358 to=exp.DataType(this=exp.DType.TEXT), 359 ), 360 "GENERATE_DATE_ARRAY": lambda args: exp.GenerateDateArray( 361 start=seq_get(args, 0), 362 end=seq_get(args, 1), 363 step=seq_get(args, 2) or exp.Interval(this=exp.Literal.string(1), unit=exp.var("DAY")), 364 ), 365 "GENERATE_UUID": lambda args, dialect: exp.Uuid( 366 is_string=dialect.UUID_IS_STRING_TYPE or None 367 ), 368 "GLOB": lambda args: exp.Glob(this=seq_get(args, 1), expression=seq_get(args, 0)), 369 "GREATEST": lambda args, dialect: exp.Greatest( 370 this=seq_get(args, 0), 371 expressions=args[1:], 372 ignore_nulls=dialect.LEAST_GREATEST_IGNORES_NULLS, 373 ), 374 "LEAST": lambda args, dialect: exp.Least( 375 this=seq_get(args, 0), 376 expressions=args[1:], 377 ignore_nulls=dialect.LEAST_GREATEST_IGNORES_NULLS, 378 ), 379 "HEX": build_hex, 380 "JSON_EXTRACT": build_extract_json_with_path(exp.JSONExtract), 381 "JSON_EXTRACT_SCALAR": build_extract_json_with_path(exp.JSONExtractScalar), 382 "JSON_EXTRACT_PATH_TEXT": build_extract_json_with_path(exp.JSONExtractScalar), 383 "JSON_KEYS": lambda args, dialect: exp.JSONKeys( 384 this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1)) 385 ), 386 "LIKE": build_like, 387 "LOG": build_logarithm, 388 "LOG2": lambda args: exp.Log(this=exp.Literal.number(2), expression=seq_get(args, 0)), 389 "LOG10": lambda args: exp.Log(this=exp.Literal.number(10), expression=seq_get(args, 0)), 390 "LOWER": build_lower, 391 "LPAD": lambda args: build_pad(args), 392 "LEFTPAD": lambda args: build_pad(args), 393 "LTRIM": lambda args: build_trim(args), 394 "MOD": build_mod, 395 "RIGHTPAD": lambda args: build_pad(args, is_left=False), 396 "RPAD": lambda args: build_pad(args, is_left=False), 397 "RTRIM": lambda args: build_trim(args, is_left=False), 398 "SCOPE_RESOLUTION": lambda args: ( 399 exp.ScopeResolution(expression=seq_get(args, 0)) 400 if len(args) != 2 401 else exp.ScopeResolution(this=seq_get(args, 0), expression=seq_get(args, 1)) 402 ), 403 "STRPOS": exp.StrPosition.from_arg_list, 404 "CHARINDEX": lambda args: build_locate_strposition(args), 405 "INSTR": exp.StrPosition.from_arg_list, 406 "LOCATE": lambda args: build_locate_strposition(args), 407 "TIME_TO_TIME_STR": lambda args: exp.Cast( 408 this=seq_get(args, 0), 409 to=exp.DataType(this=exp.DType.TEXT), 410 ), 411 "TO_HEX": build_hex, 412 "TS_OR_DS_TO_DATE_STR": lambda args: exp.Substring( 413 this=exp.Cast( 414 this=seq_get(args, 0), 415 to=exp.DataType(this=exp.DType.TEXT), 416 ), 417 start=exp.Literal.number(1), 418 length=exp.Literal.number(10), 419 ), 420 "UNNEST": lambda args: exp.Unnest(expressions=ensure_list(seq_get(args, 0))), 421 "UPPER": build_upper, 422 "UUID": lambda args, dialect: exp.Uuid(is_string=dialect.UUID_IS_STRING_TYPE or None), 423 "UUID_STRING": lambda args, dialect: exp.Uuid( 424 this=seq_get(args, 0), 425 name=seq_get(args, 1), 426 is_string=dialect.UUID_IS_STRING_TYPE or None, 427 ), 428 "VAR_MAP": build_var_map, 429 } 430 431 NO_PAREN_FUNCTIONS: t.ClassVar[dict] = { 432 TokenType.CURRENT_DATE: exp.CurrentDate, 433 TokenType.CURRENT_DATETIME: exp.CurrentDate, 434 TokenType.CURRENT_TIME: exp.CurrentTime, 435 TokenType.CURRENT_TIMESTAMP: exp.CurrentTimestamp, 436 TokenType.CURRENT_USER: exp.CurrentUser, 437 TokenType.CURRENT_ROLE: exp.CurrentRole, 438 } 439 440 STRUCT_TYPE_TOKENS: t.ClassVar = { 441 TokenType.NESTED, 442 TokenType.OBJECT, 443 TokenType.STRUCT, 444 TokenType.UNION, 445 } 446 447 NESTED_TYPE_TOKENS: t.ClassVar = { 448 TokenType.ARRAY, 449 TokenType.LIST, 450 TokenType.LOWCARDINALITY, 451 TokenType.MAP, 452 TokenType.NULLABLE, 453 TokenType.RANGE, 454 *STRUCT_TYPE_TOKENS, 455 } 456 457 ENUM_TYPE_TOKENS: t.ClassVar = { 458 TokenType.DYNAMIC, 459 TokenType.ENUM, 460 TokenType.ENUM8, 461 TokenType.ENUM16, 462 } 463 464 AGGREGATE_TYPE_TOKENS: t.ClassVar = { 465 TokenType.AGGREGATEFUNCTION, 466 TokenType.SIMPLEAGGREGATEFUNCTION, 467 } 468 469 TYPE_TOKENS: t.ClassVar = { 470 TokenType.BIT, 471 TokenType.BOOLEAN, 472 TokenType.TINYINT, 473 TokenType.UTINYINT, 474 TokenType.SMALLINT, 475 TokenType.USMALLINT, 476 TokenType.INT, 477 TokenType.UINT, 478 TokenType.BIGINT, 479 TokenType.UBIGINT, 480 TokenType.BIGNUM, 481 TokenType.INT128, 482 TokenType.UINT128, 483 TokenType.INT256, 484 TokenType.UINT256, 485 TokenType.MEDIUMINT, 486 TokenType.UMEDIUMINT, 487 TokenType.FIXEDSTRING, 488 TokenType.FLOAT, 489 TokenType.DOUBLE, 490 TokenType.UDOUBLE, 491 TokenType.CHAR, 492 TokenType.NCHAR, 493 TokenType.VARCHAR, 494 TokenType.NVARCHAR, 495 TokenType.BPCHAR, 496 TokenType.TEXT, 497 TokenType.MEDIUMTEXT, 498 TokenType.LONGTEXT, 499 TokenType.BLOB, 500 TokenType.MEDIUMBLOB, 501 TokenType.LONGBLOB, 502 TokenType.BINARY, 503 TokenType.VARBINARY, 504 TokenType.JSON, 505 TokenType.JSONB, 506 TokenType.INTERVAL, 507 TokenType.TINYBLOB, 508 TokenType.TINYTEXT, 509 TokenType.TIME, 510 TokenType.TIMETZ, 511 TokenType.TIME_NS, 512 TokenType.TIMESTAMP, 513 TokenType.TIMESTAMP_S, 514 TokenType.TIMESTAMP_MS, 515 TokenType.TIMESTAMP_NS, 516 TokenType.TIMESTAMPTZ, 517 TokenType.TIMESTAMPLTZ, 518 TokenType.TIMESTAMPNTZ, 519 TokenType.DATETIME, 520 TokenType.DATETIME2, 521 TokenType.DATETIME64, 522 TokenType.SMALLDATETIME, 523 TokenType.DATE, 524 TokenType.DATE32, 525 TokenType.INT4RANGE, 526 TokenType.INT4MULTIRANGE, 527 TokenType.INT8RANGE, 528 TokenType.INT8MULTIRANGE, 529 TokenType.NUMRANGE, 530 TokenType.NUMMULTIRANGE, 531 TokenType.TSRANGE, 532 TokenType.TSMULTIRANGE, 533 TokenType.TSTZRANGE, 534 TokenType.TSTZMULTIRANGE, 535 TokenType.DATERANGE, 536 TokenType.DATEMULTIRANGE, 537 TokenType.DECIMAL, 538 TokenType.DECIMAL32, 539 TokenType.DECIMAL64, 540 TokenType.DECIMAL128, 541 TokenType.DECIMAL256, 542 TokenType.DECFLOAT, 543 TokenType.UDECIMAL, 544 TokenType.BIGDECIMAL, 545 TokenType.UUID, 546 TokenType.GEOGRAPHY, 547 TokenType.GEOGRAPHYPOINT, 548 TokenType.GEOMETRY, 549 TokenType.POINT, 550 TokenType.RING, 551 TokenType.LINESTRING, 552 TokenType.MULTILINESTRING, 553 TokenType.POLYGON, 554 TokenType.MULTIPOLYGON, 555 TokenType.HLLSKETCH, 556 TokenType.HSTORE, 557 TokenType.PSEUDO_TYPE, 558 TokenType.SUPER, 559 TokenType.SERIAL, 560 TokenType.SMALLSERIAL, 561 TokenType.BIGSERIAL, 562 TokenType.XML, 563 TokenType.YEAR, 564 TokenType.USERDEFINED, 565 TokenType.MONEY, 566 TokenType.SMALLMONEY, 567 TokenType.ROWVERSION, 568 TokenType.IMAGE, 569 TokenType.VARIANT, 570 TokenType.VECTOR, 571 TokenType.VOID, 572 TokenType.OBJECT, 573 TokenType.OBJECT_IDENTIFIER, 574 TokenType.INET, 575 TokenType.IPADDRESS, 576 TokenType.IPPREFIX, 577 TokenType.IPV4, 578 TokenType.IPV6, 579 TokenType.UNKNOWN, 580 TokenType.NOTHING, 581 TokenType.NULL, 582 TokenType.NAME, 583 TokenType.TDIGEST, 584 TokenType.DYNAMIC, 585 *ENUM_TYPE_TOKENS, 586 *NESTED_TYPE_TOKENS, 587 *AGGREGATE_TYPE_TOKENS, 588 } 589 590 SIGNED_TO_UNSIGNED_TYPE_TOKEN: t.ClassVar = { 591 TokenType.BIGINT: TokenType.UBIGINT, 592 TokenType.INT: TokenType.UINT, 593 TokenType.MEDIUMINT: TokenType.UMEDIUMINT, 594 TokenType.SMALLINT: TokenType.USMALLINT, 595 TokenType.TINYINT: TokenType.UTINYINT, 596 TokenType.DECIMAL: TokenType.UDECIMAL, 597 TokenType.DOUBLE: TokenType.UDOUBLE, 598 } 599 600 SUBQUERY_PREDICATES: t.ClassVar = { 601 TokenType.ANY: exp.Any, 602 TokenType.ALL: exp.All, 603 TokenType.EXISTS: exp.Exists, 604 TokenType.SOME: exp.Any, 605 } 606 607 SUBQUERY_TOKENS: t.ClassVar = { 608 TokenType.SELECT, 609 TokenType.WITH, 610 TokenType.FROM, 611 } 612 613 RESERVED_TOKENS: t.ClassVar = { 614 *Tokenizer.SINGLE_TOKENS.values(), 615 TokenType.SELECT, 616 } - {TokenType.IDENTIFIER} 617 618 DB_CREATABLES: t.ClassVar = { 619 TokenType.DATABASE, 620 TokenType.DICTIONARY, 621 TokenType.FILE_FORMAT, 622 TokenType.MODEL, 623 TokenType.NAMESPACE, 624 TokenType.SCHEMA, 625 TokenType.SEMANTIC_VIEW, 626 TokenType.SEQUENCE, 627 TokenType.SINK, 628 TokenType.SOURCE, 629 TokenType.STAGE, 630 TokenType.STORAGE_INTEGRATION, 631 TokenType.STREAMLIT, 632 TokenType.TABLE, 633 TokenType.TAG, 634 TokenType.VIEW, 635 TokenType.WAREHOUSE, 636 } 637 638 CREATABLES: t.ClassVar = { 639 TokenType.COLUMN, 640 TokenType.CONSTRAINT, 641 TokenType.FOREIGN_KEY, 642 TokenType.FUNCTION, 643 TokenType.INDEX, 644 TokenType.PROCEDURE, 645 TokenType.TRIGGER, 646 TokenType.TYPE, 647 *DB_CREATABLES, 648 } 649 650 TRIGGER_EVENTS: t.ClassVar = { 651 TokenType.INSERT, 652 TokenType.UPDATE, 653 TokenType.DELETE, 654 TokenType.TRUNCATE, 655 } 656 657 ALTERABLES: t.ClassVar = { 658 TokenType.INDEX, 659 TokenType.TABLE, 660 TokenType.VIEW, 661 TokenType.SESSION, 662 } 663 664 # Tokens that can represent identifiers 665 ID_VAR_TOKENS: t.ClassVar[set] = { 666 TokenType.ALL, 667 TokenType.ANALYZE, 668 TokenType.ATTACH, 669 TokenType.VAR, 670 TokenType.ANTI, 671 TokenType.APPLY, 672 TokenType.ASC, 673 TokenType.ASOF, 674 TokenType.AUTO_INCREMENT, 675 TokenType.BEGIN, 676 TokenType.BPCHAR, 677 TokenType.CACHE, 678 TokenType.CASE, 679 TokenType.COLLATE, 680 TokenType.COMMAND, 681 TokenType.COMMENT, 682 TokenType.COMMIT, 683 TokenType.CONSTRAINT, 684 TokenType.COPY, 685 TokenType.CUBE, 686 TokenType.CURRENT_SCHEMA, 687 TokenType.DEFAULT, 688 TokenType.DELETE, 689 TokenType.DESC, 690 TokenType.DESCRIBE, 691 TokenType.DETACH, 692 TokenType.DICTIONARY, 693 TokenType.DIV, 694 TokenType.END, 695 TokenType.EXECUTE, 696 TokenType.EXPORT, 697 TokenType.ESCAPE, 698 TokenType.FALSE, 699 TokenType.FIRST, 700 TokenType.FILE, 701 TokenType.FILTER, 702 TokenType.FINAL, 703 TokenType.FORMAT, 704 TokenType.FULL, 705 TokenType.GET, 706 TokenType.IDENTIFIER, 707 TokenType.INOUT, 708 TokenType.IS, 709 TokenType.ISNULL, 710 TokenType.INTERVAL, 711 TokenType.KEEP, 712 TokenType.KILL, 713 TokenType.LEFT, 714 TokenType.LIMIT, 715 TokenType.LOAD, 716 TokenType.LOCK, 717 TokenType.MATCH, 718 TokenType.MERGE, 719 TokenType.NATURAL, 720 TokenType.NEXT, 721 TokenType.OFFSET, 722 TokenType.OPERATOR, 723 TokenType.ORDINALITY, 724 TokenType.OVER, 725 TokenType.OVERLAPS, 726 TokenType.OVERWRITE, 727 TokenType.PARTITION, 728 TokenType.PERCENT, 729 TokenType.PIVOT, 730 TokenType.PRAGMA, 731 TokenType.PUT, 732 TokenType.RANGE, 733 TokenType.RECURSIVE, 734 TokenType.REFERENCES, 735 TokenType.REFRESH, 736 TokenType.RENAME, 737 TokenType.REPLACE, 738 TokenType.RIGHT, 739 TokenType.ROLLUP, 740 TokenType.ROW, 741 TokenType.ROWS, 742 TokenType.SEMI, 743 TokenType.SET, 744 TokenType.SETTINGS, 745 TokenType.SHOW, 746 TokenType.STREAM, 747 TokenType.STREAMLIT, 748 TokenType.TEMPORARY, 749 TokenType.TOP, 750 TokenType.TRUE, 751 TokenType.TRUNCATE, 752 TokenType.UNIQUE, 753 TokenType.UNNEST, 754 TokenType.UNPIVOT, 755 TokenType.UPDATE, 756 TokenType.USE, 757 TokenType.VOLATILE, 758 TokenType.WINDOW, 759 TokenType.CURRENT_CATALOG, 760 TokenType.LOCALTIME, 761 TokenType.LOCALTIMESTAMP, 762 TokenType.SESSION_USER, 763 TokenType.STRAIGHT_JOIN, 764 *ALTERABLES, 765 *CREATABLES, 766 *SUBQUERY_PREDICATES, 767 *TYPE_TOKENS, 768 *NO_PAREN_FUNCTIONS, 769 } - {TokenType.UNION} 770 771 TABLE_ALIAS_TOKENS: t.ClassVar[set] = ID_VAR_TOKENS - { 772 TokenType.ANTI, 773 TokenType.ASOF, 774 TokenType.FULL, 775 TokenType.LEFT, 776 TokenType.LOCK, 777 TokenType.NATURAL, 778 TokenType.RIGHT, 779 TokenType.SEMI, 780 TokenType.WINDOW, 781 } 782 783 ALIAS_TOKENS: t.ClassVar = ID_VAR_TOKENS 784 785 COLON_PLACEHOLDER_TOKENS: t.ClassVar = ID_VAR_TOKENS 786 787 ARRAY_CONSTRUCTORS: t.ClassVar = { 788 "ARRAY": exp.Array, 789 "LIST": exp.List, 790 } 791 792 COMMENT_TABLE_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - {TokenType.IS} 793 794 UPDATE_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - {TokenType.SET} 795 796 TRIM_TYPES: t.ClassVar = {"LEADING", "TRAILING", "BOTH"} 797 798 # Tokens that indicate a simple column reference 799 IDENTIFIER_TOKENS: t.ClassVar[frozenset] = frozenset({TokenType.VAR, TokenType.IDENTIFIER}) 800 801 BRACKETS: t.ClassVar[frozenset] = frozenset({TokenType.L_BRACKET, TokenType.L_BRACE}) 802 803 # Postfix tokens that prevent the bare column fast path 804 COLUMN_POSTFIX_TOKENS: t.ClassVar[frozenset] = frozenset( 805 { 806 TokenType.L_PAREN, 807 TokenType.L_BRACKET, 808 TokenType.L_BRACE, 809 TokenType.COLON, 810 TokenType.JOIN_MARKER, 811 } 812 ) 813 814 TABLE_POSTFIX_TOKENS: t.ClassVar[frozenset] = frozenset( 815 { 816 TokenType.L_PAREN, 817 TokenType.L_BRACKET, 818 TokenType.L_BRACE, 819 TokenType.PIVOT, 820 TokenType.UNPIVOT, 821 TokenType.TABLE_SAMPLE, 822 } 823 ) 824 825 FUNC_TOKENS: t.ClassVar = { 826 TokenType.COLLATE, 827 TokenType.COMMAND, 828 TokenType.CURRENT_DATE, 829 TokenType.CURRENT_DATETIME, 830 TokenType.CURRENT_SCHEMA, 831 TokenType.CURRENT_TIMESTAMP, 832 TokenType.CURRENT_TIME, 833 TokenType.CURRENT_USER, 834 TokenType.CURRENT_CATALOG, 835 TokenType.FILTER, 836 TokenType.FIRST, 837 TokenType.FORMAT, 838 TokenType.GET, 839 TokenType.GLOB, 840 TokenType.IDENTIFIER, 841 TokenType.INDEX, 842 TokenType.ISNULL, 843 TokenType.ILIKE, 844 TokenType.INSERT, 845 TokenType.LIKE, 846 TokenType.LOCALTIME, 847 TokenType.LOCALTIMESTAMP, 848 TokenType.MERGE, 849 TokenType.NEXT, 850 TokenType.OFFSET, 851 TokenType.PRIMARY_KEY, 852 TokenType.RANGE, 853 TokenType.REPLACE, 854 TokenType.RLIKE, 855 TokenType.ROW, 856 TokenType.SESSION_USER, 857 TokenType.UNNEST, 858 TokenType.VAR, 859 TokenType.LEFT, 860 TokenType.RIGHT, 861 TokenType.SEQUENCE, 862 TokenType.DATE, 863 TokenType.DATETIME, 864 TokenType.TABLE, 865 TokenType.TIMESTAMP, 866 TokenType.TIMESTAMPTZ, 867 TokenType.TRUNCATE, 868 TokenType.UTC_DATE, 869 TokenType.UTC_TIME, 870 TokenType.UTC_TIMESTAMP, 871 TokenType.WINDOW, 872 TokenType.XOR, 873 *TYPE_TOKENS, 874 *SUBQUERY_PREDICATES, 875 } 876 877 CONJUNCTION: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 878 TokenType.AND: exp.And, 879 } 880 881 ASSIGNMENT: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 882 TokenType.COLON_EQ: exp.PropertyEQ, 883 } 884 885 DISJUNCTION: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 886 TokenType.OR: exp.Or, 887 } 888 889 EQUALITY: t.ClassVar = { 890 TokenType.EQ: exp.EQ, 891 TokenType.NEQ: exp.NEQ, 892 TokenType.NULLSAFE_EQ: exp.NullSafeEQ, 893 } 894 895 COMPARISON: t.ClassVar = { 896 TokenType.GT: exp.GT, 897 TokenType.GTE: exp.GTE, 898 TokenType.LT: exp.LT, 899 TokenType.LTE: exp.LTE, 900 } 901 902 BITWISE: t.ClassVar = { 903 TokenType.AMP: exp.BitwiseAnd, 904 TokenType.CARET: exp.BitwiseXor, 905 TokenType.PIPE: exp.BitwiseOr, 906 } 907 908 TERM: t.ClassVar = { 909 TokenType.DASH: exp.Sub, 910 TokenType.PLUS: exp.Add, 911 TokenType.MOD: exp.Mod, 912 TokenType.COLLATE: exp.Collate, 913 } 914 915 FACTOR: t.ClassVar = { 916 TokenType.DIV: exp.IntDiv, 917 TokenType.LR_ARROW: exp.Distance, 918 TokenType.LLRR_ARROW: exp.DistanceNd, 919 TokenType.SLASH: exp.Div, 920 TokenType.STAR: exp.Mul, 921 } 922 923 EXPONENT: t.ClassVar[dict[TokenType, type[exp.Expr]]] = {} 924 925 TIMES: t.ClassVar = { 926 TokenType.TIME, 927 TokenType.TIMETZ, 928 } 929 930 TIMESTAMPS: t.ClassVar = { 931 TokenType.TIMESTAMP, 932 TokenType.TIMESTAMPNTZ, 933 TokenType.TIMESTAMPTZ, 934 TokenType.TIMESTAMPLTZ, 935 *TIMES, 936 } 937 938 SET_OPERATIONS: t.ClassVar = { 939 TokenType.UNION, 940 TokenType.INTERSECT, 941 TokenType.EXCEPT, 942 } 943 944 JOIN_METHODS: t.ClassVar = { 945 TokenType.ASOF, 946 TokenType.NATURAL, 947 TokenType.POSITIONAL, 948 } 949 950 JOIN_SIDES: t.ClassVar = { 951 TokenType.LEFT, 952 TokenType.RIGHT, 953 TokenType.FULL, 954 } 955 956 JOIN_KINDS: t.ClassVar = { 957 TokenType.ANTI, 958 TokenType.CROSS, 959 TokenType.INNER, 960 TokenType.OUTER, 961 TokenType.SEMI, 962 TokenType.STRAIGHT_JOIN, 963 } 964 965 JOIN_HINTS: t.ClassVar[set[str]] = set() 966 967 # Tokens that unambiguously end a table reference on the fast path 968 TABLE_TERMINATORS: t.ClassVar[frozenset] = frozenset( 969 { 970 TokenType.COMMA, 971 TokenType.GROUP_BY, 972 TokenType.HAVING, 973 TokenType.JOIN, 974 TokenType.LIMIT, 975 TokenType.ON, 976 TokenType.ORDER_BY, 977 TokenType.R_PAREN, 978 TokenType.SEMICOLON, 979 TokenType.SENTINEL, 980 TokenType.WHERE, 981 *SET_OPERATIONS, 982 *JOIN_KINDS, 983 *JOIN_METHODS, 984 *JOIN_SIDES, 985 } 986 ) 987 988 LAMBDAS: t.ClassVar = { 989 TokenType.ARROW: lambda self, expressions: self.expression( 990 exp.Lambda( 991 this=self._replace_lambda( 992 self._parse_disjunction(), 993 expressions, 994 ), 995 expressions=expressions, 996 ) 997 ), 998 TokenType.FARROW: lambda self, expressions: self.expression( 999 exp.Kwarg(this=exp.var(expressions[0].name), expression=self._parse_disjunction()) 1000 ), 1001 } 1002 1003 # Whether lambda args include type annotations, e.g. TRANSFORM(arr, x INT -> x + 1) in Snowflake 1004 TYPED_LAMBDA_ARGS: t.ClassVar[bool] = False 1005 1006 LAMBDA_ARG_TERMINATORS: t.ClassVar[frozenset] = frozenset({TokenType.COMMA, TokenType.R_PAREN}) 1007 1008 COLUMN_OPERATORS: t.ClassVar = { 1009 TokenType.DOT: None, 1010 TokenType.DOTCOLON: lambda self, this, to: self.expression(exp.JSONCast(this=this, to=to)), 1011 TokenType.DCOLON: lambda self, this, to: self.build_cast( 1012 strict=self.STRICT_CAST, this=this, to=to 1013 ), 1014 TokenType.ARROW: lambda self, this, path: self.expression( 1015 exp.JSONExtract( 1016 this=this, 1017 expression=self.dialect.to_json_path(path), 1018 only_json_types=self.JSON_ARROWS_REQUIRE_JSON_TYPE, 1019 ) 1020 ), 1021 TokenType.DARROW: lambda self, this, path: self.expression( 1022 exp.JSONExtractScalar( 1023 this=this, 1024 expression=self.dialect.to_json_path(path), 1025 only_json_types=self.JSON_ARROWS_REQUIRE_JSON_TYPE, 1026 scalar_only=self.dialect.JSON_EXTRACT_SCALAR_SCALAR_ONLY, 1027 ) 1028 ), 1029 TokenType.HASH_ARROW: lambda self, this, path: self.expression( 1030 exp.JSONBExtract(this=this, expression=path) 1031 ), 1032 TokenType.DHASH_ARROW: lambda self, this, path: self.expression( 1033 exp.JSONBExtractScalar(this=this, expression=path) 1034 ), 1035 TokenType.PLACEHOLDER: lambda self, this, key: self.expression( 1036 exp.JSONBContains(this=this, expression=key) 1037 ), 1038 } 1039 1040 CAST_COLUMN_OPERATORS: t.ClassVar = { 1041 TokenType.DOTCOLON, 1042 TokenType.DCOLON, 1043 } 1044 1045 EXPRESSION_PARSERS: t.ClassVar = { 1046 exp.Cluster: lambda self: self._parse_sort(exp.Cluster, TokenType.CLUSTER_BY), 1047 exp.Column: lambda self: self._parse_column(), 1048 exp.ColumnDef: lambda self: self._parse_column_def(self._parse_column()), 1049 exp.Condition: lambda self: self._parse_disjunction(), 1050 exp.DataType: lambda self: self._parse_types(allow_identifiers=False, schema=True), 1051 exp.Expr: lambda self: self._parse_expression(), 1052 exp.From: lambda self: self._parse_from(joins=True), 1053 exp.GrantPrincipal: lambda self: self._parse_grant_principal(), 1054 exp.GrantPrivilege: lambda self: self._parse_grant_privilege(), 1055 exp.Group: lambda self: self._parse_group(), 1056 exp.Having: lambda self: self._parse_having(), 1057 exp.Hint: lambda self: self._parse_hint_body(), 1058 exp.Identifier: lambda self: self._parse_id_var(), 1059 exp.Join: lambda self: self._parse_join(), 1060 exp.Lambda: lambda self: self._parse_lambda(), 1061 exp.Lateral: lambda self: self._parse_lateral(), 1062 exp.Limit: lambda self: self._parse_limit(), 1063 exp.Offset: lambda self: self._parse_offset(), 1064 exp.Order: lambda self: self._parse_order(), 1065 exp.Ordered: lambda self: self._parse_ordered(), 1066 exp.Properties: lambda self: self._parse_properties(), 1067 exp.PartitionedByProperty: lambda self: self._parse_partitioned_by(), 1068 exp.Qualify: lambda self: self._parse_qualify(), 1069 exp.Returning: lambda self: self._parse_returning(), 1070 exp.Select: lambda self: self._parse_select(), 1071 exp.Sort: lambda self: self._parse_sort(exp.Sort, TokenType.SORT_BY), 1072 exp.Table: lambda self: self._parse_table_parts(), 1073 exp.TableAlias: lambda self: self._parse_table_alias(), 1074 exp.Tuple: lambda self: self._parse_value(values=False), 1075 exp.Whens: lambda self: self._parse_when_matched(), 1076 exp.Where: lambda self: self._parse_where(), 1077 exp.Window: lambda self: self._parse_named_window(), 1078 exp.With: lambda self: self._parse_with(), 1079 } 1080 1081 STATEMENT_PARSERS: t.ClassVar = { 1082 TokenType.ALTER: lambda self: self._parse_alter(), 1083 TokenType.ANALYZE: lambda self: self._parse_analyze(), 1084 TokenType.BEGIN: lambda self: self._parse_transaction(), 1085 TokenType.CACHE: lambda self: self._parse_cache(), 1086 TokenType.COMMENT: lambda self: self._parse_comment(), 1087 TokenType.COMMIT: lambda self: self._parse_commit_or_rollback(), 1088 TokenType.COPY: lambda self: self._parse_copy(), 1089 TokenType.CREATE: lambda self: self._parse_create(), 1090 TokenType.DELETE: lambda self: self._parse_delete(), 1091 TokenType.DESC: lambda self: self._parse_describe(), 1092 TokenType.DESCRIBE: lambda self: self._parse_describe(), 1093 TokenType.DROP: lambda self: self._parse_drop(), 1094 TokenType.GRANT: lambda self: self._parse_grant(), 1095 TokenType.REVOKE: lambda self: self._parse_revoke(), 1096 TokenType.INSERT: lambda self: self._parse_insert(), 1097 TokenType.KILL: lambda self: self._parse_kill(), 1098 TokenType.LOAD: lambda self: self._parse_load(), 1099 TokenType.MERGE: lambda self: self._parse_merge(), 1100 TokenType.PIVOT: lambda self: self._parse_simplified_pivot(), 1101 TokenType.PRAGMA: lambda self: self.expression(exp.Pragma(this=self._parse_expression())), 1102 TokenType.REFRESH: lambda self: self._parse_refresh(), 1103 TokenType.ROLLBACK: lambda self: self._parse_commit_or_rollback(), 1104 TokenType.SET: lambda self: self._parse_set(), 1105 TokenType.TRUNCATE: lambda self: self._parse_truncate_table(), 1106 TokenType.UNCACHE: lambda self: self._parse_uncache(), 1107 TokenType.UNPIVOT: lambda self: self._parse_simplified_pivot(is_unpivot=True), 1108 TokenType.UPDATE: lambda self: self._parse_update(), 1109 TokenType.USE: lambda self: self._parse_use(), 1110 TokenType.SEMICOLON: lambda self: exp.Semicolon(), 1111 } 1112 1113 UNARY_PARSERS: t.ClassVar = { 1114 TokenType.PLUS: lambda self: self._parse_unary(), # Unary + is handled as a no-op 1115 TokenType.NOT: lambda self: self.expression(exp.Not(this=self._parse_equality())), 1116 TokenType.TILDE: lambda self: self.expression(exp.BitwiseNot(this=self._parse_unary())), 1117 TokenType.DASH: lambda self: self.expression(exp.Neg(this=self._parse_unary())), 1118 TokenType.PIPE_SLASH: lambda self: self.expression(exp.Sqrt(this=self._parse_unary())), 1119 TokenType.DPIPE_SLASH: lambda self: self.expression(exp.Cbrt(this=self._parse_unary())), 1120 } 1121 1122 STRING_PARSERS: t.ClassVar = { 1123 TokenType.HEREDOC_STRING: lambda self, token: self.expression( 1124 exp.RawString(this=token.text), token 1125 ), 1126 TokenType.NATIONAL_STRING: lambda self, token: self.expression( 1127 exp.National(this=token.text), token 1128 ), 1129 TokenType.RAW_STRING: lambda self, token: self.expression( 1130 exp.RawString(this=token.text), token 1131 ), 1132 TokenType.STRING: lambda self, token: self.expression( 1133 exp.Literal(this=token.text, is_string=True), token 1134 ), 1135 TokenType.UNICODE_STRING: lambda self, token: self.expression( 1136 exp.UnicodeString( 1137 this=token.text, escape=self._match_text_seq("UESCAPE") and self._parse_string() 1138 ), 1139 token, 1140 ), 1141 } 1142 1143 NUMERIC_PARSERS: t.ClassVar = { 1144 TokenType.BIT_STRING: lambda self, token: self.expression( 1145 exp.BitString(this=token.text), token 1146 ), 1147 TokenType.BYTE_STRING: lambda self, token: self.expression( 1148 exp.ByteString( 1149 this=token.text, is_bytes=self.dialect.BYTE_STRING_IS_BYTES_TYPE or None 1150 ), 1151 token, 1152 ), 1153 TokenType.HEX_STRING: lambda self, token: self.expression( 1154 exp.HexString( 1155 this=token.text, is_integer=self.dialect.HEX_STRING_IS_INTEGER_TYPE or None 1156 ), 1157 token, 1158 ), 1159 TokenType.NUMBER: lambda self, token: self.expression( 1160 exp.Literal(this=token.text, is_string=False), token 1161 ), 1162 } 1163 1164 PRIMARY_PARSERS: t.ClassVar = { 1165 **STRING_PARSERS, 1166 **NUMERIC_PARSERS, 1167 TokenType.INTRODUCER: lambda self, token: self._parse_introducer(token), 1168 TokenType.NULL: lambda self, _: self.expression(exp.Null()), 1169 TokenType.TRUE: lambda self, _: self.expression(exp.Boolean(this=True)), 1170 TokenType.FALSE: lambda self, _: self.expression(exp.Boolean(this=False)), 1171 TokenType.SESSION_PARAMETER: lambda self, _: self._parse_session_parameter(), 1172 TokenType.STAR: lambda self, _: self._parse_star_ops(), 1173 } 1174 1175 PLACEHOLDER_PARSERS: t.ClassVar = { 1176 TokenType.PLACEHOLDER: lambda self: self.expression(exp.Placeholder()), 1177 TokenType.PARAMETER: lambda self: self._parse_parameter(), 1178 TokenType.COLON: lambda self: ( 1179 self.expression(exp.Placeholder(this=self._prev.text)) 1180 if self._match_set(self.COLON_PLACEHOLDER_TOKENS) 1181 else None 1182 ), 1183 } 1184 1185 RANGE_PARSERS: t.ClassVar = { 1186 TokenType.AT_GT: binary_range_parser(exp.ArrayContainsAll), 1187 TokenType.BETWEEN: lambda self, this: self._parse_between(this), 1188 TokenType.GLOB: binary_range_parser(exp.Glob), 1189 TokenType.ILIKE: binary_range_parser(exp.ILike), 1190 TokenType.IN: lambda self, this: self._parse_in(this), 1191 TokenType.IRLIKE: binary_range_parser(exp.RegexpILike), 1192 TokenType.IS: lambda self, this: self._parse_is(this), 1193 TokenType.LIKE: binary_range_parser(exp.Like), 1194 TokenType.LT_AT: binary_range_parser(exp.ArrayContainedBy), 1195 TokenType.OVERLAPS: binary_range_parser(exp.Overlaps), 1196 TokenType.RLIKE: binary_range_parser(exp.RegexpLike), 1197 TokenType.SIMILAR_TO: binary_range_parser(exp.SimilarTo), 1198 TokenType.FOR: lambda self, this: self._parse_comprehension(this), 1199 TokenType.QMARK_AMP: binary_range_parser(exp.JSONBContainsAllTopKeys), 1200 TokenType.QMARK_PIPE: binary_range_parser(exp.JSONBContainsAnyTopKeys), 1201 TokenType.HASH_DASH: binary_range_parser(exp.JSONBDeleteAtPath), 1202 TokenType.AT_QMARK: binary_range_parser(exp.JSONBPathExists), 1203 TokenType.ADJACENT: binary_range_parser(exp.Adjacent), 1204 TokenType.OPERATOR: lambda self, this: self._parse_operator(this), 1205 TokenType.AMP_LT: binary_range_parser(exp.ExtendsLeft), 1206 TokenType.AMP_GT: binary_range_parser(exp.ExtendsRight), 1207 } 1208 1209 PIPE_SYNTAX_TRANSFORM_PARSERS: t.ClassVar = { 1210 "AGGREGATE": lambda self, query: self._parse_pipe_syntax_aggregate(query), 1211 "AS": lambda self, query: self._build_pipe_cte( 1212 query, [exp.Star()], self._parse_table_alias() 1213 ), 1214 "DISTINCT": lambda self, query: self._advance() or query.distinct(copy=False), 1215 "EXTEND": lambda self, query: self._parse_pipe_syntax_extend(query), 1216 "LIMIT": lambda self, query: self._parse_pipe_syntax_limit(query), 1217 "ORDER BY": lambda self, query: query.order_by( 1218 self._parse_order(), append=False, copy=False 1219 ), 1220 "PIVOT": lambda self, query: self._parse_pipe_syntax_pivot(query), 1221 "SELECT": lambda self, query: self._parse_pipe_syntax_select(query), 1222 "TABLESAMPLE": lambda self, query: self._parse_pipe_syntax_tablesample(query), 1223 "UNPIVOT": lambda self, query: self._parse_pipe_syntax_pivot(query), 1224 "WHERE": lambda self, query: query.where(self._parse_where(), copy=False), 1225 } 1226 1227 PROPERTY_PARSERS: t.ClassVar[dict[str, t.Callable]] = { 1228 "ALLOWED_VALUES": lambda self: self.expression( 1229 exp.AllowedValuesProperty(expressions=self._parse_csv(self._parse_primary)) 1230 ), 1231 "ALGORITHM": lambda self: self._parse_property_assignment(exp.AlgorithmProperty), 1232 "AUTO": lambda self: self._parse_auto_property(), 1233 "AUTO_INCREMENT": lambda self: self._parse_property_assignment(exp.AutoIncrementProperty), 1234 "BACKUP": lambda self: self.expression( 1235 exp.BackupProperty(this=self._parse_var(any_token=True)) 1236 ), 1237 "BLOCKCOMPRESSION": lambda self: self._parse_blockcompression(), 1238 "CALLED": lambda self: self._parse_called_on_null_input_property(), 1239 "CHARSET": lambda self, **kwargs: self._parse_character_set(**kwargs), 1240 "CHARACTER SET": lambda self, **kwargs: self._parse_character_set(**kwargs), 1241 "CHECKSUM": lambda self: self._parse_checksum(), 1242 "CLUSTER BY": lambda self: self._parse_cluster_property(), 1243 "CLUSTERED": lambda self: self._parse_clustered_by(), 1244 "COLLATE": lambda self, **kwargs: self._parse_property_assignment( 1245 exp.CollateProperty, **kwargs 1246 ), 1247 "COMMENT": lambda self: self._parse_property_assignment(exp.SchemaCommentProperty), 1248 "CONTAINS": lambda self: self._parse_contains_property(), 1249 "COPY": lambda self: self._parse_copy_property(), 1250 "DATABLOCKSIZE": lambda self, **kwargs: self._parse_datablocksize(**kwargs), 1251 "DATA_DELETION": lambda self: self._parse_data_deletion_property(), 1252 "DEFINER": lambda self: self._parse_definer(), 1253 "DETERMINISTIC": lambda self: self.expression( 1254 exp.StabilityProperty(this=exp.Literal.string("IMMUTABLE")) 1255 ), 1256 "DISTRIBUTED": lambda self: self._parse_distributed_property(), 1257 "DUPLICATE": lambda self: self._parse_composite_key_property(exp.DuplicateKeyProperty), 1258 "DYNAMIC": lambda self: self.expression(exp.DynamicProperty()), 1259 "DISTKEY": lambda self: self._parse_distkey(), 1260 "DISTSTYLE": lambda self: self._parse_property_assignment(exp.DistStyleProperty), 1261 "EMPTY": lambda self: self.expression(exp.EmptyProperty()), 1262 "ENGINE": lambda self: self._parse_property_assignment(exp.EngineProperty), 1263 "ENVIRONMENT": lambda self: self.expression( 1264 exp.EnviromentProperty(expressions=self._parse_wrapped_csv(self._parse_assignment)) 1265 ), 1266 "HANDLER": lambda self: self._parse_property_assignment(exp.HandlerProperty), 1267 "EXECUTE": lambda self: self._parse_property_assignment(exp.ExecuteAsProperty), 1268 "EXTERNAL": lambda self: self.expression(exp.ExternalProperty()), 1269 "FALLBACK": lambda self, **kwargs: self._parse_fallback(**kwargs), 1270 "FORMAT": lambda self: self._parse_property_assignment(exp.FileFormatProperty), 1271 "FREESPACE": lambda self: self._parse_freespace(), 1272 "GLOBAL": lambda self: self.expression(exp.GlobalProperty()), 1273 "HEAP": lambda self: self.expression(exp.HeapProperty()), 1274 "ICEBERG": lambda self: self.expression(exp.IcebergProperty()), 1275 "IMMUTABLE": lambda self: self.expression( 1276 exp.StabilityProperty(this=exp.Literal.string("IMMUTABLE")) 1277 ), 1278 "INHERITS": lambda self: self.expression( 1279 exp.InheritsProperty(expressions=self._parse_wrapped_csv(self._parse_table)) 1280 ), 1281 "INPUT": lambda self: self.expression(exp.InputModelProperty(this=self._parse_schema())), 1282 "JOURNAL": lambda self, **kwargs: self._parse_journal(**kwargs), 1283 "LANGUAGE": lambda self: self._parse_property_assignment(exp.LanguageProperty), 1284 "LAYOUT": lambda self: self._parse_dict_property(this="LAYOUT"), 1285 "LIFETIME": lambda self: self._parse_dict_range(this="LIFETIME"), 1286 "LIKE": lambda self: self._parse_create_like(), 1287 "LOCATION": lambda self: self._parse_property_assignment(exp.LocationProperty), 1288 "LOCK": lambda self: self._parse_locking(), 1289 "LOCKING": lambda self: self._parse_locking(), 1290 "LOG": lambda self, **kwargs: self._parse_log(**kwargs), 1291 "MATERIALIZED": lambda self: self.expression(exp.MaterializedProperty()), 1292 "MERGEBLOCKRATIO": lambda self, **kwargs: self._parse_mergeblockratio(**kwargs), 1293 "MODIFIES": lambda self: self._parse_modifies_property(), 1294 "MULTISET": lambda self: self.expression(exp.SetProperty(multi=True)), 1295 "NO": lambda self: self._parse_no_property(), 1296 "ON": lambda self: self._parse_on_property(), 1297 "ORDER BY": lambda self: self._parse_order(skip_order_token=True), 1298 "OUTPUT": lambda self: self.expression(exp.OutputModelProperty(this=self._parse_schema())), 1299 "PARTITION": lambda self: self._parse_partitioned_of(), 1300 "PARTITION BY": lambda self: self._parse_partitioned_by(), 1301 "PARTITIONED BY": lambda self: self._parse_partitioned_by(), 1302 "PARTITIONED_BY": lambda self: self._parse_partitioned_by(), 1303 "PRIMARY KEY": lambda self: self._parse_primary_key(in_props=True), 1304 "RANGE": lambda self: self._parse_dict_range(this="RANGE"), 1305 "READS": lambda self: self._parse_reads_property(), 1306 "REMOTE": lambda self: self._parse_remote_with_connection(), 1307 "RETURNS": lambda self: self._parse_returns(), 1308 "STRICT": lambda self: self.expression(exp.StrictProperty()), 1309 "STREAMING": lambda self: self.expression(exp.StreamingTableProperty()), 1310 "ROW": lambda self: self._parse_row(), 1311 "ROW_FORMAT": lambda self: self._parse_property_assignment(exp.RowFormatProperty), 1312 "SAMPLE": lambda self: self.expression( 1313 exp.SampleProperty(this=self._match_text_seq("BY") and self._parse_bitwise()) 1314 ), 1315 "SECURE": lambda self: self.expression(exp.SecureProperty()), 1316 "SECURITY": lambda self: self._parse_sql_security(), 1317 "SQL SECURITY": lambda self: self._parse_sql_security(), 1318 "SET": lambda self: self.expression(exp.SetProperty(multi=False)), 1319 "SETTINGS": lambda self: self._parse_settings_property(), 1320 "SHARING": lambda self: self._parse_property_assignment(exp.SharingProperty), 1321 "SORTKEY": lambda self: self._parse_sortkey(), 1322 "SOURCE": lambda self: self._parse_dict_property(this="SOURCE"), 1323 "STABLE": lambda self: self.expression( 1324 exp.StabilityProperty(this=exp.Literal.string("STABLE")) 1325 ), 1326 "STORED": lambda self: self._parse_stored(), 1327 "SYSTEM_VERSIONING": lambda self: self._parse_system_versioning_property(), 1328 "TBLPROPERTIES": lambda self: self._parse_wrapped_properties(), 1329 "TEMP": lambda self: self.expression(exp.TemporaryProperty()), 1330 "TEMPORARY": lambda self: self.expression(exp.TemporaryProperty()), 1331 "TO": lambda self: self._parse_to_table(), 1332 "TRANSIENT": lambda self: self.expression(exp.TransientProperty()), 1333 "TRANSFORM": lambda self: self.expression( 1334 exp.TransformModelProperty(expressions=self._parse_wrapped_csv(self._parse_expression)) 1335 ), 1336 "TTL": lambda self: self._parse_ttl(), 1337 "USING": lambda self: self._parse_property_assignment(exp.FileFormatProperty), 1338 "UNLOGGED": lambda self: self.expression(exp.UnloggedProperty()), 1339 "VOLATILE": lambda self: self._parse_volatile_property(), 1340 "WITH": lambda self: self._parse_with_property(), 1341 } 1342 1343 CONSTRAINT_PARSERS: t.ClassVar = { 1344 "AUTOINCREMENT": lambda self: self._parse_auto_increment(), 1345 "AUTO_INCREMENT": lambda self: self._parse_auto_increment(), 1346 "CASESPECIFIC": lambda self: self.expression(exp.CaseSpecificColumnConstraint(not_=False)), 1347 "CHARACTER SET": lambda self: self.expression( 1348 exp.CharacterSetColumnConstraint(this=self._parse_var_or_string()) 1349 ), 1350 "CHECK": lambda self: self._parse_check_constraint(), 1351 "COLLATE": lambda self: self.expression( 1352 exp.CollateColumnConstraint(this=self._parse_identifier() or self._parse_column()) 1353 ), 1354 "COMMENT": lambda self: self.expression( 1355 exp.CommentColumnConstraint(this=self._parse_string()) 1356 ), 1357 "COMPRESS": lambda self: self._parse_compress(), 1358 "CLUSTERED": lambda self: self.expression( 1359 exp.ClusteredColumnConstraint(this=self._parse_wrapped_csv(self._parse_ordered)) 1360 ), 1361 "NONCLUSTERED": lambda self: self.expression( 1362 exp.NonClusteredColumnConstraint(this=self._parse_wrapped_csv(self._parse_ordered)) 1363 ), 1364 "DEFAULT": lambda self: self.expression( 1365 exp.DefaultColumnConstraint(this=self._parse_bitwise()) 1366 ), 1367 "ENCODE": lambda self: self.expression(exp.EncodeColumnConstraint(this=self._parse_var())), 1368 "EPHEMERAL": lambda self: self.expression( 1369 exp.EphemeralColumnConstraint(this=self._parse_bitwise()) 1370 ), 1371 "EXCLUDE": lambda self: self.expression( 1372 exp.ExcludeColumnConstraint(this=self._parse_index_params()) 1373 ), 1374 "FOREIGN KEY": lambda self: self._parse_foreign_key(), 1375 "FORMAT": lambda self: self.expression( 1376 exp.DateFormatColumnConstraint(this=self._parse_var_or_string()) 1377 ), 1378 "GENERATED": lambda self: self._parse_generated_as_identity(), 1379 "IDENTITY": lambda self: self._parse_auto_increment(), 1380 "INLINE": lambda self: self._parse_inline(), 1381 "LIKE": lambda self: self._parse_create_like(), 1382 "NOT": lambda self: self._parse_not_constraint(), 1383 "NULL": lambda self: self.expression(exp.NotNullColumnConstraint(allow_null=True)), 1384 "ON": lambda self: ( 1385 ( 1386 self._match(TokenType.UPDATE) 1387 and self.expression(exp.OnUpdateColumnConstraint(this=self._parse_function())) 1388 ) 1389 or self.expression(exp.OnProperty(this=self._parse_id_var())) 1390 ), 1391 "PATH": lambda self: self.expression(exp.PathColumnConstraint(this=self._parse_string())), 1392 "PERIOD": lambda self: self._parse_period_for_system_time(), 1393 "PRIMARY KEY": lambda self: self._parse_primary_key(), 1394 "REFERENCES": lambda self: self._parse_references(match=False), 1395 "TITLE": lambda self: self.expression( 1396 exp.TitleColumnConstraint(this=self._parse_var_or_string()) 1397 ), 1398 "TTL": lambda self: self.expression(exp.MergeTreeTTL(expressions=[self._parse_bitwise()])), 1399 "UNIQUE": lambda self: self._parse_unique(), 1400 "UPPERCASE": lambda self: self.expression(exp.UppercaseColumnConstraint()), 1401 "WITH": lambda self: self.expression( 1402 exp.Properties(expressions=self._parse_wrapped_properties()) 1403 ), 1404 "BUCKET": lambda self: self._parse_partitioned_by_bucket_or_truncate(), 1405 "TRUNCATE": lambda self: self._parse_partitioned_by_bucket_or_truncate(), 1406 } 1407 1408 def _parse_partitioned_by_bucket_or_truncate(self) -> exp.Expr | None: 1409 if not self._match(TokenType.L_PAREN, advance=False): 1410 # Partitioning by bucket or truncate follows the syntax: 1411 # PARTITION BY (BUCKET(..) | TRUNCATE(..)) 1412 # If we don't have parenthesis after each keyword, we should instead parse this as an identifier 1413 self._retreat(self._index - 1) 1414 return None 1415 1416 klass = ( 1417 exp.PartitionedByBucket 1418 if self._prev.text.upper() == "BUCKET" 1419 else exp.PartitionByTruncate 1420 ) 1421 1422 args = self._parse_wrapped_csv(lambda: self._parse_primary() or self._parse_column()) 1423 this, expression = seq_get(args, 0), seq_get(args, 1) 1424 1425 if isinstance(this, exp.Literal): 1426 # Check for Iceberg partition transforms (bucket / truncate) and ensure their arguments are in the right order 1427 # - For Hive, it's `bucket(<num buckets>, <col name>)` or `truncate(<num_chars>, <col_name>)` 1428 # - For Trino, it's reversed - `bucket(<col name>, <num buckets>)` or `truncate(<col_name>, <num_chars>)` 1429 # Both variants are canonicalized in the latter i.e `bucket(<col name>, <num buckets>)` 1430 # 1431 # Hive ref: https://docs.aws.amazon.com/athena/latest/ug/querying-iceberg-creating-tables.html#querying-iceberg-partitioning 1432 # Trino ref: https://docs.aws.amazon.com/athena/latest/ug/create-table-as.html#ctas-table-properties 1433 this, expression = expression, this 1434 1435 return self.expression(klass(this=this, expression=expression)) 1436 1437 ALTER_PARSERS: t.ClassVar = { 1438 "ADD": lambda self: self._parse_alter_table_add(), 1439 "AS": lambda self: self._parse_select(), 1440 "ALTER": lambda self: self._parse_alter_table_alter(), 1441 "CLUSTER BY": lambda self: self._parse_cluster_property(), 1442 "DELETE": lambda self: self.expression(exp.Delete(where=self._parse_where())), 1443 "DROP": lambda self: self._parse_alter_table_drop(), 1444 "RENAME": lambda self: self._parse_alter_table_rename(), 1445 "SET": lambda self: self._parse_alter_table_set(), 1446 "SWAP": lambda self: self.expression( 1447 exp.SwapTable(this=self._match(TokenType.WITH) and self._parse_table(schema=True)) 1448 ), 1449 } 1450 1451 ALTER_ALTER_PARSERS: t.ClassVar = { 1452 "DISTKEY": lambda self: self._parse_alter_diststyle(), 1453 "DISTSTYLE": lambda self: self._parse_alter_diststyle(), 1454 "SORTKEY": lambda self: self._parse_alter_sortkey(), 1455 "COMPOUND": lambda self: self._parse_alter_sortkey(compound=True), 1456 } 1457 1458 SCHEMA_UNNAMED_CONSTRAINTS: t.ClassVar = { 1459 "CHECK", 1460 "EXCLUDE", 1461 "FOREIGN KEY", 1462 "LIKE", 1463 "PERIOD", 1464 "PRIMARY KEY", 1465 "UNIQUE", 1466 "BUCKET", 1467 "TRUNCATE", 1468 } 1469 1470 NO_PAREN_FUNCTION_PARSERS: t.ClassVar = { 1471 "ANY": lambda self: self.expression(exp.Any(this=self._parse_bitwise())), 1472 "CASE": lambda self: self._parse_case(), 1473 "CONNECT_BY_ROOT": lambda self: self.expression( 1474 exp.ConnectByRoot(this=self._parse_column()) 1475 ), 1476 "IF": lambda self: self._parse_if(), 1477 } 1478 1479 INVALID_FUNC_NAME_TOKENS: t.ClassVar = { 1480 TokenType.IDENTIFIER, 1481 TokenType.STRING, 1482 } 1483 1484 FUNCTIONS_WITH_ALIASED_ARGS: t.ClassVar = {"STRUCT"} 1485 1486 KEY_VALUE_DEFINITIONS: t.ClassVar = (exp.Alias, exp.EQ, exp.PropertyEQ, exp.Slice) 1487 1488 FUNCTION_PARSERS: t.ClassVar[dict[str, t.Callable]] = { 1489 **{ 1490 name: lambda self: self._parse_max_min_by(exp.ArgMax) for name in exp.ArgMax.sql_names() 1491 }, 1492 **{ 1493 name: lambda self: self._parse_max_min_by(exp.ArgMin) for name in exp.ArgMin.sql_names() 1494 }, 1495 "CAST": lambda self: self._parse_cast(self.STRICT_CAST), 1496 "CEIL": lambda self: self._parse_ceil_floor(exp.Ceil), 1497 "CONVERT": lambda self: self._parse_convert(self.STRICT_CAST), 1498 "CHAR": lambda self: self._parse_char(), 1499 "CHR": lambda self: self._parse_char(), 1500 "DECODE": lambda self: self._parse_decode(), 1501 "EXTRACT": lambda self: self._parse_extract(), 1502 "FLOOR": lambda self: self._parse_ceil_floor(exp.Floor), 1503 "GAP_FILL": lambda self: self._parse_gap_fill(), 1504 "INITCAP": lambda self: self._parse_initcap(), 1505 "JSON_OBJECT": lambda self: self._parse_json_object(), 1506 "JSON_OBJECTAGG": lambda self: self._parse_json_object(agg=True), 1507 "JSON_TABLE": lambda self: self._parse_json_table(), 1508 "MATCH": lambda self: self._parse_match_against(), 1509 "NORMALIZE": lambda self: self._parse_normalize(), 1510 "OPENJSON": lambda self: self._parse_open_json(), 1511 "OVERLAY": lambda self: self._parse_overlay(), 1512 "POSITION": lambda self: self._parse_position(), 1513 "SAFE_CAST": lambda self: self._parse_cast(False, safe=True), 1514 "STRING_AGG": lambda self: self._parse_string_agg(), 1515 "SUBSTRING": lambda self: self._parse_substring(), 1516 "TRIM": lambda self: self._parse_trim(), 1517 "TRY_CAST": lambda self: self._parse_cast(False, safe=True), 1518 "TRY_CONVERT": lambda self: self._parse_convert(False, safe=True), 1519 "XMLELEMENT": lambda self: self._parse_xml_element(), 1520 "XMLTABLE": lambda self: self._parse_xml_table(), 1521 } 1522 1523 QUERY_MODIFIER_PARSERS: t.ClassVar = { 1524 TokenType.MATCH_RECOGNIZE: lambda self: ("match", self._parse_match_recognize()), 1525 TokenType.PREWHERE: lambda self: ("prewhere", self._parse_prewhere()), 1526 TokenType.WHERE: lambda self: ("where", self._parse_where()), 1527 TokenType.GROUP_BY: lambda self: ("group", self._parse_group()), 1528 TokenType.HAVING: lambda self: ("having", self._parse_having()), 1529 TokenType.QUALIFY: lambda self: ("qualify", self._parse_qualify()), 1530 TokenType.WINDOW: lambda self: ("windows", self._parse_window_clause()), 1531 TokenType.ORDER_BY: lambda self: ("order", self._parse_order()), 1532 TokenType.LIMIT: lambda self: ("limit", self._parse_limit()), 1533 TokenType.FETCH: lambda self: ("limit", self._parse_limit()), 1534 TokenType.OFFSET: lambda self: ("offset", self._parse_offset()), 1535 TokenType.FOR: lambda self: ("locks", self._parse_locks()), 1536 TokenType.LOCK: lambda self: ("locks", self._parse_locks()), 1537 TokenType.TABLE_SAMPLE: lambda self: ("sample", self._parse_table_sample(as_modifier=True)), 1538 TokenType.USING: lambda self: ("sample", self._parse_table_sample(as_modifier=True)), 1539 TokenType.CLUSTER_BY: lambda self: ( 1540 "cluster", 1541 self._parse_cluster(), 1542 ), 1543 TokenType.DISTRIBUTE_BY: lambda self: ( 1544 "distribute", 1545 self._parse_sort(exp.Distribute, TokenType.DISTRIBUTE_BY), 1546 ), 1547 TokenType.SORT_BY: lambda self: ("sort", self._parse_sort(exp.Sort, TokenType.SORT_BY)), 1548 TokenType.CONNECT_BY: lambda self: ("connect", self._parse_connect(skip_start_token=True)), 1549 TokenType.START_WITH: lambda self: ("connect", self._parse_connect()), 1550 } 1551 QUERY_MODIFIER_TOKENS: t.ClassVar = set(QUERY_MODIFIER_PARSERS) 1552 1553 SET_PARSERS: t.ClassVar = { 1554 "GLOBAL": lambda self: self._parse_set_item_assignment("GLOBAL"), 1555 "LOCAL": lambda self: self._parse_set_item_assignment("LOCAL"), 1556 "SESSION": lambda self: self._parse_set_item_assignment("SESSION"), 1557 "TRANSACTION": lambda self: self._parse_set_transaction(), 1558 } 1559 1560 SHOW_PARSERS: t.ClassVar[dict[str, t.Callable]] = {} 1561 1562 TYPE_LITERAL_PARSERS: t.ClassVar = { 1563 exp.DType.JSON: lambda self, this, _: self.expression(exp.ParseJSON(this=this)), 1564 } 1565 1566 TYPE_CONVERTERS: t.ClassVar[dict[exp.DType, t.Callable[[exp.DataType], exp.DataType]]] = {} 1567 1568 DDL_SELECT_TOKENS: t.ClassVar = {TokenType.SELECT, TokenType.WITH, TokenType.L_PAREN} 1569 1570 PRE_VOLATILE_TOKENS: t.ClassVar = {TokenType.CREATE, TokenType.REPLACE, TokenType.UNIQUE} 1571 1572 TRANSACTION_KIND: t.ClassVar = {"DEFERRED", "IMMEDIATE", "EXCLUSIVE"} 1573 TRANSACTION_CHARACTERISTICS: t.ClassVar[OPTIONS_TYPE] = { 1574 "ISOLATION": ( 1575 ("LEVEL", "REPEATABLE", "READ"), 1576 ("LEVEL", "READ", "COMMITTED"), 1577 ("LEVEL", "READ", "UNCOMITTED"), 1578 ("LEVEL", "SERIALIZABLE"), 1579 ), 1580 "READ": ("WRITE", "ONLY"), 1581 } 1582 1583 CONFLICT_ACTIONS: t.ClassVar[OPTIONS_TYPE] = { 1584 **dict.fromkeys(("ABORT", "FAIL", "IGNORE", "REPLACE", "ROLLBACK", "UPDATE"), tuple()), 1585 "DO": ("NOTHING", "UPDATE"), 1586 } 1587 1588 TRIGGER_TIMING: t.ClassVar[OPTIONS_TYPE] = { 1589 "INSTEAD": (("OF",),), 1590 "BEFORE": tuple(), 1591 "AFTER": tuple(), 1592 } 1593 1594 TRIGGER_DEFERRABLE: t.ClassVar[OPTIONS_TYPE] = { 1595 "NOT": (("DEFERRABLE",),), 1596 "DEFERRABLE": tuple(), 1597 } 1598 1599 CREATE_SEQUENCE: t.ClassVar[OPTIONS_TYPE] = { 1600 "SCALE": ("EXTEND", "NOEXTEND"), 1601 "SHARD": ("EXTEND", "NOEXTEND"), 1602 "NO": ("CYCLE", "CACHE", "MAXVALUE", "MINVALUE"), 1603 **dict.fromkeys( 1604 ( 1605 "SESSION", 1606 "GLOBAL", 1607 "KEEP", 1608 "NOKEEP", 1609 "ORDER", 1610 "NOORDER", 1611 "NOCACHE", 1612 "CYCLE", 1613 "NOCYCLE", 1614 "NOMINVALUE", 1615 "NOMAXVALUE", 1616 "NOSCALE", 1617 "NOSHARD", 1618 ), 1619 tuple(), 1620 ), 1621 } 1622 1623 ISOLATED_LOADING_OPTIONS: t.ClassVar[OPTIONS_TYPE] = {"FOR": ("ALL", "INSERT", "NONE")} 1624 1625 USABLES: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys( 1626 ("ROLE", "WAREHOUSE", "DATABASE", "SCHEMA", "CATALOG"), tuple() 1627 ) 1628 1629 CAST_ACTIONS: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys(("RENAME", "ADD"), ("FIELDS",)) 1630 1631 SCHEMA_BINDING_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1632 "TYPE": ("EVOLUTION",), 1633 **dict.fromkeys(("BINDING", "COMPENSATION", "EVOLUTION"), tuple()), 1634 } 1635 1636 PROCEDURE_OPTIONS: t.ClassVar[OPTIONS_TYPE] = {} 1637 1638 EXECUTE_AS_OPTIONS: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys( 1639 ("CALLER", "SELF", "OWNER"), tuple() 1640 ) 1641 1642 KEY_CONSTRAINT_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1643 "NOT": ("ENFORCED",), 1644 "MATCH": ( 1645 "FULL", 1646 "PARTIAL", 1647 "SIMPLE", 1648 ), 1649 "INITIALLY": ("DEFERRED", "IMMEDIATE"), 1650 "USING": ( 1651 "BTREE", 1652 "HASH", 1653 ), 1654 **dict.fromkeys(("DEFERRABLE", "NORELY", "RELY"), tuple()), 1655 } 1656 1657 WINDOW_EXCLUDE_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1658 "NO": ("OTHERS",), 1659 "CURRENT": ("ROW",), 1660 **dict.fromkeys(("GROUP", "TIES"), tuple()), 1661 } 1662 1663 INSERT_ALTERNATIVES: t.ClassVar = {"ABORT", "FAIL", "IGNORE", "REPLACE", "ROLLBACK"} 1664 1665 CLONE_KEYWORDS: t.ClassVar = {"CLONE", "COPY"} 1666 HISTORICAL_DATA_PREFIX: t.ClassVar = {"AT", "BEFORE", "END"} 1667 HISTORICAL_DATA_KIND: t.ClassVar = {"OFFSET", "STATEMENT", "STREAM", "TIMESTAMP", "VERSION"} 1668 1669 OPCLASS_FOLLOW_KEYWORDS: t.ClassVar = {"ASC", "DESC", "NULLS", "WITH"} 1670 1671 OPTYPE_FOLLOW_TOKENS: t.ClassVar = {TokenType.COMMA, TokenType.R_PAREN} 1672 1673 TABLE_INDEX_HINT_TOKENS: t.ClassVar = {TokenType.FORCE, TokenType.IGNORE, TokenType.USE} 1674 1675 VIEW_ATTRIBUTES: t.ClassVar = {"ENCRYPTION", "SCHEMABINDING", "VIEW_METADATA"} 1676 1677 WINDOW_ALIAS_TOKENS: t.ClassVar = ID_VAR_TOKENS - {TokenType.RANGE, TokenType.ROWS} 1678 WINDOW_BEFORE_PAREN_TOKENS: t.ClassVar = {TokenType.OVER} 1679 WINDOW_SIDES: t.ClassVar = {"FOLLOWING", "PRECEDING"} 1680 1681 JSON_KEY_VALUE_SEPARATOR_TOKENS: t.ClassVar = {TokenType.COLON, TokenType.COMMA, TokenType.IS} 1682 1683 FETCH_TOKENS: t.ClassVar = ID_VAR_TOKENS - {TokenType.ROW, TokenType.ROWS, TokenType.PERCENT} 1684 1685 ADD_CONSTRAINT_TOKENS: t.ClassVar = { 1686 TokenType.CONSTRAINT, 1687 TokenType.FOREIGN_KEY, 1688 TokenType.INDEX, 1689 TokenType.KEY, 1690 TokenType.PRIMARY_KEY, 1691 TokenType.UNIQUE, 1692 } 1693 1694 DISTINCT_TOKENS: t.ClassVar = {TokenType.DISTINCT} 1695 1696 UNNEST_OFFSET_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - SET_OPERATIONS 1697 1698 SELECT_START_TOKENS: t.ClassVar = {TokenType.L_PAREN, TokenType.WITH, TokenType.SELECT} 1699 1700 COPY_INTO_VARLEN_OPTIONS: t.ClassVar = { 1701 "FILE_FORMAT", 1702 "COPY_OPTIONS", 1703 "FORMAT_OPTIONS", 1704 "CREDENTIAL", 1705 } 1706 1707 IS_JSON_PREDICATE_KIND: t.ClassVar = {"VALUE", "SCALAR", "ARRAY", "OBJECT"} 1708 1709 ODBC_DATETIME_LITERALS: t.ClassVar[dict[str, type[exp.Expr]]] = {} 1710 1711 ON_CONDITION_TOKENS: t.ClassVar = {"ERROR", "NULL", "TRUE", "FALSE", "EMPTY"} 1712 1713 PRIVILEGE_FOLLOW_TOKENS: t.ClassVar = {TokenType.ON, TokenType.COMMA, TokenType.L_PAREN} 1714 1715 # The style options for the DESCRIBE statement 1716 DESCRIBE_STYLES: t.ClassVar = {"ANALYZE", "EXTENDED", "FORMATTED", "HISTORY"} 1717 1718 SET_ASSIGNMENT_DELIMITERS: t.ClassVar = {"=", ":=", "TO"} 1719 1720 # The style options for the ANALYZE statement 1721 ANALYZE_STYLES: t.ClassVar = { 1722 "BUFFER_USAGE_LIMIT", 1723 "FULL", 1724 "LOCAL", 1725 "NO_WRITE_TO_BINLOG", 1726 "SAMPLE", 1727 "SKIP_LOCKED", 1728 "VERBOSE", 1729 } 1730 1731 ANALYZE_EXPRESSION_PARSERS: t.ClassVar = { 1732 "ALL": lambda self: self._parse_analyze_columns(), 1733 "COMPUTE": lambda self: self._parse_analyze_statistics(), 1734 "DELETE": lambda self: self._parse_analyze_delete(), 1735 "DROP": lambda self: self._parse_analyze_histogram(), 1736 "ESTIMATE": lambda self: self._parse_analyze_statistics(), 1737 "LIST": lambda self: self._parse_analyze_list(), 1738 "PREDICATE": lambda self: self._parse_analyze_columns(), 1739 "UPDATE": lambda self: self._parse_analyze_histogram(), 1740 "VALIDATE": lambda self: self._parse_analyze_validate(), 1741 } 1742 1743 PARTITION_KEYWORDS: t.ClassVar = {"PARTITION", "SUBPARTITION"} 1744 1745 AMBIGUOUS_ALIAS_TOKENS: t.ClassVar = (TokenType.LIMIT, TokenType.OFFSET) 1746 1747 OPERATION_MODIFIERS: t.ClassVar[set[str]] = set() 1748 1749 RECURSIVE_CTE_SEARCH_KIND: t.ClassVar = {"BREADTH", "DEPTH", "CYCLE"} 1750 1751 SECURITY_PROPERTY_KEYWORDS: t.ClassVar = {"DEFINER", "INVOKER", "NONE"} 1752 1753 MODIFIABLES: t.ClassVar = (exp.Query, exp.Table, exp.TableFromRows, exp.Values) 1754 1755 STRICT_CAST: t.ClassVar = True 1756 1757 PREFIXED_PIVOT_COLUMNS: t.ClassVar = False 1758 IDENTIFY_PIVOT_STRINGS: t.ClassVar = False 1759 # Controls when an aggregation's name is included in a pivoted column's name: 1760 # "agg_name_if_aliased" - only for aggregations that carry an explicit alias 1761 # "agg_name_if_aliased_or_multiple" - if aliased, or whenever there are multiple aggregations 1762 # "agg_name_if_multiple" - only when there are multiple aggregations (a lone agg is value-only) 1763 PIVOT_COLUMN_NAMING: t.ClassVar[str] = "agg_name_if_aliased" 1764 1765 LOG_DEFAULTS_TO_LN: t.ClassVar = False 1766 1767 # Whether the table sample clause expects CSV syntax 1768 TABLESAMPLE_CSV: t.ClassVar = False 1769 1770 # The default method used for table sampling 1771 DEFAULT_SAMPLING_METHOD: t.ClassVar[str | None] = None 1772 1773 # Whether the SET command needs a delimiter (e.g. "=") for assignments 1774 SET_REQUIRES_ASSIGNMENT_DELIMITER: t.ClassVar = True 1775 1776 # Whether the TRIM function expects the characters to trim as its first argument 1777 TRIM_PATTERN_FIRST: t.ClassVar = False 1778 1779 # Whether string aliases are supported `SELECT COUNT(*) 'count'` 1780 STRING_ALIASES: t.ClassVar = False 1781 1782 # Whether query modifiers such as LIMIT are attached to the UNION node (vs its right operand) 1783 MODIFIERS_ATTACHED_TO_SET_OP: t.ClassVar = True 1784 SET_OP_MODIFIERS: t.ClassVar = {"order", "limit", "offset"} 1785 1786 # Whether to parse IF statements that aren't followed by a left parenthesis as commands 1787 NO_PAREN_IF_COMMANDS: t.ClassVar = True 1788 1789 # Whether the -> and ->> operators expect documents of type JSON (e.g. Postgres) 1790 JSON_ARROWS_REQUIRE_JSON_TYPE: t.ClassVar = False 1791 1792 # Whether the `:` operator is used to extract a value from a VARIANT column 1793 COLON_IS_VARIANT_EXTRACT: t.ClassVar = False 1794 1795 # Whether a chain of colon extractions (x:y:z) is a single extraction with a merged 1796 # path (x:y.z, e.g. Snowflake) or each colon extracts from the previous result (e.g. Databricks) 1797 COLON_CHAIN_IS_SINGLE_EXTRACT: t.ClassVar = True 1798 1799 # Whether or not a VALUES keyword needs to be followed by '(' to form a VALUES clause. 1800 # If this is True and '(' is not found, the keyword will be treated as an identifier 1801 VALUES_FOLLOWED_BY_PAREN: t.ClassVar = True 1802 1803 # Whether implicit unnesting is supported, e.g. SELECT 1 FROM y.z AS z, z.a (Redshift) 1804 SUPPORTS_IMPLICIT_UNNEST: t.ClassVar = False 1805 1806 # Whether or not interval spans are supported, INTERVAL 1 YEAR TO MONTHS 1807 INTERVAL_SPANS: t.ClassVar = True 1808 1809 # Whether a PARTITION clause can follow a table reference 1810 SUPPORTS_PARTITION_SELECTION: t.ClassVar = False 1811 1812 # Whether the `name AS expr` schema/column constraint requires parentheses around `expr` 1813 WRAPPED_TRANSFORM_COLUMN_CONSTRAINT: t.ClassVar = True 1814 1815 # Whether the 'AS' keyword is optional in the CTE definition syntax 1816 OPTIONAL_ALIAS_TOKEN_CTE: t.ClassVar = True 1817 1818 # Whether renaming a column with an ALTER statement requires the presence of the COLUMN keyword 1819 ALTER_RENAME_REQUIRES_COLUMN: t.ClassVar = True 1820 1821 # Whether Alter statements are allowed to contain Partition specifications 1822 ALTER_TABLE_PARTITIONS: t.ClassVar = False 1823 1824 # Whether all join types have the same precedence, i.e., they "naturally" produce a left-deep tree. 1825 # In standard SQL, joins that use the JOIN keyword take higher precedence than comma-joins. That is 1826 # to say, JOIN operators happen before comma operators. This is not the case in some dialects, such 1827 # as BigQuery, where all joins have the same precedence. 1828 JOINS_HAVE_EQUAL_PRECEDENCE: t.ClassVar = False 1829 1830 # Whether TIMESTAMP <literal> can produce a zone-aware timestamp 1831 ZONE_AWARE_TIMESTAMP_CONSTRUCTOR: t.ClassVar = False 1832 1833 # Whether map literals support arbitrary expressions as keys. 1834 # When True, allows complex keys like arrays or literals: {[1, 2]: 3}, {1: 2} (e.g. DuckDB). 1835 # When False, keys are typically restricted to identifiers. 1836 MAP_KEYS_ARE_ARBITRARY_EXPRESSIONS: t.ClassVar = False 1837 1838 # Whether JSON_EXTRACT requires a JSON expression as the first argument, e.g this 1839 # is true for Snowflake but not for BigQuery which can also process strings 1840 JSON_EXTRACT_REQUIRES_JSON_EXPRESSION: t.ClassVar = False 1841 1842 # Dialects like Databricks support JOINS without join criteria 1843 # Adding an ON TRUE, makes transpilation semantically correct for other dialects 1844 ADD_JOIN_ON_TRUE: t.ClassVar = False 1845 1846 # Whether INTERVAL spans with literal format '\d+ hh:[mm:[ss[.ff]]]' 1847 # can omit the span unit `DAY TO MINUTE` or `DAY TO SECOND` 1848 SUPPORTS_OMITTED_INTERVAL_SPAN_UNIT: t.ClassVar = False 1849 1850 # Whether adjacent string literals like 'foo' 'bar' require a whitespace or comment between them 1851 # to be considered valid syntactically. Such expressions evaluate to the strings' concatenation. 1852 ADJACENT_STRINGS_CANNOT_BE_CONNECTED: t.ClassVar = False 1853 1854 SHOW_TRIE: t.ClassVar[dict] = new_trie(key.split(" ") for key in SHOW_PARSERS) 1855 SET_TRIE: t.ClassVar[dict] = new_trie(key.split(" ") for key in SET_PARSERS) 1856 1857 def __init__( 1858 self, 1859 error_level: ErrorLevel | None = None, 1860 error_message_context: int = 100, 1861 max_errors: int = 3, 1862 max_nodes: int = -1, 1863 dialect: DialectType = None, 1864 ): 1865 self.error_level: ErrorLevel = error_level or ErrorLevel.IMMEDIATE 1866 self.error_message_context: int = error_message_context 1867 self.max_errors: int = max_errors 1868 self.max_nodes: int = max_nodes 1869 self.dialect: t.Any = _resolve_dialect(dialect) 1870 self.sql: str = "" 1871 self.errors: list[ParseError] = [] 1872 self._tokens: list[Token] = [] 1873 self._tokens_size: i64 = 0 1874 self._index: i64 = 0 1875 self._curr: Token = SENTINEL_NONE 1876 self._next: Token = SENTINEL_NONE 1877 self._prev: Token = SENTINEL_NONE 1878 self._prev_comments: list[str] = [] 1879 self._pipe_cte_counter: int = 0 1880 self._chunks: list[list[Token]] = [] 1881 self._chunk_index: i64 = 0 1882 self._node_count: int = 0 1883 1884 def reset(self) -> None: 1885 self.sql = "" 1886 self.errors = [] 1887 self._tokens = [] 1888 self._tokens_size = 0 1889 self._index = 0 1890 self._curr = SENTINEL_NONE 1891 self._next = SENTINEL_NONE 1892 self._prev = SENTINEL_NONE 1893 self._prev_comments = [] 1894 self._pipe_cte_counter = 0 1895 self._chunks = [] 1896 self._chunk_index = 0 1897 self._node_count = 0 1898 1899 def _advance(self, times: i64 = 1) -> None: 1900 index = self._index + times 1901 self._index = index 1902 tokens = self._tokens 1903 size = self._tokens_size 1904 self._curr = tokens[index] if index < size else SENTINEL_NONE 1905 self._next = tokens[index + 1] if index + 1 < size else SENTINEL_NONE 1906 1907 if index > 0: 1908 prev = tokens[index - 1] 1909 self._prev = prev 1910 self._prev_comments = prev.comments 1911 else: 1912 self._prev = SENTINEL_NONE 1913 self._prev_comments = [] 1914 1915 def _advance_chunk(self) -> None: 1916 self._index = -1 1917 self._tokens = self._chunks[self._chunk_index] 1918 self._tokens_size = i64(len(self._tokens)) 1919 self._chunk_index += 1 1920 self._advance() 1921 1922 def _retreat(self, index: i64) -> None: 1923 if index != self._index: 1924 self._advance(index - self._index) 1925 1926 def _add_comments(self, expression: exp.Expr | None) -> None: 1927 if expression and self._prev_comments: 1928 expression.add_comments(self._prev_comments) 1929 self._prev_comments = [] 1930 1931 def _match( 1932 self, token_type: TokenType, advance: bool = True, expression: exp.Expr | None = None 1933 ) -> bool: 1934 if self._curr.token_type == token_type: 1935 if advance: 1936 self._advance() 1937 self._add_comments(expression) 1938 return True 1939 return False 1940 1941 def _match_set(self, types: t.Collection[TokenType], advance: bool = True) -> bool: 1942 if self._curr.token_type in types: 1943 if advance: 1944 self._advance() 1945 return True 1946 return False 1947 1948 def _match_pair( 1949 self, token_type_a: TokenType, token_type_b: TokenType, advance: bool = True 1950 ) -> bool: 1951 if self._curr.token_type == token_type_a and self._next.token_type == token_type_b: 1952 if advance: 1953 self._advance(2) 1954 return True 1955 return False 1956 1957 def _match_texts(self, texts: t.Collection[str], advance: bool = True) -> bool: 1958 if self._curr.token_type != TokenType.STRING and self._curr.text.upper() in texts: 1959 if advance: 1960 self._advance() 1961 return True 1962 return False 1963 1964 def _match_text_seq(self, *texts: str, advance: bool = True) -> bool: 1965 index = self._index 1966 string_type = TokenType.STRING 1967 for text in texts: 1968 if self._curr.token_type != string_type and self._curr.text.upper() == text: 1969 self._advance() 1970 else: 1971 self._retreat(index) 1972 return False 1973 1974 if not advance: 1975 self._retreat(index) 1976 1977 return True 1978 1979 def _is_connected(self) -> bool: 1980 prev = self._prev 1981 curr = self._curr 1982 return bool(prev and curr and prev.end + 1 == curr.start) 1983 1984 def _find_sql(self, start: Token, end: Token) -> str: 1985 return self.sql[start.start : end.end + 1] 1986 1987 def raise_error(self, message: str, token: Token = SENTINEL_NONE) -> None: 1988 token = token or self._curr or self._prev or Token.string("") 1989 formatted_sql, start_context, highlight, end_context = highlight_sql( 1990 sql=self.sql, 1991 positions=[(token.start, token.end)], 1992 context_length=self.error_message_context, 1993 ) 1994 formatted_message = f"{message}. Line {token.line}, Col: {token.col}.\n {formatted_sql}" 1995 1996 error = ParseError.new( 1997 formatted_message, 1998 description=message, 1999 line=token.line, 2000 col=token.col, 2001 start_context=start_context, 2002 highlight=highlight, 2003 end_context=end_context, 2004 ) 2005 2006 if self.error_level == ErrorLevel.IMMEDIATE: 2007 raise error 2008 2009 self.errors.append(error) 2010 2011 def validate_expression(self, expression: E, args: list | None = None) -> E: 2012 if self.max_nodes > -1: 2013 self._node_count += 1 2014 if self._node_count > self.max_nodes: 2015 self.raise_error(f"Maximum number of AST nodes ({self.max_nodes}) exceeded") 2016 if self.error_level != ErrorLevel.IGNORE: 2017 for error_message in expression.error_messages(args): 2018 self.raise_error(error_message) 2019 return expression 2020 2021 def _try_parse(self, parse_method: t.Callable[[], T], retreat: bool = False) -> T | None: 2022 index = self._index 2023 error_level = self.error_level 2024 this: T | None = None 2025 2026 self.error_level = ErrorLevel.IMMEDIATE 2027 try: 2028 this = parse_method() 2029 except ParseError: 2030 this = None 2031 finally: 2032 if not this or retreat: 2033 self._retreat(index) 2034 self.error_level = error_level 2035 2036 return this 2037 2038 def parse(self, raw_tokens: list[Token], sql: str) -> list[exp.Expr | None]: 2039 """ 2040 Parses a list of tokens and returns a list of syntax trees, one tree 2041 per parsed SQL statement. 2042 2043 Args: 2044 raw_tokens: The list of tokens. 2045 sql: The original SQL string. 2046 2047 Returns: 2048 The list of the produced syntax trees. 2049 """ 2050 return self._parse( 2051 parse_method=self.__class__._parse_statement, raw_tokens=raw_tokens, sql=sql 2052 ) 2053 2054 def parse_into( 2055 self, 2056 expression_types: exp.IntoType, 2057 raw_tokens: list[Token], 2058 sql: str | None = None, 2059 ) -> list[exp.Expr | None]: 2060 """ 2061 Parses a list of tokens into a given Expr type. If a collection of Expr 2062 types is given instead, this method will try to parse the token list into each one 2063 of them, stopping at the first for which the parsing succeeds. 2064 2065 Args: 2066 expression_types: The expression type(s) to try and parse the token list into. 2067 raw_tokens: The list of tokens. 2068 sql: The original SQL string, used to produce helpful debug messages. 2069 2070 Returns: 2071 The target Expr. 2072 """ 2073 errors = [] 2074 for expression_type in ensure_list(expression_types): 2075 parser = self.EXPRESSION_PARSERS.get(t.cast(type[exp.Expr], expression_type)) 2076 if not parser: 2077 raise TypeError(f"No parser registered for {expression_type}") 2078 2079 try: 2080 return self._parse(parser, raw_tokens, sql) 2081 except ParseError as e: 2082 e.errors[0]["into_expression"] = expression_type 2083 errors.append(e) 2084 2085 raise ParseError( 2086 f"Failed to parse '{sql or raw_tokens}' into {expression_types}", 2087 errors=merge_errors(errors), 2088 ) from errors[-1] 2089 2090 def check_errors(self) -> None: 2091 """Logs or raises any found errors, depending on the chosen error level setting.""" 2092 if self.error_level == ErrorLevel.WARN: 2093 for error in self.errors: 2094 logger.error(str(error)) 2095 elif self.error_level == ErrorLevel.RAISE and self.errors: 2096 raise ParseError( 2097 concat_messages(self.errors, self.max_errors), 2098 errors=merge_errors(self.errors), 2099 ) 2100 2101 def expression( 2102 self, 2103 instance: E, 2104 token: Token | None = None, 2105 comments: list[str] | None = None, 2106 ) -> E: 2107 if token: 2108 instance.update_positions(token) 2109 instance.add_comments(comments) if comments else self._add_comments(instance) 2110 if not instance.is_primitive: 2111 instance = self.validate_expression(instance) 2112 return instance 2113 2114 def _parse_batch_statements( 2115 self, 2116 parse_method: t.Callable[[Parser], exp.Expr | None], 2117 sep_first_statement: bool = True, 2118 ) -> list[exp.Expr | None]: 2119 expressions = [] 2120 2121 # Chunkification binds if/while statements with the first statement of the body 2122 if sep_first_statement: 2123 self._match(TokenType.BEGIN) 2124 expressions.append(parse_method(self)) 2125 2126 chunks_length = len(self._chunks) 2127 while self._chunk_index < chunks_length: 2128 self._advance_chunk() 2129 2130 if self._match(TokenType.ELSE, advance=False): 2131 return expressions 2132 2133 if expressions and not self._next and self._match(TokenType.END): 2134 expressions.append(exp.EndStatement()) 2135 continue 2136 2137 expressions.append(parse_method(self)) 2138 2139 if self._index < self._tokens_size: 2140 self.raise_error("Invalid expression / Unexpected token") 2141 2142 self.check_errors() 2143 2144 return expressions 2145 2146 def _parse( 2147 self, 2148 parse_method: t.Callable[[Parser], exp.Expr | None], 2149 raw_tokens: list[Token], 2150 sql: str | None = None, 2151 ) -> list[exp.Expr | None]: 2152 self.reset() 2153 self.sql = sql or "" 2154 2155 total = len(raw_tokens) 2156 chunks: list[list[Token]] = [[]] 2157 2158 for i, token in enumerate(raw_tokens): 2159 if token.token_type == TokenType.SEMICOLON: 2160 if token.comments: 2161 chunks.append([token]) 2162 2163 if i < total - 1: 2164 chunks.append([]) 2165 else: 2166 chunks[-1].append(token) 2167 2168 self._chunks = chunks 2169 2170 return self._parse_batch_statements(parse_method=parse_method, sep_first_statement=False) 2171 2172 def _warn_unsupported(self) -> None: 2173 if self._tokens_size <= 1: 2174 return 2175 2176 # We use _find_sql because self.sql may comprise multiple chunks, and we're only 2177 # interested in emitting a warning for the one being currently processed. 2178 sql = self._find_sql(self._tokens[0], self._tokens[-1])[: self.error_message_context] 2179 2180 logger.warning( 2181 f"'{sql}' contains unsupported syntax. Falling back to parsing as a 'Command'." 2182 ) 2183 2184 def _parse_command(self) -> exp.Command: 2185 self._warn_unsupported() 2186 comments = self._prev_comments 2187 return self.expression( 2188 exp.Command(this=self._prev.text.upper(), expression=self._parse_string()), 2189 comments=comments, 2190 ) 2191 2192 def _parse_comment(self, allow_exists: bool = True) -> exp.Expr: 2193 start = self._prev 2194 exists = self._parse_exists() if allow_exists else None 2195 2196 self._match(TokenType.ON) 2197 2198 materialized = self._match_text_seq("MATERIALIZED") 2199 kind = self._match_set(self.CREATABLES) and self._prev 2200 if not kind: 2201 return self._parse_as_command(start) 2202 2203 if kind.token_type in (TokenType.FUNCTION, TokenType.PROCEDURE): 2204 this = self._parse_user_defined_function(kind=kind.token_type) 2205 elif kind.token_type == TokenType.TABLE: 2206 this = self._parse_table(alias_tokens=self.COMMENT_TABLE_ALIAS_TOKENS) 2207 elif kind.token_type == TokenType.COLUMN: 2208 this = self._parse_column() 2209 else: 2210 this = self._parse_table_parts(schema=True) 2211 2212 self._match(TokenType.IS) 2213 2214 return self.expression( 2215 exp.Comment( 2216 this=this, 2217 kind=kind.text, 2218 expression=self._parse_string(), 2219 exists=exists, 2220 materialized=materialized, 2221 ) 2222 ) 2223 2224 def _parse_to_table( 2225 self, 2226 ) -> exp.ToTableProperty: 2227 table = self._parse_table_parts(schema=True) 2228 return self.expression(exp.ToTableProperty(this=table)) 2229 2230 # https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#mergetree-table-ttl 2231 def _parse_ttl(self) -> exp.Expr: 2232 def _parse_ttl_action() -> exp.Expr | None: 2233 this = self._parse_bitwise() 2234 2235 if self._match_text_seq("DELETE"): 2236 return self.expression(exp.MergeTreeTTLAction(this=this, delete=True)) 2237 if self._match_text_seq("RECOMPRESS"): 2238 return self.expression( 2239 exp.MergeTreeTTLAction(this=this, recompress=self._parse_bitwise()) 2240 ) 2241 if self._match_text_seq("TO", "DISK"): 2242 return self.expression( 2243 exp.MergeTreeTTLAction(this=this, to_disk=self._parse_string()) 2244 ) 2245 if self._match_text_seq("TO", "VOLUME"): 2246 return self.expression( 2247 exp.MergeTreeTTLAction(this=this, to_volume=self._parse_string()) 2248 ) 2249 2250 return this 2251 2252 expressions = self._parse_csv(_parse_ttl_action) 2253 where = self._parse_where() 2254 group = self._parse_group() 2255 2256 aggregates = None 2257 if group and self._match(TokenType.SET): 2258 aggregates = self._parse_csv(self._parse_set_item) 2259 2260 return self.expression( 2261 exp.MergeTreeTTL( 2262 expressions=expressions, where=where, group=group, aggregates=aggregates 2263 ) 2264 ) 2265 2266 def _parse_condition(self) -> exp.Expr | None: 2267 return self._parse_wrapped(parse_method=self._parse_expression, optional=True) 2268 2269 def _parse_block(self) -> exp.Block: 2270 return self.expression( 2271 exp.Block( 2272 expressions=self._parse_batch_statements( 2273 parse_method=lambda self: self._parse_statement() 2274 ) 2275 ) 2276 ) 2277 2278 def _parse_whileblock(self) -> exp.WhileBlock: 2279 return self.expression( 2280 exp.WhileBlock(this=self._parse_condition(), body=self._parse_block()) 2281 ) 2282 2283 def _parse_statement(self) -> exp.Expr | None: 2284 if not self._curr: 2285 return None 2286 2287 if self._match_set(self.STATEMENT_PARSERS): 2288 comments = self._prev_comments 2289 stmt = self.STATEMENT_PARSERS[self._prev.token_type](self) 2290 stmt.add_comments(comments, prepend=True) 2291 return stmt 2292 2293 if self._match_set(self.dialect.tokenizer_class.COMMANDS): 2294 return self._parse_command() 2295 2296 if self._match_text_seq("WHILE"): 2297 return self._parse_whileblock() 2298 2299 expression = self._parse_expression() 2300 expression = self._parse_set_operations(expression) if expression else self._parse_select() 2301 2302 if isinstance(expression, exp.Subquery) and self._match(TokenType.PIPE_GT, advance=False): 2303 expression = self._parse_pipe_syntax_query(expression) 2304 2305 return self._parse_query_modifiers(expression) 2306 2307 def _parse_drop(self, exists: bool = False) -> exp.Drop | exp.Command: 2308 start = self._prev 2309 temporary = self._match(TokenType.TEMPORARY) 2310 materialized = self._match_text_seq("MATERIALIZED") 2311 iceberg = self._match_text_seq("ICEBERG") 2312 2313 kind = self._match_set(self.CREATABLES) and self._prev.text.upper() 2314 if not kind or (iceberg and kind and kind != "TABLE"): 2315 return self._parse_as_command(start) 2316 2317 concurrently = self._match_text_seq("CONCURRENTLY") 2318 if_exists = exists or self._parse_exists() 2319 2320 if kind == "COLUMN": 2321 this = self._parse_column() 2322 else: 2323 this = self._parse_table_parts(schema=True, is_db_reference=kind == "SCHEMA") 2324 2325 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 2326 2327 if self._match(TokenType.L_PAREN, advance=False): 2328 expressions = self._parse_wrapped_csv(self._parse_types) 2329 else: 2330 expressions = None 2331 2332 cascade_or_restrict = self._match_texts(("CASCADE", "RESTRICT")) and self._prev.text.upper() 2333 2334 return self.expression( 2335 exp.Drop( 2336 exists=if_exists, 2337 this=this, 2338 expressions=expressions, 2339 kind=self.dialect.CREATABLE_KIND_MAPPING.get(kind) or kind, 2340 temporary=temporary, 2341 materialized=materialized, 2342 cascade=cascade_or_restrict == "CASCADE", 2343 restrict=cascade_or_restrict == "RESTRICT", 2344 constraints=self._match_text_seq("CONSTRAINTS"), 2345 purge=self._match_text_seq("PURGE"), 2346 cluster=cluster, 2347 concurrently=concurrently, 2348 sync=self._match_text_seq("SYNC"), 2349 iceberg=iceberg, 2350 ) 2351 ) 2352 2353 def _parse_exists(self, not_: bool = False) -> bool | None: 2354 return ( 2355 self._match_text_seq("IF") 2356 and (not not_ or self._match(TokenType.NOT)) 2357 and self._match(TokenType.EXISTS) 2358 ) 2359 2360 def _parse_create(self) -> exp.Create | exp.Command: 2361 # Note: this can't be None because we've matched a statement parser 2362 start = self._prev 2363 2364 replace = ( 2365 start.token_type == TokenType.REPLACE 2366 or self._match_pair(TokenType.OR, TokenType.REPLACE) 2367 or self._match_pair(TokenType.OR, TokenType.ALTER) 2368 ) 2369 refresh = self._match_pair(TokenType.OR, TokenType.REFRESH) 2370 2371 unique = self._match(TokenType.UNIQUE) 2372 2373 if self._match_text_seq("CLUSTERED", "COLUMNSTORE"): 2374 clustered = True 2375 elif self._match_text_seq("NONCLUSTERED", "COLUMNSTORE") or self._match_text_seq( 2376 "COLUMNSTORE" 2377 ): 2378 clustered = False 2379 else: 2380 clustered = None 2381 2382 if self._match_pair(TokenType.TABLE, TokenType.FUNCTION, advance=False): 2383 self._advance() 2384 2385 properties = None 2386 create_token = self._match_set(self.CREATABLES) and self._prev 2387 2388 if not create_token: 2389 # exp.Properties.Location.POST_CREATE 2390 properties = self._parse_properties() 2391 create_token = self._match_set(self.CREATABLES) and self._prev 2392 2393 if not properties or not create_token: 2394 return self._parse_as_command(start) 2395 2396 create_token_type = t.cast(Token, create_token).token_type 2397 2398 concurrently = self._match_text_seq("CONCURRENTLY") 2399 exists = self._parse_exists(not_=True) 2400 this = None 2401 expression: exp.Expr | None = None 2402 indexes = None 2403 no_schema_binding = None 2404 begin = None 2405 clone = None 2406 2407 def extend_props(temp_props: exp.Properties | None) -> None: 2408 nonlocal properties 2409 if properties and temp_props: 2410 properties.expressions.extend(temp_props.expressions) 2411 elif temp_props: 2412 properties = temp_props 2413 2414 if create_token_type in (TokenType.FUNCTION, TokenType.PROCEDURE): 2415 this = self._parse_user_defined_function(kind=create_token_type) 2416 2417 # exp.Properties.Location.POST_SCHEMA ("schema" here is the UDF's type signature) 2418 extend_props(self._parse_properties()) 2419 2420 expression = self._parse_heredoc() if self._match(TokenType.ALIAS) else None 2421 2422 if ( 2423 not expression 2424 and create_token_type == TokenType.FUNCTION 2425 and isinstance(this, exp.UserDefinedFunction) 2426 and this.args.get("wrapped") 2427 ): 2428 pre_table_index = self._index 2429 is_table = self._match(TokenType.TABLE) 2430 2431 expression = self._parse_expression() 2432 overload_mode = bool( 2433 expression 2434 and self._curr.token_type == TokenType.COMMA 2435 and self._next.token_type == TokenType.L_PAREN 2436 ) 2437 if not overload_mode: 2438 self._retreat(pre_table_index) 2439 is_table = False 2440 expression = None 2441 else: 2442 is_table = False 2443 overload_mode = False 2444 2445 extend_props(self._parse_function_properties()) 2446 2447 if not expression: 2448 if self._match(TokenType.COMMAND): 2449 expression = self._parse_as_command(self._prev) 2450 else: 2451 begin = self._match(TokenType.BEGIN) 2452 return_ = self._match_text_seq("RETURN") 2453 2454 if self._match(TokenType.STRING, advance=False): 2455 # Takes care of BigQuery's JavaScript UDF definitions that end in an OPTIONS property 2456 # # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_function_statement 2457 expression = self._parse_string() 2458 extend_props(self._parse_properties()) 2459 else: 2460 expression = ( 2461 self._parse_user_defined_function_expression() 2462 if create_token_type == TokenType.FUNCTION 2463 else self._parse_block() 2464 ) 2465 2466 if return_: 2467 expression = self.expression(exp.Return(this=expression)) 2468 2469 if overload_mode and expression: 2470 expression = self._parse_macro_overloads( 2471 t.cast(exp.UserDefinedFunction, this), expression, is_table 2472 ) 2473 elif create_token_type == TokenType.INDEX: 2474 # Postgres allows anonymous indexes, eg. CREATE INDEX IF NOT EXISTS ON t(c) 2475 if not self._match(TokenType.ON): 2476 index = self._parse_id_var() 2477 anonymous = False 2478 else: 2479 index = None 2480 anonymous = True 2481 2482 this = self._parse_index(index=index, anonymous=anonymous) 2483 elif ( 2484 create_token_type == TokenType.CONSTRAINT and self._match(TokenType.TRIGGER) 2485 ) or create_token_type == TokenType.TRIGGER: 2486 if is_constraint := (create_token_type == TokenType.CONSTRAINT): 2487 create_token = self._prev 2488 2489 trigger_name = self._parse_id_var() 2490 if not trigger_name: 2491 return self._parse_as_command(start) 2492 2493 timing_var = self._parse_var_from_options(self.TRIGGER_TIMING, raise_unmatched=False) 2494 timing = timing_var.this if timing_var else None 2495 if not timing: 2496 return self._parse_as_command(start) 2497 2498 events = self._parse_trigger_events() 2499 if not self._match(TokenType.ON): 2500 self.raise_error("Expected ON in trigger definition") 2501 2502 table = self._parse_table_parts() 2503 referenced_table = self._parse_table_parts() if self._match(TokenType.FROM) else None 2504 deferrable, initially = self._parse_trigger_deferrable() 2505 referencing = self._parse_trigger_referencing() 2506 for_each = self._parse_trigger_for_each() 2507 when = self._match_text_seq("WHEN") and self._parse_wrapped( 2508 self._parse_disjunction, optional=True 2509 ) 2510 execute = self._parse_trigger_execute() 2511 2512 if execute is None: 2513 return self._parse_as_command(start) 2514 2515 trigger_props = self.expression( 2516 exp.TriggerProperties( 2517 table=table, 2518 timing=timing, 2519 events=events, 2520 execute=execute, 2521 constraint=is_constraint, 2522 referenced_table=referenced_table, 2523 deferrable=deferrable, 2524 initially=initially, 2525 referencing=referencing, 2526 for_each=for_each, 2527 when=when, 2528 ) 2529 ) 2530 2531 this = trigger_name 2532 extend_props(exp.Properties(expressions=[trigger_props] if trigger_props else [])) 2533 elif create_token_type == TokenType.TYPE: 2534 this = self._parse_table_parts(schema=True) 2535 if not this or not self._match(TokenType.ALIAS): 2536 return self._parse_as_command(start) 2537 2538 if self._match(TokenType.ENUM): 2539 expression = exp.DataType( 2540 this=exp.DType.ENUM, 2541 expressions=self._parse_wrapped_csv(self._parse_string), 2542 ) 2543 elif self._match(TokenType.L_PAREN, advance=False): 2544 expression = self._parse_schema() 2545 else: 2546 return self._parse_as_command(start) 2547 elif create_token_type in self.DB_CREATABLES: 2548 table_parts = self._parse_table_parts( 2549 schema=True, is_db_reference=create_token_type == TokenType.SCHEMA 2550 ) 2551 2552 # exp.Properties.Location.POST_NAME 2553 self._match(TokenType.COMMA) 2554 extend_props(self._parse_properties(before=True)) 2555 2556 this = self._parse_schema(this=table_parts) 2557 2558 # exp.Properties.Location.POST_SCHEMA and POST_WITH 2559 extend_props(self._parse_properties()) 2560 2561 has_alias = self._match(TokenType.ALIAS) 2562 if not self._match_set(self.DDL_SELECT_TOKENS, advance=False): 2563 # exp.Properties.Location.POST_ALIAS 2564 extend_props(self._parse_properties()) 2565 2566 if create_token_type == TokenType.SEQUENCE: 2567 expression = self._parse_types() 2568 props = self._parse_properties() 2569 if props: 2570 sequence_props = exp.SequenceProperties() 2571 options = [] 2572 for prop in props: 2573 if isinstance(prop, exp.SequenceProperties): 2574 for arg, value in prop.args.items(): 2575 if arg == "options": 2576 options.extend(value) 2577 else: 2578 sequence_props.set(arg, value) 2579 prop.pop() 2580 2581 if options: 2582 sequence_props.set("options", options) 2583 2584 props.append("expressions", sequence_props) 2585 extend_props(props) 2586 else: 2587 expression = self._parse_ddl_select() 2588 2589 # Some dialects also support using a table as an alias instead of a SELECT. 2590 # Here we fallback to this as an alternative. 2591 if not expression and has_alias: 2592 expression = self._try_parse(self._parse_table_parts) 2593 2594 if create_token_type == TokenType.TABLE: 2595 # exp.Properties.Location.POST_EXPRESSION 2596 extend_props(self._parse_properties()) 2597 2598 indexes = [] 2599 while True: 2600 index = self._parse_index() 2601 2602 # exp.Properties.Location.POST_INDEX 2603 extend_props(self._parse_properties()) 2604 if not index: 2605 break 2606 else: 2607 self._match(TokenType.COMMA) 2608 indexes.append(index) 2609 elif create_token_type == TokenType.VIEW: 2610 if self._match_text_seq("WITH", "NO", "SCHEMA", "BINDING"): 2611 no_schema_binding = True 2612 elif create_token_type in (TokenType.SINK, TokenType.SOURCE): 2613 extend_props(self._parse_properties()) 2614 2615 shallow = self._match_text_seq("SHALLOW") 2616 2617 if self._match_texts(self.CLONE_KEYWORDS): 2618 copy = self._prev.text.lower() == "copy" 2619 clone = self.expression( 2620 exp.Clone(this=self._parse_table(schema=True), shallow=shallow, copy=copy) 2621 ) 2622 2623 if self._curr and not self._match_set((TokenType.R_PAREN, TokenType.COMMA), advance=False): 2624 return self._parse_as_command(start) 2625 2626 create_kind_text = create_token.text.upper() 2627 return self.expression( 2628 exp.Create( 2629 this=this, 2630 kind=self.dialect.CREATABLE_KIND_MAPPING.get(create_kind_text) or create_kind_text, 2631 replace=replace, 2632 refresh=refresh, 2633 unique=unique, 2634 expression=expression, 2635 exists=exists, 2636 properties=properties, 2637 indexes=indexes, 2638 no_schema_binding=no_schema_binding, 2639 begin=begin, 2640 clone=clone, 2641 concurrently=concurrently, 2642 clustered=clustered, 2643 ) 2644 ) 2645 2646 def _parse_sequence_properties(self) -> exp.SequenceProperties | None: 2647 seq = exp.SequenceProperties() 2648 2649 options = [] 2650 index = self._index 2651 2652 while self._curr: 2653 self._match(TokenType.COMMA) 2654 if self._match_text_seq("INCREMENT"): 2655 self._match_text_seq("BY") 2656 self._match_text_seq("=") 2657 seq.set("increment", self._parse_term()) 2658 elif self._match_text_seq("MINVALUE"): 2659 seq.set("minvalue", self._parse_term()) 2660 elif self._match_text_seq("MAXVALUE"): 2661 seq.set("maxvalue", self._parse_term()) 2662 elif self._match(TokenType.START_WITH) or self._match_text_seq("START"): 2663 self._match_text_seq("=") 2664 seq.set("start", self._parse_term()) 2665 elif self._match_text_seq("CACHE"): 2666 # T-SQL allows empty CACHE which is initialized dynamically 2667 seq.set("cache", self._parse_number() or True) 2668 elif self._match_text_seq("OWNED", "BY"): 2669 # "OWNED BY NONE" is the default 2670 seq.set("owned", None if self._match_text_seq("NONE") else self._parse_column()) 2671 else: 2672 opt = self._parse_var_from_options(self.CREATE_SEQUENCE, raise_unmatched=False) 2673 if opt: 2674 options.append(opt) 2675 else: 2676 break 2677 2678 seq.set("options", options if options else None) 2679 return None if self._index == index else seq 2680 2681 def _parse_trigger_events(self) -> list[exp.TriggerEvent]: 2682 events = [] 2683 2684 while True: 2685 event_type = self._match_set(self.TRIGGER_EVENTS) and self._prev.text.upper() 2686 2687 if not event_type: 2688 self.raise_error("Expected trigger event (INSERT, UPDATE, DELETE, TRUNCATE)") 2689 2690 columns = ( 2691 self._parse_csv(self._parse_column) 2692 if event_type == "UPDATE" and self._match_text_seq("OF") 2693 else None 2694 ) 2695 2696 events.append(self.expression(exp.TriggerEvent(this=event_type, columns=columns))) 2697 2698 if not self._match(TokenType.OR): 2699 break 2700 2701 return events 2702 2703 def _parse_trigger_deferrable( 2704 self, 2705 ) -> tuple[str | None, str | None]: 2706 deferrable_var = self._parse_var_from_options( 2707 self.TRIGGER_DEFERRABLE, raise_unmatched=False 2708 ) 2709 deferrable = deferrable_var.this if deferrable_var else None 2710 2711 initially = None 2712 if deferrable and self._match_text_seq("INITIALLY"): 2713 initially = ( 2714 self._prev.text.upper() if self._match_texts(("IMMEDIATE", "DEFERRED")) else None 2715 ) 2716 2717 return deferrable, initially 2718 2719 def _parse_trigger_referencing_clause(self, keyword: str) -> exp.Expr | None: 2720 if not self._match_text_seq(keyword): 2721 return None 2722 if not self._match_text_seq("TABLE"): 2723 self.raise_error(f"Expected TABLE after {keyword} in REFERENCING clause") 2724 self._match_text_seq("AS") 2725 return self._parse_id_var() 2726 2727 def _parse_trigger_referencing(self) -> exp.TriggerReferencing | None: 2728 if not self._match_text_seq("REFERENCING"): 2729 return None 2730 2731 old_alias = None 2732 new_alias = None 2733 2734 while True: 2735 if alias := self._parse_trigger_referencing_clause("OLD"): 2736 if old_alias is not None: 2737 self.raise_error("Duplicate OLD clause in REFERENCING") 2738 old_alias = alias 2739 elif alias := self._parse_trigger_referencing_clause("NEW"): 2740 if new_alias is not None: 2741 self.raise_error("Duplicate NEW clause in REFERENCING") 2742 new_alias = alias 2743 else: 2744 break 2745 2746 if old_alias is None and new_alias is None: 2747 self.raise_error("REFERENCING clause requires at least OLD TABLE or NEW TABLE") 2748 2749 return self.expression(exp.TriggerReferencing(old=old_alias, new=new_alias)) 2750 2751 def _parse_trigger_for_each(self) -> str | None: 2752 if not self._match_text_seq("FOR", "EACH"): 2753 return None 2754 2755 return self._prev.text.upper() if self._match_texts(("ROW", "STATEMENT")) else None 2756 2757 def _parse_trigger_execute(self) -> exp.TriggerExecute | None: 2758 if not self._match(TokenType.EXECUTE): 2759 return None 2760 2761 if not self._match_set((TokenType.FUNCTION, TokenType.PROCEDURE)): 2762 self.raise_error("Expected FUNCTION or PROCEDURE after EXECUTE") 2763 2764 func_call = self._parse_column() 2765 return self.expression(exp.TriggerExecute(this=func_call)) 2766 2767 def _parse_property_before(self) -> exp.Expr | list[exp.Expr] | None: 2768 # only used for teradata currently 2769 self._match(TokenType.COMMA) 2770 2771 kwargs = { 2772 "no": self._match_text_seq("NO"), 2773 "dual": self._match_text_seq("DUAL"), 2774 "before": self._match_text_seq("BEFORE"), 2775 "default": self._match_text_seq("DEFAULT"), 2776 "local": (self._match_text_seq("LOCAL") and "LOCAL") 2777 or (self._match_text_seq("NOT", "LOCAL") and "NOT LOCAL"), 2778 "after": self._match_text_seq("AFTER"), 2779 "minimum": self._match_texts(("MIN", "MINIMUM")), 2780 "maximum": self._match_texts(("MAX", "MAXIMUM")), 2781 } 2782 2783 if self._match_texts(self.PROPERTY_PARSERS): 2784 parser = self.PROPERTY_PARSERS[self._prev.text.upper()] 2785 try: 2786 return parser(self, **{k: v for k, v in kwargs.items() if v}) 2787 except TypeError: 2788 self.raise_error(f"Cannot parse property '{self._prev.text}'") 2789 2790 return None 2791 2792 def _parse_wrapped_properties(self) -> list[exp.Expr | list[exp.Expr]]: 2793 return self._parse_wrapped_csv(self._parse_property) 2794 2795 def _parse_property(self) -> exp.Expr | list[exp.Expr] | None: 2796 if self._match_texts(self.PROPERTY_PARSERS): 2797 return self.PROPERTY_PARSERS[self._prev.text.upper()](self) 2798 2799 if self._match(TokenType.DEFAULT) and self._match_texts(self.PROPERTY_PARSERS): 2800 return self.PROPERTY_PARSERS[self._prev.text.upper()](self, default=True) 2801 2802 if self._match_text_seq("COMPOUND", "SORTKEY"): 2803 return self._parse_sortkey(compound=True) 2804 2805 if self._match_text_seq("PARAMETER", "STYLE", "PANDAS"): 2806 return self.expression(exp.ParameterStyleProperty(this="PANDAS")) 2807 2808 index = self._index 2809 2810 seq_props = self._parse_sequence_properties() 2811 if seq_props: 2812 return seq_props 2813 2814 self._retreat(index) 2815 return self._parse_key_value_property() 2816 2817 def _parse_key_value_property( 2818 self, parse_value: t.Callable[[], exp.Expr | None] | None = None 2819 ) -> exp.Property | None: 2820 index = self._index 2821 key = self._parse_column() 2822 2823 if not self._match(TokenType.EQ): 2824 self._retreat(index) 2825 return None 2826 2827 # Transform the key to exp.Dot if it's dotted identifiers wrapped in exp.Column or to exp.Var otherwise 2828 if isinstance(key, exp.Column): 2829 key = key.to_dot() if len(key.parts) > 1 else exp.var(key.name) 2830 2831 value = ( 2832 parse_value() 2833 if parse_value 2834 else self._parse_bitwise() or self._parse_var(any_token=True) 2835 ) 2836 2837 # Transform the value to exp.Var if it was parsed as exp.Column(exp.Identifier()) 2838 if isinstance(value, exp.Column): 2839 value = exp.var(value.name) 2840 2841 return self.expression(exp.Property(this=key, value=value)) 2842 2843 def _parse_stored(self) -> exp.FileFormatProperty | exp.StorageHandlerProperty: 2844 if self._match_text_seq("BY"): 2845 return self.expression(exp.StorageHandlerProperty(this=self._parse_var_or_string())) 2846 2847 self._match(TokenType.ALIAS) 2848 input_format = self._parse_string() if self._match_text_seq("INPUTFORMAT") else None 2849 output_format = self._parse_string() if self._match_text_seq("OUTPUTFORMAT") else None 2850 2851 return self.expression( 2852 exp.FileFormatProperty( 2853 this=( 2854 self.expression( 2855 exp.InputOutputFormat( 2856 input_format=input_format, output_format=output_format 2857 ) 2858 ) 2859 if input_format or output_format 2860 else self._parse_var_or_string() or self._parse_number() or self._parse_id_var() 2861 ), 2862 hive_format=True, 2863 ) 2864 ) 2865 2866 def _parse_unquoted_field(self) -> exp.Expr | None: 2867 field = self._parse_field() 2868 if isinstance(field, exp.Identifier) and not field.quoted: 2869 field = exp.var(field) 2870 2871 return field 2872 2873 def _parse_property_assignment(self, exp_class: type[E], **kwargs: t.Any) -> E: 2874 self._match(TokenType.EQ) 2875 self._match(TokenType.ALIAS) 2876 2877 return self.expression(exp_class(this=self._parse_unquoted_field(), **kwargs)) 2878 2879 def _parse_properties(self, before: bool | None = None) -> exp.Properties | None: 2880 properties = [] 2881 while True: 2882 if before: 2883 prop = self._parse_property_before() 2884 else: 2885 prop = self._parse_property() 2886 if not prop: 2887 break 2888 for p in ensure_list(prop): 2889 properties.append(p) 2890 2891 if properties: 2892 return self.expression(exp.Properties(expressions=properties)) 2893 2894 return None 2895 2896 def _parse_fallback(self, no: bool = False) -> exp.FallbackProperty: 2897 return self.expression( 2898 exp.FallbackProperty(no=no, protection=self._match_text_seq("PROTECTION")) 2899 ) 2900 2901 def _parse_sql_security(self) -> exp.SqlSecurityProperty: 2902 return self.expression( 2903 exp.SqlSecurityProperty( 2904 this=self._match_texts(self.SECURITY_PROPERTY_KEYWORDS) and self._prev.text.upper() 2905 ) 2906 ) 2907 2908 def _parse_settings_property(self) -> exp.SettingsProperty: 2909 return self.expression( 2910 exp.SettingsProperty(expressions=self._parse_csv(self._parse_assignment)) 2911 ) 2912 2913 def _parse_called_on_null_input_property(self) -> exp.CalledOnNullInputProperty | None: 2914 if not self._match_text_seq("ON", "NULL", "INPUT"): 2915 self._retreat(self._index - 1) 2916 return None 2917 2918 return self.expression(exp.CalledOnNullInputProperty()) 2919 2920 def _parse_volatile_property(self) -> exp.VolatileProperty | exp.StabilityProperty: 2921 if self._index >= 2: 2922 pre_volatile_token = self._tokens[self._index - 2] 2923 else: 2924 pre_volatile_token = None 2925 2926 if pre_volatile_token and pre_volatile_token.token_type in self.PRE_VOLATILE_TOKENS: 2927 return exp.VolatileProperty() 2928 2929 return self.expression(exp.StabilityProperty(this=exp.Literal.string("VOLATILE"))) 2930 2931 def _parse_retention_period(self) -> exp.Var: 2932 # Parse TSQL's HISTORY_RETENTION_PERIOD: {INFINITE | <number> DAY | DAYS | MONTH ...} 2933 number = self._parse_number() 2934 number_str = f"{number} " if number else "" 2935 unit = self._parse_var(any_token=True) 2936 return exp.var(f"{number_str}{unit}") 2937 2938 def _parse_system_versioning_property( 2939 self, with_: bool = False 2940 ) -> exp.WithSystemVersioningProperty: 2941 self._match(TokenType.EQ) 2942 prop = self.expression(exp.WithSystemVersioningProperty(on=True, with_=with_)) 2943 2944 if self._match_text_seq("OFF"): 2945 prop.set("on", False) 2946 return prop 2947 2948 self._match(TokenType.ON) 2949 if self._match(TokenType.L_PAREN): 2950 while self._curr and not self._match(TokenType.R_PAREN): 2951 if self._match_text_seq("HISTORY_TABLE", "="): 2952 prop.set("this", self._parse_table_parts()) 2953 elif self._match_text_seq("DATA_CONSISTENCY_CHECK", "="): 2954 prop.set("data_consistency", self._advance_any() and self._prev.text.upper()) 2955 elif self._match_text_seq("HISTORY_RETENTION_PERIOD", "="): 2956 prop.set("retention_period", self._parse_retention_period()) 2957 2958 self._match(TokenType.COMMA) 2959 2960 return prop 2961 2962 def _parse_data_deletion_property(self) -> exp.DataDeletionProperty: 2963 self._match(TokenType.EQ) 2964 on = self._match_text_seq("ON") or not self._match_text_seq("OFF") 2965 prop = self.expression(exp.DataDeletionProperty(on=on)) 2966 2967 if self._match(TokenType.L_PAREN): 2968 while self._curr and not self._match(TokenType.R_PAREN): 2969 if self._match_text_seq("FILTER_COLUMN", "="): 2970 prop.set("filter_column", self._parse_column()) 2971 elif self._match_text_seq("RETENTION_PERIOD", "="): 2972 prop.set("retention_period", self._parse_retention_period()) 2973 2974 self._match(TokenType.COMMA) 2975 2976 return prop 2977 2978 def _parse_distributed_property(self) -> exp.DistributedByProperty: 2979 kind = "HASH" 2980 expressions: list[exp.Expr] | None = None 2981 if self._match_text_seq("BY", "HASH"): 2982 expressions = self._parse_wrapped_csv(self._parse_id_var) 2983 elif self._match_text_seq("BY", "RANDOM"): 2984 kind = "RANDOM" 2985 2986 # If the BUCKETS keyword is not present, the number of buckets is AUTO 2987 buckets: exp.Expr | None = None 2988 if self._match_text_seq("BUCKETS") and not self._match_text_seq("AUTO"): 2989 buckets = self._parse_number() 2990 2991 return self.expression( 2992 exp.DistributedByProperty( 2993 expressions=expressions, kind=kind, buckets=buckets, order=self._parse_order() 2994 ) 2995 ) 2996 2997 def _parse_composite_key_property(self, expr_type: type[E]) -> E: 2998 self._match_text_seq("KEY") 2999 expressions = self._parse_wrapped_id_vars() 3000 return self.expression(expr_type(expressions=expressions)) 3001 3002 def _parse_with_property(self) -> exp.Expr | None | list[exp.Expr]: 3003 if self._match_text_seq("(", "SYSTEM_VERSIONING"): 3004 prop = self._parse_system_versioning_property(with_=True) 3005 self._match_r_paren() 3006 return prop 3007 3008 if self._match(TokenType.L_PAREN, advance=False): 3009 result: list[exp.Expr] = [] 3010 for i in self._parse_wrapped_properties(): 3011 result.extend(i) if isinstance(i, list) else result.append(i) 3012 return result 3013 3014 if self._match_text_seq("JOURNAL"): 3015 return self._parse_withjournaltable() 3016 3017 if self._match_texts(self.VIEW_ATTRIBUTES): 3018 return self.expression(exp.ViewAttributeProperty(this=self._prev.text.upper())) 3019 3020 if self._match_text_seq("DATA"): 3021 return self._parse_withdata(no=False) 3022 elif self._match_text_seq("NO", "DATA"): 3023 return self._parse_withdata(no=True) 3024 3025 if self._match(TokenType.SERDE_PROPERTIES, advance=False): 3026 return self._parse_serde_properties(with_=True) 3027 3028 if self._match(TokenType.SCHEMA): 3029 return self.expression( 3030 exp.WithSchemaBindingProperty( 3031 this=self._parse_var_from_options(self.SCHEMA_BINDING_OPTIONS) 3032 ) 3033 ) 3034 3035 if self._match_texts(self.PROCEDURE_OPTIONS, advance=False): 3036 return self.expression( 3037 exp.WithProcedureOptions(expressions=self._parse_csv(self._parse_procedure_option)) 3038 ) 3039 3040 if not self._next: 3041 return None 3042 3043 return self._parse_withisolatedloading() 3044 3045 def _parse_procedure_option(self) -> exp.Expr | None: 3046 if self._match_text_seq("EXECUTE", "AS"): 3047 return self.expression( 3048 exp.ExecuteAsProperty( 3049 this=self._parse_var_from_options( 3050 self.EXECUTE_AS_OPTIONS, raise_unmatched=False 3051 ) 3052 or self._parse_string() 3053 ) 3054 ) 3055 3056 return self._parse_var_from_options(self.PROCEDURE_OPTIONS) 3057 3058 # https://dev.mysql.com/doc/refman/8.0/en/create-view.html 3059 def _parse_definer(self) -> exp.DefinerProperty | None: 3060 self._match(TokenType.EQ) 3061 3062 user = self._parse_id_var() 3063 self._match(TokenType.PARAMETER) 3064 host = self._parse_id_var() or (self._match(TokenType.MOD) and self._prev.text) 3065 3066 if not user or not host: 3067 return None 3068 3069 return exp.DefinerProperty(this=f"{user}@{host}") 3070 3071 def _parse_withjournaltable(self) -> exp.WithJournalTableProperty: 3072 self._match(TokenType.TABLE) 3073 self._match(TokenType.EQ) 3074 return self.expression(exp.WithJournalTableProperty(this=self._parse_table_parts())) 3075 3076 def _parse_log(self, no: bool = False) -> exp.LogProperty: 3077 return self.expression(exp.LogProperty(no=no)) 3078 3079 def _parse_journal(self, **kwargs) -> exp.JournalProperty: 3080 return self.expression(exp.JournalProperty(**kwargs)) 3081 3082 def _parse_checksum(self) -> exp.ChecksumProperty: 3083 self._match(TokenType.EQ) 3084 3085 on = None 3086 if self._match(TokenType.ON): 3087 on = True 3088 elif self._match_text_seq("OFF"): 3089 on = False 3090 3091 return self.expression(exp.ChecksumProperty(on=on, default=self._match(TokenType.DEFAULT))) 3092 3093 def _parse_cluster(self) -> exp.Cluster: 3094 self._match(TokenType.CLUSTER_BY) 3095 return self.expression( 3096 exp.Cluster( 3097 expressions=self._parse_csv(self._parse_column), 3098 ) 3099 ) 3100 3101 def _parse_cluster_property(self) -> exp.ClusterProperty: 3102 return self.expression( 3103 exp.ClusterProperty( 3104 expressions=self._parse_wrapped_csv(self._parse_column), 3105 ) 3106 ) 3107 3108 def _parse_clustered_by(self) -> exp.ClusteredByProperty: 3109 self._match_text_seq("BY") 3110 3111 self._match_l_paren() 3112 expressions = self._parse_csv(self._parse_column) 3113 self._match_r_paren() 3114 3115 if self._match_text_seq("SORTED", "BY"): 3116 self._match_l_paren() 3117 sorted_by = self._parse_csv(self._parse_ordered) 3118 self._match_r_paren() 3119 else: 3120 sorted_by = None 3121 3122 self._match(TokenType.INTO) 3123 buckets = self._parse_number() 3124 self._match_text_seq("BUCKETS") 3125 3126 return self.expression( 3127 exp.ClusteredByProperty(expressions=expressions, sorted_by=sorted_by, buckets=buckets) 3128 ) 3129 3130 def _parse_copy_property(self) -> exp.CopyGrantsProperty | None: 3131 if not self._match_text_seq("GRANTS"): 3132 self._retreat(self._index - 1) 3133 return None 3134 3135 return self.expression(exp.CopyGrantsProperty()) 3136 3137 def _parse_freespace(self) -> exp.FreespaceProperty: 3138 self._match(TokenType.EQ) 3139 return self.expression( 3140 exp.FreespaceProperty(this=self._parse_number(), percent=self._match(TokenType.PERCENT)) 3141 ) 3142 3143 def _parse_mergeblockratio( 3144 self, no: bool = False, default: bool = False 3145 ) -> exp.MergeBlockRatioProperty: 3146 if self._match(TokenType.EQ): 3147 return self.expression( 3148 exp.MergeBlockRatioProperty( 3149 this=self._parse_number(), percent=self._match(TokenType.PERCENT) 3150 ) 3151 ) 3152 3153 return self.expression(exp.MergeBlockRatioProperty(no=no, default=default)) 3154 3155 def _parse_datablocksize( 3156 self, 3157 default: bool | None = None, 3158 minimum: bool | None = None, 3159 maximum: bool | None = None, 3160 ) -> exp.DataBlocksizeProperty: 3161 self._match(TokenType.EQ) 3162 size = self._parse_number() 3163 3164 units = None 3165 if self._match_texts(("BYTES", "KBYTES", "KILOBYTES")): 3166 units = self._prev.text 3167 3168 return self.expression( 3169 exp.DataBlocksizeProperty( 3170 size=size, units=units, default=default, minimum=minimum, maximum=maximum 3171 ) 3172 ) 3173 3174 def _parse_blockcompression(self) -> exp.BlockCompressionProperty: 3175 self._match(TokenType.EQ) 3176 always = self._match_text_seq("ALWAYS") 3177 manual = self._match_text_seq("MANUAL") 3178 never = self._match_text_seq("NEVER") 3179 default = self._match_text_seq("DEFAULT") 3180 3181 autotemp = None 3182 if self._match_text_seq("AUTOTEMP"): 3183 autotemp = self._parse_schema() 3184 3185 return self.expression( 3186 exp.BlockCompressionProperty( 3187 always=always, manual=manual, never=never, default=default, autotemp=autotemp 3188 ) 3189 ) 3190 3191 def _parse_withisolatedloading(self) -> exp.IsolatedLoadingProperty | None: 3192 index = self._index 3193 no = self._match_text_seq("NO") 3194 concurrent = self._match_text_seq("CONCURRENT") 3195 3196 if not self._match_text_seq("ISOLATED", "LOADING"): 3197 self._retreat(index) 3198 return None 3199 3200 target = self._parse_var_from_options(self.ISOLATED_LOADING_OPTIONS, raise_unmatched=False) 3201 return self.expression( 3202 exp.IsolatedLoadingProperty(no=no, concurrent=concurrent, target=target) 3203 ) 3204 3205 def _parse_locking(self) -> exp.LockingProperty: 3206 if self._match(TokenType.TABLE): 3207 kind = "TABLE" 3208 elif self._match(TokenType.VIEW): 3209 kind = "VIEW" 3210 elif self._match(TokenType.ROW): 3211 kind = "ROW" 3212 elif self._match_text_seq("DATABASE"): 3213 kind = "DATABASE" 3214 else: 3215 kind = None 3216 3217 if kind in ("DATABASE", "TABLE", "VIEW"): 3218 this = self._parse_table_parts() 3219 else: 3220 this = None 3221 3222 if self._match(TokenType.FOR): 3223 for_or_in = "FOR" 3224 elif self._match(TokenType.IN): 3225 for_or_in = "IN" 3226 else: 3227 for_or_in = None 3228 3229 if self._match_text_seq("ACCESS"): 3230 lock_type = "ACCESS" 3231 elif self._match_texts(("EXCL", "EXCLUSIVE")): 3232 lock_type = "EXCLUSIVE" 3233 elif self._match_text_seq("SHARE"): 3234 lock_type = "SHARE" 3235 elif self._match_text_seq("READ"): 3236 lock_type = "READ" 3237 elif self._match_text_seq("WRITE"): 3238 lock_type = "WRITE" 3239 elif self._match_text_seq("CHECKSUM"): 3240 lock_type = "CHECKSUM" 3241 else: 3242 lock_type = None 3243 3244 override = self._match_text_seq("OVERRIDE") 3245 3246 return self.expression( 3247 exp.LockingProperty( 3248 this=this, kind=kind, for_or_in=for_or_in, lock_type=lock_type, override=override 3249 ) 3250 ) 3251 3252 def _parse_partition_by(self) -> list[exp.Expr]: 3253 if self._match(TokenType.PARTITION_BY): 3254 return self._parse_csv(self._parse_disjunction) 3255 return [] 3256 3257 def _parse_partition_bound_spec(self) -> exp.PartitionBoundSpec: 3258 def _parse_partition_bound_expr() -> exp.Expr | None: 3259 if self._match_text_seq("MINVALUE"): 3260 return exp.var("MINVALUE") 3261 if self._match_text_seq("MAXVALUE"): 3262 return exp.var("MAXVALUE") 3263 return self._parse_bitwise() 3264 3265 this: exp.Expr | list[exp.Expr] | None = None 3266 expression = None 3267 from_expressions = None 3268 to_expressions = None 3269 3270 if self._match(TokenType.IN): 3271 this = self._parse_wrapped_csv(self._parse_bitwise) 3272 elif self._match(TokenType.FROM): 3273 from_expressions = self._parse_wrapped_csv(_parse_partition_bound_expr) 3274 self._match_text_seq("TO") 3275 to_expressions = self._parse_wrapped_csv(_parse_partition_bound_expr) 3276 elif self._match_text_seq("WITH", "(", "MODULUS"): 3277 this = self._parse_number() 3278 self._match_text_seq(",", "REMAINDER") 3279 expression = self._parse_number() 3280 self._match_r_paren() 3281 else: 3282 self.raise_error("Failed to parse partition bound spec.") 3283 3284 return self.expression( 3285 exp.PartitionBoundSpec( 3286 this=this, 3287 expression=expression, 3288 from_expressions=from_expressions, 3289 to_expressions=to_expressions, 3290 ) 3291 ) 3292 3293 # https://www.postgresql.org/docs/current/sql-createtable.html 3294 def _parse_partitioned_of(self) -> exp.PartitionedOfProperty | None: 3295 if not self._match_text_seq("OF"): 3296 self._retreat(self._index - 1) 3297 return None 3298 3299 this = self._parse_table(schema=True) 3300 3301 if self._match(TokenType.DEFAULT): 3302 expression: exp.Var | exp.PartitionBoundSpec = exp.var("DEFAULT") 3303 elif self._match_text_seq("FOR", "VALUES"): 3304 expression = self._parse_partition_bound_spec() 3305 else: 3306 self.raise_error("Expecting either DEFAULT or FOR VALUES clause.") 3307 3308 return self.expression(exp.PartitionedOfProperty(this=this, expression=expression)) 3309 3310 def _parse_partitioned_by(self) -> exp.PartitionedByProperty: 3311 self._match(TokenType.EQ) 3312 return self.expression( 3313 exp.PartitionedByProperty( 3314 this=self._parse_schema() or self._parse_bracket(self._parse_field()) 3315 ) 3316 ) 3317 3318 def _parse_withdata(self, no: bool = False) -> exp.WithDataProperty: 3319 if self._match_text_seq("AND", "STATISTICS"): 3320 statistics = True 3321 elif self._match_text_seq("AND", "NO", "STATISTICS"): 3322 statistics = False 3323 else: 3324 statistics = None 3325 3326 return self.expression(exp.WithDataProperty(no=no, statistics=statistics)) 3327 3328 def _parse_contains_property(self) -> exp.SqlReadWriteProperty | None: 3329 if self._match_text_seq("SQL"): 3330 return self.expression(exp.SqlReadWriteProperty(this="CONTAINS SQL")) 3331 return None 3332 3333 def _parse_modifies_property(self) -> exp.SqlReadWriteProperty | None: 3334 if self._match_text_seq("SQL", "DATA"): 3335 return self.expression(exp.SqlReadWriteProperty(this="MODIFIES SQL DATA")) 3336 return None 3337 3338 def _parse_no_property(self) -> exp.Expr | None: 3339 if self._match_text_seq("PRIMARY", "INDEX"): 3340 return exp.NoPrimaryIndexProperty() 3341 if self._match_text_seq("SQL"): 3342 return self.expression(exp.SqlReadWriteProperty(this="NO SQL")) 3343 return None 3344 3345 def _parse_on_property(self) -> exp.Expr | None: 3346 if self._match_text_seq("COMMIT", "PRESERVE", "ROWS"): 3347 return exp.OnCommitProperty() 3348 if self._match_text_seq("COMMIT", "DELETE", "ROWS"): 3349 return exp.OnCommitProperty(delete=True) 3350 return self.expression(exp.OnProperty(this=self._parse_schema(self._parse_id_var()))) 3351 3352 def _parse_reads_property(self) -> exp.SqlReadWriteProperty | None: 3353 if self._match_text_seq("SQL", "DATA"): 3354 return self.expression(exp.SqlReadWriteProperty(this="READS SQL DATA")) 3355 return None 3356 3357 def _parse_distkey(self) -> exp.DistKeyProperty: 3358 return self.expression(exp.DistKeyProperty(this=self._parse_wrapped(self._parse_id_var))) 3359 3360 def _parse_create_like(self) -> exp.LikeProperty | None: 3361 table = self._parse_table(schema=True) 3362 3363 options = [] 3364 while self._match_texts(("INCLUDING", "EXCLUDING")): 3365 this = self._prev.text.upper() 3366 3367 id_var = self._parse_id_var() 3368 if not id_var: 3369 return None 3370 3371 options.append( 3372 self.expression(exp.Property(this=this, value=exp.var(id_var.this.upper()))) 3373 ) 3374 3375 return self.expression(exp.LikeProperty(this=table, expressions=options)) 3376 3377 def _parse_sortkey(self, compound: bool = False) -> exp.SortKeyProperty: 3378 return self.expression( 3379 exp.SortKeyProperty(this=self._parse_wrapped_id_vars(), compound=compound) 3380 ) 3381 3382 def _parse_character_set(self, default: bool = False) -> exp.CharacterSetProperty: 3383 self._match(TokenType.EQ) 3384 return self.expression( 3385 exp.CharacterSetProperty(this=self._parse_var_or_string(), default=default) 3386 ) 3387 3388 def _parse_remote_with_connection(self) -> exp.RemoteWithConnectionModelProperty: 3389 self._match_text_seq("WITH", "CONNECTION") 3390 return self.expression( 3391 exp.RemoteWithConnectionModelProperty(this=self._parse_table_parts()) 3392 ) 3393 3394 def _parse_returns(self) -> exp.ReturnsProperty: 3395 value: exp.Expr | None 3396 null = None 3397 is_table = self._match(TokenType.TABLE) 3398 3399 if is_table: 3400 if self._match(TokenType.LT): 3401 value = self.expression( 3402 exp.Schema(this="TABLE", expressions=self._parse_csv(self._parse_struct_types)) 3403 ) 3404 if not self._match(TokenType.GT): 3405 self.raise_error("Expecting >") 3406 else: 3407 value = self._parse_schema(exp.var("TABLE")) 3408 elif self._match_text_seq("NULL", "ON", "NULL", "INPUT"): 3409 null = True 3410 value = None 3411 else: 3412 value = self._parse_types() 3413 3414 return self.expression(exp.ReturnsProperty(this=value, is_table=is_table, null=null)) 3415 3416 def _parse_describe(self) -> exp.Describe: 3417 kind = self._prev.text if self._match_set(self.CREATABLES) else None 3418 style: str | None = ( 3419 self._prev.text.upper() if self._match_texts(self.DESCRIBE_STYLES) else None 3420 ) 3421 if self._match(TokenType.DOT): 3422 style = None 3423 self._retreat(self._index - 2) 3424 3425 format = self._parse_property() if self._match(TokenType.FORMAT, advance=False) else None 3426 3427 if self._match_set(self.STATEMENT_PARSERS, advance=False): 3428 this = self._parse_statement() 3429 else: 3430 this = self._parse_table(schema=True) 3431 3432 properties = self._parse_properties() 3433 expressions = properties.expressions if properties else None 3434 partition = self._parse_partition() 3435 return self.expression( 3436 exp.Describe( 3437 this=this, 3438 style=style, 3439 kind=kind, 3440 expressions=expressions, 3441 partition=partition, 3442 format=format, 3443 as_json=self._match_text_seq("AS", "JSON"), 3444 ) 3445 ) 3446 3447 def _parse_multitable_inserts(self, comments: list[str] | None) -> exp.MultitableInserts: 3448 kind = self._prev.text.upper() 3449 expressions = [] 3450 3451 def parse_conditional_insert() -> exp.ConditionalInsert | None: 3452 if self._match(TokenType.WHEN): 3453 expression = self._parse_disjunction() 3454 self._match(TokenType.THEN) 3455 else: 3456 expression = None 3457 3458 else_ = self._match(TokenType.ELSE) 3459 3460 if not self._match(TokenType.INTO): 3461 return None 3462 3463 return self.expression( 3464 exp.ConditionalInsert( 3465 this=self.expression( 3466 exp.Insert( 3467 this=self._parse_table(schema=True), 3468 expression=self._parse_derived_table_values(), 3469 ) 3470 ), 3471 expression=expression, 3472 else_=else_, 3473 ) 3474 ) 3475 3476 expression = parse_conditional_insert() 3477 while expression is not None: 3478 expressions.append(expression) 3479 expression = parse_conditional_insert() 3480 3481 return self.expression( 3482 exp.MultitableInserts(kind=kind, expressions=expressions, source=self._parse_table()), 3483 comments=comments, 3484 ) 3485 3486 def _parse_insert(self) -> exp.Insert | exp.MultitableInserts: 3487 comments: list[str] = [] 3488 hint = self._parse_hint() 3489 overwrite = self._match(TokenType.OVERWRITE) 3490 ignore = self._match(TokenType.IGNORE) 3491 local = self._match_text_seq("LOCAL") 3492 alternative = None 3493 is_function = None 3494 3495 if self._match_text_seq("DIRECTORY"): 3496 this: exp.Expr | None = self.expression( 3497 exp.Directory( 3498 this=self._parse_var_or_string(), 3499 local=local, 3500 row_format=self._parse_row_format(match_row=True), 3501 ) 3502 ) 3503 else: 3504 if self._match_set((TokenType.FIRST, TokenType.ALL)): 3505 comments += ensure_list(self._prev_comments) 3506 return self._parse_multitable_inserts(comments) 3507 3508 if self._match(TokenType.OR): 3509 alternative = self._match_texts(self.INSERT_ALTERNATIVES) and self._prev.text 3510 3511 self._match(TokenType.INTO) 3512 comments += ensure_list(self._prev_comments) 3513 self._match(TokenType.TABLE) 3514 is_function = self._match(TokenType.FUNCTION) 3515 3516 this = self._parse_function() if is_function else self._parse_insert_table() 3517 3518 returning = self._parse_returning() # TSQL allows RETURNING before source 3519 3520 return self.expression( 3521 exp.Insert( 3522 hint=hint, 3523 is_function=is_function, 3524 this=this, 3525 stored=self._match_text_seq("STORED") and self._parse_stored(), 3526 by_name=self._match_text_seq("BY", "NAME"), 3527 exists=self._parse_exists(), 3528 where=self._match_pair(TokenType.REPLACE, TokenType.WHERE) 3529 and self._parse_disjunction(), 3530 partition=self._match(TokenType.PARTITION_BY) and self._parse_partitioned_by(), 3531 settings=self._match_text_seq("SETTINGS") and self._parse_settings_property(), 3532 default=self._match_text_seq("DEFAULT", "VALUES"), 3533 expression=self._parse_derived_table_values() or self._parse_ddl_select(), 3534 conflict=self._parse_on_conflict(), 3535 returning=returning or self._parse_returning(), 3536 overwrite=overwrite, 3537 alternative=alternative, 3538 ignore=ignore, 3539 source=self._match(TokenType.TABLE) and self._parse_table(), 3540 ), 3541 comments=comments, 3542 ) 3543 3544 def _parse_insert_table(self) -> exp.Expr | None: 3545 this = self._parse_table(schema=True, parse_partition=True) 3546 if isinstance(this, exp.Table) and self._match(TokenType.ALIAS, advance=False): 3547 this.set("alias", self._parse_table_alias()) 3548 return this 3549 3550 def _parse_kill(self) -> exp.Kill: 3551 kind = exp.var(self._prev.text) if self._match_texts(("CONNECTION", "QUERY")) else None 3552 3553 return self.expression(exp.Kill(this=self._parse_primary(), kind=kind)) 3554 3555 def _parse_on_conflict(self) -> exp.OnConflict | None: 3556 conflict = self._match_text_seq("ON", "CONFLICT") 3557 duplicate = self._match_text_seq("ON", "DUPLICATE", "KEY") 3558 3559 if not conflict and not duplicate: 3560 return None 3561 3562 conflict_keys = None 3563 constraint = None 3564 3565 if conflict: 3566 if self._match_text_seq("ON", "CONSTRAINT"): 3567 constraint = self._parse_id_var() 3568 elif self._match(TokenType.L_PAREN): 3569 conflict_keys = self._parse_csv(self._parse_indexed_column) 3570 self._match_r_paren() 3571 3572 index_predicate = self._parse_where() 3573 3574 action = self._parse_var_from_options(self.CONFLICT_ACTIONS) 3575 if self._prev.token_type == TokenType.UPDATE: 3576 self._match(TokenType.SET) 3577 expressions = self._parse_csv(self._parse_equality) 3578 else: 3579 expressions = None 3580 3581 return self.expression( 3582 exp.OnConflict( 3583 duplicate=duplicate, 3584 expressions=expressions, 3585 action=action, 3586 conflict_keys=conflict_keys, 3587 index_predicate=index_predicate, 3588 constraint=constraint, 3589 where=self._parse_where(), 3590 ) 3591 ) 3592 3593 def _parse_returning(self) -> exp.Returning | None: 3594 if not self._match(TokenType.RETURNING): 3595 return None 3596 return self.expression( 3597 exp.Returning( 3598 expressions=self._parse_csv(self._parse_expression), 3599 into=self._match(TokenType.INTO) and self._parse_table_part(), 3600 ) 3601 ) 3602 3603 def _parse_row(self) -> exp.RowFormatSerdeProperty | exp.RowFormatDelimitedProperty | None: 3604 if not self._match(TokenType.FORMAT): 3605 return None 3606 return self._parse_row_format() 3607 3608 def _parse_serde_properties(self, with_: bool = False) -> exp.SerdeProperties | None: 3609 index = self._index 3610 with_ = with_ or self._match_text_seq("WITH") 3611 3612 if not self._match(TokenType.SERDE_PROPERTIES): 3613 self._retreat(index) 3614 return None 3615 return self.expression( 3616 exp.SerdeProperties(expressions=self._parse_wrapped_properties(), with_=with_) 3617 ) 3618 3619 def _parse_row_format( 3620 self, match_row: bool = False 3621 ) -> exp.RowFormatSerdeProperty | exp.RowFormatDelimitedProperty | None: 3622 if match_row and not self._match_pair(TokenType.ROW, TokenType.FORMAT): 3623 return None 3624 3625 if self._match_text_seq("SERDE"): 3626 this = self._parse_string() 3627 3628 serde_properties = self._parse_serde_properties() 3629 3630 return self.expression( 3631 exp.RowFormatSerdeProperty(this=this, serde_properties=serde_properties) 3632 ) 3633 3634 self._match_text_seq("DELIMITED") 3635 3636 kwargs = {} 3637 3638 if self._match_text_seq("FIELDS", "TERMINATED", "BY"): 3639 kwargs["fields"] = self._parse_string() 3640 if self._match_text_seq("ESCAPED", "BY"): 3641 kwargs["escaped"] = self._parse_string() 3642 if self._match_text_seq("COLLECTION", "ITEMS", "TERMINATED", "BY"): 3643 kwargs["collection_items"] = self._parse_string() 3644 if self._match_text_seq("MAP", "KEYS", "TERMINATED", "BY"): 3645 kwargs["map_keys"] = self._parse_string() 3646 if self._match_text_seq("LINES", "TERMINATED", "BY"): 3647 kwargs["lines"] = self._parse_string() 3648 if self._match_text_seq("NULL", "DEFINED", "AS"): 3649 kwargs["null"] = self._parse_string() 3650 3651 return self.expression(exp.RowFormatDelimitedProperty(**kwargs)) # type: ignore 3652 3653 def _parse_load(self) -> exp.LoadData | exp.Command: 3654 if self._match_text_seq("DATA"): 3655 local = self._match_text_seq("LOCAL") 3656 self._match_text_seq("INPATH") 3657 inpath = self._parse_string() 3658 overwrite = self._match(TokenType.OVERWRITE) 3659 temp: bool | None = None 3660 if self._match(TokenType.INTO): 3661 temp = self._match(TokenType.TEMPORARY) 3662 self._match(TokenType.TABLE) 3663 3664 return self.expression( 3665 exp.LoadData( 3666 this=self._parse_table(schema=True), 3667 local=local, 3668 overwrite=overwrite, 3669 temp=temp, 3670 inpath=inpath, 3671 files=self._match_text_seq("FROM", "FILES") 3672 and exp.Properties(expressions=self._parse_wrapped_properties()), 3673 partition=self._parse_partition(), 3674 input_format=self._match_text_seq("INPUTFORMAT") and self._parse_string(), 3675 serde=self._match_text_seq("SERDE") and self._parse_string(), 3676 ) 3677 ) 3678 return self._parse_as_command(self._prev) 3679 3680 def _parse_delete(self) -> exp.Delete: 3681 hint = self._parse_hint() 3682 3683 # This handles MySQL's "Multiple-Table Syntax" 3684 # https://dev.mysql.com/doc/refman/8.0/en/delete.html 3685 tables = None 3686 if not self._match(TokenType.FROM, advance=False): 3687 tables = self._parse_csv(self._parse_table) or None 3688 3689 returning = self._parse_returning() 3690 3691 return self.expression( 3692 exp.Delete( 3693 hint=hint, 3694 tables=tables, 3695 this=self._match(TokenType.FROM) and self._parse_table(joins=True), 3696 using=self._match(TokenType.USING) 3697 and self._parse_csv(lambda: self._parse_table(joins=True)), 3698 cluster=self._match(TokenType.ON) and self._parse_on_property(), 3699 where=self._parse_where(), 3700 returning=returning or self._parse_returning(), 3701 order=self._parse_order(), 3702 limit=self._parse_limit(), 3703 ) 3704 ) 3705 3706 def _parse_update(self) -> exp.Update: 3707 hint = self._parse_hint() 3708 kwargs: dict[str, object] = { 3709 "hint": hint, 3710 "this": self._parse_table(joins=True, alias_tokens=self.UPDATE_ALIAS_TOKENS), 3711 } 3712 while self._curr: 3713 if self._match(TokenType.SET): 3714 kwargs["expressions"] = self._parse_csv(self._parse_equality) 3715 elif self._match(TokenType.RETURNING, advance=False): 3716 kwargs["returning"] = self._parse_returning() 3717 elif self._match(TokenType.FROM, advance=False): 3718 from_ = self._parse_from(joins=True) 3719 table = from_.this if from_ else None 3720 if isinstance(table, exp.Subquery) and self._match(TokenType.JOIN, advance=False): 3721 table.set("joins", list(self._parse_joins()) or None) 3722 3723 kwargs["from_"] = from_ 3724 elif self._match(TokenType.WHERE, advance=False): 3725 kwargs["where"] = self._parse_where() 3726 elif self._match(TokenType.ORDER_BY, advance=False): 3727 kwargs["order"] = self._parse_order() 3728 elif self._match(TokenType.LIMIT, advance=False): 3729 kwargs["limit"] = self._parse_limit() 3730 else: 3731 break 3732 3733 return self.expression(exp.Update(**kwargs)) 3734 3735 def _parse_use(self) -> exp.Use: 3736 return self.expression( 3737 exp.Use( 3738 kind=self._parse_var_from_options(self.USABLES, raise_unmatched=False), 3739 this=self._parse_table(schema=False), 3740 ) 3741 ) 3742 3743 def _parse_uncache(self) -> exp.Uncache: 3744 if not self._match(TokenType.TABLE): 3745 self.raise_error("Expecting TABLE after UNCACHE") 3746 3747 return self.expression( 3748 exp.Uncache(exists=self._parse_exists(), this=self._parse_table(schema=True)) 3749 ) 3750 3751 def _parse_cache(self) -> exp.Cache: 3752 lazy = self._match_text_seq("LAZY") 3753 self._match(TokenType.TABLE) 3754 table = self._parse_table(schema=True) 3755 3756 options = [] 3757 if self._match_text_seq("OPTIONS"): 3758 self._match_l_paren() 3759 k = self._parse_string() 3760 self._match(TokenType.EQ) 3761 v = self._parse_string() 3762 options = [k, v] 3763 self._match_r_paren() 3764 3765 self._match(TokenType.ALIAS) 3766 return self.expression( 3767 exp.Cache( 3768 this=table, lazy=lazy, options=options, expression=self._parse_select(nested=True) 3769 ) 3770 ) 3771 3772 def _parse_partition(self) -> exp.Partition | None: 3773 if not self._match_texts(self.PARTITION_KEYWORDS): 3774 return None 3775 3776 return self.expression( 3777 exp.Partition( 3778 subpartition=self._prev.text.upper() == "SUBPARTITION", 3779 expressions=self._parse_wrapped_csv(self._parse_disjunction), 3780 ) 3781 ) 3782 3783 def _parse_value(self, values: bool = True) -> exp.Tuple | None: 3784 def _parse_value_expression() -> exp.Expr | None: 3785 if self.dialect.SUPPORTS_VALUES_DEFAULT and self._match(TokenType.DEFAULT): 3786 return exp.var(self._prev.text.upper()) 3787 return self._parse_expression() 3788 3789 if self._match(TokenType.L_PAREN): 3790 expressions = self._parse_csv(_parse_value_expression) 3791 self._match_r_paren() 3792 return self.expression(exp.Tuple(expressions=expressions)) 3793 3794 # In some dialects we can have VALUES 1, 2 which results in 1 column & 2 rows. 3795 expression = self._parse_expression() 3796 if expression: 3797 return self.expression(exp.Tuple(expressions=[expression])) 3798 return None 3799 3800 def _parse_projections( 3801 self, 3802 ) -> tuple[list[exp.Expr], list[exp.Expr] | None]: 3803 return self._parse_expressions(), None 3804 3805 def _parse_wrapped_select(self, table: bool = False) -> exp.Expr | None: 3806 if self._match_set((TokenType.PIVOT, TokenType.UNPIVOT)): 3807 this: exp.Expr | None = self._parse_simplified_pivot( 3808 is_unpivot=self._prev.token_type == TokenType.UNPIVOT 3809 ) 3810 elif self._match(TokenType.FROM): 3811 from_ = self._parse_from(joins=True, skip_from_token=True, consume_pipe=True) 3812 # Support parentheses for duckdb FROM-first syntax 3813 select = self._parse_select(from_=from_) 3814 if select: 3815 if not select.args.get("from_"): 3816 select.set("from_", from_) 3817 this = select 3818 else: 3819 this = exp.select("*").from_(t.cast(exp.From, from_)) 3820 this = self._parse_query_modifiers(self._parse_set_operations(this)) 3821 else: 3822 this = ( 3823 self._parse_table(consume_pipe=True) 3824 if table 3825 else self._parse_select(nested=True, parse_set_operation=False) 3826 ) 3827 3828 # Transform exp.Values into a exp.Table to pass through parse_query_modifiers 3829 # in case a modifier (e.g. join) is following 3830 if table and isinstance(this, exp.Values) and this.alias: 3831 alias = this.args["alias"].pop() 3832 this = exp.Table(this=this, alias=alias) 3833 3834 this = self._parse_query_modifiers(self._parse_set_operations(this)) 3835 3836 return this 3837 3838 def _parse_select( 3839 self, 3840 nested: bool = False, 3841 table: bool = False, 3842 parse_subquery_alias: bool = True, 3843 parse_set_operation: bool = True, 3844 consume_pipe: bool = True, 3845 from_: exp.From | None = None, 3846 ) -> exp.Expr | None: 3847 query = self._parse_select_query( 3848 nested=nested, 3849 table=table, 3850 parse_subquery_alias=parse_subquery_alias, 3851 parse_set_operation=parse_set_operation, 3852 ) 3853 3854 if consume_pipe and self._match(TokenType.PIPE_GT, advance=False): 3855 if not query and from_: 3856 query = exp.select("*").from_(from_) 3857 if isinstance(query, exp.Query): 3858 query = self._parse_pipe_syntax_query(query) 3859 query = query.subquery(copy=False) if query and table else query 3860 3861 return query 3862 3863 def _parse_select_query( 3864 self, 3865 nested: bool = False, 3866 table: bool = False, 3867 parse_subquery_alias: bool = True, 3868 parse_set_operation: bool = True, 3869 ) -> exp.Expr | None: 3870 cte = self._parse_with() 3871 3872 if cte: 3873 this = self._parse_statement() 3874 3875 if not this: 3876 self.raise_error("Failed to parse any statement following CTE") 3877 return cte 3878 3879 while isinstance(this, exp.Subquery) and this.is_wrapper: 3880 this = this.this 3881 3882 assert this is not None 3883 if "with_" in this.arg_types: 3884 if inner_cte := this.args.get("with_"): 3885 cte.set("expressions", cte.expressions + inner_cte.expressions) 3886 if inner_cte.args.get("recursive"): 3887 cte.set("recursive", True) 3888 this.set("with_", cte) 3889 else: 3890 self.raise_error(f"{this.key} does not support CTE") 3891 this = cte 3892 3893 return this 3894 3895 # duckdb supports leading with FROM x 3896 from_ = ( 3897 self._parse_from(joins=True, consume_pipe=True) 3898 if self._match(TokenType.FROM, advance=False) 3899 else None 3900 ) 3901 3902 if self._match(TokenType.SELECT): 3903 comments = self._prev_comments 3904 3905 hint = self._parse_hint() 3906 3907 if self._next and not self._next.token_type == TokenType.DOT: 3908 all_ = self._match(TokenType.ALL) 3909 matched_distinct = self._match_set(self.DISTINCT_TOKENS) 3910 else: 3911 all_, matched_distinct = None, False 3912 3913 kind = ( 3914 self._prev.text.upper() 3915 if self._match(TokenType.ALIAS) and self._match_texts(("STRUCT", "VALUE")) 3916 else None 3917 ) 3918 3919 distinct: exp.Expr | None = ( 3920 self.expression( 3921 exp.Distinct( 3922 on=self._parse_value(values=False) if self._match(TokenType.ON) else None 3923 ) 3924 ) 3925 if matched_distinct 3926 else None 3927 ) 3928 3929 operation_modifiers = [] 3930 while self._curr and self._match_texts(self.OPERATION_MODIFIERS): 3931 operation_modifiers.append(exp.var(self._prev.text.upper())) 3932 3933 limit = self._parse_limit(top=True) 3934 3935 # Some dialects (e.g. Redshift, T-SQL) allow SELECT TOP N DISTINCT ... 3936 if limit and not matched_distinct and not all_: 3937 matched_distinct = self._match_set(self.DISTINCT_TOKENS) 3938 if matched_distinct: 3939 distinct = self.expression( 3940 exp.Distinct( 3941 on=self._parse_value(values=False) 3942 if self._match(TokenType.ON) 3943 else None 3944 ) 3945 ) 3946 else: 3947 all_ = self._match(TokenType.ALL) 3948 3949 if all_ and distinct: 3950 self.raise_error("Cannot specify both ALL and DISTINCT after SELECT") 3951 3952 projections, exclude = self._parse_projections() 3953 3954 this = self.expression( 3955 exp.Select( 3956 kind=kind, 3957 hint=hint, 3958 distinct=distinct, 3959 expressions=projections, 3960 limit=limit, 3961 exclude=exclude, 3962 operation_modifiers=operation_modifiers or None, 3963 ) 3964 ) 3965 this.comments = comments 3966 3967 into = self._parse_into() 3968 if into: 3969 this.set("into", into) 3970 3971 if not from_: 3972 from_ = self._parse_from() 3973 3974 if from_: 3975 this.set("from_", from_) 3976 3977 this = self._parse_query_modifiers(this) 3978 elif (table or nested) and self._match(TokenType.L_PAREN): 3979 comments = self._prev_comments 3980 this = self._parse_wrapped_select(table=table) 3981 3982 if this: 3983 this.add_comments(comments, prepend=True) 3984 3985 # We return early here so that the UNION isn't attached to the subquery by the 3986 # following call to _parse_set_operations, but instead becomes the parent node 3987 self._match_r_paren() 3988 return self._parse_subquery(this, parse_alias=parse_subquery_alias) 3989 elif self._match(TokenType.VALUES, advance=False): 3990 this = self._parse_derived_table_values() 3991 elif from_: 3992 this = exp.select("*").from_(from_.this, copy=False) 3993 this = self._parse_query_modifiers(this) 3994 elif self._match(TokenType.SUMMARIZE): 3995 table = self._match(TokenType.TABLE) 3996 this = self._parse_select() or self._parse_string() or self._parse_table() 3997 return self.expression(exp.Summarize(this=this, table=table)) 3998 elif self._match(TokenType.DESCRIBE): 3999 this = self._parse_describe() 4000 else: 4001 this = None 4002 4003 return self._parse_set_operations(this) if parse_set_operation else this 4004 4005 def _parse_recursive_with_search(self) -> exp.RecursiveWithSearch | None: 4006 self._match_text_seq("SEARCH") 4007 4008 kind = self._match_texts(self.RECURSIVE_CTE_SEARCH_KIND) and self._prev.text.upper() 4009 4010 if not kind: 4011 return None 4012 4013 self._match_text_seq("FIRST", "BY") 4014 4015 return self.expression( 4016 exp.RecursiveWithSearch( 4017 kind=kind, 4018 this=self._parse_id_var(), 4019 expression=self._match_text_seq("SET") and self._parse_id_var(), 4020 using=self._match_text_seq("USING") and self._parse_id_var(), 4021 ) 4022 ) 4023 4024 def _parse_with(self, skip_with_token: bool = False) -> exp.With | None: 4025 if not skip_with_token and not self._match(TokenType.WITH): 4026 return None 4027 4028 comments = self._prev_comments 4029 recursive = self._match(TokenType.RECURSIVE) 4030 4031 last_comments = None 4032 expressions = [] 4033 while True: 4034 cte = self._parse_cte() 4035 if isinstance(cte, exp.CTE): 4036 expressions.append(cte) 4037 if last_comments: 4038 cte.add_comments(last_comments) 4039 4040 if not self._match(TokenType.COMMA) and not self._match(TokenType.WITH): 4041 break 4042 else: 4043 self._match(TokenType.WITH) 4044 4045 last_comments = self._prev_comments 4046 4047 return self.expression( 4048 exp.With( 4049 expressions=expressions, 4050 recursive=recursive or None, 4051 search=self._parse_recursive_with_search(), 4052 ), 4053 comments=comments, 4054 ) 4055 4056 def _parse_cte(self) -> exp.CTE | None: 4057 index = self._index 4058 4059 alias = self._parse_table_alias(self.ID_VAR_TOKENS) 4060 if not alias or not alias.this: 4061 self.raise_error("Expected CTE to have alias") 4062 4063 key_expressions = ( 4064 self._parse_wrapped_id_vars() if self._match_text_seq("USING", "KEY") else None 4065 ) 4066 4067 if not self._match(TokenType.ALIAS) and not self.OPTIONAL_ALIAS_TOKEN_CTE: 4068 self._retreat(index) 4069 return None 4070 4071 comments = self._prev_comments 4072 4073 if self._match_text_seq("NOT", "MATERIALIZED"): 4074 materialized = False 4075 elif self._match_text_seq("MATERIALIZED"): 4076 materialized = True 4077 else: 4078 materialized = None 4079 4080 cte = self.expression( 4081 exp.CTE( 4082 this=self._parse_wrapped(self._parse_statement), 4083 alias=alias, 4084 materialized=materialized, 4085 key_expressions=key_expressions, 4086 ), 4087 comments=comments, 4088 ) 4089 4090 values = cte.this 4091 if isinstance(values, exp.Values): 4092 if values.alias: 4093 cte.set("this", exp.select("*").from_(values)) 4094 else: 4095 cte.set("this", exp.select("*").from_(exp.alias_(values, "_values", table=True))) 4096 4097 return cte 4098 4099 def _parse_table_alias( 4100 self, alias_tokens: t.Collection[TokenType] | None = None 4101 ) -> exp.TableAlias | None: 4102 # In some dialects, LIMIT and OFFSET can act as both identifiers and keywords (clauses) 4103 # so this section tries to parse the clause version and if it fails, it treats the token 4104 # as an identifier (alias) 4105 if self._can_parse_limit_or_offset(): 4106 return None 4107 4108 any_token = self._match(TokenType.ALIAS) 4109 alias = ( 4110 self._parse_id_var(any_token=any_token, tokens=alias_tokens or self.TABLE_ALIAS_TOKENS) 4111 or self._parse_string_as_identifier() 4112 ) 4113 4114 index = self._index 4115 if self._match(TokenType.L_PAREN): 4116 columns = self._parse_csv(self._parse_function_parameter) 4117 self._match_r_paren() if columns else self._retreat(index) 4118 else: 4119 columns = None 4120 4121 if not alias and not columns: 4122 return None 4123 4124 table_alias = self.expression(exp.TableAlias(this=alias, columns=columns)) 4125 4126 # We bubble up comments from the Identifier to the TableAlias 4127 if isinstance(alias, exp.Identifier): 4128 table_alias.add_comments(alias.pop_comments()) 4129 4130 return table_alias 4131 4132 def _parse_subquery( 4133 self, this: exp.Expr | None, parse_alias: bool = True 4134 ) -> exp.Subquery | None: 4135 if not this: 4136 return None 4137 4138 return self.expression( 4139 exp.Subquery( 4140 this=this, 4141 pivots=self._parse_pivots(), 4142 alias=self._parse_table_alias() if parse_alias else None, 4143 sample=self._parse_table_sample(), 4144 ) 4145 ) 4146 4147 def _implicit_unnests_to_explicit(self, this: E) -> E: 4148 from sqlglot.optimizer.normalize_identifiers import normalize_identifiers as _norm 4149 4150 refs = {_norm(this.args["from_"].this.copy(), dialect=self.dialect).alias_or_name} 4151 for i, join in enumerate(this.args.get("joins") or []): 4152 table = join.this 4153 normalized_table = table.copy() 4154 normalized_table.meta["maybe_column"] = True 4155 normalized_table = _norm(normalized_table, dialect=self.dialect) 4156 4157 if isinstance(table, exp.Table) and not join.args.get("on"): 4158 if len(normalized_table.parts) > 1 and normalized_table.parts[0].name in refs: 4159 table_as_column = table.to_column() 4160 unnest = exp.Unnest(expressions=[table_as_column]) 4161 4162 # Table.to_column creates a parent Alias node that we want to convert to 4163 # a TableAlias and attach to the Unnest, so it matches the parser's output 4164 if isinstance(table.args.get("alias"), exp.TableAlias): 4165 table_as_column.replace(table_as_column.this) 4166 exp.alias_(unnest, None, table=[table.args["alias"].this], copy=False) 4167 4168 table.replace(unnest) 4169 4170 refs.add(normalized_table.alias_or_name) 4171 4172 return this 4173 4174 @t.overload 4175 def _parse_query_modifiers(self, this: E) -> E: ... 4176 4177 @t.overload 4178 def _parse_query_modifiers(self, this: None) -> None: ... 4179 4180 def _parse_query_modifiers(self, this): 4181 if isinstance(this, self.MODIFIABLES): 4182 for join in self._parse_joins(): 4183 this.append("joins", join) 4184 for lateral in iter(self._parse_lateral, None): 4185 this.append("laterals", lateral) 4186 4187 while True: 4188 if self._match_set(self.QUERY_MODIFIER_PARSERS, advance=False): 4189 modifier_token = self._curr 4190 parser = self.QUERY_MODIFIER_PARSERS[modifier_token.token_type] 4191 key, expression = parser(self) 4192 4193 if expression: 4194 if this.args.get(key): 4195 self.raise_error( 4196 f"Found multiple '{modifier_token.text.upper()}' clauses", 4197 token=modifier_token, 4198 ) 4199 4200 this.set(key, expression) 4201 if key == "limit": 4202 offset = expression.args.get("offset") 4203 expression.set("offset", None) 4204 4205 if offset: 4206 offset = exp.Offset(expression=offset) 4207 this.set("offset", offset) 4208 4209 limit_by_expressions = expression.expressions 4210 expression.set("expressions", None) 4211 offset.set("expressions", limit_by_expressions) 4212 continue 4213 break 4214 4215 if self.SUPPORTS_IMPLICIT_UNNEST and this and this.args.get("from_"): 4216 this = self._implicit_unnests_to_explicit(this) 4217 4218 return this 4219 4220 def _parse_hint_fallback_to_string(self) -> exp.Hint | None: 4221 start = self._curr 4222 while self._curr: 4223 self._advance() 4224 4225 end = self._tokens[self._index - 1] 4226 return exp.Hint(expressions=[self._find_sql(start, end)]) 4227 4228 def _parse_hint_function_call(self) -> exp.Expr | None: 4229 return self._parse_function_call() 4230 4231 def _parse_hint_body(self) -> exp.Hint | None: 4232 start_index = self._index 4233 should_fallback_to_string = False 4234 4235 hints = [] 4236 try: 4237 for hint in iter( 4238 lambda: self._parse_csv( 4239 lambda: self._parse_hint_function_call() or self._parse_var(upper=True), 4240 ), 4241 [], 4242 ): 4243 hints.extend(hint) 4244 except ParseError: 4245 should_fallback_to_string = True 4246 4247 if should_fallback_to_string or self._curr: 4248 self._retreat(start_index) 4249 return self._parse_hint_fallback_to_string() 4250 4251 return self.expression(exp.Hint(expressions=hints)) 4252 4253 def _parse_hint(self) -> exp.Hint | None: 4254 if self._match(TokenType.HINT) and self._prev_comments: 4255 return exp.maybe_parse(self._prev_comments[0], into=exp.Hint, dialect=self.dialect) 4256 4257 return None 4258 4259 def _parse_into(self) -> exp.Into | None: 4260 if not self._match(TokenType.INTO): 4261 return None 4262 4263 temp = self._match(TokenType.TEMPORARY) 4264 unlogged = self._match_text_seq("UNLOGGED") 4265 self._match(TokenType.TABLE) 4266 4267 return self.expression( 4268 exp.Into(this=self._parse_table(schema=True), temporary=temp, unlogged=unlogged) 4269 ) 4270 4271 def _parse_from( 4272 self, 4273 joins: bool = False, 4274 skip_from_token: bool = False, 4275 consume_pipe: bool = False, 4276 ) -> exp.From | None: 4277 if not skip_from_token and not self._match(TokenType.FROM): 4278 return None 4279 4280 comments = self._prev_comments 4281 return self.expression( 4282 exp.From(this=self._parse_table(joins=joins, consume_pipe=consume_pipe)), 4283 comments=comments, 4284 ) 4285 4286 def _parse_match_recognize_measure(self) -> exp.MatchRecognizeMeasure: 4287 return self.expression( 4288 exp.MatchRecognizeMeasure( 4289 window_frame=self._match_texts(("FINAL", "RUNNING")) and self._prev.text.upper(), 4290 this=self._parse_expression(), 4291 ) 4292 ) 4293 4294 def _parse_match_recognize(self) -> exp.MatchRecognize | None: 4295 if not self._match(TokenType.MATCH_RECOGNIZE): 4296 return None 4297 4298 self._match_l_paren() 4299 4300 partition = self._parse_partition_by() 4301 order = self._parse_order() 4302 4303 measures = ( 4304 self._parse_csv(self._parse_match_recognize_measure) 4305 if self._match_text_seq("MEASURES") 4306 else None 4307 ) 4308 4309 if self._match_text_seq("ONE", "ROW", "PER", "MATCH"): 4310 rows = exp.var("ONE ROW PER MATCH") 4311 elif self._match_text_seq("ALL", "ROWS", "PER", "MATCH"): 4312 text = "ALL ROWS PER MATCH" 4313 if self._match_text_seq("SHOW", "EMPTY", "MATCHES"): 4314 text += " SHOW EMPTY MATCHES" 4315 elif self._match_text_seq("OMIT", "EMPTY", "MATCHES"): 4316 text += " OMIT EMPTY MATCHES" 4317 elif self._match_text_seq("WITH", "UNMATCHED", "ROWS"): 4318 text += " WITH UNMATCHED ROWS" 4319 rows = exp.var(text) 4320 else: 4321 rows = None 4322 4323 if self._match_text_seq("AFTER", "MATCH", "SKIP"): 4324 text = "AFTER MATCH SKIP" 4325 if self._match_text_seq("PAST", "LAST", "ROW"): 4326 text += " PAST LAST ROW" 4327 elif self._match_text_seq("TO", "NEXT", "ROW"): 4328 text += " TO NEXT ROW" 4329 elif self._match_text_seq("TO", "FIRST"): 4330 text += f" TO FIRST {self._advance_any().text}" # type: ignore 4331 elif self._match_text_seq("TO", "LAST"): 4332 text += f" TO LAST {self._advance_any().text}" # type: ignore 4333 after = exp.var(text) 4334 else: 4335 after = None 4336 4337 if self._match_text_seq("PATTERN"): 4338 self._match_l_paren() 4339 4340 if not self._curr: 4341 self.raise_error("Expecting )", self._curr) 4342 4343 paren = 1 4344 start = self._curr 4345 4346 while self._curr and paren > 0: 4347 if self._curr.token_type == TokenType.L_PAREN: 4348 paren += 1 4349 if self._curr.token_type == TokenType.R_PAREN: 4350 paren -= 1 4351 4352 end = self._prev 4353 self._advance() 4354 4355 if paren > 0: 4356 self.raise_error("Expecting )", self._curr) 4357 4358 pattern = exp.var(self._find_sql(start, end)) 4359 else: 4360 pattern = None 4361 4362 define = ( 4363 self._parse_csv(self._parse_name_as_expression) 4364 if self._match_text_seq("DEFINE") 4365 else None 4366 ) 4367 4368 self._match_r_paren() 4369 4370 return self.expression( 4371 exp.MatchRecognize( 4372 partition_by=partition, 4373 order=order, 4374 measures=measures, 4375 rows=rows, 4376 after=after, 4377 pattern=pattern, 4378 define=define, 4379 alias=self._parse_table_alias(), 4380 ) 4381 ) 4382 4383 def _parse_lateral(self) -> exp.Lateral | None: 4384 cross_apply: bool | None = None 4385 if self._match_pair(TokenType.CROSS, TokenType.APPLY): 4386 cross_apply = True 4387 elif self._match_pair(TokenType.OUTER, TokenType.APPLY): 4388 cross_apply = False 4389 4390 if cross_apply is not None: 4391 this = self._parse_select(table=True) 4392 view = None 4393 outer = None 4394 elif self._match(TokenType.LATERAL): 4395 this = self._parse_select(table=True) 4396 view = self._match(TokenType.VIEW) 4397 outer = self._match(TokenType.OUTER) 4398 else: 4399 return None 4400 4401 if not this: 4402 this = ( 4403 self._parse_unnest() 4404 or self._parse_function() 4405 or self._parse_id_var(any_token=False) 4406 ) 4407 4408 while self._match(TokenType.DOT): 4409 this = exp.Dot( 4410 this=this, 4411 expression=self._parse_function() or self._parse_id_var(any_token=False), 4412 ) 4413 4414 ordinality: bool | None = None 4415 4416 if view: 4417 table = self._parse_id_var(any_token=False) 4418 columns = self._parse_csv(self._parse_id_var) if self._match(TokenType.ALIAS) else [] 4419 table_alias: exp.TableAlias | None = self.expression( 4420 exp.TableAlias(this=table, columns=columns) 4421 ) 4422 elif isinstance(this, (exp.Subquery, exp.Unnest)) and this.alias: 4423 # We move the alias from the lateral's child node to the lateral itself 4424 table_alias = this.args["alias"].pop() 4425 else: 4426 ordinality = self._match_pair(TokenType.WITH, TokenType.ORDINALITY) 4427 table_alias = self._parse_table_alias() 4428 4429 return self.expression( 4430 exp.Lateral( 4431 this=this, 4432 view=view, 4433 outer=outer, 4434 alias=table_alias, 4435 cross_apply=cross_apply, 4436 ordinality=ordinality, 4437 ) 4438 ) 4439 4440 def _parse_stream(self) -> exp.Stream | None: 4441 index = self._index 4442 if self._match(TokenType.STREAM): 4443 if this := self._try_parse(self._parse_table): 4444 return self.expression(exp.Stream(this=this)) 4445 self._retreat(index) 4446 return None 4447 4448 def _parse_join_parts( 4449 self, 4450 ) -> tuple[Token | None, Token | None, Token | None]: 4451 return ( 4452 self._prev if self._match_set(self.JOIN_METHODS) else None, 4453 self._prev if self._match_set(self.JOIN_SIDES) else None, 4454 self._prev if self._match_set(self.JOIN_KINDS) else None, 4455 ) 4456 4457 def _parse_using_identifiers(self) -> list[exp.Expr]: 4458 def _parse_column_as_identifier() -> exp.Expr | None: 4459 this = self._parse_column() 4460 if isinstance(this, exp.Column): 4461 return this.this 4462 return this 4463 4464 return self._parse_wrapped_csv(_parse_column_as_identifier, optional=True) 4465 4466 def _parse_join( 4467 self, 4468 skip_join_token: bool = False, 4469 parse_bracket: bool = False, 4470 alias_tokens: t.Collection[TokenType] | None = None, 4471 ) -> exp.Join | None: 4472 if self._match(TokenType.COMMA): 4473 table = self._try_parse(lambda: self._parse_table(alias_tokens=alias_tokens)) 4474 cross_join = self.expression(exp.Join(this=table)) if table else None 4475 4476 if cross_join and self.JOINS_HAVE_EQUAL_PRECEDENCE: 4477 cross_join.set("kind", "CROSS") 4478 4479 return cross_join 4480 4481 index = self._index 4482 method, side, kind = self._parse_join_parts() 4483 directed = self._match_text_seq("DIRECTED") 4484 hint = self._prev.text if self._match_texts(self.JOIN_HINTS) else None 4485 join = self._match(TokenType.JOIN) or (kind and kind.token_type == TokenType.STRAIGHT_JOIN) 4486 join_comments = self._prev_comments 4487 4488 if not skip_join_token and not join: 4489 self._retreat(index) 4490 kind = None 4491 method = None 4492 side = None 4493 4494 outer_apply = self._match_pair(TokenType.OUTER, TokenType.APPLY, False) 4495 cross_apply = self._match_pair(TokenType.CROSS, TokenType.APPLY, False) 4496 4497 if not skip_join_token and not join and not outer_apply and not cross_apply: 4498 return None 4499 4500 kwargs: dict[str, t.Any] = { 4501 "this": self._parse_table(parse_bracket=parse_bracket, alias_tokens=alias_tokens) 4502 } 4503 if kind and kind.token_type == TokenType.ARRAY and self._match(TokenType.COMMA): 4504 kwargs["expressions"] = self._parse_csv( 4505 lambda: self._parse_table(parse_bracket=parse_bracket, alias_tokens=alias_tokens) 4506 ) 4507 4508 if method: 4509 kwargs["method"] = method.text.upper() 4510 if side: 4511 kwargs["side"] = side.text.upper() 4512 if kind: 4513 kwargs["kind"] = kind.text.upper() 4514 if hint: 4515 kwargs["hint"] = hint 4516 4517 if self._match(TokenType.MATCH_CONDITION): 4518 kwargs["match_condition"] = self._parse_wrapped(self._parse_comparison) 4519 4520 if self._match(TokenType.ON): 4521 kwargs["on"] = self._parse_disjunction() 4522 elif self._match(TokenType.USING): 4523 kwargs["using"] = self._parse_using_identifiers() 4524 elif ( 4525 not method 4526 and not (outer_apply or cross_apply) 4527 and not isinstance(kwargs["this"], exp.Unnest) 4528 and not (kind and kind.token_type in (TokenType.CROSS, TokenType.ARRAY)) 4529 ): 4530 index = self._index 4531 joins: list | None = list(self._parse_joins(alias_tokens=alias_tokens)) 4532 4533 if joins and self._match(TokenType.ON): 4534 kwargs["on"] = self._parse_disjunction() 4535 elif joins and self._match(TokenType.USING): 4536 kwargs["using"] = self._parse_using_identifiers() 4537 else: 4538 joins = None 4539 self._retreat(index) 4540 4541 kwargs["this"].set("joins", joins if joins else None) 4542 4543 kwargs["pivots"] = self._parse_pivots() 4544 4545 comments = [c for token in (method, side, kind) if token for c in token.comments] 4546 comments = (join_comments or []) + comments 4547 4548 if ( 4549 self.ADD_JOIN_ON_TRUE 4550 and not kwargs.get("on") 4551 and not kwargs.get("using") 4552 and not kwargs.get("method") 4553 and kwargs.get("kind") in (None, "INNER", "OUTER") 4554 ): 4555 kwargs["on"] = exp.true() 4556 4557 if directed: 4558 kwargs["directed"] = directed 4559 4560 return self.expression(exp.Join(**kwargs), comments=comments) 4561 4562 def _parse_opclass(self) -> exp.Expr | None: 4563 this = self._parse_disjunction() 4564 4565 if self._match_texts(self.OPCLASS_FOLLOW_KEYWORDS, advance=False): 4566 return this 4567 4568 if not self._match_set(self.OPTYPE_FOLLOW_TOKENS, advance=False): 4569 return self.expression(exp.Opclass(this=this, expression=self._parse_table_parts())) 4570 4571 return this 4572 4573 def _parse_index_params(self) -> exp.IndexParameters: 4574 using = self._parse_var(any_token=True) if self._match(TokenType.USING) else None 4575 4576 if self._match(TokenType.L_PAREN, advance=False): 4577 columns = self._parse_wrapped_csv(self._parse_with_operator) 4578 else: 4579 columns = None 4580 4581 include = self._parse_wrapped_id_vars() if self._match_text_seq("INCLUDE") else None 4582 partition_by = self._parse_partition_by() 4583 with_storage = self._match(TokenType.WITH) and self._parse_wrapped_properties() 4584 tablespace = ( 4585 self._parse_var(any_token=True) 4586 if self._match_text_seq("USING", "INDEX", "TABLESPACE") 4587 else None 4588 ) 4589 where = self._parse_where() 4590 4591 on = self._parse_field() if self._match(TokenType.ON) else None 4592 4593 return self.expression( 4594 exp.IndexParameters( 4595 using=using, 4596 columns=columns, 4597 include=include, 4598 partition_by=partition_by, 4599 where=where, 4600 with_storage=with_storage, 4601 tablespace=tablespace, 4602 on=on, 4603 ) 4604 ) 4605 4606 def _parse_index( 4607 self, index: exp.Expr | None = None, anonymous: bool = False 4608 ) -> exp.Index | None: 4609 if index or anonymous: 4610 unique = None 4611 primary = None 4612 amp = None 4613 4614 self._match(TokenType.ON) 4615 self._match(TokenType.TABLE) # hive 4616 table = self._parse_table_parts(schema=True) 4617 else: 4618 unique = self._match(TokenType.UNIQUE) 4619 primary = self._match_text_seq("PRIMARY") 4620 amp = self._match_text_seq("AMP") 4621 4622 if not self._match(TokenType.INDEX): 4623 return None 4624 4625 index = self._parse_id_var() 4626 table = None 4627 4628 params = self._parse_index_params() 4629 4630 return self.expression( 4631 exp.Index( 4632 this=index, table=table, unique=unique, primary=primary, amp=amp, params=params 4633 ) 4634 ) 4635 4636 def _parse_table_hints(self) -> list[exp.Expr] | None: 4637 hints: list[exp.Expr] = [] 4638 if self._match_pair(TokenType.WITH, TokenType.L_PAREN): 4639 # https://learn.microsoft.com/en-us/sql/t-sql/queries/hints-transact-sql-table?view=sql-server-ver16 4640 hints.append( 4641 self.expression( 4642 exp.WithTableHint( 4643 expressions=self._parse_csv( 4644 lambda: self._parse_function() or self._parse_var(any_token=True) 4645 ) 4646 ) 4647 ) 4648 ) 4649 self._match_r_paren() 4650 else: 4651 # https://dev.mysql.com/doc/refman/8.0/en/index-hints.html 4652 while self._match_set(self.TABLE_INDEX_HINT_TOKENS): 4653 hint = exp.IndexTableHint(this=self._prev.text.upper()) 4654 4655 self._match_set((TokenType.INDEX, TokenType.KEY)) 4656 if self._match(TokenType.FOR): 4657 hint.set("target", self._advance_any() and self._prev.text.upper()) 4658 4659 hint.set("expressions", self._parse_wrapped_id_vars()) 4660 hints.append(hint) 4661 4662 return hints or None 4663 4664 def _parse_table_part(self, schema: bool = False) -> exp.Expr | None: 4665 return ( 4666 (not schema and self._parse_function(optional_parens=False)) 4667 or self._parse_id_var(any_token=False) 4668 or self._parse_string_as_identifier() 4669 or self._parse_placeholder() 4670 ) 4671 4672 def _parse_table_parts_fast(self) -> exp.Table | None: 4673 index = self._index 4674 parts: list[exp.Identifier] | None = None 4675 all_comments: list[str] | None = None 4676 4677 while self._match_set(self.IDENTIFIER_TOKENS): 4678 token = self._prev 4679 comments = self._prev_comments 4680 4681 has_dot = self._match(TokenType.DOT) 4682 curr_tt = self._curr.token_type 4683 4684 if not has_dot: 4685 if curr_tt in self.TABLE_POSTFIX_TOKENS: 4686 self._retreat(index) 4687 return None 4688 elif curr_tt not in self.IDENTIFIER_TOKENS: 4689 self._retreat(index) 4690 return None 4691 4692 if parts is None: 4693 parts = [] 4694 4695 if comments: 4696 if all_comments is None: 4697 all_comments = [] 4698 all_comments.extend(comments) 4699 self._prev_comments = [] 4700 4701 parts.append( 4702 self.expression( 4703 exp.Identifier( 4704 this=token.text, quoted=token.token_type == TokenType.IDENTIFIER 4705 ), 4706 token, 4707 ) 4708 ) 4709 4710 if not has_dot: 4711 break 4712 4713 if parts is None: 4714 return None 4715 4716 n = len(parts) 4717 4718 if n == 1: 4719 table: exp.Table = exp.Table(this=parts[0]) 4720 elif n == 2: 4721 table = exp.Table(this=parts[1], db=parts[0]) 4722 elif n >= 3: 4723 this: exp.Identifier | exp.Dot = parts[2] 4724 for i in range(3, n): 4725 this = exp.Dot(this=this, expression=parts[i]) 4726 4727 table = exp.Table(this=this, db=parts[1], catalog=parts[0]) 4728 4729 if table is None: 4730 self._retreat(index) 4731 elif all_comments: 4732 table.add_comments(all_comments) 4733 return table 4734 4735 def _parse_table_parts( 4736 self, 4737 schema: bool = False, 4738 is_db_reference: bool = False, 4739 wildcard: bool = False, 4740 fast: bool = False, 4741 ) -> exp.Table | exp.Dot | None: 4742 if fast: 4743 return self._parse_table_parts_fast() 4744 4745 catalog: exp.Expr | str | None = None 4746 db: exp.Expr | str | None = None 4747 table: exp.Expr | str | None = self._parse_table_part(schema=schema) 4748 4749 while self._match(TokenType.DOT): 4750 if catalog: 4751 # This allows nesting the table in arbitrarily many dot expressions if needed 4752 table = self.expression( 4753 exp.Dot(this=table, expression=self._parse_table_part(schema=schema)) 4754 ) 4755 else: 4756 catalog = db 4757 db = table 4758 # "" used for tsql FROM a..b case 4759 table = self._parse_table_part(schema=schema) or "" 4760 4761 if ( 4762 wildcard 4763 and self._is_connected() 4764 and (isinstance(table, exp.Identifier) or not table) 4765 and self._match(TokenType.STAR) 4766 ): 4767 if isinstance(table, exp.Identifier): 4768 table.args["this"] += "*" 4769 else: 4770 table = exp.Identifier(this="*") 4771 4772 if is_db_reference: 4773 catalog = db 4774 db = table 4775 table = None 4776 4777 if not table and not is_db_reference: 4778 self.raise_error(f"Expected table name but got {self._curr}") 4779 if not db and is_db_reference: 4780 self.raise_error(f"Expected database name but got {self._curr}") 4781 4782 table = self.expression(exp.Table(this=table, db=db, catalog=catalog)) 4783 4784 # Bubble up comments from identifier parts to the Table 4785 comments = [] 4786 for part in table.parts: 4787 if part_comments := part.pop_comments(): 4788 comments.extend(part_comments) 4789 if comments: 4790 table.add_comments(comments) 4791 4792 changes = self._parse_changes() 4793 if changes: 4794 table.set("changes", changes) 4795 4796 at_before = self._parse_historical_data() 4797 if at_before: 4798 table.set("when", at_before) 4799 4800 pivots = self._parse_pivots() 4801 if pivots: 4802 table.set("pivots", pivots) 4803 4804 return table 4805 4806 def _parse_table( 4807 self, 4808 schema: bool = False, 4809 joins: bool = False, 4810 alias_tokens: t.Collection[TokenType] | None = None, 4811 parse_bracket: bool = False, 4812 is_db_reference: bool = False, 4813 parse_partition: bool = False, 4814 consume_pipe: bool = False, 4815 ) -> exp.Expr | None: 4816 if not schema and not is_db_reference and not consume_pipe and not joins: 4817 index = self._index 4818 table = self._parse_table_parts(fast=True) 4819 4820 if table is not None: 4821 curr_tt = self._curr.token_type 4822 next_tt = self._next.token_type 4823 4824 fast_terminators = self.TABLE_TERMINATORS 4825 4826 # only return the table if we're sure there are no other operators 4827 # MATCH_CONDITION is a special case because it accepts any alias before it like LIMIT 4828 if curr_tt in fast_terminators and next_tt != TokenType.MATCH_CONDITION: 4829 return table 4830 4831 postfix_tokens = self.TABLE_POSTFIX_TOKENS 4832 4833 if curr_tt not in postfix_tokens and next_tt not in postfix_tokens: 4834 if alias := self._parse_table_alias( 4835 alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS 4836 ): 4837 table.set("alias", alias) 4838 4839 if self._curr.token_type in fast_terminators: 4840 return table 4841 4842 self._retreat(index) 4843 4844 if stream := self._parse_stream(): 4845 return stream 4846 4847 if lateral := self._parse_lateral(): 4848 return lateral 4849 4850 if unnest := self._parse_unnest(): 4851 return unnest 4852 4853 if values := self._parse_derived_table_values(): 4854 return values 4855 4856 if subquery := self._parse_select(table=True, consume_pipe=consume_pipe): 4857 if not subquery.args.get("pivots"): 4858 subquery.set("pivots", self._parse_pivots()) 4859 if joins: 4860 for join in self._parse_joins(): 4861 subquery.append("joins", join) 4862 return subquery 4863 4864 bracket = parse_bracket and self._parse_bracket(None) 4865 bracket = self.expression(exp.Table(this=bracket)) if bracket else None 4866 4867 rows_from_tables = ( 4868 self._parse_wrapped_csv(self._parse_table) 4869 if self._match_text_seq("ROWS", "FROM") 4870 else None 4871 ) 4872 rows_from = ( 4873 self.expression(exp.Table(rows_from=rows_from_tables)) if rows_from_tables else None 4874 ) 4875 4876 only = self._match(TokenType.ONLY) 4877 4878 this = t.cast( 4879 exp.Expr, 4880 bracket 4881 or rows_from 4882 or self._parse_bracket( 4883 self._parse_table_parts(schema=schema, is_db_reference=is_db_reference) 4884 ), 4885 ) 4886 4887 if only: 4888 this.set("only", only) 4889 4890 # Postgres supports a wildcard (table) suffix operator, which is a no-op in this context 4891 self._match(TokenType.STAR) 4892 4893 parse_partition = parse_partition or self.SUPPORTS_PARTITION_SELECTION 4894 if parse_partition and self._match(TokenType.PARTITION, advance=False): 4895 this.set("partition", self._parse_partition()) 4896 4897 if schema: 4898 return self._parse_schema(this=this) 4899 4900 if self.dialect.ALIAS_POST_VERSION: 4901 this.set("version", self._parse_version()) 4902 4903 if self.dialect.ALIAS_POST_TABLESAMPLE: 4904 this.set("sample", self._parse_table_sample()) 4905 4906 alias = self._parse_table_alias(alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS) 4907 if alias: 4908 this.set("alias", alias) 4909 4910 if self._match(TokenType.INDEXED_BY): 4911 this.set("indexed", self._parse_table_parts()) 4912 elif self._match_text_seq("NOT", "INDEXED"): 4913 this.set("indexed", False) 4914 4915 if isinstance(this, exp.Table) and self._match_text_seq("AT"): 4916 return self.expression( 4917 exp.AtIndex(this=this.to_column(copy=False), expression=self._parse_id_var()) 4918 ) 4919 4920 this.set("hints", self._parse_table_hints()) 4921 4922 if not this.args.get("pivots"): 4923 this.set("pivots", self._parse_pivots()) 4924 4925 if not self.dialect.ALIAS_POST_TABLESAMPLE: 4926 this.set("sample", self._parse_table_sample()) 4927 4928 if not self.dialect.ALIAS_POST_VERSION: 4929 this.set("version", self._parse_version()) 4930 4931 if joins: 4932 for join in self._parse_joins(alias_tokens=alias_tokens): 4933 this.append("joins", join) 4934 4935 if self._match_pair(TokenType.WITH, TokenType.ORDINALITY): 4936 this.set("ordinality", True) 4937 this.set("alias", self._parse_table_alias()) 4938 4939 return this 4940 4941 def _parse_version(self) -> exp.Version | None: 4942 if self._match(TokenType.TIMESTAMP_SNAPSHOT): 4943 this = "TIMESTAMP" 4944 elif self._match(TokenType.VERSION_SNAPSHOT): 4945 this = "VERSION" 4946 else: 4947 return None 4948 4949 if self._match_set((TokenType.FROM, TokenType.BETWEEN)): 4950 kind = self._prev.text.upper() 4951 start = self._parse_bitwise() 4952 self._match_texts(("TO", "AND")) 4953 end = self._parse_bitwise() 4954 expression: exp.Expr | None = self.expression(exp.Tuple(expressions=[start, end])) 4955 elif self._match_text_seq("CONTAINED", "IN"): 4956 kind = "CONTAINED IN" 4957 expression = self.expression( 4958 exp.Tuple(expressions=self._parse_wrapped_csv(self._parse_bitwise)) 4959 ) 4960 elif self._match(TokenType.ALL): 4961 kind = "ALL" 4962 expression = None 4963 else: 4964 self._match_text_seq("AS", "OF") 4965 kind = "AS OF" 4966 expression = self._parse_type() 4967 4968 return self.expression(exp.Version(this=this, expression=expression, kind=kind)) 4969 4970 def _parse_historical_data(self) -> exp.HistoricalData | None: 4971 # https://docs.snowflake.com/en/sql-reference/constructs/at-before 4972 index = self._index 4973 historical_data = None 4974 if self._match_texts(self.HISTORICAL_DATA_PREFIX): 4975 this = self._prev.text.upper() 4976 kind = ( 4977 self._match(TokenType.L_PAREN) 4978 and self._match_texts(self.HISTORICAL_DATA_KIND) 4979 and self._prev.text.upper() 4980 ) 4981 expression = self._match(TokenType.FARROW) and self._parse_bitwise() 4982 4983 if expression: 4984 self._match_r_paren() 4985 historical_data = self.expression( 4986 exp.HistoricalData(this=this, kind=kind, expression=expression) 4987 ) 4988 else: 4989 self._retreat(index) 4990 4991 return historical_data 4992 4993 def _parse_changes(self) -> exp.Changes | None: 4994 if not self._match_text_seq("CHANGES", "(", "INFORMATION", "=>"): 4995 return None 4996 4997 information = self._parse_var(any_token=True) 4998 self._match_r_paren() 4999 5000 return self.expression( 5001 exp.Changes( 5002 information=information, 5003 at_before=self._parse_historical_data(), 5004 end=self._parse_historical_data(), 5005 ) 5006 ) 5007 5008 def _parse_unnest(self, with_alias: bool = True) -> exp.Unnest | None: 5009 if not self._match_pair(TokenType.UNNEST, TokenType.L_PAREN, advance=False): 5010 return None 5011 5012 self._advance() 5013 5014 expressions = self._parse_wrapped_csv(self._parse_equality) 5015 offset: bool | exp.Expr = self._match_pair(TokenType.WITH, TokenType.ORDINALITY) 5016 5017 alias = self._parse_table_alias() if with_alias else None 5018 5019 if alias: 5020 if self.dialect.UNNEST_COLUMN_ONLY: 5021 if alias.args.get("columns"): 5022 self.raise_error("Unexpected extra column alias in unnest.") 5023 5024 alias.set("columns", [alias.this]) 5025 alias.set("this", None) 5026 5027 columns = alias.args.get("columns") or [] 5028 if offset and len(expressions) < len(columns): 5029 offset = columns.pop() 5030 5031 if not offset and self._match_pair(TokenType.WITH, TokenType.OFFSET): 5032 self._match(TokenType.ALIAS) 5033 offset = self._parse_id_var( 5034 any_token=False, tokens=self.UNNEST_OFFSET_ALIAS_TOKENS 5035 ) or exp.to_identifier("offset") 5036 5037 return self.expression(exp.Unnest(expressions=expressions, alias=alias, offset=offset)) 5038 5039 def _parse_derived_table_values(self) -> exp.Values | None: 5040 is_derived = self._match_pair(TokenType.L_PAREN, TokenType.VALUES) 5041 if not is_derived and not ( 5042 # ClickHouse's `FORMAT Values` is equivalent to `VALUES` 5043 self._match_text_seq("VALUES") or self._match_text_seq("FORMAT", "VALUES") 5044 ): 5045 return None 5046 5047 expressions = self._parse_csv(self._parse_value) 5048 alias = self._parse_table_alias() 5049 5050 if is_derived: 5051 self._match_r_paren() 5052 5053 return self.expression( 5054 exp.Values(expressions=expressions, alias=alias or self._parse_table_alias()) 5055 ) 5056 5057 def _parse_table_sample(self, as_modifier: bool = False) -> exp.TableSample | None: 5058 if not self._match(TokenType.TABLE_SAMPLE) and not ( 5059 as_modifier and self._match_text_seq("USING", "SAMPLE") 5060 ): 5061 return None 5062 5063 bucket_numerator = None 5064 bucket_denominator = None 5065 bucket_field = None 5066 percent = None 5067 size = None 5068 seed = None 5069 5070 method = self._parse_var(tokens=(TokenType.ROW,), upper=True) 5071 matched_l_paren = self._match(TokenType.L_PAREN) 5072 5073 if self.TABLESAMPLE_CSV: 5074 num = None 5075 expressions = self._parse_csv(self._parse_primary) 5076 else: 5077 expressions = None 5078 num = ( 5079 self._parse_factor() 5080 if self._match(TokenType.NUMBER, advance=False) 5081 else self._parse_primary() or self._parse_placeholder() 5082 ) 5083 5084 if self._match_text_seq("BUCKET"): 5085 bucket_numerator = self._parse_number() 5086 self._match_text_seq("OUT", "OF") 5087 bucket_denominator = bucket_denominator = self._parse_number() 5088 self._match(TokenType.ON) 5089 bucket_field = self._parse_field() 5090 elif self._match_set((TokenType.PERCENT, TokenType.MOD)): 5091 percent = num 5092 elif self._match(TokenType.ROWS) or not self.dialect.TABLESAMPLE_SIZE_IS_PERCENT: 5093 size = num 5094 else: 5095 percent = num 5096 5097 if matched_l_paren: 5098 self._match_r_paren() 5099 5100 if self._match(TokenType.L_PAREN): 5101 method = self._parse_var(upper=True) 5102 seed = self._match(TokenType.COMMA) and self._parse_number() 5103 self._match_r_paren() 5104 elif self._match_texts(("SEED", "REPEATABLE")): 5105 seed = self._parse_wrapped(self._parse_number) 5106 5107 if not method and self.DEFAULT_SAMPLING_METHOD: 5108 method = exp.var(self.DEFAULT_SAMPLING_METHOD) 5109 5110 return self.expression( 5111 exp.TableSample( 5112 expressions=expressions, 5113 method=method, 5114 bucket_numerator=bucket_numerator, 5115 bucket_denominator=bucket_denominator, 5116 bucket_field=bucket_field, 5117 percent=percent, 5118 size=size, 5119 seed=seed, 5120 ) 5121 ) 5122 5123 def _parse_pivots(self) -> list[exp.Pivot] | None: 5124 if self._curr.token_type not in (TokenType.PIVOT, TokenType.UNPIVOT): 5125 return None 5126 return list(iter(self._parse_pivot, None)) or None 5127 5128 def _parse_joins( 5129 self, alias_tokens: t.Collection[TokenType] | None = None 5130 ) -> t.Iterator[exp.Join]: 5131 return iter(lambda: self._parse_join(alias_tokens=alias_tokens), None) 5132 5133 def _parse_unpivot_columns(self) -> exp.UnpivotColumns | None: 5134 if not self._match(TokenType.INTO): 5135 return None 5136 5137 return self.expression( 5138 exp.UnpivotColumns( 5139 this=self._match_text_seq("NAME") and self._parse_column(), 5140 expressions=self._match_text_seq("VALUE") and self._parse_csv(self._parse_column), 5141 ) 5142 ) 5143 5144 # https://duckdb.org/docs/sql/statements/pivot 5145 def _parse_simplified_pivot(self, is_unpivot: bool | None = None) -> exp.Pivot: 5146 def _parse_on() -> exp.Expr | None: 5147 this = self._parse_bitwise() 5148 5149 if self._match(TokenType.IN): 5150 # PIVOT ... ON col IN (row_val1, row_val2) 5151 return self._parse_in(this) 5152 if self._match(TokenType.ALIAS, advance=False): 5153 # UNPIVOT ... ON (col1, col2, col3) AS row_val 5154 return self._parse_alias(this) 5155 5156 return this 5157 5158 this = self._parse_table() 5159 expressions = self._match(TokenType.ON) and self._parse_csv(_parse_on) 5160 into = self._parse_unpivot_columns() 5161 using = self._match(TokenType.USING) and self._parse_csv( 5162 lambda: self._parse_alias(self._parse_column()) 5163 ) 5164 group = self._parse_group() 5165 5166 return self.expression( 5167 exp.Pivot( 5168 this=this, 5169 expressions=expressions, 5170 using=using, 5171 group=group, 5172 unpivot=is_unpivot, 5173 into=into, 5174 ) 5175 ) 5176 5177 def _parse_pivot_in(self) -> exp.In: 5178 def _parse_aliased_expression() -> exp.Expr | None: 5179 this = self._parse_select_or_expression() 5180 5181 self._match(TokenType.ALIAS) 5182 alias = self._parse_bitwise() 5183 if alias: 5184 if isinstance(alias, exp.Column) and not alias.db: 5185 alias = alias.this 5186 return self.expression(exp.PivotAlias(this=this, alias=alias)) 5187 5188 return this 5189 5190 value = self._parse_column() 5191 5192 if not self._match(TokenType.IN): 5193 self.raise_error("Expecting IN") 5194 5195 if self._match(TokenType.L_PAREN): 5196 if self._match(TokenType.ANY): 5197 exprs: list[exp.Expr] = ensure_list(exp.PivotAny(this=self._parse_order())) 5198 else: 5199 exprs = self._parse_csv(_parse_aliased_expression) 5200 self._match_r_paren() 5201 return self.expression(exp.In(this=value, expressions=exprs)) 5202 5203 return self.expression(exp.In(this=value, field=self._parse_id_var())) 5204 5205 def _parse_pivot_aggregation(self) -> exp.Expr | None: 5206 func = self._parse_function() 5207 if not func: 5208 if self._prev.token_type == TokenType.COMMA: 5209 return None 5210 self.raise_error("Expecting an aggregation function in PIVOT") 5211 5212 return self._parse_alias(func) 5213 5214 def _parse_pivot(self) -> exp.Pivot | None: 5215 index = self._index 5216 include_nulls = None 5217 5218 if self._match(TokenType.PIVOT): 5219 unpivot = False 5220 elif self._match(TokenType.UNPIVOT): 5221 unpivot = True 5222 5223 # https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-qry-select-unpivot.html#syntax 5224 if self._match_text_seq("INCLUDE", "NULLS"): 5225 include_nulls = True 5226 elif self._match_text_seq("EXCLUDE", "NULLS"): 5227 include_nulls = False 5228 else: 5229 return None 5230 5231 expressions = [] 5232 5233 if not self._match(TokenType.L_PAREN): 5234 self._retreat(index) 5235 return None 5236 5237 if unpivot: 5238 expressions = self._parse_csv(self._parse_column) 5239 else: 5240 expressions = self._parse_csv(self._parse_pivot_aggregation) 5241 5242 if not expressions: 5243 self.raise_error("Failed to parse PIVOT's aggregation list") 5244 5245 if not self._match(TokenType.FOR): 5246 self.raise_error("Expecting FOR") 5247 5248 fields = [] 5249 while True: 5250 field = self._try_parse(self._parse_pivot_in) 5251 if not field: 5252 break 5253 fields.append(field) 5254 5255 default_on_null = self._match_text_seq("DEFAULT", "ON", "NULL") and self._parse_wrapped( 5256 self._parse_bitwise 5257 ) 5258 5259 group = self._parse_group() 5260 5261 self._match_r_paren() 5262 5263 pivot = self.expression( 5264 exp.Pivot( 5265 expressions=expressions, 5266 fields=fields, 5267 unpivot=unpivot, 5268 include_nulls=include_nulls, 5269 default_on_null=default_on_null, 5270 group=group, 5271 ) 5272 ) 5273 5274 if unpivot: 5275 pivot.set("expressions", [_unpivot_target(e) for e in pivot.expressions]) 5276 for pivot_field in pivot.fields: 5277 if isinstance(pivot_field, exp.In): 5278 pivot_field.set("this", _unpivot_target(pivot_field.this)) 5279 5280 if not self._match_set((TokenType.PIVOT, TokenType.UNPIVOT), advance=False): 5281 pivot.set("alias", self._parse_table_alias()) 5282 5283 if not unpivot: 5284 names = self._pivot_column_names(t.cast(list[exp.Expr], expressions)) 5285 5286 columns: list[exp.Expr] = [] 5287 all_fields = [] 5288 for pivot_field in pivot.fields: 5289 pivot_field_expressions = pivot_field.expressions 5290 5291 # The `PivotAny` expression corresponds to `ANY ORDER BY <column>`; we can't infer in this case. 5292 if isinstance(seq_get(pivot_field_expressions, 0), exp.PivotAny): 5293 continue 5294 5295 all_fields.append( 5296 [ 5297 fld.sql() if self.IDENTIFY_PIVOT_STRINGS else fld.alias_or_name 5298 for fld in pivot_field_expressions 5299 ] 5300 ) 5301 5302 if all_fields: 5303 if names: 5304 all_fields.append(names) 5305 5306 # Generate all possible combinations of the pivot columns 5307 # e.g PIVOT(sum(...) as total FOR year IN (2000, 2010) FOR country IN ('NL', 'US')) 5308 # generates the product between [[2000, 2010], ['NL', 'US'], ['total']] 5309 for fld_parts_tuple in itertools.product(*all_fields): 5310 fld_parts = list(fld_parts_tuple) 5311 5312 if names and self.PREFIXED_PIVOT_COLUMNS: 5313 # Move the "name" to the front of the list 5314 fld_parts.insert(0, fld_parts.pop(-1)) 5315 5316 columns.append(exp.to_identifier("_".join(fld_parts))) 5317 5318 pivot.set("columns", columns) 5319 pivot.set("identify_pivot_strings", self.IDENTIFY_PIVOT_STRINGS) 5320 pivot.set("prefixed_pivot_columns", self.PREFIXED_PIVOT_COLUMNS) 5321 pivot.set("pivot_column_naming", self.PIVOT_COLUMN_NAMING) 5322 5323 return pivot 5324 5325 def _pivot_column_names(self, aggregations: list[exp.Expr]) -> list[str]: 5326 return [agg.alias for agg in aggregations if agg.alias] 5327 5328 def _parse_prewhere(self, skip_where_token: bool = False) -> exp.PreWhere | None: 5329 if not skip_where_token and not self._match(TokenType.PREWHERE): 5330 return None 5331 5332 comments = self._prev_comments 5333 return self.expression( 5334 exp.PreWhere(this=self._parse_disjunction()), 5335 comments=comments, 5336 ) 5337 5338 def _parse_where(self, skip_where_token: bool = False) -> exp.Where | None: 5339 if not skip_where_token and not self._match(TokenType.WHERE): 5340 return None 5341 5342 comments = self._prev_comments 5343 return self.expression( 5344 exp.Where(this=self._parse_disjunction()), 5345 comments=comments, 5346 ) 5347 5348 def _parse_group(self, skip_group_by_token: bool = False) -> exp.Group | None: 5349 if not skip_group_by_token and not self._match(TokenType.GROUP_BY): 5350 return None 5351 comments = self._prev_comments 5352 5353 elements: dict[str, t.Any] = defaultdict(list) 5354 5355 if self._match(TokenType.ALL): 5356 elements["all"] = True 5357 elif self._match(TokenType.DISTINCT): 5358 elements["all"] = False 5359 5360 if self._match_set(self.QUERY_MODIFIER_TOKENS, advance=False): 5361 return self.expression(exp.Group(**elements), comments=comments) # type: ignore 5362 5363 while True: 5364 index = self._index 5365 5366 elements["expressions"].extend( 5367 self._parse_csv( 5368 lambda: ( 5369 None 5370 if self._match_set((TokenType.CUBE, TokenType.ROLLUP), advance=False) 5371 else self._parse_disjunction() 5372 ) 5373 ) 5374 ) 5375 5376 before_with_index = self._index 5377 with_prefix = self._match(TokenType.WITH) 5378 5379 if cube_or_rollup := self._parse_cube_or_rollup(with_prefix=with_prefix): 5380 key = "rollup" if isinstance(cube_or_rollup, exp.Rollup) else "cube" 5381 elements[key].append(cube_or_rollup) 5382 elif grouping_sets := self._parse_grouping_sets(): 5383 elements["grouping_sets"].append(grouping_sets) 5384 elif self._match_text_seq("TOTALS"): 5385 elements["totals"] = True # type: ignore 5386 5387 if before_with_index <= self._index <= before_with_index + 1: 5388 self._retreat(before_with_index) 5389 break 5390 5391 if index == self._index: 5392 break 5393 5394 return self.expression(exp.Group(**elements), comments=comments) # type: ignore 5395 5396 def _parse_cube_or_rollup(self, with_prefix: bool = False) -> exp.Cube | exp.Rollup | None: 5397 if self._match(TokenType.CUBE): 5398 kind: type[exp.Cube | exp.Rollup] = exp.Cube 5399 elif self._match(TokenType.ROLLUP): 5400 kind = exp.Rollup 5401 else: 5402 return None 5403 5404 return self.expression( 5405 kind(expressions=[] if with_prefix else self._parse_wrapped_csv(self._parse_bitwise)) 5406 ) 5407 5408 def _parse_grouping_sets(self) -> exp.GroupingSets | None: 5409 if self._match(TokenType.GROUPING_SETS): 5410 return self.expression( 5411 exp.GroupingSets(expressions=self._parse_wrapped_csv(self._parse_grouping_set)) 5412 ) 5413 return None 5414 5415 def _parse_grouping_set(self) -> exp.Expr | None: 5416 return self._parse_grouping_sets() or self._parse_cube_or_rollup() or self._parse_bitwise() 5417 5418 def _parse_having(self, skip_having_token: bool = False) -> exp.Having | None: 5419 if not skip_having_token and not self._match(TokenType.HAVING): 5420 return None 5421 comments = self._prev_comments 5422 return self.expression( 5423 exp.Having(this=self._parse_disjunction()), 5424 comments=comments, 5425 ) 5426 5427 def _parse_qualify(self) -> exp.Qualify | None: 5428 if not self._match(TokenType.QUALIFY): 5429 return None 5430 return self.expression(exp.Qualify(this=self._parse_disjunction())) 5431 5432 def _parse_connect_with_prior(self) -> exp.Expr | None: 5433 self.NO_PAREN_FUNCTION_PARSERS["PRIOR"] = lambda self: self.expression( 5434 exp.Prior(this=self._parse_bitwise()) 5435 ) 5436 connect = self._parse_disjunction() 5437 self.NO_PAREN_FUNCTION_PARSERS.pop("PRIOR") 5438 return connect 5439 5440 def _parse_connect(self, skip_start_token: bool = False) -> exp.Connect | None: 5441 if skip_start_token: 5442 start = None 5443 elif self._match(TokenType.START_WITH): 5444 start = self._parse_disjunction() 5445 else: 5446 return None 5447 5448 self._match(TokenType.CONNECT_BY) 5449 nocycle = self._match_text_seq("NOCYCLE") 5450 connect = self._parse_connect_with_prior() 5451 5452 if not start and self._match(TokenType.START_WITH): 5453 start = self._parse_disjunction() 5454 5455 return self.expression(exp.Connect(start=start, connect=connect, nocycle=nocycle)) 5456 5457 def _parse_name_as_expression(self) -> exp.Expr | None: 5458 this = self._parse_id_var(any_token=True) 5459 if self._match(TokenType.ALIAS): 5460 this = self.expression(exp.Alias(alias=this, this=self._parse_disjunction())) 5461 return this 5462 5463 def _parse_interpolate(self) -> list[exp.Expr] | None: 5464 if self._match_text_seq("INTERPOLATE"): 5465 return self._parse_wrapped_csv(self._parse_name_as_expression) 5466 return None 5467 5468 def _parse_order( 5469 self, this: exp.Expr | None = None, skip_order_token: bool = False 5470 ) -> exp.Expr | None: 5471 siblings = None 5472 if not skip_order_token and not self._match(TokenType.ORDER_BY): 5473 if not self._match(TokenType.ORDER_SIBLINGS_BY): 5474 return this 5475 5476 siblings = True 5477 5478 comments = self._prev_comments 5479 return self.expression( 5480 exp.Order( 5481 this=this, 5482 expressions=self._parse_csv(self._parse_ordered), 5483 siblings=siblings, 5484 ), 5485 comments=comments, 5486 ) 5487 5488 def _parse_sort(self, exp_class: type[E], token: TokenType) -> E | None: 5489 if not self._match(token): 5490 return None 5491 return self.expression(exp_class(expressions=self._parse_csv(self._parse_ordered))) 5492 5493 def _parse_ordered( 5494 self, parse_method: t.Callable[[], exp.Expr | None] | None = None 5495 ) -> exp.Ordered | None: 5496 this = parse_method() if parse_method else self._parse_disjunction() 5497 if not this: 5498 return None 5499 5500 if this.name.upper() == "ALL" and self.dialect.SUPPORTS_ORDER_BY_ALL: 5501 this = exp.var("ALL") 5502 5503 asc = self._match(TokenType.ASC) 5504 desc: bool | None = True if self._match(TokenType.DESC) else (False if asc else None) 5505 5506 is_nulls_first = self._match_text_seq("NULLS", "FIRST") 5507 is_nulls_last = self._match_text_seq("NULLS", "LAST") 5508 5509 nulls_first = is_nulls_first or False 5510 explicitly_null_ordered = is_nulls_first or is_nulls_last 5511 5512 if ( 5513 not explicitly_null_ordered 5514 and ( 5515 (not desc and self.dialect.NULL_ORDERING == "nulls_are_small") 5516 or (desc and self.dialect.NULL_ORDERING != "nulls_are_small") 5517 ) 5518 and self.dialect.NULL_ORDERING != "nulls_are_last" 5519 ): 5520 nulls_first = True 5521 5522 if self._match_text_seq("WITH", "FILL"): 5523 with_fill = self.expression( 5524 exp.WithFill( 5525 from_=self._match(TokenType.FROM) and self._parse_bitwise(), 5526 to=self._match_text_seq("TO") and self._parse_bitwise(), 5527 step=self._match_text_seq("STEP") and self._parse_bitwise(), 5528 interpolate=self._parse_interpolate(), 5529 ) 5530 ) 5531 else: 5532 with_fill = None 5533 5534 return self.expression( 5535 exp.Ordered(this=this, desc=desc, nulls_first=nulls_first, with_fill=with_fill) 5536 ) 5537 5538 def _parse_limit_options(self) -> exp.LimitOptions | None: 5539 percent = self._match_set((TokenType.PERCENT, TokenType.MOD)) 5540 rows = self._match_set((TokenType.ROW, TokenType.ROWS)) 5541 self._match_text_seq("ONLY") 5542 with_ties = self._match_text_seq("WITH", "TIES") 5543 5544 if not (percent or rows or with_ties): 5545 return None 5546 5547 return self.expression(exp.LimitOptions(percent=percent, rows=rows, with_ties=with_ties)) 5548 5549 def _parse_limit( 5550 self, 5551 this: exp.Expr | None = None, 5552 top: bool = False, 5553 skip_limit_token: bool = False, 5554 ) -> exp.Expr | None: 5555 if skip_limit_token or self._match(TokenType.TOP if top else TokenType.LIMIT): 5556 comments = self._prev_comments 5557 if top: 5558 limit_paren = self._match(TokenType.L_PAREN) 5559 expression = ( 5560 self._parse_term() or self._parse_select() 5561 if limit_paren 5562 else self._parse_number() 5563 ) 5564 5565 if limit_paren: 5566 self._match_r_paren() 5567 5568 else: 5569 if self.dialect.SUPPORTS_LIMIT_ALL and self._match(TokenType.ALL): 5570 return this 5571 5572 # Parsing LIMIT x% (i.e x PERCENT) as a term leads to an error, since 5573 # we try to build an exp.Mod expr. For that matter, we backtrack and instead 5574 # consume the factor plus parse the percentage separately 5575 index = self._index 5576 expression = self._try_parse(self._parse_term) 5577 if isinstance(expression, exp.Mod): 5578 self._retreat(index) 5579 expression = self._parse_factor() 5580 elif not expression: 5581 expression = self._parse_factor() 5582 limit_options = self._parse_limit_options() 5583 5584 if self._match(TokenType.COMMA): 5585 offset = expression 5586 expression = self._parse_term() 5587 else: 5588 offset = None 5589 5590 limit_exp = self.expression( 5591 exp.Limit( 5592 this=this, 5593 expression=expression, 5594 offset=offset, 5595 limit_options=limit_options, 5596 expressions=self._parse_limit_by(), 5597 ), 5598 comments=comments, 5599 ) 5600 5601 return limit_exp 5602 5603 if self._match(TokenType.FETCH): 5604 direction = ( 5605 self._prev.text.upper() 5606 if self._match_set((TokenType.FIRST, TokenType.NEXT)) 5607 else "FIRST" 5608 ) 5609 5610 count = self._parse_field(tokens=self.FETCH_TOKENS) 5611 5612 return self.expression( 5613 exp.Fetch( 5614 direction=direction, count=count, limit_options=self._parse_limit_options() 5615 ) 5616 ) 5617 5618 return this 5619 5620 def _parse_offset(self, this: exp.Expr | None = None) -> exp.Expr | None: 5621 if not self._match(TokenType.OFFSET): 5622 return this 5623 5624 count = self._parse_term() 5625 self._match_set((TokenType.ROW, TokenType.ROWS)) 5626 5627 return self.expression( 5628 exp.Offset(this=this, expression=count, expressions=self._parse_limit_by()) 5629 ) 5630 5631 def _can_parse_limit_or_offset(self) -> bool: 5632 if not self._match_set(self.AMBIGUOUS_ALIAS_TOKENS, advance=False): 5633 return False 5634 5635 index = self._index 5636 result = bool( 5637 self._try_parse(self._parse_limit, retreat=True) 5638 or self._try_parse(self._parse_offset, retreat=True) 5639 ) 5640 self._retreat(index) 5641 5642 # MATCH_CONDITION (...) is a special construct that should not be consumed by limit/offset 5643 if self._next.token_type == TokenType.MATCH_CONDITION: 5644 result = False 5645 5646 return result 5647 5648 def _can_parse_named_window(self) -> bool: 5649 # `WINDOW` is in ID_VAR_TOKENS so it could be mistakenly consumed as an implicit alias. 5650 # Refuse only when the following tokens look like a named-window clause: `WINDOW <id> AS (`. 5651 if not self._match(TokenType.WINDOW, advance=False): 5652 return False 5653 5654 name = self._tokens[self._index + 1] if self._index + 1 < len(self._tokens) else None 5655 if name is None or name.token_type not in self.ID_VAR_TOKENS: 5656 return False 5657 5658 alias_tok = self._tokens[self._index + 2] if self._index + 2 < len(self._tokens) else None 5659 if alias_tok is None or alias_tok.token_type != TokenType.ALIAS: 5660 return False 5661 5662 body = self._tokens[self._index + 3] if self._index + 3 < len(self._tokens) else None 5663 return body is not None and body.token_type == TokenType.L_PAREN 5664 5665 def _parse_limit_by(self) -> list[exp.Expr] | None: 5666 return self._parse_csv(self._parse_bitwise) if self._match_text_seq("BY") else None 5667 5668 def _parse_locks(self) -> list[exp.Lock]: 5669 locks = [] 5670 while True: 5671 update, key = None, None 5672 if self._match_text_seq("FOR", "UPDATE"): 5673 update = True 5674 elif self._match_text_seq("FOR", "SHARE") or self._match_text_seq( 5675 "LOCK", "IN", "SHARE", "MODE" 5676 ): 5677 update = False 5678 elif self._match_text_seq("FOR", "KEY", "SHARE"): 5679 update, key = False, True 5680 elif self._match_text_seq("FOR", "NO", "KEY", "UPDATE"): 5681 update, key = True, True 5682 else: 5683 break 5684 5685 expressions = None 5686 if self._match_text_seq("OF"): 5687 expressions = self._parse_csv(lambda: self._parse_table(schema=True)) 5688 5689 wait: bool | exp.Expr | None = None 5690 if self._match_text_seq("NOWAIT"): 5691 wait = True 5692 elif self._match_text_seq("WAIT"): 5693 wait = self._parse_primary() 5694 elif self._match_text_seq("SKIP", "LOCKED"): 5695 wait = False 5696 5697 locks.append( 5698 self.expression( 5699 exp.Lock(update=update, expressions=expressions, wait=wait, key=key) 5700 ) 5701 ) 5702 5703 return locks 5704 5705 def parse_set_operation( 5706 self, this: exp.Expr | None, consume_pipe: bool = False 5707 ) -> exp.Expr | None: 5708 start = self._index 5709 _, side_token, kind_token = self._parse_join_parts() 5710 5711 side = side_token.text if side_token else None 5712 kind = kind_token.text if kind_token else None 5713 5714 if not self._match_set(self.SET_OPERATIONS): 5715 self._retreat(start) 5716 return None 5717 5718 token_type = self._prev.token_type 5719 5720 if token_type == TokenType.UNION: 5721 operation: type[exp.SetOperation] = exp.Union 5722 elif token_type == TokenType.EXCEPT: 5723 operation = exp.Except 5724 else: 5725 operation = exp.Intersect 5726 5727 comments = self._prev.comments 5728 5729 if self._match(TokenType.DISTINCT): 5730 distinct: bool | None = True 5731 elif self._match(TokenType.ALL): 5732 distinct = False 5733 else: 5734 distinct = self.dialect.SET_OP_DISTINCT_BY_DEFAULT[operation] 5735 if distinct is None: 5736 self.raise_error(f"Expected DISTINCT or ALL for {operation.__name__}") 5737 5738 by_name = ( 5739 self._match_text_seq("BY", "NAME") 5740 or self._match_text_seq("STRICT", "CORRESPONDING") 5741 or None 5742 ) 5743 if self._match_text_seq("CORRESPONDING"): 5744 by_name = True 5745 if not side and not kind: 5746 kind = "INNER" 5747 5748 on_column_list = None 5749 if by_name and self._match_texts(("ON", "BY")): 5750 on_column_list = self._parse_wrapped_csv(self._parse_column) 5751 5752 expression = self._parse_select( 5753 nested=True, parse_set_operation=False, consume_pipe=consume_pipe 5754 ) 5755 5756 return self.expression( 5757 operation( 5758 this=this, 5759 distinct=distinct, 5760 by_name=by_name, 5761 expression=expression, 5762 side=side, 5763 kind=kind, 5764 on=on_column_list, 5765 ), 5766 comments=comments, 5767 ) 5768 5769 def _parse_set_operations(self, this: exp.Expr | None) -> exp.Expr | None: 5770 while this: 5771 setop = self.parse_set_operation(this) 5772 if not setop: 5773 break 5774 this = setop 5775 5776 if isinstance(this, exp.SetOperation) and self.MODIFIERS_ATTACHED_TO_SET_OP: 5777 expression = this.expression 5778 5779 if expression: 5780 for arg in self.SET_OP_MODIFIERS: 5781 expr = expression.args.get(arg) 5782 if expr: 5783 this.set(arg, expr.pop()) 5784 5785 return this 5786 5787 def _parse_expression(self) -> exp.Expr | None: 5788 return self._parse_alias(self._parse_assignment()) 5789 5790 def _parse_assignment(self) -> exp.Expr | None: 5791 this = self._parse_disjunction() 5792 if not this and self._next.token_type in self.ASSIGNMENT: 5793 # This allows us to parse <non-identifier token> := <expr> 5794 this = exp.column( 5795 t.cast(str, self._advance_any(ignore_reserved=True) and self._prev.text) 5796 ) 5797 5798 while self._match_set(self.ASSIGNMENT): 5799 if isinstance(this, exp.Column) and len(this.parts) == 1: 5800 this = this.this 5801 5802 comments = self._prev_comments 5803 this = self.expression( 5804 self.ASSIGNMENT[self._prev.token_type]( 5805 this=this, expression=self._parse_assignment() 5806 ), 5807 comments=comments, 5808 ) 5809 5810 return this 5811 5812 def _parse_disjunction(self) -> exp.Expr | None: 5813 this = self._parse_conjunction() 5814 while self._match_set(self.DISJUNCTION): 5815 comments = self._prev_comments 5816 this = self.expression( 5817 self.DISJUNCTION[self._prev.token_type]( 5818 this=this, expression=self._parse_conjunction() 5819 ), 5820 comments=comments, 5821 ) 5822 return this 5823 5824 def _parse_conjunction(self) -> exp.Expr | None: 5825 this = self._parse_equality() 5826 while self._match_set(self.CONJUNCTION): 5827 comments = self._prev_comments 5828 this = self.expression( 5829 self.CONJUNCTION[self._prev.token_type]( 5830 this=this, expression=self._parse_equality() 5831 ), 5832 comments=comments, 5833 ) 5834 return this 5835 5836 def _parse_equality(self) -> exp.Expr | None: 5837 this = self._parse_comparison() 5838 while self._match_set(self.EQUALITY): 5839 comments = self._prev_comments 5840 this = self.expression( 5841 self.EQUALITY[self._prev.token_type]( 5842 this=this, expression=self._parse_comparison() 5843 ), 5844 comments=comments, 5845 ) 5846 return this 5847 5848 def _parse_comparison(self) -> exp.Expr | None: 5849 this = self._parse_range() 5850 while self._match_set(self.COMPARISON): 5851 comments = self._prev_comments 5852 this = self.expression( 5853 self.COMPARISON[self._prev.token_type](this=this, expression=self._parse_range()), 5854 comments=comments, 5855 ) 5856 return this 5857 5858 def _parse_range(self, this: exp.Expr | None = None) -> exp.Expr | None: 5859 this = this or self._parse_bitwise() 5860 negate = self._match(TokenType.NOT) 5861 5862 if self._match_set(self.RANGE_PARSERS): 5863 expression = self.RANGE_PARSERS[self._prev.token_type](self, this) 5864 if not expression: 5865 return this 5866 5867 this = expression 5868 elif self._match(TokenType.ISNULL) or (negate and self._match(TokenType.NULL)): 5869 this = self.expression(exp.Is(this=this, expression=exp.Null())) 5870 5871 # Postgres supports ISNULL and NOTNULL for conditions. 5872 # https://blog.andreiavram.ro/postgresql-null-composite-type/ 5873 if self._match(TokenType.NOTNULL): 5874 this = self.expression(exp.Is(this=this, expression=exp.Null())) 5875 this = self.expression(exp.Not(this=this)) 5876 5877 if negate: 5878 this = self._negate_range(this) 5879 5880 if self._match(TokenType.IS): 5881 this = self._parse_is(this) 5882 5883 return this 5884 5885 def _negate_range(self, this: exp.Expr | None = None) -> exp.Expr | None: 5886 if not this: 5887 return this 5888 5889 expression = this.this if isinstance(this, exp.Escape) else this 5890 if isinstance(expression, (exp.Like, exp.ILike)): 5891 expression.set("negate", True) 5892 return this 5893 5894 return self.expression(exp.Not(this=this)) 5895 5896 def _parse_is(self, this: exp.Expr | None) -> exp.Expr | None: 5897 index = self._index - 1 5898 negate = self._match(TokenType.NOT) 5899 5900 if self._match_text_seq("DISTINCT", "FROM"): 5901 klass = exp.NullSafeEQ if negate else exp.NullSafeNEQ 5902 return self.expression(klass(this=this, expression=self._parse_bitwise())) 5903 5904 if self._match(TokenType.JSON): 5905 kind = self._match_texts(self.IS_JSON_PREDICATE_KIND) and self._prev.text.upper() 5906 5907 if self._match_text_seq("WITH"): 5908 _with = True 5909 elif self._match_text_seq("WITHOUT"): 5910 _with = False 5911 else: 5912 _with = None 5913 5914 unique = self._match(TokenType.UNIQUE) 5915 self._match_text_seq("KEYS") 5916 expression: exp.Expr | None = self.expression( 5917 exp.JSON(this=kind, with_=_with, unique=unique) 5918 ) 5919 else: 5920 expression = self._parse_null() or self._parse_bitwise() 5921 if not expression: 5922 self._retreat(index) 5923 return None 5924 5925 this = self.expression(exp.Is(this=this, expression=expression)) 5926 this = self.expression(exp.Not(this=this)) if negate else this 5927 return self._parse_column_ops(this) 5928 5929 def _parse_in(self, this: exp.Expr | None, alias: bool = False) -> exp.In: 5930 unnest = self._parse_unnest(with_alias=False) 5931 if unnest: 5932 this = self.expression(exp.In(this=this, unnest=unnest)) 5933 elif self._match_set((TokenType.L_PAREN, TokenType.L_BRACKET)): 5934 matched_l_paren = self._prev.token_type == TokenType.L_PAREN 5935 expressions = self._parse_csv(lambda: self._parse_select_or_expression(alias=alias)) 5936 5937 if len(expressions) == 1 and isinstance(query := expressions[0], exp.Query): 5938 this = self.expression( 5939 exp.In(this=this, query=self._parse_query_modifiers(query).subquery(copy=False)) 5940 ) 5941 else: 5942 this = self.expression(exp.In(this=this, expressions=expressions)) 5943 5944 if matched_l_paren: 5945 self._match_r_paren(this) 5946 elif not self._match(TokenType.R_BRACKET, expression=this): 5947 self.raise_error("Expecting ]") 5948 else: 5949 this = self.expression(exp.In(this=this, field=self._parse_column())) 5950 5951 return this 5952 5953 def _parse_between(self, this: exp.Expr | None) -> exp.Between: 5954 symmetric = None 5955 if self._match_text_seq("SYMMETRIC"): 5956 symmetric = True 5957 elif self._match_text_seq("ASYMMETRIC"): 5958 symmetric = False 5959 5960 low = self._parse_bitwise() 5961 self._match(TokenType.AND) 5962 high = self._parse_bitwise() 5963 5964 return self.expression(exp.Between(this=this, low=low, high=high, symmetric=symmetric)) 5965 5966 def _parse_escape(self, this: exp.Expr | None) -> exp.Expr | None: 5967 if not self._match(TokenType.ESCAPE): 5968 return this 5969 return self.expression( 5970 exp.Escape(this=this, expression=self._parse_string() or self._parse_null()) 5971 ) 5972 5973 def _parse_interval_span(self, this: exp.Expr) -> exp.Interval: 5974 # handle day-time format interval span with omitted units: 5975 # INTERVAL '<number days> hh[:][mm[:ss[.ff]]]' <maybe `unit TO unit`> 5976 interval_span_units_omitted = None 5977 if ( 5978 this 5979 and this.is_string 5980 and self.SUPPORTS_OMITTED_INTERVAL_SPAN_UNIT 5981 and exp.INTERVAL_DAY_TIME_RE.match(this.name) 5982 ): 5983 index = self._index 5984 5985 # Var "TO" Var 5986 first_unit = self._parse_var(any_token=True, upper=True) 5987 second_unit = None 5988 if first_unit and self._match_text_seq("TO"): 5989 second_unit = self._parse_var(any_token=True, upper=True) 5990 5991 interval_span_units_omitted = not (first_unit and second_unit) 5992 5993 self._retreat(index) 5994 5995 if interval_span_units_omitted: 5996 unit = None 5997 else: 5998 unit = self._parse_function() 5999 if not unit and ( 6000 self._curr.token_type == TokenType.VAR 6001 or self._curr.text.upper() in self.dialect.VALID_INTERVAL_UNITS 6002 ): 6003 unit = self._parse_var(any_token=True, upper=True) 6004 6005 # Most dialects support, e.g., the form INTERVAL '5' day, thus we try to parse 6006 # each INTERVAL expression into this canonical form so it's easy to transpile 6007 if this and this.is_number: 6008 this = exp.Literal.string(this.to_py()) 6009 elif this and this.is_string: 6010 parts = exp.INTERVAL_STRING_RE.findall(this.name) 6011 if parts and unit: 6012 # Unconsume the eagerly-parsed unit, since the real unit was part of the string 6013 unit = None 6014 self._retreat(self._index - 1) 6015 6016 if len(parts) == 1: 6017 this = exp.Literal.string(parts[0][0]) 6018 unit = self.expression(exp.Var(this=parts[0][1].upper())) 6019 6020 if self.INTERVAL_SPANS and self._match_text_seq("TO"): 6021 unit = self.expression( 6022 exp.IntervalSpan( 6023 this=unit, 6024 expression=self._parse_function() 6025 or self._parse_var(any_token=True, upper=True), 6026 ) 6027 ) 6028 6029 return self.expression(exp.Interval(this=this, unit=unit)) 6030 6031 def _parse_interval(self, require_interval: bool = True) -> exp.Add | exp.Interval | None: 6032 index = self._index 6033 6034 if not self._match(TokenType.INTERVAL) and require_interval: 6035 return None 6036 6037 if self._match(TokenType.STRING, advance=False): 6038 this = self._parse_primary() 6039 else: 6040 this = self._parse_term() 6041 6042 if not this or ( 6043 isinstance(this, exp.Column) 6044 and not this.table 6045 and not this.this.quoted 6046 and self._curr 6047 and self._curr.text.upper() not in self.dialect.VALID_INTERVAL_UNITS 6048 ): 6049 self._retreat(index) 6050 return None 6051 6052 interval = self._parse_interval_span(this) 6053 6054 index = self._index 6055 self._match(TokenType.PLUS) 6056 6057 # Convert INTERVAL 'val_1' unit_1 [+] ... [+] 'val_n' unit_n into a sum of intervals 6058 if self._match_set((TokenType.STRING, TokenType.NUMBER), advance=False): 6059 return self.expression(exp.Add(this=interval, expression=self._parse_interval(False))) 6060 6061 self._retreat(index) 6062 return interval 6063 6064 def _parse_bitwise(self) -> exp.Expr | None: 6065 this = self._parse_term() 6066 6067 while True: 6068 if self._match_set(self.BITWISE): 6069 this = self.expression( 6070 self.BITWISE[self._prev.token_type](this=this, expression=self._parse_term()) 6071 ) 6072 elif self.dialect.DPIPE_IS_STRING_CONCAT and self._match(TokenType.DPIPE): 6073 this = self.expression( 6074 exp.DPipe( 6075 this=this, 6076 expression=self._parse_term(), 6077 safe=not self.dialect.STRICT_STRING_CONCAT, 6078 ) 6079 ) 6080 elif self._match(TokenType.DQMARK): 6081 this = self.expression( 6082 exp.Coalesce(this=this, expressions=ensure_list(self._parse_term())) 6083 ) 6084 elif self._match_pair(TokenType.LT, TokenType.LT): 6085 this = self.expression( 6086 exp.BitwiseLeftShift(this=this, expression=self._parse_term()) 6087 ) 6088 elif self._match_pair(TokenType.GT, TokenType.GT): 6089 this = self.expression( 6090 exp.BitwiseRightShift(this=this, expression=self._parse_term()) 6091 ) 6092 else: 6093 break 6094 6095 return this 6096 6097 def _parse_term(self) -> exp.Expr | None: 6098 this = self._parse_factor() 6099 6100 while self._match_set(self.TERM): 6101 klass = self.TERM[self._prev.token_type] 6102 comments = self._prev_comments 6103 expression = self._parse_factor() 6104 6105 this = self.expression(klass(this=this, expression=expression), comments=comments) 6106 6107 if isinstance(this, exp.Collate): 6108 expr = this.expression 6109 6110 # Preserve collations such as pg_catalog."default" (Postgres) as columns, otherwise 6111 # fallback to Identifier / Var 6112 if isinstance(expr, exp.Column) and len(expr.parts) == 1: 6113 ident = expr.this 6114 if isinstance(ident, exp.Identifier): 6115 this.set("expression", ident if ident.quoted else exp.var(ident.name)) 6116 6117 return this 6118 6119 def _parse_factor(self) -> exp.Expr | None: 6120 parse_method = self._parse_exponent if self.EXPONENT else self._parse_unary 6121 this = self._parse_at_time_zone(parse_method()) 6122 6123 while self._match_set(self.FACTOR): 6124 klass = self.FACTOR[self._prev.token_type] 6125 comments = self._prev_comments 6126 expression = parse_method() 6127 6128 if not expression and klass is exp.IntDiv and self._prev.text.isalpha(): 6129 self._retreat(self._index - 1) 6130 return this 6131 6132 this = self.expression(klass(this=this, expression=expression), comments=comments) 6133 6134 if isinstance(this, exp.Div): 6135 this.set("typed", self.dialect.TYPED_DIVISION) 6136 this.set("safe", self.dialect.SAFE_DIVISION) 6137 6138 return this 6139 6140 def _parse_exponent(self) -> exp.Expr | None: 6141 this = self._parse_unary() 6142 while self._match_set(self.EXPONENT): 6143 comments = self._prev_comments 6144 this = self.expression( 6145 self.EXPONENT[self._prev.token_type](this=this, expression=self._parse_unary()), 6146 comments=comments, 6147 ) 6148 return this 6149 6150 def _parse_unary(self) -> exp.Expr | None: 6151 if self._match_set(self.UNARY_PARSERS): 6152 return self.UNARY_PARSERS[self._prev.token_type](self) 6153 return self._parse_type() 6154 6155 def _parse_type( 6156 self, parse_interval: bool = True, fallback_to_identifier: bool = False 6157 ) -> exp.Expr | None: 6158 if not fallback_to_identifier and (atom := self._parse_atom()) is not None: 6159 return atom 6160 6161 if interval := parse_interval and self._parse_interval(): 6162 return self._parse_column_ops(interval) 6163 6164 index = self._index 6165 data_type = self._parse_types(check_func=True, allow_identifiers=False) 6166 6167 # parse_types() returns a Cast if we parsed BQ's inline constructor <type>(<values>) e.g. 6168 # STRUCT<a INT, b STRING>(1, 'foo'), which is canonicalized to CAST(<values> AS <type>) 6169 if isinstance(data_type, exp.Cast): 6170 # This constructor can contain ops directly after it, for instance struct unnesting: 6171 # STRUCT<a INT, b STRING>(1, 'foo').* --> CAST(STRUCT(1, 'foo') AS STRUCT<a iNT, b STRING).* 6172 return self._parse_column_ops(data_type) 6173 6174 if data_type: 6175 index2 = self._index 6176 this = self._parse_primary() 6177 6178 if isinstance(this, exp.Literal): 6179 literal = this.name 6180 this = self._parse_column_ops(this) 6181 6182 parser = self.TYPE_LITERAL_PARSERS.get(data_type.this) 6183 if parser: 6184 return parser(self, this, data_type) 6185 6186 if ( 6187 self.ZONE_AWARE_TIMESTAMP_CONSTRUCTOR 6188 and data_type.is_type(exp.DType.TIMESTAMP) 6189 and TIME_ZONE_RE.search(literal) 6190 ): 6191 data_type = exp.DType.TIMESTAMPTZ.into_expr() 6192 6193 return self.expression(exp.Cast(this=this, to=data_type)) 6194 6195 # The expressions arg gets set by the parser when we have something like DECIMAL(38, 0) 6196 # in the input SQL. In that case, we'll produce these tokens: DECIMAL ( 38 , 0 ) 6197 # 6198 # If the index difference here is greater than 1, that means the parser itself must have 6199 # consumed additional tokens such as the DECIMAL scale and precision in the above example. 6200 # 6201 # If it's not greater than 1, then it must be 1, because we've consumed at least the type 6202 # keyword, meaning that the expressions arg of the DataType must have gotten set by a 6203 # callable in the TYPE_CONVERTERS mapping. For example, Snowflake converts DECIMAL to 6204 # DECIMAL(38, 0)) in order to facilitate the data type's transpilation. 6205 # 6206 # In these cases, we don't really want to return the converted type, but instead retreat 6207 # and try to parse a Column or Identifier in the section below. 6208 if data_type.expressions and index2 - index > 1: 6209 self._retreat(index2) 6210 return self._parse_column_ops(data_type) 6211 6212 self._retreat(index) 6213 6214 if fallback_to_identifier: 6215 return self._parse_id_var() 6216 6217 return self._parse_column() 6218 6219 def _parse_type_size(self) -> exp.DataTypeParam | None: 6220 this = self._parse_type() 6221 if not this: 6222 return None 6223 6224 if isinstance(this, exp.Column) and not this.table: 6225 this = exp.var(this.name.upper()) 6226 6227 return self.expression( 6228 exp.DataTypeParam(this=this, expression=self._parse_var(any_token=True)) 6229 ) 6230 6231 def _parse_user_defined_type(self, identifier: exp.Identifier) -> exp.Expr | None: 6232 type_name = identifier.name 6233 6234 while self._match(TokenType.DOT): 6235 type_name = f"{type_name}.{self._advance_any() and self._prev.text}" 6236 6237 return exp.DataType.from_str(type_name, dialect=self.dialect, udt=True) 6238 6239 def _parse_types( 6240 self, 6241 check_func: bool = False, 6242 schema: bool = False, 6243 allow_identifiers: bool = True, 6244 with_collation: bool = False, 6245 ) -> exp.Expr | None: 6246 index = self._index 6247 this: exp.Expr | None = None 6248 6249 if self._match_set(self.TYPE_TOKENS): 6250 type_token = self._prev.token_type 6251 else: 6252 type_token = None 6253 identifier = allow_identifiers and self._parse_id_var( 6254 any_token=False, tokens=(TokenType.VAR,) 6255 ) 6256 if isinstance(identifier, exp.Identifier): 6257 try: 6258 tokens = self.dialect.tokenize(identifier.name) 6259 except TokenError: 6260 tokens = None 6261 6262 if tokens and (type_token := tokens[0].token_type) in self.TYPE_TOKENS: 6263 if len(tokens) > 1: 6264 return exp.DataType.from_str(identifier.name, dialect=self.dialect) 6265 elif self.dialect.SUPPORTS_USER_DEFINED_TYPES: 6266 this = self._parse_user_defined_type(identifier) 6267 else: 6268 self._retreat(self._index - 1) 6269 return None 6270 else: 6271 return None 6272 6273 if type_token == TokenType.PSEUDO_TYPE: 6274 return self.expression(exp.PseudoType(this=self._prev.text.upper())) 6275 6276 if type_token == TokenType.OBJECT_IDENTIFIER: 6277 return self.expression(exp.ObjectIdentifier(this=self._prev.text.upper())) 6278 6279 # https://materialize.com/docs/sql/types/map/ 6280 if type_token == TokenType.MAP and self._match(TokenType.L_BRACKET): 6281 key_type = self._parse_types( 6282 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6283 ) 6284 if not self._match(TokenType.FARROW): 6285 self._retreat(index) 6286 return None 6287 6288 value_type = self._parse_types( 6289 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6290 ) 6291 if not self._match(TokenType.R_BRACKET): 6292 self._retreat(index) 6293 return None 6294 6295 return exp.DataType( 6296 this=exp.DType.MAP, 6297 expressions=[key_type, value_type], 6298 nested=True, 6299 ) 6300 6301 nested = type_token in self.NESTED_TYPE_TOKENS 6302 is_struct = type_token in self.STRUCT_TYPE_TOKENS 6303 is_aggregate = type_token in self.AGGREGATE_TYPE_TOKENS 6304 expressions = None 6305 maybe_func = False 6306 6307 if self._match(TokenType.L_PAREN): 6308 if is_struct: 6309 expressions = self._parse_csv(lambda: self._parse_struct_types(type_required=True)) 6310 elif nested: 6311 expressions = self._parse_csv( 6312 lambda: self._parse_types( 6313 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6314 ) 6315 ) 6316 if type_token == TokenType.NULLABLE and len(expressions) == 1: 6317 this = expressions[0] 6318 this.set("nullable", True) 6319 self._match_r_paren() 6320 return this 6321 elif type_token in self.ENUM_TYPE_TOKENS: 6322 expressions = self._parse_csv(self._parse_equality) 6323 elif type_token == TokenType.JSON: 6324 # ClickHouse JSON type supports arguments: JSON(col Type, SKIP col, param=value) 6325 # https://clickhouse.com/docs/sql-reference/data-types/newjson 6326 expressions = self._parse_csv(self._parse_json_type_arg) 6327 elif is_aggregate: 6328 func_or_ident = self._parse_function(anonymous=True) or self._parse_id_var( 6329 any_token=False, tokens=(TokenType.VAR, TokenType.ANY) 6330 ) 6331 if not func_or_ident: 6332 return None 6333 expressions = [func_or_ident] 6334 if self._match(TokenType.COMMA): 6335 expressions.extend( 6336 self._parse_csv( 6337 lambda: self._parse_types( 6338 check_func=check_func, 6339 schema=schema, 6340 allow_identifiers=allow_identifiers, 6341 ) 6342 ) 6343 ) 6344 else: 6345 expressions = self._parse_csv(self._parse_type_size) 6346 6347 # https://docs.snowflake.com/en/sql-reference/data-types-vector 6348 if type_token == TokenType.VECTOR and len(expressions) == 2: 6349 expressions = self._parse_vector_expressions(expressions) 6350 6351 if not self._match(TokenType.R_PAREN): 6352 self._retreat(index) 6353 return None 6354 6355 maybe_func = True 6356 6357 values: list[exp.Expr] | None = None 6358 6359 if nested and self._match(TokenType.LT): 6360 if is_struct: 6361 expressions = self._parse_csv(lambda: self._parse_struct_types(type_required=True)) 6362 else: 6363 expressions = self._parse_csv( 6364 lambda: self._parse_types( 6365 check_func=check_func, 6366 schema=schema, 6367 allow_identifiers=allow_identifiers, 6368 with_collation=True, 6369 ) 6370 ) 6371 6372 if not self._match(TokenType.GT): 6373 self.raise_error("Expecting >") 6374 6375 if self._match_set((TokenType.L_BRACKET, TokenType.L_PAREN)): 6376 values = self._parse_csv(self._parse_disjunction) 6377 if not values and is_struct: 6378 values = None 6379 self._retreat(self._index - 1) 6380 else: 6381 self._match_set((TokenType.R_BRACKET, TokenType.R_PAREN)) 6382 6383 if type_token in self.TIMESTAMPS: 6384 if self._match_text_seq("WITH", "TIME", "ZONE"): 6385 maybe_func = False 6386 tz_type = exp.DType.TIMETZ if type_token in self.TIMES else exp.DType.TIMESTAMPTZ 6387 this = exp.DataType(this=tz_type, expressions=expressions) 6388 elif self._match_text_seq("WITH", "LOCAL", "TIME", "ZONE"): 6389 maybe_func = False 6390 this = exp.DataType(this=exp.DType.TIMESTAMPLTZ, expressions=expressions) 6391 elif self._match_text_seq("WITHOUT", "TIME", "ZONE"): 6392 maybe_func = False 6393 elif type_token == TokenType.INTERVAL: 6394 if self._curr.text.upper() in self.dialect.VALID_INTERVAL_UNITS: 6395 unit = self._parse_var(upper=True) 6396 if self._match_text_seq("TO"): 6397 unit = exp.IntervalSpan(this=unit, expression=self._parse_var(upper=True)) 6398 6399 this = self.expression(exp.DataType(this=self.expression(exp.Interval(unit=unit)))) 6400 else: 6401 this = self.expression(exp.DataType(this=exp.DType.INTERVAL)) 6402 elif type_token == TokenType.VOID: 6403 this = exp.DataType(this=exp.DType.NULL) 6404 6405 if maybe_func and check_func: 6406 index2 = self._index 6407 peek = self._parse_string() 6408 6409 if not peek: 6410 self._retreat(index) 6411 return None 6412 6413 self._retreat(index2) 6414 6415 if not this: 6416 assert type_token is not None 6417 if self._match_text_seq("UNSIGNED"): 6418 unsigned_type_token = self.SIGNED_TO_UNSIGNED_TYPE_TOKEN.get(type_token) 6419 if not unsigned_type_token: 6420 self.raise_error(f"Cannot convert {type_token.name} to unsigned.") 6421 6422 type_token = unsigned_type_token or type_token 6423 6424 # NULLABLE without parentheses can be a column (Presto/Trino) 6425 if type_token == TokenType.NULLABLE and not expressions: 6426 self._retreat(index) 6427 return None 6428 6429 this = exp.DataType( 6430 this=exp.DType[type_token.name], 6431 expressions=expressions, 6432 nested=nested, 6433 ) 6434 6435 # Empty arrays/structs are allowed 6436 if values is not None: 6437 cls = exp.Struct if is_struct else exp.Array 6438 this = exp.cast(cls(expressions=values), this, copy=False) 6439 6440 elif expressions: 6441 this.set("expressions", expressions) 6442 6443 # https://materialize.com/docs/sql/types/list/#type-name 6444 while self._match(TokenType.LIST): 6445 this = exp.DataType(this=exp.DType.LIST, expressions=[this], nested=True) 6446 6447 index = self._index 6448 6449 # Postgres supports the INT ARRAY[3] syntax as a synonym for INT[3] 6450 matched_array = self._match(TokenType.ARRAY) 6451 6452 while self._curr: 6453 datatype_token = self._prev.token_type 6454 matched_l_bracket = self._match(TokenType.L_BRACKET) 6455 6456 if (not matched_l_bracket and not matched_array) or ( 6457 datatype_token == TokenType.ARRAY and self._match(TokenType.R_BRACKET) 6458 ): 6459 # Postgres allows casting empty arrays such as ARRAY[]::INT[], 6460 # not to be confused with the fixed size array parsing 6461 break 6462 6463 matched_array = False 6464 values = self._parse_csv(self._parse_disjunction) or None 6465 if ( 6466 values 6467 and not schema 6468 and ( 6469 not self.dialect.SUPPORTS_FIXED_SIZE_ARRAYS 6470 or datatype_token == TokenType.ARRAY 6471 or not self._match(TokenType.R_BRACKET, advance=False) 6472 ) 6473 ): 6474 # Retreating here means that we should not parse the following values as part of the data type, e.g. in DuckDB 6475 # ARRAY[1] should retreat and instead be parsed into exp.Array in contrast to INT[x][y] which denotes a fixed-size array data type 6476 self._retreat(index) 6477 break 6478 6479 this = exp.DataType( 6480 this=exp.DType.ARRAY, expressions=[this], values=values, nested=True 6481 ) 6482 self._match(TokenType.R_BRACKET) 6483 6484 if self.TYPE_CONVERTERS and isinstance(this.this, exp.DType): 6485 converter = self.TYPE_CONVERTERS.get(this.this) 6486 if converter: 6487 this = converter(t.cast(exp.DataType, this)) 6488 6489 if with_collation and isinstance(this, exp.DataType) and self._match(TokenType.COLLATE): 6490 this.set("collate", self._parse_identifier() or self._parse_column()) 6491 6492 return this 6493 6494 def _parse_json_type_arg(self) -> exp.Expr | None: 6495 """Parse a single argument to ClickHouse's JSON type.""" 6496 6497 # SKIP col or SKIP REGEXP 'pattern' 6498 if self._match_text_seq("SKIP"): 6499 regexp = self._match(TokenType.RLIKE) 6500 arg = self._parse_column() 6501 if isinstance(arg, exp.Column): 6502 arg = arg.to_dot() 6503 return self.expression(exp.SkipJSONColumn(regexp=regexp, expression=arg)) 6504 6505 param_or_col = self._parse_column() 6506 if not isinstance(param_or_col, exp.Column): 6507 return None 6508 6509 # Parameter: name=value (e.g., max_dynamic_paths=2) 6510 if len(param_or_col.parts) == 1 and self._match(TokenType.EQ): 6511 param = param_or_col.name 6512 value = self._parse_primary() 6513 return self.expression(exp.EQ(this=exp.var(param), expression=value)) 6514 6515 # Column type hint: col_name Type 6516 col = param_or_col.to_dot() 6517 kind = self._parse_types(check_func=False, allow_identifiers=False) 6518 return self.expression(exp.ColumnDef(this=col, kind=kind)) 6519 6520 def _parse_vector_expressions(self, expressions: list[exp.Expr]) -> list[exp.Expr]: 6521 return [exp.DataType.from_str(expressions[0].name, dialect=self.dialect), *expressions[1:]] 6522 6523 def _parse_struct_types(self, type_required: bool = False) -> exp.Expr | None: 6524 index = self._index 6525 6526 if ( 6527 self._curr 6528 and self._next 6529 and self._curr.token_type in self.TYPE_TOKENS 6530 and self._next.token_type in self.TYPE_TOKENS 6531 ): 6532 # Takes care of special cases like `STRUCT<list ARRAY<...>>` where the identifier is also a 6533 # type token. Without this, the list will be parsed as a type and we'll eventually crash 6534 this = self._parse_id_var() 6535 else: 6536 this = ( 6537 self._parse_type(parse_interval=False, fallback_to_identifier=True) 6538 or self._parse_id_var() 6539 ) 6540 6541 self._match(TokenType.COLON) 6542 6543 if ( 6544 type_required 6545 and not isinstance(this, exp.DataType) 6546 and not self._match_set(self.TYPE_TOKENS, advance=False) 6547 ): 6548 self._retreat(index) 6549 return self._parse_types() 6550 6551 return self._parse_column_def(this) 6552 6553 def _parse_at_time_zone(self, this: exp.Expr | None) -> exp.Expr | None: 6554 if not self._match_text_seq("AT", "TIME", "ZONE"): 6555 return this 6556 return self._parse_at_time_zone( 6557 self.expression(exp.AtTimeZone(this=this, zone=self._parse_unary())) 6558 ) 6559 6560 def _parse_atom(self) -> exp.Expr | None: 6561 if ( 6562 self._curr.token_type in self.IDENTIFIER_TOKENS 6563 and (column := self._parse_column()) is not None 6564 ): 6565 return column 6566 6567 token = self._curr 6568 token_type = token.token_type 6569 6570 if not (primary_parser := self.PRIMARY_PARSERS.get(token_type)): 6571 return None 6572 6573 next_type = self._next.token_type 6574 6575 if ( 6576 next_type in self.COLUMN_OPERATORS 6577 or next_type in self.COLUMN_POSTFIX_TOKENS 6578 or (token_type == TokenType.STRING and next_type == TokenType.STRING) 6579 ): 6580 return None 6581 6582 self._advance() 6583 return primary_parser(self, token) 6584 6585 def _parse_column(self) -> exp.Expr | None: 6586 column: exp.Expr | None = self._parse_column_parts_fast() 6587 if column is None: 6588 this = self._parse_column_reference() 6589 if not this: 6590 this = self._parse_bracket(this) 6591 column = self._parse_column_ops(this) if this else this 6592 6593 if column: 6594 if self.dialect.SUPPORTS_COLUMN_JOIN_MARKS: 6595 column.set("join_mark", self._match(TokenType.JOIN_MARKER)) 6596 if self.COLON_IS_VARIANT_EXTRACT: 6597 column = self._parse_colon_as_variant_extract(column) 6598 6599 return column 6600 6601 def _parse_column_parts_fast(self) -> exp.Column | exp.Dot | None: 6602 """Fast path for simple column and dot references (a, a.b, ...). 6603 6604 Greedily consumes VAR/IDENTIFIER tokens separated by DOTs, then checks 6605 that nothing complex follows. If it does, retreats and returns None so 6606 the slow path can handle it. For >4 parts, wraps in exp.Dot nodes. 6607 """ 6608 index = self._index 6609 parts: list[exp.Identifier] | None = None 6610 all_comments: list[str] | None = None 6611 6612 while self._match_set(self.IDENTIFIER_TOKENS): 6613 token = self._prev 6614 comments = self._prev_comments 6615 6616 if parts is None and token.text.upper() in self.NO_PAREN_FUNCTION_PARSERS: 6617 self._retreat(index) 6618 return None 6619 6620 has_dot = self._match(TokenType.DOT) 6621 curr_tt = self._curr.token_type 6622 6623 if not has_dot: 6624 if curr_tt in self.COLUMN_OPERATORS or curr_tt in self.COLUMN_POSTFIX_TOKENS: 6625 self._retreat(index) 6626 return None 6627 elif curr_tt not in self.IDENTIFIER_TOKENS: 6628 self._retreat(index) 6629 return None 6630 6631 if parts is None: 6632 parts = [] 6633 6634 if comments: 6635 if all_comments is None: 6636 all_comments = [] 6637 all_comments.extend(comments) 6638 self._prev_comments = [] 6639 6640 parts.append( 6641 self.expression( 6642 exp.Identifier( 6643 this=token.text, quoted=token.token_type == TokenType.IDENTIFIER 6644 ), 6645 token, 6646 ) 6647 ) 6648 6649 if not has_dot: 6650 break 6651 6652 if parts is None: 6653 return None 6654 6655 n = len(parts) 6656 6657 if n == 1: 6658 column: exp.Column | exp.Dot = exp.Column(this=parts[0]) 6659 elif n == 2: 6660 column = exp.Column(this=parts[1], table=parts[0]) 6661 elif n == 3: 6662 column = exp.Column(this=parts[2], table=parts[1], db=parts[0]) 6663 else: 6664 column = exp.Column(this=parts[3], table=parts[2], db=parts[1], catalog=parts[0]) 6665 6666 for i in range(4, n): 6667 column = exp.Dot(this=column, expression=parts[i]) 6668 6669 if all_comments: 6670 column.add_comments(all_comments) 6671 6672 return column 6673 6674 def _parse_column_reference(self) -> exp.Expr | None: 6675 this = self._parse_field() 6676 if ( 6677 not this 6678 and self._match(TokenType.VALUES, advance=False) 6679 and self.VALUES_FOLLOWED_BY_PAREN 6680 and (not self._next or self._next.token_type != TokenType.L_PAREN) 6681 ): 6682 this = self._parse_id_var() 6683 6684 if isinstance(this, exp.Identifier): 6685 # We bubble up comments from the Identifier to the Column 6686 this = self.expression(exp.Column(this=this), comments=this.pop_comments()) 6687 6688 return this 6689 6690 def _build_json_extract( 6691 self, 6692 this: exp.Expr | None, 6693 path_parts: list[exp.JSONPathPart], 6694 ) -> tuple[exp.Expr | None, list[exp.JSONPathPart]]: 6695 if len(path_parts) > 1: 6696 this = self.expression( 6697 exp.JSONExtract( 6698 this=this, 6699 expression=exp.JSONPath(expressions=path_parts), 6700 variant_extract=True, 6701 requires_json=self.JSON_EXTRACT_REQUIRES_JSON_EXPRESSION, 6702 ) 6703 ) 6704 path_parts = [exp.JSONPathRoot()] 6705 6706 return this, path_parts 6707 6708 def _parse_colon_as_variant_extract(self, this: exp.Expr | None) -> exp.Expr | None: 6709 path_parts: list[exp.JSONPathPart] = [exp.JSONPathRoot()] 6710 6711 while self._match(TokenType.COLON): 6712 if not self.COLON_CHAIN_IS_SINGLE_EXTRACT: 6713 this, path_parts = self._build_json_extract(this, path_parts) 6714 6715 key = self._parse_id_var(any_token=True, tokens=(TokenType.SELECT,)) 6716 6717 if key: 6718 quoted = isinstance(key, exp.Identifier) and key.quoted 6719 path_parts.append(exp.JSONPathKey(this=key.name, quoted=quoted)) 6720 6721 while True: 6722 if self._match(TokenType.DOT): 6723 next_key = self._parse_id_var(any_token=True, tokens=(TokenType.SELECT,)) 6724 6725 if next_key: 6726 quoted = isinstance(next_key, exp.Identifier) and next_key.quoted 6727 path_parts.append(exp.JSONPathKey(this=next_key.name, quoted=quoted)) 6728 elif self._match(TokenType.L_BRACKET): 6729 bracket_expr = self._parse_bracket_key_value() 6730 6731 if not self._match(TokenType.R_BRACKET): 6732 self.raise_error("Expected ]") 6733 6734 if bracket_expr: 6735 if bracket_expr.is_string: 6736 path_parts.append(exp.JSONPathKey(this=bracket_expr.name, quoted=True)) 6737 elif bracket_expr.is_star: 6738 path_parts.append(exp.JSONPathSubscript(this=exp.JSONPathWildcard())) 6739 elif bracket_expr.is_number: 6740 path_parts.append(exp.JSONPathSubscript(this=bracket_expr.to_py())) 6741 else: 6742 this, path_parts = self._build_json_extract(this, path_parts) 6743 6744 this = self.expression( 6745 exp.Bracket( 6746 this=this, expressions=[bracket_expr], json_access=True 6747 ), 6748 ) 6749 6750 elif self._match(TokenType.DCOLON): 6751 this, path_parts = self._build_json_extract(this, path_parts) 6752 6753 cast_type = self._parse_types() 6754 if cast_type: 6755 this = self.expression(exp.Cast(this=this, to=cast_type)) 6756 else: 6757 self.raise_error("Expected type after '::'") 6758 else: 6759 break 6760 6761 this, _ = self._build_json_extract(this, path_parts) 6762 6763 return this 6764 6765 def _parse_dcolon(self) -> exp.Expr | None: 6766 return self._parse_types() 6767 6768 def _parse_column_ops(self, this: exp.Expr | None) -> exp.Expr | None: 6769 while self._curr.token_type in self.BRACKETS: 6770 this = self._parse_bracket(this) 6771 6772 column_operators = self.COLUMN_OPERATORS 6773 cast_column_operators = self.CAST_COLUMN_OPERATORS 6774 while self._curr: 6775 op_token = self._curr.token_type 6776 6777 if op_token not in column_operators: 6778 break 6779 op = column_operators[op_token] 6780 self._advance() 6781 6782 if op_token in cast_column_operators: 6783 field = self._parse_dcolon() 6784 if not field: 6785 self.raise_error("Expected type") 6786 elif op and self._curr: 6787 field = self._parse_column_reference() or self._parse_bitwise() 6788 if isinstance(field, exp.Column) and self._match(TokenType.DOT, advance=False): 6789 field = self._parse_column_ops(field) 6790 else: 6791 field = self._parse_field(any_token=True, anonymous_func=True) 6792 6793 # Function calls can be qualified, e.g., x.y.FOO() 6794 # This converts the final AST to a series of Dots leading to the function call 6795 # https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-reference#function_call_rules 6796 if isinstance(field, (exp.Func, exp.Window)) and this: 6797 this = this.transform( 6798 lambda n: n.to_dot(include_dots=False) if isinstance(n, exp.Column) else n 6799 ) 6800 6801 if op: 6802 this = op(self, this, field) 6803 elif isinstance(this, exp.Column) and not this.args.get("catalog"): 6804 this = self.expression( 6805 exp.Column( 6806 this=field, 6807 table=this.this, 6808 db=this.args.get("table"), 6809 catalog=this.args.get("db"), 6810 ), 6811 comments=this.comments, 6812 ) 6813 elif isinstance(field, exp.Window): 6814 # Move the exp.Dot's to the window's function 6815 window_func = self.expression(exp.Dot(this=this, expression=field.this)) 6816 field.set("this", window_func) 6817 this = field 6818 else: 6819 this = self.expression(exp.Dot(this=this, expression=field)) 6820 6821 if field and field.comments: 6822 t.cast(exp.Expr, this).add_comments(field.pop_comments()) 6823 6824 this = self._parse_bracket(this) 6825 6826 return this 6827 6828 def _parse_paren(self) -> exp.Expr | None: 6829 if not self._match(TokenType.L_PAREN): 6830 return None 6831 6832 comments = self._prev_comments 6833 query = self._parse_select() 6834 6835 if query: 6836 expressions = [query] 6837 else: 6838 expressions = self._parse_expressions() 6839 6840 this = seq_get(expressions, 0) 6841 6842 if not this and self._match(TokenType.R_PAREN, advance=False): 6843 this = self.expression(exp.Tuple()) 6844 elif isinstance(this, exp.UNWRAPPED_QUERIES): 6845 this = self._parse_subquery(this=this, parse_alias=False) 6846 elif isinstance(this, (exp.Subquery, exp.Values)): 6847 this = self._parse_subquery( 6848 this=self._parse_query_modifiers(self._parse_set_operations(this)), 6849 parse_alias=False, 6850 ) 6851 elif len(expressions) > 1 or self._prev.token_type == TokenType.COMMA: 6852 this = self.expression(exp.Tuple(expressions=expressions)) 6853 else: 6854 this = self.expression(exp.Paren(this=this)) 6855 6856 if this: 6857 this.add_comments(comments) 6858 6859 self._match_r_paren(expression=this) 6860 6861 if isinstance(this, exp.Paren) and isinstance(this.this, exp.AggFunc): 6862 return self._parse_window(this) 6863 6864 return this 6865 6866 def _parse_primary(self) -> exp.Expr | None: 6867 if self._match_set(self.PRIMARY_PARSERS): 6868 token_type = self._prev.token_type 6869 primary = self.PRIMARY_PARSERS[token_type](self, self._prev) 6870 6871 if token_type == TokenType.STRING: 6872 expressions = [primary] 6873 while self._match(TokenType.STRING, advance=False): 6874 if self._is_connected() and self.ADJACENT_STRINGS_CANNOT_BE_CONNECTED: 6875 self.raise_error( 6876 "Adjacent string literals need to be separated by whitespace or comments" 6877 ) 6878 6879 self._advance() 6880 expressions.append(exp.Literal.string(self._prev.text)) 6881 6882 if len(expressions) > 1: 6883 return self.expression( 6884 exp.Concat(expressions=expressions, coalesce=self.dialect.CONCAT_COALESCE) 6885 ) 6886 6887 return primary 6888 6889 if self._match_pair(TokenType.DOT, TokenType.NUMBER): 6890 return exp.Literal.number(f"0.{self._prev.text}") 6891 6892 return self._parse_paren() 6893 6894 def _parse_field( 6895 self, 6896 any_token: bool = False, 6897 tokens: t.Collection[TokenType] | None = None, 6898 anonymous_func: bool = False, 6899 ) -> exp.Expr | None: 6900 if anonymous_func: 6901 field = ( 6902 self._parse_function(anonymous=anonymous_func, any_token=any_token) 6903 or self._parse_primary() 6904 ) 6905 else: 6906 field = self._parse_primary() or self._parse_function( 6907 anonymous=anonymous_func, any_token=any_token 6908 ) 6909 return field or self._parse_id_var(any_token=any_token, tokens=tokens) 6910 6911 def _parse_function( 6912 self, 6913 functions: dict[str, t.Callable] | None = None, 6914 anonymous: bool = False, 6915 optional_parens: bool = True, 6916 any_token: bool = False, 6917 ) -> exp.Expr | None: 6918 # This allows us to also parse {fn <function>} syntax (Snowflake, MySQL support this) 6919 # See: https://community.snowflake.com/s/article/SQL-Escape-Sequences 6920 fn_syntax = False 6921 if ( 6922 self._match(TokenType.L_BRACE, advance=False) 6923 and self._next 6924 and self._next.text.upper() == "FN" 6925 ): 6926 self._advance(2) 6927 fn_syntax = True 6928 6929 func = self._parse_function_call( 6930 functions=functions, 6931 anonymous=anonymous, 6932 optional_parens=optional_parens, 6933 any_token=any_token, 6934 ) 6935 6936 if fn_syntax: 6937 self._match(TokenType.R_BRACE) 6938 6939 return func 6940 6941 def _parse_function_args(self, alias: bool = False) -> list[exp.Expr]: 6942 return self._parse_csv(lambda: self._parse_lambda(alias=alias)) 6943 6944 def _parse_function_call( 6945 self, 6946 functions: dict[str, t.Callable] | None = None, 6947 anonymous: bool = False, 6948 optional_parens: bool = True, 6949 any_token: bool = False, 6950 ) -> exp.Expr | None: 6951 if not self._curr: 6952 return None 6953 6954 comments = self._curr.comments 6955 prev = self._prev 6956 token = self._curr 6957 token_type = self._curr.token_type 6958 this: str | exp.Expr = self._curr.text 6959 upper = self._curr.text.upper() 6960 6961 after_dot = prev.token_type == TokenType.DOT 6962 parser = self.NO_PAREN_FUNCTION_PARSERS.get(upper) 6963 if ( 6964 optional_parens 6965 and parser 6966 and token_type not in self.INVALID_FUNC_NAME_TOKENS 6967 and not after_dot 6968 ): 6969 self._advance() 6970 return self._parse_window(parser(self)) 6971 6972 if self._next.token_type != TokenType.L_PAREN: 6973 if optional_parens and token_type in self.NO_PAREN_FUNCTIONS and not after_dot: 6974 self._advance() 6975 return self.expression(self.NO_PAREN_FUNCTIONS[token_type]()) 6976 6977 return None 6978 6979 if any_token: 6980 if token_type in self.RESERVED_TOKENS: 6981 return None 6982 elif token_type not in self.FUNC_TOKENS: 6983 return None 6984 6985 self._advance(2) 6986 6987 parser = self.FUNCTION_PARSERS.get(upper) 6988 if parser and not anonymous: 6989 result = parser(self) 6990 else: 6991 subquery_predicate = self.SUBQUERY_PREDICATES.get(token_type) 6992 6993 if subquery_predicate: 6994 expr = None 6995 if self._curr.token_type in self.SUBQUERY_TOKENS: 6996 expr = self._parse_select() 6997 self._match_r_paren() 6998 elif prev and prev.token_type in (TokenType.LIKE, TokenType.ILIKE): 6999 # Backtrack one token since we've consumed the L_PAREN here. Instead, we'd like 7000 # to parse "LIKE [ANY | ALL] (...)" as a whole into an exp.Tuple or exp.Paren 7001 self._advance(-1) 7002 expr = self._parse_bitwise() 7003 7004 if expr: 7005 return self.expression(subquery_predicate(this=expr), comments=comments) 7006 7007 if functions is None: 7008 functions = self.FUNCTIONS 7009 7010 function = functions.get(upper) 7011 known_function = function and not anonymous 7012 7013 alias = not known_function or upper in self.FUNCTIONS_WITH_ALIASED_ARGS 7014 args = self._parse_function_args(alias) 7015 7016 post_func_comments = self._curr.comments if self._curr else None 7017 if known_function and post_func_comments: 7018 # If the user-inputted comment "/* sqlglot.anonymous */" is following the function 7019 # call we'll construct it as exp.Anonymous, even if it's "known" 7020 if any( 7021 comment.lstrip().startswith(exp.SQLGLOT_ANONYMOUS) 7022 for comment in post_func_comments 7023 ): 7024 known_function = False 7025 7026 if alias and known_function: 7027 args = self._kv_to_prop_eq(args) 7028 7029 if known_function: 7030 func_builder = t.cast(t.Callable, function) 7031 7032 # mypyc compiled functions don't have __code__, so we use 7033 # try/except to check if func_builder accepts 'dialect'. 7034 try: 7035 func = func_builder(args) 7036 except TypeError: 7037 func = func_builder(args, dialect=self.dialect) 7038 7039 func = self.validate_expression(func, args) 7040 if self.dialect.PRESERVE_ORIGINAL_NAMES: 7041 func.meta["name"] = this 7042 7043 result = func 7044 else: 7045 if token_type == TokenType.IDENTIFIER: 7046 this = exp.Identifier(this=this, quoted=True).update_positions(token) 7047 7048 result = self.expression(exp.Anonymous(this=this, expressions=args)) 7049 7050 result = result.update_positions(token) 7051 7052 if isinstance(result, exp.Expr): 7053 result.add_comments(comments) 7054 7055 if parser: 7056 self._match(TokenType.R_PAREN, expression=result) 7057 else: 7058 self._match_r_paren(result) 7059 return self._parse_window(result) 7060 7061 def _to_prop_eq(self, expression: exp.Expr, index: int) -> exp.Expr: 7062 return expression 7063 7064 def _kv_to_prop_eq( 7065 self, expressions: list[exp.Expr], parse_map: bool = False 7066 ) -> list[exp.Expr]: 7067 transformed = [] 7068 7069 for index, e in enumerate(expressions): 7070 if isinstance(e, self.KEY_VALUE_DEFINITIONS): 7071 if isinstance(e, exp.Alias): 7072 e = self.expression(exp.PropertyEQ(this=e.args.get("alias"), expression=e.this)) 7073 7074 if not isinstance(e, exp.PropertyEQ): 7075 e = self.expression( 7076 exp.PropertyEQ( 7077 this=e.this if parse_map else exp.to_identifier(e.this.name), 7078 expression=e.expression, 7079 ) 7080 ) 7081 7082 if isinstance(e.this, exp.Column): 7083 e.this.replace(e.this.this) 7084 else: 7085 e = self._to_prop_eq(e, index) 7086 7087 transformed.append(e) 7088 7089 return transformed 7090 7091 def _parse_function_properties(self) -> exp.Properties | None: 7092 # Skip the generic `key = value` fallback in _parse_property since this 7093 # runs post-AS where a function body like `name = expr` can be misread 7094 # as a property. 7095 properties = [] 7096 while True: 7097 if self._match_texts(self.PROPERTY_PARSERS): 7098 prop = self.PROPERTY_PARSERS[self._prev.text.upper()](self) 7099 elif self._match(TokenType.DEFAULT) and self._match_texts(self.PROPERTY_PARSERS): 7100 prop = self.PROPERTY_PARSERS[self._prev.text.upper()](self, default=True) 7101 else: 7102 break 7103 for p in ensure_list(prop): 7104 properties.append(p) 7105 7106 return self.expression(exp.Properties(expressions=properties)) if properties else None 7107 7108 def _parse_user_defined_function_expression(self) -> exp.Expr | None: 7109 return self._parse_statement() 7110 7111 def _parse_function_parameter(self) -> exp.Expr | None: 7112 return self._parse_column_def(this=self._parse_id_var(), computed_column=False) 7113 7114 def _parse_user_defined_function(self, kind: TokenType | None = None) -> exp.Expr | None: 7115 this = self._parse_table_parts(schema=True) 7116 7117 if not self._match(TokenType.L_PAREN): 7118 return this 7119 7120 expressions = self._parse_csv(self._parse_function_parameter) 7121 self._match_r_paren() 7122 return self.expression( 7123 exp.UserDefinedFunction(this=this, expressions=expressions, wrapped=True) 7124 ) 7125 7126 def _parse_macro_overloads( 7127 self, 7128 this: exp.UserDefinedFunction, 7129 first_body: exp.Expr, 7130 first_is_table: bool = False, 7131 ) -> exp.MacroOverloads: 7132 overloads = [ 7133 self.expression( 7134 exp.MacroOverload( 7135 this=first_body, 7136 expressions=this.expressions or None, 7137 is_table=first_is_table, 7138 ) 7139 ) 7140 ] 7141 this.set("expressions", None) 7142 this.set("wrapped", False) 7143 7144 while self._match(TokenType.COMMA): 7145 if not self._match(TokenType.L_PAREN): 7146 break 7147 7148 params = self._parse_csv(self._parse_function_parameter) 7149 self._match_r_paren() 7150 7151 if not self._match(TokenType.ALIAS): 7152 break 7153 7154 is_table = self._match(TokenType.TABLE) 7155 body = self._parse_expression() 7156 macro = exp.MacroOverload(this=body, expressions=params, is_table=is_table) 7157 overloads.append(self.expression(macro)) 7158 7159 return self.expression(exp.MacroOverloads(expressions=overloads)) 7160 7161 def _parse_introducer(self, token: Token) -> exp.Introducer | exp.Identifier: 7162 literal = self._parse_primary() 7163 if literal: 7164 return self.expression(exp.Introducer(this=token.text, expression=literal), token) 7165 7166 return self._identifier_expression(token) 7167 7168 def _parse_session_parameter(self) -> exp.SessionParameter: 7169 kind = None 7170 this = self._parse_id_var() or self._parse_primary() 7171 7172 if this and self._match(TokenType.DOT): 7173 kind = this.name 7174 this = self._parse_var() or self._parse_primary() 7175 7176 return self.expression(exp.SessionParameter(this=this, kind=kind)) 7177 7178 def _parse_lambda_arg(self) -> exp.Expr | None: 7179 return self._parse_id_var() 7180 7181 def _parse_lambda(self, alias: bool = False) -> exp.Expr | None: 7182 next_token_type = self._next.token_type 7183 7184 # Fast path: simple atom (column, literal, null, bool) followed by , or ) 7185 if ( 7186 next_token_type in self.LAMBDA_ARG_TERMINATORS 7187 and (atom := self._parse_atom()) is not None 7188 ): 7189 return atom 7190 7191 index = self._index 7192 7193 if self._match(TokenType.L_PAREN): 7194 expressions = t.cast( 7195 list[t.Optional[exp.Expr]], self._parse_csv(self._parse_lambda_arg) 7196 ) 7197 7198 if not self._match(TokenType.R_PAREN): 7199 self._retreat(index) 7200 elif self._match_set(self.LAMBDAS): 7201 return self.LAMBDAS[self._prev.token_type](self, expressions) 7202 else: 7203 self._retreat(index) 7204 elif self.TYPED_LAMBDA_ARGS or next_token_type in self.LAMBDAS: 7205 expressions = [self._parse_lambda_arg()] 7206 7207 if self._match_set(self.LAMBDAS): 7208 return self.LAMBDAS[self._prev.token_type](self, expressions) 7209 7210 self._retreat(index) 7211 7212 this: exp.Expr | None 7213 7214 if self._match(TokenType.DISTINCT): 7215 this = self.expression( 7216 exp.Distinct(expressions=self._parse_csv(self._parse_disjunction)) 7217 ) 7218 else: 7219 self._match(TokenType.ALL) # ALL is the default/no-op aggregate modifier (SQL-92) 7220 this = self._parse_select_or_expression(alias=alias) 7221 7222 return self._parse_limit( 7223 self._parse_respect_or_ignore_nulls( 7224 self._parse_order(self._parse_having_max(self._parse_respect_or_ignore_nulls(this))) 7225 ) 7226 ) 7227 7228 def _parse_schema(self, this: exp.Expr | None = None) -> exp.Expr | None: 7229 index = self._index 7230 if not self._match(TokenType.L_PAREN): 7231 return this 7232 7233 # Disambiguate between schema and subquery/CTE, e.g. in INSERT INTO table (<expr>), 7234 # expr can be of both types 7235 if self._match_set(self.SELECT_START_TOKENS): 7236 self._retreat(index) 7237 return this 7238 args = self._parse_csv(lambda: self._parse_constraint() or self._parse_field_def()) 7239 self._match_r_paren() 7240 return self.expression(exp.Schema(this=this, expressions=args)) 7241 7242 def _parse_field_def(self) -> exp.Expr | None: 7243 return self._parse_column_def(self._parse_field(any_token=True)) 7244 7245 def _parse_column_def( 7246 self, this: exp.Expr | None, computed_column: bool = True 7247 ) -> exp.Expr | None: 7248 # column defs are not really columns, they're identifiers 7249 if isinstance(this, exp.Column): 7250 this = this.this 7251 7252 if not computed_column: 7253 self._match(TokenType.ALIAS) 7254 7255 kind = self._parse_types(schema=True) 7256 7257 if self._match_text_seq("FOR", "ORDINALITY"): 7258 return self.expression(exp.ColumnDef(this=this, ordinality=True)) 7259 7260 constraints: list[exp.Expr] = [] 7261 7262 if (not kind and self._match(TokenType.ALIAS)) or self._match_texts( 7263 ("ALIAS", "MATERIALIZED") 7264 ): 7265 persisted = self._prev.text.upper() == "MATERIALIZED" 7266 constraint_kind = exp.ComputedColumnConstraint( 7267 this=self._parse_disjunction(), 7268 persisted=persisted or self._match_text_seq("PERSISTED"), 7269 data_type=exp.Var(this="AUTO") 7270 if self._match_text_seq("AUTO") 7271 else self._parse_types(), 7272 not_null=self._match_pair(TokenType.NOT, TokenType.NULL), 7273 ) 7274 constraints.append(self.expression(exp.ColumnConstraint(kind=constraint_kind))) 7275 elif not kind and self._match_set({TokenType.IN, TokenType.OUT}, advance=False): 7276 in_out_constraint = self.expression( 7277 exp.InOutColumnConstraint( 7278 input_=self._match(TokenType.IN), output=self._match(TokenType.OUT) 7279 ) 7280 ) 7281 constraints.append(in_out_constraint) 7282 kind = self._parse_types() 7283 elif ( 7284 kind 7285 and self._match(TokenType.ALIAS, advance=False) 7286 and ( 7287 not self.WRAPPED_TRANSFORM_COLUMN_CONSTRAINT 7288 or self._next.token_type == TokenType.L_PAREN 7289 ) 7290 ): 7291 self._advance() 7292 constraints.append( 7293 self.expression( 7294 exp.ColumnConstraint( 7295 kind=exp.ComputedColumnConstraint( 7296 this=self._parse_disjunction(), 7297 persisted=self._match_texts(("STORED", "VIRTUAL")) 7298 and self._prev.text.upper() == "STORED", 7299 ) 7300 ) 7301 ) 7302 ) 7303 7304 while True: 7305 constraint = self._parse_column_constraint() 7306 if not constraint: 7307 break 7308 constraints.append(constraint) 7309 7310 if not kind and not constraints: 7311 return this 7312 7313 position = None 7314 if self._match_texts(("FIRST", "AFTER")): 7315 pos = self._prev.text 7316 position = self.expression(exp.ColumnPosition(this=self._parse_column(), position=pos)) 7317 7318 return self.expression( 7319 exp.ColumnDef(this=this, kind=kind, constraints=constraints, position=position) 7320 ) 7321 7322 def _parse_auto_increment( 7323 self, 7324 ) -> exp.GeneratedAsIdentityColumnConstraint | exp.AutoIncrementColumnConstraint: 7325 start = None 7326 increment = None 7327 order = None 7328 7329 if self._match(TokenType.L_PAREN, advance=False): 7330 args = self._parse_wrapped_csv(self._parse_bitwise) 7331 start = seq_get(args, 0) 7332 increment = seq_get(args, 1) 7333 elif self._match_text_seq("START"): 7334 start = self._parse_bitwise() 7335 self._match_text_seq("INCREMENT") 7336 increment = self._parse_bitwise() 7337 if self._match_text_seq("ORDER"): 7338 order = True 7339 elif self._match_text_seq("NOORDER"): 7340 order = False 7341 7342 if start and increment: 7343 return exp.GeneratedAsIdentityColumnConstraint( 7344 start=start, increment=increment, this=False, order=order 7345 ) 7346 7347 return exp.AutoIncrementColumnConstraint() 7348 7349 def _parse_check_constraint(self) -> exp.CheckColumnConstraint | None: 7350 if not self._match(TokenType.L_PAREN, advance=False): 7351 return None 7352 7353 return self.expression( 7354 exp.CheckColumnConstraint( 7355 this=self._parse_wrapped(self._parse_assignment), 7356 enforced=self._match_text_seq("ENFORCED"), 7357 ) 7358 ) 7359 7360 def _parse_auto_property(self) -> exp.AutoRefreshProperty | None: 7361 if not self._match_text_seq("REFRESH"): 7362 self._retreat(self._index - 1) 7363 return None 7364 return self.expression(exp.AutoRefreshProperty(this=self._parse_var(upper=True))) 7365 7366 def _parse_compress(self) -> exp.CompressColumnConstraint: 7367 if self._match(TokenType.L_PAREN, advance=False): 7368 return self.expression( 7369 exp.CompressColumnConstraint(this=self._parse_wrapped_csv(self._parse_bitwise)) 7370 ) 7371 7372 return self.expression(exp.CompressColumnConstraint(this=self._parse_bitwise())) 7373 7374 def _parse_generated_as_identity( 7375 self, 7376 ) -> ( 7377 exp.GeneratedAsIdentityColumnConstraint 7378 | exp.ComputedColumnConstraint 7379 | exp.GeneratedAsRowColumnConstraint 7380 ): 7381 if self._match_text_seq("BY", "DEFAULT"): 7382 on_null = self._match_pair(TokenType.ON, TokenType.NULL) 7383 this = self.expression( 7384 exp.GeneratedAsIdentityColumnConstraint(this=False, on_null=on_null) 7385 ) 7386 else: 7387 self._match_text_seq("ALWAYS") 7388 this = self.expression(exp.GeneratedAsIdentityColumnConstraint(this=True)) 7389 7390 self._match(TokenType.ALIAS) 7391 7392 if self._match_text_seq("ROW"): 7393 start = self._match_text_seq("START") 7394 if not start: 7395 self._match(TokenType.END) 7396 hidden = self._match_text_seq("HIDDEN") 7397 return self.expression(exp.GeneratedAsRowColumnConstraint(start=start, hidden=hidden)) 7398 7399 identity = self._match_text_seq("IDENTITY") 7400 7401 if self._match(TokenType.L_PAREN): 7402 if self._match(TokenType.START_WITH): 7403 this.set("start", self._parse_bitwise()) 7404 if self._match_text_seq("INCREMENT", "BY"): 7405 this.set("increment", self._parse_bitwise()) 7406 if self._match_text_seq("MINVALUE"): 7407 this.set("minvalue", self._parse_bitwise()) 7408 if self._match_text_seq("MAXVALUE"): 7409 this.set("maxvalue", self._parse_bitwise()) 7410 7411 if self._match_text_seq("CYCLE"): 7412 this.set("cycle", True) 7413 elif self._match_text_seq("NO", "CYCLE"): 7414 this.set("cycle", False) 7415 7416 if not identity: 7417 this.set("expression", self._parse_range()) 7418 elif not this.args.get("start") and self._match(TokenType.NUMBER, advance=False): 7419 args = self._parse_csv(self._parse_bitwise) 7420 this.set("start", seq_get(args, 0)) 7421 this.set("increment", seq_get(args, 1)) 7422 7423 self._match_r_paren() 7424 7425 return this 7426 7427 def _parse_inline(self) -> exp.InlineLengthColumnConstraint: 7428 self._match_text_seq("LENGTH") 7429 return self.expression(exp.InlineLengthColumnConstraint(this=self._parse_bitwise())) 7430 7431 def _parse_not_constraint(self) -> exp.Expr | None: 7432 if self._match_text_seq("NULL"): 7433 return self.expression(exp.NotNullColumnConstraint()) 7434 if self._match_text_seq("CASESPECIFIC"): 7435 return self.expression(exp.CaseSpecificColumnConstraint(not_=True)) 7436 if self._match_text_seq("FOR", "REPLICATION"): 7437 return self.expression(exp.NotForReplicationColumnConstraint()) 7438 7439 # Unconsume the `NOT` token 7440 self._retreat(self._index - 1) 7441 return None 7442 7443 def _parse_column_constraint(self) -> exp.Expr | None: 7444 this = self._parse_id_var() if self._match(TokenType.CONSTRAINT) else None 7445 7446 procedure_option_follows = ( 7447 self._match(TokenType.WITH, advance=False) 7448 and self._next 7449 and self._next.text.upper() in self.PROCEDURE_OPTIONS 7450 ) 7451 7452 if not procedure_option_follows and self._match_texts(self.CONSTRAINT_PARSERS): 7453 constraint = self.CONSTRAINT_PARSERS[self._prev.text.upper()](self) 7454 if not constraint: 7455 self._retreat(self._index - 1) 7456 return None 7457 7458 return self.expression(exp.ColumnConstraint(this=this, kind=constraint)) 7459 7460 return this 7461 7462 def _parse_constraint(self) -> exp.Expr | None: 7463 if not self._match(TokenType.CONSTRAINT): 7464 return self._parse_unnamed_constraint(constraints=self.SCHEMA_UNNAMED_CONSTRAINTS) 7465 7466 return self.expression( 7467 exp.Constraint(this=self._parse_id_var(), expressions=self._parse_unnamed_constraints()) 7468 ) 7469 7470 def _parse_unnamed_constraints(self) -> list[exp.Expr]: 7471 constraints = [] 7472 while True: 7473 constraint = self._parse_unnamed_constraint() or self._parse_function() 7474 if not constraint: 7475 break 7476 constraints.append(constraint) 7477 7478 return constraints 7479 7480 def _parse_unnamed_constraint( 7481 self, constraints: t.Collection[str] | None = None 7482 ) -> exp.Expr | None: 7483 index = self._index 7484 7485 if self._match(TokenType.IDENTIFIER, advance=False) or not self._match_texts( 7486 constraints or self.CONSTRAINT_PARSERS 7487 ): 7488 return None 7489 7490 constraint_key = self._prev.text.upper() 7491 if constraint_key not in self.CONSTRAINT_PARSERS: 7492 self.raise_error(f"No parser found for schema constraint {constraint_key}.") 7493 7494 result = self.CONSTRAINT_PARSERS[constraint_key](self) 7495 if not result: 7496 self._retreat(index) 7497 7498 return result 7499 7500 def _parse_unique_key(self) -> exp.Expr | None: 7501 if ( 7502 self._curr 7503 and self._curr.token_type != TokenType.IDENTIFIER 7504 and self._curr.text.upper() in self.CONSTRAINT_PARSERS 7505 ): 7506 return None 7507 return self._parse_id_var(any_token=False) 7508 7509 def _parse_unique(self) -> exp.UniqueColumnConstraint: 7510 self._match_texts(("KEY", "INDEX")) 7511 return self.expression( 7512 exp.UniqueColumnConstraint( 7513 nulls=self._match_text_seq("NULLS", "NOT", "DISTINCT"), 7514 this=self._parse_schema(self._parse_unique_key()), 7515 index_type=self._match(TokenType.USING) and self._advance_any() and self._prev.text, 7516 on_conflict=self._parse_on_conflict(), 7517 options=self._parse_key_constraint_options(), 7518 ) 7519 ) 7520 7521 def _parse_key_constraint_options(self) -> list[str]: 7522 options = [] 7523 while True: 7524 if not self._curr: 7525 break 7526 7527 if self._match(TokenType.ON): 7528 action = None 7529 on = self._advance_any() and self._prev.text 7530 7531 if self._match_text_seq("NO", "ACTION"): 7532 action = "NO ACTION" 7533 elif self._match_text_seq("CASCADE"): 7534 action = "CASCADE" 7535 elif self._match_text_seq("RESTRICT"): 7536 action = "RESTRICT" 7537 elif self._match_pair(TokenType.SET, TokenType.NULL): 7538 action = "SET NULL" 7539 elif self._match_pair(TokenType.SET, TokenType.DEFAULT): 7540 action = "SET DEFAULT" 7541 else: 7542 self.raise_error("Invalid key constraint") 7543 7544 options.append(f"ON {on} {action}") 7545 else: 7546 var = self._parse_var_from_options( 7547 self.KEY_CONSTRAINT_OPTIONS, raise_unmatched=False 7548 ) 7549 if not var: 7550 break 7551 options.append(var.name) 7552 7553 return options 7554 7555 def _parse_references(self, match: bool = True) -> exp.Reference | None: 7556 if match and not self._match(TokenType.REFERENCES): 7557 return None 7558 7559 expressions: list | None = None 7560 this = self._parse_table(schema=True) 7561 options = self._parse_key_constraint_options() 7562 return self.expression(exp.Reference(this=this, expressions=expressions, options=options)) 7563 7564 def _parse_foreign_key(self) -> exp.ForeignKey: 7565 expressions = ( 7566 self._parse_wrapped_id_vars() 7567 if not self._match(TokenType.REFERENCES, advance=False) 7568 else None 7569 ) 7570 reference = self._parse_references() 7571 on_options = {} 7572 7573 while self._match(TokenType.ON): 7574 if not self._match_set((TokenType.DELETE, TokenType.UPDATE)): 7575 self.raise_error("Expected DELETE or UPDATE") 7576 7577 kind = self._prev.text.lower() 7578 7579 if self._match_text_seq("NO", "ACTION"): 7580 action = "NO ACTION" 7581 elif self._match(TokenType.SET): 7582 self._match_set((TokenType.NULL, TokenType.DEFAULT)) 7583 action = "SET " + self._prev.text.upper() 7584 else: 7585 self._advance() 7586 action = self._prev.text.upper() 7587 7588 on_options[kind] = action 7589 7590 return self.expression( 7591 exp.ForeignKey( 7592 expressions=expressions, 7593 reference=reference, 7594 options=self._parse_key_constraint_options(), 7595 **on_options, 7596 ) 7597 ) 7598 7599 def _parse_primary_key_part(self) -> exp.Expr | None: 7600 return self._parse_field() 7601 7602 def _parse_period_for_system_time(self) -> exp.PeriodForSystemTimeConstraint | None: 7603 if not self._match(TokenType.TIMESTAMP_SNAPSHOT): 7604 self._retreat(self._index - 1) 7605 return None 7606 7607 id_vars = self._parse_wrapped_id_vars() 7608 return self.expression( 7609 exp.PeriodForSystemTimeConstraint( 7610 this=seq_get(id_vars, 0), expression=seq_get(id_vars, 1) 7611 ) 7612 ) 7613 7614 def _parse_primary_key( 7615 self, 7616 wrapped_optional: bool = False, 7617 in_props: bool = False, 7618 named_primary_key: bool = False, 7619 ) -> exp.PrimaryKeyColumnConstraint | exp.PrimaryKey: 7620 desc = ( 7621 self._prev.token_type == TokenType.DESC 7622 if self._match_set((TokenType.ASC, TokenType.DESC)) 7623 else None 7624 ) 7625 7626 this = None 7627 if ( 7628 named_primary_key 7629 and self._curr.text.upper() not in self.CONSTRAINT_PARSERS 7630 and self._next 7631 and self._next.token_type == TokenType.L_PAREN 7632 ): 7633 this = self._parse_id_var() 7634 7635 if not in_props and not self._match(TokenType.L_PAREN, advance=False): 7636 return self.expression( 7637 exp.PrimaryKeyColumnConstraint( 7638 desc=desc, options=self._parse_key_constraint_options() 7639 ) 7640 ) 7641 7642 expressions = self._parse_wrapped_csv( 7643 self._parse_primary_key_part, optional=wrapped_optional 7644 ) 7645 7646 return self.expression( 7647 exp.PrimaryKey( 7648 this=this, 7649 expressions=expressions, 7650 include=self._parse_index_params(), 7651 options=self._parse_key_constraint_options(), 7652 ) 7653 ) 7654 7655 def _parse_bracket_key_value(self, is_map: bool = False) -> exp.Expr | None: 7656 return self._parse_slice(self._parse_alias(self._parse_disjunction(), explicit=True)) 7657 7658 def _parse_odbc_datetime_literal(self) -> exp.Expr: 7659 """ 7660 Parses a datetime column in ODBC format. We parse the column into the corresponding 7661 types, for example `{d'yyyy-mm-dd'}` will be parsed as a `Date` column, exactly the 7662 same as we did for `DATE('yyyy-mm-dd')`. 7663 7664 Reference: 7665 https://learn.microsoft.com/en-us/sql/odbc/reference/develop-app/date-time-and-timestamp-literals 7666 """ 7667 self._match(TokenType.VAR) 7668 exp_class = self.ODBC_DATETIME_LITERALS[self._prev.text.lower()] 7669 expression = self.expression(exp_class(this=self._parse_string())) 7670 if not self._match(TokenType.R_BRACE): 7671 self.raise_error("Expected }") 7672 return expression 7673 7674 def _parse_bracket(self, this: exp.Expr | None = None) -> exp.Expr | None: 7675 if not self._match_set(self.BRACKETS): 7676 return this 7677 7678 if self.MAP_KEYS_ARE_ARBITRARY_EXPRESSIONS: 7679 map_token = seq_get(self._tokens, self._index - 2) 7680 parse_map = map_token is not None and map_token.text.upper() == "MAP" 7681 else: 7682 parse_map = False 7683 7684 bracket_kind = self._prev.token_type 7685 if ( 7686 bracket_kind == TokenType.L_BRACE 7687 and self._curr 7688 and self._curr.token_type == TokenType.VAR 7689 and self._curr.text.lower() in self.ODBC_DATETIME_LITERALS 7690 ): 7691 return self._parse_odbc_datetime_literal() 7692 7693 expressions = self._parse_csv( 7694 lambda: self._parse_bracket_key_value(is_map=bracket_kind == TokenType.L_BRACE) 7695 ) 7696 7697 if bracket_kind == TokenType.L_BRACKET and not self._match(TokenType.R_BRACKET): 7698 self.raise_error("Expected ]") 7699 elif bracket_kind == TokenType.L_BRACE and not self._match(TokenType.R_BRACE): 7700 self.raise_error("Expected }") 7701 7702 # https://duckdb.org/docs/sql/data_types/struct.html#creating-structs 7703 if bracket_kind == TokenType.L_BRACE: 7704 this = self.expression( 7705 exp.Struct( 7706 expressions=self._kv_to_prop_eq(expressions=expressions, parse_map=parse_map) 7707 ) 7708 ) 7709 elif not this: 7710 this = build_array_constructor( 7711 exp.Array, args=expressions, bracket_kind=bracket_kind, dialect=self.dialect 7712 ) 7713 else: 7714 constructor_type = self.ARRAY_CONSTRUCTORS.get(this.name.upper()) 7715 if constructor_type: 7716 return build_array_constructor( 7717 constructor_type, 7718 args=expressions, 7719 bracket_kind=bracket_kind, 7720 dialect=self.dialect, 7721 ) 7722 7723 expressions = apply_index_offset( 7724 this, expressions, -self.dialect.INDEX_OFFSET, dialect=self.dialect 7725 ) 7726 this = self.expression( 7727 exp.Bracket(this=this, expressions=expressions), comments=this.pop_comments() 7728 ) 7729 7730 self._add_comments(this) 7731 return self._parse_bracket(this) 7732 7733 def _parse_slice(self, this: exp.Expr | None) -> exp.Expr | None: 7734 if not self._match(TokenType.COLON): 7735 return this 7736 7737 if self._match_pair(TokenType.DASH, TokenType.COLON, advance=False): 7738 self._advance() 7739 end: exp.Expr | None = -exp.Literal.number("1") 7740 else: 7741 end = self._parse_assignment() 7742 step = self._parse_unary() if self._match(TokenType.COLON) else None 7743 return self.expression(exp.Slice(this=this, expression=end, step=step)) 7744 7745 def _parse_case(self) -> exp.Expr | None: 7746 if self._match(TokenType.DOT, advance=False): 7747 # Avoid raising on valid expressions like case.*, supported by, e.g., spark & snowflake 7748 self._retreat(self._index - 1) 7749 return None 7750 7751 ifs = [] 7752 default = None 7753 7754 comments = self._prev_comments 7755 expression = self._parse_disjunction() 7756 7757 while self._match(TokenType.WHEN): 7758 this = self._parse_disjunction() 7759 self._match(TokenType.THEN) 7760 then = self._parse_disjunction() 7761 ifs.append(self.expression(exp.If(this=this, true=then))) 7762 7763 if self._match(TokenType.ELSE): 7764 default = self._parse_disjunction() 7765 7766 if not self._match(TokenType.END): 7767 if isinstance(default, exp.Interval) and default.this.sql().upper() == "END": 7768 default = exp.column("interval") 7769 else: 7770 self.raise_error("Expected END after CASE", self._prev) 7771 7772 return self.expression( 7773 exp.Case(this=expression, ifs=ifs, default=default), comments=comments 7774 ) 7775 7776 def _parse_if(self) -> exp.Expr | None: 7777 if self._match(TokenType.L_PAREN): 7778 args = self._parse_csv( 7779 lambda: self._parse_alias(self._parse_assignment(), explicit=True) 7780 ) 7781 this = self.validate_expression(exp.If.from_arg_list(args), args) 7782 self._match_r_paren() 7783 else: 7784 index = self._index - 1 7785 7786 if self.NO_PAREN_IF_COMMANDS and index == 0: 7787 return self._parse_as_command(self._prev) 7788 7789 condition = self._parse_disjunction() 7790 7791 if not condition: 7792 self._retreat(index) 7793 return None 7794 7795 self._match(TokenType.THEN) 7796 true = self._parse_disjunction() 7797 false = self._parse_disjunction() if self._match(TokenType.ELSE) else None 7798 self._match(TokenType.END) 7799 this = self.expression(exp.If(this=condition, true=true, false=false)) 7800 7801 return this 7802 7803 def _parse_next_value_for(self) -> exp.Expr | None: 7804 if not self._match_text_seq("VALUE", "FOR"): 7805 self._retreat(self._index - 1) 7806 return None 7807 7808 return self.expression( 7809 exp.NextValueFor( 7810 this=self._parse_column(), 7811 order=self._match(TokenType.OVER) and self._parse_wrapped(self._parse_order), 7812 ) 7813 ) 7814 7815 def _parse_extract(self) -> exp.Extract | exp.Anonymous: 7816 this = self._parse_function() or self._parse_var_or_string(upper=True) 7817 7818 if self._match(TokenType.FROM): 7819 return self.expression(exp.Extract(this=this, expression=self._parse_bitwise())) 7820 7821 if not self._match(TokenType.COMMA): 7822 self.raise_error("Expected FROM or comma after EXTRACT", self._prev) 7823 7824 return self.expression(exp.Extract(this=this, expression=self._parse_bitwise())) 7825 7826 def _parse_gap_fill(self) -> exp.GapFill: 7827 self._match(TokenType.TABLE) 7828 this = self._parse_table() 7829 7830 self._match(TokenType.COMMA) 7831 args = [this, *self._parse_csv(self._parse_lambda)] 7832 7833 gap_fill = exp.GapFill.from_arg_list(args) 7834 return self.validate_expression(gap_fill, args) 7835 7836 def _parse_char(self) -> exp.Chr: 7837 return self.expression( 7838 exp.Chr( 7839 expressions=self._parse_csv(self._parse_assignment), 7840 charset=self._match(TokenType.USING) and self._parse_charset_name(), 7841 ) 7842 ) 7843 7844 def _parse_charset_name(self) -> exp.Expr | None: 7845 """ 7846 Parse a charset name after USING or CHARACTER SET. Dialects that need to preserve quoting 7847 for specific name shapes override this. 7848 """ 7849 return self._parse_var( 7850 tokens={TokenType.BINARY, TokenType.IDENTIFIER}, 7851 ) 7852 7853 def _parse_cast(self, strict: bool, safe: bool | None = None) -> exp.Expr: 7854 this = self._parse_assignment() 7855 7856 if not self._match(TokenType.ALIAS): 7857 if self._match(TokenType.COMMA): 7858 return self.expression(exp.CastToStrType(this=this, to=self._parse_string())) 7859 7860 self.raise_error("Expected AS after CAST") 7861 7862 fmt = None 7863 to = self._parse_types(with_collation=True) 7864 7865 default = None 7866 if self._match(TokenType.DEFAULT): 7867 default = self._parse_bitwise() 7868 self._match_text_seq("ON", "CONVERSION", "ERROR") 7869 7870 if self._match_set((TokenType.FORMAT, TokenType.COMMA)): 7871 fmt_string = self._parse_wrapped(self._parse_string, optional=True) 7872 fmt = self._parse_at_time_zone(fmt_string) 7873 7874 if not to: 7875 to = exp.DType.UNKNOWN.into_expr() 7876 if to.this in exp.DataType.TEMPORAL_TYPES: 7877 this = self.expression( 7878 (exp.StrToDate if to.this == exp.DType.DATE else exp.StrToTime)( 7879 this=this, 7880 format=exp.Literal.string( 7881 format_time( 7882 fmt_string.this if fmt_string else "", 7883 self.dialect.FORMAT_MAPPING or self.dialect.TIME_MAPPING, 7884 self.dialect.FORMAT_TRIE or self.dialect.TIME_TRIE, 7885 ) 7886 ), 7887 safe=safe, 7888 ) 7889 ) 7890 7891 if isinstance(fmt, exp.AtTimeZone) and isinstance(this, exp.StrToTime): 7892 this.set("zone", fmt.args["zone"]) 7893 return this 7894 elif not to: 7895 self.raise_error("Expected TYPE after CAST") 7896 elif isinstance(to, exp.Identifier): 7897 to = exp.DataType.from_str(to.name, dialect=self.dialect, udt=True) 7898 elif to.this == exp.DType.CHAR and self._match(TokenType.CHARACTER_SET): 7899 to = exp.DType.CHARACTER_SET.into_expr(kind=self._parse_var_or_string()) 7900 7901 return self.build_cast( 7902 strict=strict, 7903 this=this, 7904 to=to, 7905 format=fmt, 7906 safe=safe, 7907 action=self._parse_var_from_options(self.CAST_ACTIONS, raise_unmatched=False), 7908 default=default, 7909 ) 7910 7911 def _parse_string_agg(self) -> exp.GroupConcat: 7912 if self._match(TokenType.DISTINCT): 7913 args: list[exp.Expr | None] = [ 7914 self.expression(exp.Distinct(expressions=[self._parse_disjunction()])) 7915 ] 7916 if self._match(TokenType.COMMA): 7917 args.extend(self._parse_csv(self._parse_disjunction)) 7918 else: 7919 args = self._parse_csv(self._parse_disjunction) # type: ignore 7920 7921 if self._match_text_seq("ON", "OVERFLOW"): 7922 # trino: LISTAGG(expression [, separator] [ON OVERFLOW overflow_behavior]) 7923 if self._match_text_seq("ERROR"): 7924 on_overflow: exp.Expr | None = exp.var("ERROR") 7925 else: 7926 self._match_text_seq("TRUNCATE") 7927 on_overflow = self.expression( 7928 exp.OverflowTruncateBehavior( 7929 this=self._parse_string(), 7930 with_count=( 7931 self._match_text_seq("WITH", "COUNT") 7932 or not self._match_text_seq("WITHOUT", "COUNT") 7933 ), 7934 ) 7935 ) 7936 else: 7937 on_overflow = None 7938 7939 index = self._index 7940 if not self._match(TokenType.R_PAREN) and args: 7941 # postgres: STRING_AGG([DISTINCT] expression, separator [ORDER BY expression1 {ASC | DESC} [, ...]]) 7942 # bigquery: STRING_AGG([DISTINCT] expression [, separator] [ORDER BY key [{ASC | DESC}] [, ... ]] [LIMIT n]) 7943 # The order is parsed through `this` as a canonicalization for WITHIN GROUPs 7944 args[0] = self._parse_limit(this=self._parse_order(this=args[0])) 7945 return self.expression(exp.GroupConcat(this=args[0], separator=seq_get(args, 1))) 7946 7947 # Checks if we can parse an order clause: WITHIN GROUP (ORDER BY <order_by_expression_list> [ASC | DESC]). 7948 # This is done "manually", instead of letting _parse_window parse it into an exp.WithinGroup node, so that 7949 # the STRING_AGG call is parsed like in MySQL / SQLite and can thus be transpiled more easily to them. 7950 if not self._match_text_seq("WITHIN", "GROUP"): 7951 self._retreat(index) 7952 return self.validate_expression(exp.GroupConcat.from_arg_list(args), args) 7953 7954 # The corresponding match_r_paren will be called in parse_function (caller) 7955 self._match_l_paren() 7956 7957 return self.expression( 7958 exp.GroupConcat( 7959 this=self._parse_order(this=seq_get(args, 0)), 7960 separator=seq_get(args, 1), 7961 on_overflow=on_overflow, 7962 ) 7963 ) 7964 7965 def _parse_convert(self, strict: bool, safe: bool | None = None) -> exp.Expr | None: 7966 this = self._parse_bitwise() 7967 7968 if self._match(TokenType.USING): 7969 to: exp.Expr | None = exp.DType.CHARACTER_SET.into_expr(kind=self._parse_charset_name()) 7970 elif self._match(TokenType.COMMA): 7971 to = self._parse_types() 7972 else: 7973 to = None 7974 7975 return self.build_cast(strict=strict, this=this, to=to, safe=safe) 7976 7977 def _parse_xml_element(self) -> exp.XMLElement: 7978 if self._match_text_seq("EVALNAME"): 7979 evalname = True 7980 this = self._parse_bitwise() 7981 else: 7982 evalname = None 7983 self._match_text_seq("NAME") 7984 this = self._parse_id_var() 7985 7986 return self.expression( 7987 exp.XMLElement( 7988 this=this, 7989 expressions=self._match(TokenType.COMMA) and self._parse_csv(self._parse_bitwise), 7990 evalname=evalname, 7991 ) 7992 ) 7993 7994 def _parse_xml_table(self) -> exp.XMLTable: 7995 namespaces = None 7996 passing = None 7997 columns = None 7998 7999 if self._match_text_seq("XMLNAMESPACES", "("): 8000 namespaces = self._parse_xml_namespace() 8001 self._match_text_seq(")", ",") 8002 8003 this = self._parse_string() 8004 8005 if self._match_text_seq("PASSING"): 8006 # The BY VALUE keywords are optional and are provided for semantic clarity 8007 self._match_text_seq("BY", "VALUE") 8008 passing = self._parse_csv(self._parse_column) 8009 8010 by_ref = self._match_text_seq("RETURNING", "SEQUENCE", "BY", "REF") 8011 8012 if self._match_text_seq("COLUMNS"): 8013 columns = self._parse_csv(self._parse_field_def) 8014 8015 return self.expression( 8016 exp.XMLTable( 8017 this=this, namespaces=namespaces, passing=passing, columns=columns, by_ref=by_ref 8018 ) 8019 ) 8020 8021 def _parse_xml_namespace(self) -> list[exp.XMLNamespace]: 8022 namespaces = [] 8023 8024 while True: 8025 if self._match(TokenType.DEFAULT): 8026 uri = self._parse_string() 8027 else: 8028 uri = self._parse_alias(self._parse_string()) 8029 namespaces.append(self.expression(exp.XMLNamespace(this=uri))) 8030 if not self._match(TokenType.COMMA): 8031 break 8032 8033 return namespaces 8034 8035 def _parse_decode(self) -> exp.Decode | exp.DecodeCase | None: 8036 args = self._parse_csv(self._parse_disjunction) 8037 8038 if len(args) < 3: 8039 return self.expression(exp.Decode(this=seq_get(args, 0), charset=seq_get(args, 1))) 8040 8041 return self.expression(exp.DecodeCase(expressions=args)) 8042 8043 def _parse_json_key_value(self) -> exp.JSONKeyValue | None: 8044 self._match_text_seq("KEY") 8045 key = self._parse_column() 8046 self._match_set(self.JSON_KEY_VALUE_SEPARATOR_TOKENS) 8047 self._match_text_seq("VALUE") 8048 value = self._parse_bitwise() 8049 8050 if not key and not value: 8051 return None 8052 return self.expression(exp.JSONKeyValue(this=key, expression=value)) 8053 8054 def _parse_format_json(self, this: exp.Expr | None) -> exp.Expr | None: 8055 if not this or not self._match_text_seq("FORMAT", "JSON"): 8056 return this 8057 8058 return self.expression(exp.FormatJson(this=this)) 8059 8060 def _parse_on_condition(self) -> exp.OnCondition | None: 8061 # MySQL uses "X ON EMPTY Y ON ERROR" (e.g. JSON_VALUE) while Oracle uses the opposite (e.g. JSON_EXISTS) 8062 if self.dialect.ON_CONDITION_EMPTY_BEFORE_ERROR: 8063 empty = self._parse_on_handling("EMPTY", *self.ON_CONDITION_TOKENS) 8064 error = self._parse_on_handling("ERROR", *self.ON_CONDITION_TOKENS) 8065 else: 8066 error = self._parse_on_handling("ERROR", *self.ON_CONDITION_TOKENS) 8067 empty = self._parse_on_handling("EMPTY", *self.ON_CONDITION_TOKENS) 8068 8069 null = self._parse_on_handling("NULL", *self.ON_CONDITION_TOKENS) 8070 8071 if not empty and not error and not null: 8072 return None 8073 8074 return self.expression(exp.OnCondition(empty=empty, error=error, null=null)) 8075 8076 def _parse_on_handling(self, on: str, *values: str) -> str | None | exp.Expr | None: 8077 # Parses the "X ON Y" or "DEFAULT <expr> ON Y syntax, e.g. NULL ON NULL (Oracle, T-SQL, MySQL) 8078 for value in values: 8079 if self._match_text_seq(value, "ON", on): 8080 return f"{value} ON {on}" 8081 8082 index = self._index 8083 if self._match(TokenType.DEFAULT): 8084 default_value = self._parse_bitwise() 8085 if self._match_text_seq("ON", on): 8086 return default_value 8087 8088 self._retreat(index) 8089 8090 return None 8091 8092 @t.overload 8093 def _parse_json_object(self, agg: t.Literal[False]) -> exp.JSONObject: ... 8094 8095 @t.overload 8096 def _parse_json_object(self, agg: t.Literal[True]) -> exp.JSONObjectAgg: ... 8097 8098 def _parse_json_object(self, agg=False): 8099 star = self._parse_star() 8100 expressions = ( 8101 [star] 8102 if star 8103 else self._parse_csv(lambda: self._parse_format_json(self._parse_json_key_value())) 8104 ) 8105 null_handling = self._parse_on_handling("NULL", "NULL", "ABSENT") 8106 8107 unique_keys = None 8108 if self._match_text_seq("WITH", "UNIQUE"): 8109 unique_keys = True 8110 elif self._match_text_seq("WITHOUT", "UNIQUE"): 8111 unique_keys = False 8112 8113 self._match_text_seq("KEYS") 8114 8115 return_type = self._match_text_seq("RETURNING") and self._parse_format_json( 8116 self._parse_type() 8117 ) 8118 encoding = self._match_text_seq("ENCODING") and self._parse_var() 8119 8120 return self.expression( 8121 (exp.JSONObjectAgg if agg else exp.JSONObject)( 8122 expressions=expressions, 8123 null_handling=null_handling, 8124 unique_keys=unique_keys, 8125 return_type=return_type, 8126 encoding=encoding, 8127 ) 8128 ) 8129 8130 # Note: this is currently incomplete; it only implements the "JSON_value_column" part 8131 def _parse_json_column_def(self) -> exp.JSONColumnDef: 8132 if not self._match_text_seq("NESTED"): 8133 this = self._parse_id_var() 8134 ordinality = self._match_pair(TokenType.FOR, TokenType.ORDINALITY) 8135 kind = self._parse_types(allow_identifiers=False) 8136 nested = None 8137 else: 8138 this = None 8139 ordinality = None 8140 kind = None 8141 nested = True 8142 8143 format_json = self._match_text_seq("FORMAT", "JSON") 8144 path = self._match_text_seq("PATH") and self._parse_string() 8145 nested_schema = nested and self._parse_json_schema() 8146 8147 return self.expression( 8148 exp.JSONColumnDef( 8149 this=this, 8150 kind=kind, 8151 path=path, 8152 nested_schema=nested_schema, 8153 ordinality=ordinality, 8154 format_json=format_json, 8155 ) 8156 ) 8157 8158 def _parse_json_schema(self) -> exp.JSONSchema: 8159 self._match_text_seq("COLUMNS") 8160 return self.expression( 8161 exp.JSONSchema( 8162 expressions=self._parse_wrapped_csv(self._parse_json_column_def, optional=True) 8163 ) 8164 ) 8165 8166 def _parse_json_table(self) -> exp.JSONTable: 8167 this = self._parse_format_json(self._parse_bitwise()) 8168 path = self._match(TokenType.COMMA) and self._parse_string() 8169 error_handling = self._parse_on_handling("ERROR", "ERROR", "NULL") 8170 empty_handling = self._parse_on_handling("EMPTY", "ERROR", "NULL") 8171 schema = self._parse_json_schema() 8172 8173 return exp.JSONTable( 8174 this=this, 8175 schema=schema, 8176 path=path, 8177 error_handling=error_handling, 8178 empty_handling=empty_handling, 8179 ) 8180 8181 def _parse_match_against(self) -> exp.MatchAgainst: 8182 if self._match_text_seq("TABLE"): 8183 # parse SingleStore MATCH(TABLE ...) syntax 8184 # https://docs.singlestore.com/cloud/reference/sql-reference/full-text-search-functions/match/ 8185 expressions = [] 8186 table = self._parse_table() 8187 if table: 8188 expressions = [table] 8189 else: 8190 expressions = self._parse_csv(self._parse_column) 8191 8192 self._match_text_seq(")", "AGAINST", "(") 8193 8194 this = self._parse_string() 8195 8196 if self._match_text_seq("IN", "NATURAL", "LANGUAGE", "MODE"): 8197 modifier = "IN NATURAL LANGUAGE MODE" 8198 if self._match_text_seq("WITH", "QUERY", "EXPANSION"): 8199 modifier = f"{modifier} WITH QUERY EXPANSION" 8200 elif self._match_text_seq("IN", "BOOLEAN", "MODE"): 8201 modifier = "IN BOOLEAN MODE" 8202 elif self._match_text_seq("WITH", "QUERY", "EXPANSION"): 8203 modifier = "WITH QUERY EXPANSION" 8204 else: 8205 modifier = None 8206 8207 return self.expression( 8208 exp.MatchAgainst(this=this, expressions=expressions, modifier=modifier) 8209 ) 8210 8211 # https://learn.microsoft.com/en-us/sql/t-sql/functions/openjson-transact-sql?view=sql-server-ver16 8212 def _parse_open_json(self) -> exp.OpenJSON: 8213 this = self._parse_bitwise() 8214 path = self._match(TokenType.COMMA) and self._parse_string() 8215 8216 def _parse_open_json_column_def() -> exp.OpenJSONColumnDef: 8217 this = self._parse_field(any_token=True) 8218 kind = self._parse_types() 8219 path = self._parse_string() 8220 as_json = self._match_pair(TokenType.ALIAS, TokenType.JSON) 8221 8222 return self.expression( 8223 exp.OpenJSONColumnDef(this=this, kind=kind, path=path, as_json=as_json) 8224 ) 8225 8226 expressions = None 8227 if self._match_pair(TokenType.R_PAREN, TokenType.WITH): 8228 self._match_l_paren() 8229 expressions = self._parse_csv(_parse_open_json_column_def) 8230 8231 return self.expression(exp.OpenJSON(this=this, path=path, expressions=expressions)) 8232 8233 def _parse_position(self, haystack_first: bool = False) -> exp.StrPosition: 8234 args = self._parse_csv(self._parse_bitwise) 8235 8236 if self._match(TokenType.IN): 8237 return self.expression( 8238 exp.StrPosition(this=self._parse_bitwise(), substr=seq_get(args, 0)) 8239 ) 8240 8241 if haystack_first: 8242 haystack = seq_get(args, 0) 8243 needle = seq_get(args, 1) 8244 else: 8245 haystack = seq_get(args, 1) 8246 needle = seq_get(args, 0) 8247 8248 return self.expression( 8249 exp.StrPosition(this=haystack, substr=needle, position=seq_get(args, 2)) 8250 ) 8251 8252 def _parse_join_hint(self, func_name: str) -> exp.JoinHint: 8253 args = self._parse_csv(self._parse_table) 8254 return exp.JoinHint(this=func_name.upper(), expressions=args) 8255 8256 def _parse_substring(self) -> exp.Substring: 8257 # Postgres supports the form: substring(string [from int] [for int]) 8258 # (despite being undocumented, the reverse order also works) 8259 # https://www.postgresql.org/docs/9.1/functions-string.html @ Table 9-6 8260 8261 args = t.cast(list[t.Optional[exp.Expr]], self._parse_csv(self._parse_bitwise)) 8262 8263 start, length = None, None 8264 8265 while self._curr: 8266 if self._match(TokenType.FROM): 8267 start = self._parse_bitwise() 8268 elif self._match(TokenType.FOR): 8269 if not start: 8270 start = exp.Literal.number(1) 8271 length = self._parse_bitwise() 8272 else: 8273 break 8274 8275 if start: 8276 args.append(start) 8277 if length: 8278 args.append(length) 8279 8280 return self.validate_expression(exp.Substring.from_arg_list(args), args) 8281 8282 def _parse_trim(self) -> exp.Trim: 8283 # https://www.w3resource.com/sql/character-functions/trim.php 8284 # https://docs.oracle.com/javadb/10.8.3.0/ref/rreftrimfunc.html 8285 8286 position = None 8287 collation = None 8288 expression = None 8289 8290 if self._match_texts(self.TRIM_TYPES): 8291 position = self._prev.text.upper() 8292 8293 this = self._parse_bitwise() 8294 if self._match_set((TokenType.FROM, TokenType.COMMA)): 8295 invert_order = self._prev.token_type == TokenType.FROM or self.TRIM_PATTERN_FIRST 8296 expression = self._parse_bitwise() 8297 8298 if invert_order: 8299 this, expression = expression, this 8300 8301 if self._match(TokenType.COLLATE): 8302 collation = self._parse_bitwise() 8303 8304 return self.expression( 8305 exp.Trim(this=this, position=position, expression=expression, collation=collation) 8306 ) 8307 8308 def _parse_window_clause(self) -> list[exp.Expr] | None: 8309 return self._parse_csv(self._parse_named_window) if self._match(TokenType.WINDOW) else None 8310 8311 def _parse_named_window(self) -> exp.Expr | None: 8312 return self._parse_window(self._parse_id_var(), alias=True) 8313 8314 def _parse_respect_or_ignore_nulls(self, this: exp.Expr | None) -> exp.Expr | None: 8315 if self._curr.token_type == TokenType.VAR: 8316 if self._match_text_seq("IGNORE", "NULLS"): 8317 return self.expression(exp.IgnoreNulls(this=this)) 8318 if self._match_text_seq("RESPECT", "NULLS"): 8319 return self.expression(exp.RespectNulls(this=this)) 8320 return this 8321 8322 def _parse_having_max(self, this: exp.Expr | None) -> exp.Expr | None: 8323 if self._match(TokenType.HAVING): 8324 self._match_texts(("MAX", "MIN")) 8325 max = self._prev.text.upper() != "MIN" 8326 return self.expression( 8327 exp.HavingMax(this=this, expression=self._parse_column(), max=max) 8328 ) 8329 8330 return this 8331 8332 def _parse_window(self, this: exp.Expr | None, alias: bool = False) -> exp.Expr | None: 8333 func = this 8334 comments = func.comments if isinstance(func, exp.Expr) else None 8335 8336 # T-SQL allows the OVER (...) syntax after WITHIN GROUP. 8337 # https://learn.microsoft.com/en-us/sql/t-sql/functions/percentile-disc-transact-sql?view=sql-server-ver16 8338 if self._match_text_seq("WITHIN", "GROUP"): 8339 order = self._parse_wrapped(self._parse_order) 8340 this = self.expression(exp.WithinGroup(this=this, expression=order)) 8341 8342 if self._match_pair(TokenType.FILTER, TokenType.L_PAREN): 8343 self._match(TokenType.WHERE) 8344 this = self.expression( 8345 exp.Filter(this=this, expression=self._parse_where(skip_where_token=True)) 8346 ) 8347 self._match_r_paren() 8348 8349 # SQL spec defines an optional [ { IGNORE | RESPECT } NULLS ] OVER 8350 # Some dialects choose to implement and some do not. 8351 # https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html 8352 8353 # There is some code above in _parse_lambda that handles 8354 # SELECT FIRST_VALUE(TABLE.COLUMN IGNORE|RESPECT NULLS) OVER ... 8355 8356 # The below changes handle 8357 # SELECT FIRST_VALUE(TABLE.COLUMN) IGNORE|RESPECT NULLS OVER ... 8358 8359 # Oracle allows both formats 8360 # (https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/img_text/first_value.html) 8361 # and Snowflake chose to do the same for familiarity 8362 # https://docs.snowflake.com/en/sql-reference/functions/first_value.html#usage-notes 8363 if isinstance(this, exp.AggFunc): 8364 ignore_respect = this.find(exp.IgnoreNulls, exp.RespectNulls) 8365 8366 if ignore_respect and ignore_respect is not this: 8367 ignore_respect.replace(ignore_respect.this) 8368 this = self.expression(ignore_respect.__class__(this=this)) 8369 8370 this = self._parse_respect_or_ignore_nulls(this) 8371 8372 # bigquery select from window x AS (partition by ...) 8373 if alias: 8374 over = None 8375 self._match(TokenType.ALIAS) 8376 elif not self._match_set(self.WINDOW_BEFORE_PAREN_TOKENS): 8377 return this 8378 else: 8379 over = self._prev.text.upper() 8380 8381 if comments and isinstance(func, exp.Expr): 8382 func.pop_comments() 8383 8384 if not self._match(TokenType.L_PAREN): 8385 return self.expression( 8386 exp.Window(this=this, alias=self._parse_id_var(False), over=over), comments=comments 8387 ) 8388 8389 window_alias = self._parse_id_var(any_token=False, tokens=self.WINDOW_ALIAS_TOKENS) 8390 8391 first: bool | None = True if self._match(TokenType.FIRST) else None 8392 if self._match_text_seq("LAST"): 8393 first = False 8394 8395 partition, order = self._parse_partition_and_order() 8396 kind = ( 8397 self._match_set((TokenType.ROWS, TokenType.RANGE)) or self._match_text_seq("GROUPS") 8398 ) and self._prev.text 8399 8400 if kind: 8401 self._match(TokenType.BETWEEN) 8402 start = self._parse_window_spec() 8403 8404 end = self._parse_window_spec() if self._match(TokenType.AND) else {} 8405 exclude = ( 8406 self._parse_var_from_options(self.WINDOW_EXCLUDE_OPTIONS) 8407 if self._match_text_seq("EXCLUDE") 8408 else None 8409 ) 8410 8411 spec = self.expression( 8412 exp.WindowSpec( 8413 kind=kind, 8414 start=start["value"], 8415 start_side=start["side"], 8416 end=end.get("value"), 8417 end_side=end.get("side"), 8418 exclude=exclude, 8419 ) 8420 ) 8421 else: 8422 spec = None 8423 8424 self._match_r_paren() 8425 8426 window = self.expression( 8427 exp.Window( 8428 this=this, 8429 partition_by=partition, 8430 order=order, 8431 spec=spec, 8432 alias=window_alias, 8433 over=over, 8434 first=first, 8435 ), 8436 comments=comments, 8437 ) 8438 8439 # This covers Oracle's FIRST/LAST syntax: aggregate KEEP (...) OVER (...) 8440 if self._match_set(self.WINDOW_BEFORE_PAREN_TOKENS, advance=False): 8441 return self._parse_window(window, alias=alias) 8442 8443 return window 8444 8445 def _parse_partition_and_order( 8446 self, 8447 ) -> tuple[list[exp.Expr], exp.Expr | None]: 8448 return self._parse_partition_by(), self._parse_order() 8449 8450 def _parse_window_spec(self) -> dict[str, str | exp.Expr | None]: 8451 self._match(TokenType.BETWEEN) 8452 8453 return { 8454 "value": ( 8455 (self._match_text_seq("UNBOUNDED") and "UNBOUNDED") 8456 or (self._match_text_seq("CURRENT", "ROW") and "CURRENT ROW") 8457 or self._parse_bitwise() 8458 ), 8459 "side": self._prev.text if self._match_texts(self.WINDOW_SIDES) else None, 8460 } 8461 8462 def _parse_alias(self, this: exp.Expr | None, explicit: bool = False) -> exp.Expr | None: 8463 # In some dialects, LIMIT and OFFSET can act as both identifiers and keywords (clauses) 8464 # so this section tries to parse the clause version and if it fails, it treats the token 8465 # as an identifier (alias) 8466 if self._can_parse_limit_or_offset(): 8467 return this 8468 8469 # WINDOW is in ID_VAR_TOKENS, so it can be consumed as an implicit alias. Detect the 8470 # named-window clause shape (`WINDOW <ident> AS (...)`) and avoid swallowing it. 8471 if self._can_parse_named_window(): 8472 return this 8473 8474 any_token = self._match(TokenType.ALIAS) 8475 comments = self._prev_comments 8476 8477 if explicit and not any_token: 8478 return this 8479 8480 if self._match(TokenType.L_PAREN): 8481 aliases = self.expression( 8482 exp.Aliases( 8483 this=this, expressions=self._parse_csv(lambda: self._parse_id_var(any_token)) 8484 ), 8485 comments=comments, 8486 ) 8487 self._match_r_paren(aliases) 8488 return aliases 8489 8490 alias = self._parse_id_var(any_token, tokens=self.ALIAS_TOKENS) or ( 8491 self.STRING_ALIASES and self._parse_string_as_identifier() 8492 ) 8493 8494 if alias: 8495 comments.extend(alias.pop_comments()) 8496 this = self.expression(exp.Alias(this=this, alias=alias), comments=comments) 8497 column = this.this 8498 8499 # Moves the comment next to the alias in `expr /* comment */ AS alias` 8500 if not this.comments and column and column.comments: 8501 this.comments = column.pop_comments() 8502 8503 return this 8504 8505 def _parse_id_var( 8506 self, 8507 any_token: bool = True, 8508 tokens: t.Collection[TokenType] | None = None, 8509 ) -> exp.Expr | None: 8510 expression = self._parse_identifier() 8511 if not expression and ( 8512 (any_token and self._advance_any()) or self._match_set(tokens or self.ID_VAR_TOKENS) 8513 ): 8514 quoted = self._prev.token_type == TokenType.STRING 8515 expression = self._identifier_expression(quoted=quoted) 8516 8517 return expression 8518 8519 def _parse_string(self) -> exp.Expr | None: 8520 if self._match_set(self.STRING_PARSERS): 8521 return self.STRING_PARSERS[self._prev.token_type](self, self._prev) 8522 return self._parse_placeholder() 8523 8524 def _parse_string_as_identifier(self) -> exp.Identifier | None: 8525 if not self._match(TokenType.STRING): 8526 return None 8527 output = exp.to_identifier(self._prev.text, quoted=True) 8528 output.update_positions(self._prev) 8529 return output 8530 8531 def _parse_number(self) -> exp.Expr | None: 8532 if self._match_set(self.NUMERIC_PARSERS): 8533 return self.NUMERIC_PARSERS[self._prev.token_type](self, self._prev) 8534 return self._parse_placeholder() 8535 8536 def _parse_identifier(self) -> exp.Expr | None: 8537 if self._match(TokenType.IDENTIFIER): 8538 return self._identifier_expression(quoted=True) 8539 return self._parse_placeholder() 8540 8541 def _parse_var( 8542 self, 8543 any_token: bool = False, 8544 tokens: t.Collection[TokenType] | None = None, 8545 upper: bool = False, 8546 ) -> exp.Expr | None: 8547 if ( 8548 (any_token and self._advance_any()) 8549 or self._match(TokenType.VAR) 8550 or (self._match_set(tokens) if tokens else False) 8551 ): 8552 return self.expression( 8553 exp.Var(this=self._prev.text.upper() if upper else self._prev.text) 8554 ) 8555 return self._parse_placeholder() 8556 8557 def _advance_any(self, ignore_reserved: bool = False) -> Token | None: 8558 if self._curr and (ignore_reserved or self._curr.token_type not in self.RESERVED_TOKENS): 8559 self._advance() 8560 return self._prev 8561 return None 8562 8563 def _parse_var_or_string(self, upper: bool = False) -> exp.Expr | None: 8564 return self._parse_string() or self._parse_var(any_token=True, upper=upper) 8565 8566 def _parse_primary_or_var(self) -> exp.Expr | None: 8567 return self._parse_primary() or self._parse_var(any_token=True) 8568 8569 def _parse_null(self) -> exp.Expr | None: 8570 if self._match_set((TokenType.NULL, TokenType.UNKNOWN)): 8571 return self.PRIMARY_PARSERS[TokenType.NULL](self, self._prev) 8572 return self._parse_placeholder() 8573 8574 def _parse_boolean(self) -> exp.Expr | None: 8575 if self._match(TokenType.TRUE): 8576 return self.PRIMARY_PARSERS[TokenType.TRUE](self, self._prev) 8577 if self._match(TokenType.FALSE): 8578 return self.PRIMARY_PARSERS[TokenType.FALSE](self, self._prev) 8579 return self._parse_placeholder() 8580 8581 def _parse_star(self) -> exp.Expr | None: 8582 if self._match(TokenType.STAR): 8583 return self.PRIMARY_PARSERS[TokenType.STAR](self, self._prev) 8584 return self._parse_placeholder() 8585 8586 def _parse_parameter(self) -> exp.Parameter: 8587 this = self._parse_identifier() or self._parse_primary_or_var() 8588 return self.expression(exp.Parameter(this=this)) 8589 8590 def _parse_placeholder(self) -> exp.Expr | None: 8591 if self._match_set(self.PLACEHOLDER_PARSERS): 8592 placeholder = self.PLACEHOLDER_PARSERS[self._prev.token_type](self) 8593 if placeholder: 8594 return placeholder 8595 self._advance(-1) 8596 return None 8597 8598 def _parse_star_op(self, *keywords: str) -> list[exp.Expr] | None: 8599 if not self._match_texts(keywords): 8600 return None 8601 if self._match(TokenType.L_PAREN, advance=False): 8602 return self._parse_wrapped_csv(self._parse_expression) 8603 8604 expression = self._parse_alias(self._parse_disjunction(), explicit=True) 8605 return [expression] if expression else None 8606 8607 def _parse_csv( 8608 self, parse_method: t.Callable[[], T | None], sep: TokenType = TokenType.COMMA 8609 ) -> list[T]: 8610 parse_result = parse_method() 8611 items = [parse_result] if parse_result is not None else [] 8612 8613 while self._match(sep): 8614 if isinstance(parse_result, exp.Expr): 8615 self._add_comments(parse_result) 8616 parse_result = parse_method() 8617 if parse_result is not None: 8618 items.append(parse_result) 8619 8620 return items 8621 8622 def _parse_wrapped_id_vars(self, optional: bool = False) -> list[exp.Expr]: 8623 return self._parse_wrapped_csv(self._parse_id_var, optional=optional) 8624 8625 def _parse_wrapped_csv( 8626 self, 8627 parse_method: t.Callable[[], T | None], 8628 sep: TokenType = TokenType.COMMA, 8629 optional: bool = False, 8630 ) -> list[T]: 8631 return self._parse_wrapped( 8632 lambda: self._parse_csv(parse_method, sep=sep), optional=optional 8633 ) 8634 8635 def _parse_wrapped(self, parse_method: t.Callable[[], T], optional: bool = False) -> T: 8636 wrapped = self._match(TokenType.L_PAREN) 8637 if not wrapped and not optional: 8638 self.raise_error("Expecting (") 8639 parse_result = parse_method() 8640 if wrapped: 8641 self._match_r_paren() 8642 return parse_result 8643 8644 def _parse_expressions(self) -> list[exp.Expr]: 8645 return self._parse_csv(self._parse_expression) 8646 8647 def _parse_select_or_expression(self, alias: bool = False) -> exp.Expr | None: 8648 return ( 8649 self._parse_set_operations( 8650 self._parse_alias(self._parse_assignment(), explicit=True) 8651 if alias 8652 else self._parse_assignment() 8653 ) 8654 or self._parse_select() 8655 ) 8656 8657 def _parse_ddl_select(self) -> exp.Expr | None: 8658 return self._parse_query_modifiers( 8659 self._parse_set_operations(self._parse_select(nested=True, parse_subquery_alias=False)) 8660 ) 8661 8662 def _parse_transaction(self) -> exp.Transaction | exp.Command: 8663 this = None 8664 if self._match_texts(self.TRANSACTION_KIND): 8665 this = self._prev.text 8666 8667 self._match_texts(("TRANSACTION", "WORK")) 8668 8669 modes = [] 8670 while True: 8671 mode = [] 8672 while self._match(TokenType.VAR) or self._match(TokenType.NOT): 8673 mode.append(self._prev.text) 8674 8675 if mode: 8676 modes.append(" ".join(mode)) 8677 if not self._match(TokenType.COMMA): 8678 break 8679 8680 return self.expression(exp.Transaction(this=this, modes=modes)) 8681 8682 def _parse_commit_or_rollback(self) -> exp.Commit | exp.Rollback: 8683 chain = None 8684 savepoint = None 8685 is_rollback = self._prev.token_type == TokenType.ROLLBACK 8686 8687 self._match_texts(("TRANSACTION", "WORK")) 8688 8689 if self._match_text_seq("TO"): 8690 self._match_text_seq("SAVEPOINT") 8691 savepoint = self._parse_id_var() 8692 8693 if self._match(TokenType.AND): 8694 chain = not self._match_text_seq("NO") 8695 self._match_text_seq("CHAIN") 8696 8697 if is_rollback: 8698 return self.expression(exp.Rollback(savepoint=savepoint)) 8699 8700 return self.expression(exp.Commit(chain=chain)) 8701 8702 def _parse_refresh(self) -> exp.Refresh | exp.Command: 8703 if self._match(TokenType.TABLE): 8704 kind = "TABLE" 8705 elif self._match_text_seq("MATERIALIZED", "VIEW"): 8706 kind = "MATERIALIZED VIEW" 8707 else: 8708 kind = "" 8709 8710 this = self._parse_string() or self._parse_table() 8711 if not kind and not isinstance(this, exp.Literal): 8712 return self._parse_as_command(self._prev) 8713 8714 return self.expression(exp.Refresh(this=this, kind=kind)) 8715 8716 def _parse_column_def_with_exists(self): 8717 start = self._index 8718 self._match(TokenType.COLUMN) 8719 8720 exists_column = self._parse_exists(not_=True) 8721 expression = self._parse_field_def() 8722 8723 if not isinstance(expression, exp.ColumnDef): 8724 self._retreat(start) 8725 return None 8726 8727 expression.set("exists", exists_column) 8728 8729 return expression 8730 8731 def _parse_add_column(self) -> exp.ColumnDef | None: 8732 if not self._prev.text.upper() == "ADD": 8733 return None 8734 8735 return self._parse_column_def_with_exists() 8736 8737 def _parse_drop_column(self) -> exp.Drop | exp.Command | None: 8738 drop = self._parse_drop() if self._match(TokenType.DROP) else None 8739 if drop and not isinstance(drop, exp.Command): 8740 drop.set("kind", drop.args.get("kind", "COLUMN")) 8741 return drop 8742 8743 def _parse_alter_drop_action(self) -> exp.Expr | None: 8744 return self._parse_drop_column() 8745 8746 # https://docs.aws.amazon.com/athena/latest/ug/alter-table-drop-partition.html 8747 def _parse_drop_partition(self, exists: bool | None = None) -> exp.DropPartition: 8748 return self.expression( 8749 exp.DropPartition(expressions=self._parse_csv(self._parse_partition), exists=exists) 8750 ) 8751 8752 def _parse_alter_table_add(self) -> list[exp.Expr]: 8753 def _parse_add_alteration() -> exp.Expr | None: 8754 self._match_text_seq("ADD") 8755 if self._match_set(self.ADD_CONSTRAINT_TOKENS, advance=False): 8756 return self.expression( 8757 exp.AddConstraint(expressions=self._parse_csv(self._parse_constraint)) 8758 ) 8759 8760 column_def = self._parse_add_column() 8761 if isinstance(column_def, exp.ColumnDef): 8762 return column_def 8763 8764 exists = self._parse_exists(not_=True) 8765 if self._match_pair(TokenType.PARTITION, TokenType.L_PAREN, advance=False): 8766 return self.expression( 8767 exp.AddPartition( 8768 exists=exists, 8769 this=self._parse_field(any_token=True), 8770 location=self._match_text_seq("LOCATION", advance=False) 8771 and self._parse_property(), 8772 ) 8773 ) 8774 8775 return None 8776 8777 if not self._match_set(self.ADD_CONSTRAINT_TOKENS, advance=False) and ( 8778 not self.dialect.ALTER_TABLE_ADD_REQUIRED_FOR_EACH_COLUMN 8779 or self._match_text_seq("COLUMNS") 8780 ): 8781 schema = self._parse_schema() 8782 8783 return ( 8784 ensure_list(schema) 8785 if schema 8786 else self._parse_csv(self._parse_column_def_with_exists) 8787 ) 8788 8789 return self._parse_csv(_parse_add_alteration) 8790 8791 def _parse_alter_table_alter(self) -> exp.Expr | None: 8792 if self._match_texts(self.ALTER_ALTER_PARSERS): 8793 return self.ALTER_ALTER_PARSERS[self._prev.text.upper()](self) 8794 8795 # Many dialects support the ALTER [COLUMN] syntax, so if there is no 8796 # keyword after ALTER we default to parsing this statement 8797 self._match(TokenType.COLUMN) 8798 column = self._parse_field(any_token=True) 8799 8800 if self._match_pair(TokenType.DROP, TokenType.DEFAULT): 8801 return self.expression(exp.AlterColumn(this=column, drop=True)) 8802 if self._match_pair(TokenType.SET, TokenType.DEFAULT): 8803 return self.expression(exp.AlterColumn(this=column, default=self._parse_disjunction())) 8804 if self._match(TokenType.COMMENT): 8805 return self.expression(exp.AlterColumn(this=column, comment=self._parse_string())) 8806 if self._match_text_seq("DROP", "NOT", "NULL"): 8807 return self.expression(exp.AlterColumn(this=column, drop=True, allow_null=True)) 8808 if self._match_text_seq("SET", "NOT", "NULL"): 8809 return self.expression(exp.AlterColumn(this=column, allow_null=False)) 8810 8811 if self._match_text_seq("SET", "VISIBLE"): 8812 return self.expression(exp.AlterColumn(this=column, visible="VISIBLE")) 8813 if self._match_text_seq("SET", "INVISIBLE"): 8814 return self.expression(exp.AlterColumn(this=column, visible="INVISIBLE")) 8815 8816 self._match_text_seq("SET", "DATA") 8817 self._match_text_seq("TYPE") 8818 return self.expression( 8819 exp.AlterColumn( 8820 this=column, 8821 dtype=self._parse_types(), 8822 collate=self._match(TokenType.COLLATE) and self._parse_term(), 8823 using=self._match(TokenType.USING) and self._parse_disjunction(), 8824 ) 8825 ) 8826 8827 def _parse_alter_diststyle(self) -> exp.AlterDistStyle: 8828 if self._match_texts(("ALL", "EVEN", "AUTO")): 8829 return self.expression(exp.AlterDistStyle(this=exp.var(self._prev.text.upper()))) 8830 8831 self._match_text_seq("KEY", "DISTKEY") 8832 return self.expression(exp.AlterDistStyle(this=self._parse_column())) 8833 8834 def _parse_alter_sortkey(self, compound: bool | None = None) -> exp.AlterSortKey: 8835 if compound: 8836 self._match_text_seq("SORTKEY") 8837 8838 if self._match(TokenType.L_PAREN, advance=False): 8839 return self.expression( 8840 exp.AlterSortKey(expressions=self._parse_wrapped_id_vars(), compound=compound) 8841 ) 8842 8843 self._match_texts(("AUTO", "NONE")) 8844 return self.expression( 8845 exp.AlterSortKey(this=exp.var(self._prev.text.upper()), compound=compound) 8846 ) 8847 8848 def _parse_alter_table_drop(self) -> list[exp.Expr]: 8849 index = self._index - 1 8850 8851 partition_exists = self._parse_exists() 8852 if self._match(TokenType.PARTITION, advance=False): 8853 return self._parse_csv(lambda: self._parse_drop_partition(exists=partition_exists)) 8854 8855 self._retreat(index) 8856 return self._parse_csv(self._parse_alter_drop_action) 8857 8858 def _parse_alter_table_rename(self) -> exp.AlterRename | exp.RenameColumn | None: 8859 if self._match(TokenType.COLUMN) or ( 8860 not self.ALTER_RENAME_REQUIRES_COLUMN and not self._match_text_seq("TO", advance=False) 8861 ): 8862 exists = self._parse_exists() 8863 old_column = self._parse_column() 8864 to = self._match_text_seq("TO") 8865 new_column = self._parse_column() 8866 8867 if old_column is None or not to or new_column is None: 8868 return None 8869 8870 return self.expression(exp.RenameColumn(this=old_column, to=new_column, exists=exists)) 8871 8872 self._match_text_seq("TO") 8873 return self.expression(exp.AlterRename(this=self._parse_table(schema=True))) 8874 8875 def _parse_alter_table_set(self) -> exp.AlterSet: 8876 alter_set = self.expression(exp.AlterSet()) 8877 8878 if self._match(TokenType.L_PAREN, advance=False) or self._match_text_seq( 8879 "TABLE", "PROPERTIES" 8880 ): 8881 alter_set.set("expressions", self._parse_wrapped_csv(self._parse_assignment)) 8882 elif self._match_text_seq("FILESTREAM_ON", advance=False): 8883 alter_set.set("expressions", [self._parse_assignment()]) 8884 elif self._match_texts(("LOGGED", "UNLOGGED")): 8885 alter_set.set("option", exp.var(self._prev.text.upper())) 8886 elif self._match_text_seq("WITHOUT") and self._match_texts(("CLUSTER", "OIDS")): 8887 alter_set.set("option", exp.var(f"WITHOUT {self._prev.text.upper()}")) 8888 elif self._match_text_seq("LOCATION"): 8889 alter_set.set("location", self._parse_field()) 8890 elif self._match_text_seq("ACCESS", "METHOD"): 8891 alter_set.set("access_method", self._parse_field()) 8892 elif self._match_text_seq("TABLESPACE"): 8893 alter_set.set("tablespace", self._parse_field()) 8894 elif self._match_text_seq("FILE", "FORMAT") or self._match_text_seq("FILEFORMAT"): 8895 alter_set.set("file_format", [self._parse_field()]) 8896 elif self._match_text_seq("STAGE_FILE_FORMAT"): 8897 alter_set.set("file_format", self._parse_wrapped_options()) 8898 elif self._match_text_seq("STAGE_COPY_OPTIONS"): 8899 alter_set.set("copy_options", self._parse_wrapped_options()) 8900 elif self._match_text_seq("TAG") or self._match_text_seq("TAGS"): 8901 alter_set.set("tag", self._parse_csv(self._parse_assignment)) 8902 else: 8903 if self._match_text_seq("SERDE"): 8904 alter_set.set("serde", self._parse_field()) 8905 8906 properties = self._parse_wrapped(self._parse_properties, optional=True) 8907 alter_set.set("expressions", [properties]) 8908 8909 return alter_set 8910 8911 def _parse_alter_session(self) -> exp.AlterSession: 8912 """Parse ALTER SESSION SET/UNSET statements.""" 8913 if self._match(TokenType.SET): 8914 expressions = self._parse_csv(lambda: self._parse_set_item_assignment()) 8915 return self.expression(exp.AlterSession(expressions=expressions, unset=False)) 8916 8917 self._match_text_seq("UNSET") 8918 expressions = self._parse_csv( 8919 lambda: self.expression(exp.SetItem(this=self._parse_id_var(any_token=True))) 8920 ) 8921 return self.expression(exp.AlterSession(expressions=expressions, unset=True)) 8922 8923 def _parse_alter(self) -> exp.Alter | exp.Command: 8924 start = self._prev 8925 8926 iceberg = self._match_text_seq("ICEBERG") 8927 8928 alter_token = self._match_set(self.ALTERABLES) and self._prev 8929 if not alter_token: 8930 return self._parse_as_command(start) 8931 if iceberg and alter_token.token_type != TokenType.TABLE: 8932 return self._parse_as_command(start) 8933 8934 exists = self._parse_exists() 8935 only = self._match_text_seq("ONLY") 8936 8937 if alter_token.token_type == TokenType.SESSION: 8938 this = None 8939 check = None 8940 cluster = None 8941 else: 8942 this = self._parse_table(schema=True, parse_partition=self.ALTER_TABLE_PARTITIONS) 8943 check = self._match_text_seq("WITH", "CHECK") 8944 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 8945 8946 if self._next: 8947 self._advance() 8948 8949 parser = self.ALTER_PARSERS.get(self._prev.text.upper()) if self._prev else None 8950 if parser: 8951 actions = ensure_list(parser(self)) 8952 not_valid = self._match_text_seq("NOT", "VALID") 8953 options = self._parse_csv(self._parse_property) 8954 cascade = self.dialect.ALTER_TABLE_SUPPORTS_CASCADE and self._match_text_seq("CASCADE") 8955 8956 if not self._curr and actions: 8957 return self.expression( 8958 exp.Alter( 8959 this=this, 8960 kind=alter_token.text.upper(), 8961 exists=exists, 8962 actions=actions, 8963 only=only, 8964 options=options, 8965 cluster=cluster, 8966 not_valid=not_valid, 8967 check=check, 8968 cascade=cascade, 8969 iceberg=iceberg, 8970 ) 8971 ) 8972 8973 return self._parse_as_command(start) 8974 8975 def _parse_analyze(self) -> exp.Analyze | exp.Command: 8976 start = self._prev 8977 # https://duckdb.org/docs/sql/statements/analyze 8978 if not self._curr: 8979 return self.expression(exp.Analyze()) 8980 8981 options = [] 8982 while self._match_texts(self.ANALYZE_STYLES): 8983 if self._prev.text.upper() == "BUFFER_USAGE_LIMIT": 8984 options.append(f"BUFFER_USAGE_LIMIT {self._parse_number()}") 8985 else: 8986 options.append(self._prev.text.upper()) 8987 8988 this: exp.Expr | None = None 8989 inner_expression: exp.Expr | None = None 8990 8991 kind = self._curr.text.upper() if self._curr else None 8992 8993 if self._match(TokenType.TABLE) or self._match(TokenType.INDEX): 8994 this = self._parse_table_parts() 8995 elif self._match_text_seq("TABLES"): 8996 if self._match_set((TokenType.FROM, TokenType.IN)): 8997 kind = f"{kind} {self._prev.text.upper()}" 8998 this = self._parse_table(schema=True, is_db_reference=True) 8999 elif self._match_text_seq("DATABASE"): 9000 this = self._parse_table(schema=True, is_db_reference=True) 9001 elif self._match_text_seq("CLUSTER"): 9002 this = self._parse_table() 9003 # Try matching inner expr keywords before fallback to parse table. 9004 elif self._match_texts(self.ANALYZE_EXPRESSION_PARSERS): 9005 kind = None 9006 inner_expression = self.ANALYZE_EXPRESSION_PARSERS[self._prev.text.upper()](self) 9007 else: 9008 # Empty kind https://prestodb.io/docs/current/sql/analyze.html 9009 kind = None 9010 this = self._parse_table_parts() 9011 9012 partition = self._try_parse(self._parse_partition) 9013 if not partition and self._match_texts(self.PARTITION_KEYWORDS): 9014 return self._parse_as_command(start) 9015 9016 # https://docs.starrocks.io/docs/sql-reference/sql-statements/cbo_stats/ANALYZE_TABLE/ 9017 if self._match_text_seq("WITH", "SYNC", "MODE") or self._match_text_seq( 9018 "WITH", "ASYNC", "MODE" 9019 ): 9020 mode = f"WITH {self._tokens[self._index - 2].text.upper()} MODE" 9021 else: 9022 mode = None 9023 9024 if self._match_texts(self.ANALYZE_EXPRESSION_PARSERS): 9025 inner_expression = self.ANALYZE_EXPRESSION_PARSERS[self._prev.text.upper()](self) 9026 9027 properties = self._parse_properties() 9028 return self.expression( 9029 exp.Analyze( 9030 kind=kind, 9031 this=this, 9032 mode=mode, 9033 partition=partition, 9034 properties=properties, 9035 expression=inner_expression, 9036 options=options, 9037 ) 9038 ) 9039 9040 # https://spark.apache.org/docs/3.5.1/sql-ref-syntax-aux-analyze-table.html 9041 def _parse_analyze_statistics(self) -> exp.AnalyzeStatistics: 9042 this = None 9043 kind = self._prev.text.upper() 9044 option = self._prev.text.upper() if self._match_text_seq("DELTA") else None 9045 expressions = [] 9046 9047 if not self._match_text_seq("STATISTICS"): 9048 self.raise_error("Expecting token STATISTICS") 9049 9050 if self._match_text_seq("NOSCAN"): 9051 this = "NOSCAN" 9052 elif self._match(TokenType.FOR): 9053 if self._match_text_seq("ALL", "COLUMNS"): 9054 this = "FOR ALL COLUMNS" 9055 if self._match_texts("COLUMNS"): 9056 this = "FOR COLUMNS" 9057 expressions = self._parse_csv(self._parse_column_reference) 9058 elif self._match_text_seq("SAMPLE"): 9059 sample = self._parse_number() 9060 expressions = [ 9061 self.expression( 9062 exp.AnalyzeSample( 9063 sample=sample, 9064 kind=self._prev.text.upper() if self._match(TokenType.PERCENT) else None, 9065 ) 9066 ) 9067 ] 9068 9069 return self.expression( 9070 exp.AnalyzeStatistics(kind=kind, option=option, this=this, expressions=expressions) 9071 ) 9072 9073 # https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/ANALYZE.html 9074 def _parse_analyze_validate(self) -> exp.AnalyzeValidate: 9075 kind = None 9076 this = None 9077 expression: exp.Expr | None = None 9078 if self._match_text_seq("REF", "UPDATE"): 9079 kind = "REF" 9080 this = "UPDATE" 9081 if self._match_text_seq("SET", "DANGLING", "TO", "NULL"): 9082 this = "UPDATE SET DANGLING TO NULL" 9083 elif self._match_text_seq("STRUCTURE"): 9084 kind = "STRUCTURE" 9085 if self._match_text_seq("CASCADE", "FAST"): 9086 this = "CASCADE FAST" 9087 elif self._match_text_seq("CASCADE", "COMPLETE") and self._match_texts( 9088 ("ONLINE", "OFFLINE") 9089 ): 9090 this = f"CASCADE COMPLETE {self._prev.text.upper()}" 9091 expression = self._parse_into() 9092 9093 return self.expression(exp.AnalyzeValidate(kind=kind, this=this, expression=expression)) 9094 9095 def _parse_analyze_columns(self) -> exp.AnalyzeColumns | None: 9096 this = self._prev.text.upper() 9097 if self._match_text_seq("COLUMNS"): 9098 return self.expression(exp.AnalyzeColumns(this=f"{this} {self._prev.text.upper()}")) 9099 return None 9100 9101 def _parse_analyze_delete(self) -> exp.AnalyzeDelete | None: 9102 kind = self._prev.text.upper() if self._match_text_seq("SYSTEM") else None 9103 if self._match_text_seq("STATISTICS"): 9104 return self.expression(exp.AnalyzeDelete(kind=kind)) 9105 return None 9106 9107 def _parse_analyze_list(self) -> exp.AnalyzeListChainedRows | None: 9108 if self._match_text_seq("CHAINED", "ROWS"): 9109 return self.expression(exp.AnalyzeListChainedRows(expression=self._parse_into())) 9110 return None 9111 9112 # https://dev.mysql.com/doc/refman/8.4/en/analyze-table.html 9113 def _parse_analyze_histogram(self) -> exp.AnalyzeHistogram: 9114 this = self._prev.text.upper() 9115 expression: exp.Expr | None = None 9116 expressions = [] 9117 update_options = None 9118 9119 if self._match_text_seq("HISTOGRAM", "ON"): 9120 expressions = self._parse_csv(self._parse_column_reference) 9121 with_expressions = [] 9122 while self._match(TokenType.WITH): 9123 # https://docs.starrocks.io/docs/sql-reference/sql-statements/cbo_stats/ANALYZE_TABLE/ 9124 if self._match_texts(("SYNC", "ASYNC")): 9125 if self._match_text_seq("MODE", advance=False): 9126 with_expressions.append(f"{self._prev.text.upper()} MODE") 9127 self._advance() 9128 else: 9129 buckets = self._parse_number() 9130 if self._match_text_seq("BUCKETS"): 9131 with_expressions.append(f"{buckets} BUCKETS") 9132 if with_expressions: 9133 expression = self.expression(exp.AnalyzeWith(expressions=with_expressions)) 9134 9135 if self._match_texts(("MANUAL", "AUTO")) and self._match( 9136 TokenType.UPDATE, advance=False 9137 ): 9138 update_options = self._prev.text.upper() 9139 self._advance() 9140 elif self._match_text_seq("USING", "DATA"): 9141 expression = self.expression(exp.UsingData(this=self._parse_string())) 9142 9143 return self.expression( 9144 exp.AnalyzeHistogram( 9145 this=this, 9146 expressions=expressions, 9147 expression=expression, 9148 update_options=update_options, 9149 ) 9150 ) 9151 9152 def _parse_merge(self) -> exp.Merge: 9153 self._match(TokenType.INTO) 9154 target = self._parse_table() 9155 9156 if target and self._match(TokenType.ALIAS, advance=False): 9157 target.set("alias", self._parse_table_alias()) 9158 9159 self._match(TokenType.USING) 9160 using = self._parse_table() 9161 9162 return self.expression( 9163 exp.Merge( 9164 this=target, 9165 using=using, 9166 on=self._match(TokenType.ON) and self._parse_disjunction(), 9167 using_cond=self._match(TokenType.USING) and self._parse_using_identifiers(), 9168 whens=self._parse_when_matched(), 9169 returning=self._parse_returning(), 9170 ) 9171 ) 9172 9173 def _parse_when_matched(self) -> exp.Whens: 9174 whens = [] 9175 9176 while self._match(TokenType.WHEN): 9177 matched = not self._match(TokenType.NOT) 9178 self._match_text_seq("MATCHED") 9179 source = ( 9180 False 9181 if self._match_text_seq("BY", "TARGET") 9182 else self._match_text_seq("BY", "SOURCE") 9183 ) 9184 condition = self._parse_disjunction() if self._match(TokenType.AND) else None 9185 9186 self._match(TokenType.THEN) 9187 9188 if self._match(TokenType.INSERT): 9189 this = self._parse_star() 9190 if this: 9191 then: exp.Expr | None = self.expression(exp.Insert(this=this)) 9192 else: 9193 then = self.expression( 9194 exp.Insert( 9195 this=exp.var("ROW") 9196 if self._match_text_seq("ROW") 9197 else self._parse_value(values=False), 9198 expression=self._match_text_seq("VALUES") and self._parse_value(), 9199 where=self._parse_where(), 9200 ) 9201 ) 9202 elif self._match(TokenType.UPDATE): 9203 expressions = self._parse_star() 9204 if expressions: 9205 then = self.expression(exp.Update(expressions=expressions)) 9206 else: 9207 then = self.expression( 9208 exp.Update( 9209 expressions=self._match(TokenType.SET) 9210 and self._parse_csv(self._parse_equality), 9211 where=self._parse_where(), 9212 ) 9213 ) 9214 elif self._match(TokenType.DELETE): 9215 then = self.expression(exp.Var(this=self._prev.text)) 9216 else: 9217 then = self._parse_var_from_options(self.CONFLICT_ACTIONS) 9218 9219 whens.append( 9220 self.expression( 9221 exp.When(matched=matched, source=source, condition=condition, then=then) 9222 ) 9223 ) 9224 return self.expression(exp.Whens(expressions=whens)) 9225 9226 def _parse_show(self) -> exp.Expr | None: 9227 parser = self._find_parser(self.SHOW_PARSERS, self.SHOW_TRIE) 9228 if parser: 9229 return parser(self) 9230 return self._parse_as_command(self._prev) 9231 9232 def _parse_set_item_assignment(self, kind: str | None = None) -> exp.Expr | None: 9233 index = self._index 9234 9235 if kind in ("GLOBAL", "SESSION") and self._match_text_seq("TRANSACTION"): 9236 return self._parse_set_transaction(global_=kind == "GLOBAL") 9237 9238 left = self._parse_primary() or self._parse_column() 9239 assignment_delimiter = self._match_texts(self.SET_ASSIGNMENT_DELIMITERS) 9240 9241 if not left or (self.SET_REQUIRES_ASSIGNMENT_DELIMITER and not assignment_delimiter): 9242 self._retreat(index) 9243 return None 9244 9245 right = self._parse_statement() or self._parse_id_var() 9246 if isinstance(right, (exp.Column, exp.Identifier)): 9247 right = exp.var(right.name) 9248 9249 this = self.expression(exp.EQ(this=left, expression=right)) 9250 return self.expression(exp.SetItem(this=this, kind=kind)) 9251 9252 def _parse_set_transaction(self, global_: bool = False) -> exp.Expr: 9253 self._match_text_seq("TRANSACTION") 9254 characteristics = self._parse_csv( 9255 lambda: self._parse_var_from_options(self.TRANSACTION_CHARACTERISTICS) 9256 ) 9257 return self.expression( 9258 exp.SetItem(expressions=characteristics, kind="TRANSACTION", global_=global_) 9259 ) 9260 9261 def _parse_set_item(self) -> exp.Expr | None: 9262 parser = self._find_parser(self.SET_PARSERS, self.SET_TRIE) 9263 return parser(self) if parser else self._parse_set_item_assignment(kind=None) 9264 9265 def _parse_set(self, unset: bool = False, tag: bool = False) -> exp.Set | exp.Command: 9266 index = self._index 9267 set_ = self.expression( 9268 exp.Set(expressions=self._parse_csv(self._parse_set_item), unset=unset, tag=tag) 9269 ) 9270 9271 if self._curr: 9272 self._retreat(index) 9273 return self._parse_as_command(self._prev) 9274 9275 return set_ 9276 9277 def _parse_var_from_options( 9278 self, options: OPTIONS_TYPE, raise_unmatched: bool = True 9279 ) -> exp.Var | None: 9280 start = self._curr 9281 if not start: 9282 return None 9283 9284 option = start.text.upper() 9285 continuations = options.get(option) 9286 9287 index = self._index 9288 self._advance() 9289 for keywords in continuations or []: 9290 if isinstance(keywords, str): 9291 keywords = (keywords,) 9292 9293 if self._match_text_seq(*keywords): 9294 option = f"{option} {' '.join(keywords)}" 9295 break 9296 else: 9297 if continuations or continuations is None: 9298 if raise_unmatched: 9299 self.raise_error(f"Unknown option {option}") 9300 9301 self._retreat(index) 9302 return None 9303 9304 return exp.var(option) 9305 9306 def _parse_as_command(self, start: Token) -> exp.Command: 9307 while self._curr: 9308 self._advance() 9309 text = self._find_sql(start, self._prev) 9310 size = len(start.text) 9311 self._warn_unsupported() 9312 return exp.Command(this=text[:size], expression=text[size:]) 9313 9314 def _parse_dict_property(self, this: str) -> exp.DictProperty: 9315 settings = [] 9316 9317 self._match_l_paren() 9318 kind = self._parse_id_var() 9319 9320 if self._match(TokenType.L_PAREN): 9321 while True: 9322 key = self._parse_id_var() 9323 value = self._parse_function() or self._parse_primary_or_var() 9324 if not key and value is None: 9325 break 9326 settings.append(self.expression(exp.DictSubProperty(this=key, value=value))) 9327 self._match(TokenType.R_PAREN) 9328 9329 self._match_r_paren() 9330 9331 return self.expression( 9332 exp.DictProperty(this=this, kind=kind.this if kind else None, settings=settings) 9333 ) 9334 9335 def _parse_dict_range(self, this: str) -> exp.DictRange: 9336 self._match_l_paren() 9337 has_min = self._match_text_seq("MIN") 9338 if has_min: 9339 min = self._parse_var() or self._parse_primary() 9340 self._match_text_seq("MAX") 9341 max = self._parse_var() or self._parse_primary() 9342 else: 9343 max = self._parse_var() or self._parse_primary() 9344 min = exp.Literal.number(0) 9345 self._match_r_paren() 9346 return self.expression(exp.DictRange(this=this, min=min, max=max)) 9347 9348 def _parse_comprehension(self, this: exp.Expr | None) -> exp.Comprehension | None: 9349 index = self._index 9350 expression = self._parse_column() 9351 position = self._match(TokenType.COMMA) and self._parse_column() 9352 9353 if not self._match(TokenType.IN): 9354 self._retreat(index - 1) 9355 return None 9356 iterator = self._parse_column() 9357 condition = self._parse_disjunction() if self._match_text_seq("IF") else None 9358 return self.expression( 9359 exp.Comprehension( 9360 this=this, 9361 expression=expression, 9362 position=position, 9363 iterator=iterator, 9364 condition=condition, 9365 ) 9366 ) 9367 9368 def _parse_heredoc(self) -> exp.Heredoc | None: 9369 if self._match(TokenType.HEREDOC_STRING): 9370 return self.expression(exp.Heredoc(this=self._prev.text)) 9371 9372 if not self._match_text_seq("$"): 9373 return None 9374 9375 tags = ["$"] 9376 tag_text = None 9377 9378 if self._is_connected(): 9379 self._advance() 9380 tags.append(self._prev.text.upper()) 9381 else: 9382 self.raise_error("No closing $ found") 9383 9384 if tags[-1] != "$": 9385 if self._is_connected() and self._match_text_seq("$"): 9386 tag_text = tags[-1] 9387 tags.append("$") 9388 else: 9389 self.raise_error("No closing $ found") 9390 9391 heredoc_start = self._curr 9392 9393 while self._curr: 9394 if self._match_text_seq(*tags, advance=False): 9395 this = self._find_sql(heredoc_start, self._prev) 9396 self._advance(len(tags)) 9397 return self.expression(exp.Heredoc(this=this, tag=tag_text)) 9398 9399 self._advance() 9400 9401 self.raise_error(f"No closing {''.join(tags)} found") 9402 return None 9403 9404 def _find_parser(self, parsers: dict[str, t.Callable], trie: dict) -> t.Callable | None: 9405 if not self._curr: 9406 return None 9407 9408 index = self._index 9409 this = [] 9410 while True: 9411 # The current token might be multiple words 9412 curr = self._curr.text.upper() 9413 key = curr.split(" ") 9414 this.append(curr) 9415 9416 self._advance() 9417 result, trie = in_trie(trie, key) 9418 if result == TrieResult.FAILED: 9419 break 9420 9421 if result == TrieResult.EXISTS: 9422 subparser = parsers[" ".join(this)] 9423 return subparser 9424 9425 self._retreat(index) 9426 return None 9427 9428 def _match_l_paren(self, expression: exp.Expr | None = None) -> None: 9429 if not self._match(TokenType.L_PAREN, expression=expression): 9430 self.raise_error("Expecting (") 9431 9432 def _match_r_paren(self, expression: exp.Expr | None = None) -> None: 9433 if not self._match(TokenType.R_PAREN, expression=expression): 9434 self.raise_error("Expecting )") 9435 9436 def _replace_lambda( 9437 self, node: exp.Expr | None, expressions: list[exp.Expr] 9438 ) -> exp.Expr | None: 9439 if not node: 9440 return node 9441 9442 lambda_types = {e.name: e.args.get("to") or False for e in expressions} 9443 9444 for column in node.find_all(exp.Column): 9445 typ = lambda_types.get(column.parts[0].name) 9446 if typ is not None: 9447 dot_or_id = column.to_dot() if column.table else column.this 9448 9449 if typ: 9450 dot_or_id = self.expression(exp.Cast(this=dot_or_id, to=typ)) 9451 9452 parent = column.parent 9453 9454 while isinstance(parent, exp.Dot): 9455 if not isinstance(parent.parent, exp.Dot): 9456 parent.replace(dot_or_id) 9457 break 9458 parent = parent.parent 9459 else: 9460 if column is node: 9461 node = dot_or_id 9462 else: 9463 column.replace(dot_or_id) 9464 return node 9465 9466 def _parse_truncate_table(self) -> exp.TruncateTable | None | exp.Expr: 9467 start = self._prev 9468 9469 # Not to be confused with TRUNCATE(number, decimals) function call 9470 if self._match(TokenType.L_PAREN): 9471 self._retreat(self._index - 2) 9472 return self._parse_function() 9473 9474 # Clickhouse supports TRUNCATE DATABASE as well 9475 is_database = self._match(TokenType.DATABASE) 9476 9477 self._match(TokenType.TABLE) 9478 9479 exists = self._parse_exists(not_=False) 9480 9481 expressions = self._parse_csv( 9482 lambda: self._parse_table(schema=True, is_db_reference=is_database) 9483 ) 9484 9485 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 9486 9487 if self._match_text_seq("RESTART", "IDENTITY"): 9488 identity = "RESTART" 9489 elif self._match_text_seq("CONTINUE", "IDENTITY"): 9490 identity = "CONTINUE" 9491 else: 9492 identity = None 9493 9494 if self._match_text_seq("CASCADE") or self._match_text_seq("RESTRICT"): 9495 option = self._prev.text 9496 else: 9497 option = None 9498 9499 partition = self._parse_partition() 9500 9501 # Fallback case 9502 if self._curr: 9503 return self._parse_as_command(start) 9504 9505 return self.expression( 9506 exp.TruncateTable( 9507 expressions=expressions, 9508 is_database=is_database, 9509 exists=exists, 9510 cluster=cluster, 9511 identity=identity, 9512 option=option, 9513 partition=partition, 9514 ) 9515 ) 9516 9517 def _parse_indexed_column(self) -> exp.Expr | None: 9518 return self._parse_ordered(self._parse_opclass) 9519 9520 def _parse_with_operator(self) -> exp.Expr | None: 9521 this = self._parse_indexed_column() 9522 9523 if not self._match(TokenType.WITH): 9524 return this 9525 9526 op = self._parse_var(any_token=True, tokens=self.RESERVED_TOKENS) 9527 9528 return self.expression(exp.WithOperator(this=this, op=op)) 9529 9530 def _parse_wrapped_options(self) -> list[exp.Expr]: 9531 self._match(TokenType.EQ) 9532 self._match(TokenType.L_PAREN) 9533 9534 opts: list[exp.Expr] = [] 9535 option: exp.Expr | list[exp.Expr] | None 9536 while self._curr and not self._match(TokenType.R_PAREN): 9537 if self._match_text_seq("FORMAT_NAME", "="): 9538 # The FORMAT_NAME can be set to an identifier for Snowflake and T-SQL 9539 option = self._parse_format_name() 9540 else: 9541 option = self._parse_property() 9542 9543 if option is None: 9544 self.raise_error("Unable to parse option") 9545 break 9546 9547 opts.extend(ensure_list(option)) 9548 9549 return opts 9550 9551 def _parse_copy_parameters(self) -> list[exp.CopyParameter]: 9552 sep = TokenType.COMMA if self.dialect.COPY_PARAMS_ARE_CSV else None 9553 9554 options = [] 9555 while self._curr and not self._match(TokenType.R_PAREN, advance=False): 9556 option = self._parse_var(any_token=True) 9557 prev = self._prev.text.upper() 9558 9559 # Different dialects might separate options and values by white space, "=" and "AS" 9560 self._match(TokenType.EQ) 9561 self._match(TokenType.ALIAS) 9562 9563 param = self.expression(exp.CopyParameter(this=option)) 9564 9565 if prev in self.COPY_INTO_VARLEN_OPTIONS and self._match( 9566 TokenType.L_PAREN, advance=False 9567 ): 9568 # Snowflake FILE_FORMAT case, Databricks COPY & FORMAT options 9569 param.set("expressions", self._parse_wrapped_options()) 9570 elif prev == "FILE_FORMAT": 9571 # T-SQL's external file format case 9572 param.set("expression", self._parse_field()) 9573 elif ( 9574 prev == "FORMAT" 9575 and self._prev.token_type == TokenType.ALIAS 9576 and self._match_texts(("AVRO", "JSON")) 9577 ): 9578 param.set("this", exp.var(f"FORMAT AS {self._prev.text.upper()}")) 9579 param.set("expression", self._parse_field()) 9580 else: 9581 param.set("expression", self._parse_unquoted_field() or self._parse_bracket()) 9582 9583 options.append(param) 9584 9585 if sep: 9586 self._match(sep) 9587 9588 return options 9589 9590 def _parse_credentials(self) -> exp.Credentials | None: 9591 expr = self.expression(exp.Credentials()) 9592 9593 if self._match_text_seq("STORAGE_INTEGRATION", "="): 9594 expr.set("storage", self._parse_field()) 9595 if self._match_text_seq("CREDENTIALS"): 9596 # Snowflake case: CREDENTIALS = (...), Redshift case: CREDENTIALS <string> 9597 creds = ( 9598 self._parse_wrapped_options() if self._match(TokenType.EQ) else self._parse_field() 9599 ) 9600 expr.set("credentials", creds) 9601 if self._match_text_seq("ENCRYPTION"): 9602 expr.set("encryption", self._parse_wrapped_options()) 9603 if self._match_text_seq("IAM_ROLE"): 9604 expr.set( 9605 "iam_role", 9606 exp.var(self._prev.text) if self._match(TokenType.DEFAULT) else self._parse_field(), 9607 ) 9608 if self._match_text_seq("REGION"): 9609 expr.set("region", self._parse_field()) 9610 9611 return expr 9612 9613 def _parse_file_location(self) -> exp.Expr | None: 9614 return self._parse_field() 9615 9616 def _parse_copy(self) -> exp.Copy | exp.Command: 9617 start = self._prev 9618 9619 self._match(TokenType.INTO) 9620 9621 this = ( 9622 self._parse_select(nested=True, parse_subquery_alias=False) 9623 if self._match(TokenType.L_PAREN, advance=False) 9624 else self._parse_table(schema=True) 9625 ) 9626 9627 kind = self._match(TokenType.FROM) or not self._match_text_seq("TO") 9628 9629 files = self._parse_csv(self._parse_file_location) 9630 if self._match(TokenType.EQ, advance=False): 9631 # Backtrack one token since we've consumed the lhs of a parameter assignment here. 9632 # This can happen for Snowflake dialect. Instead, we'd like to parse the parameter 9633 # list via `_parse_wrapped(..)` below. 9634 self._advance(-1) 9635 files = [] 9636 9637 credentials = self._parse_credentials() 9638 9639 self._match_text_seq("WITH") 9640 9641 params = self._parse_wrapped(self._parse_copy_parameters, optional=True) 9642 9643 # Fallback case 9644 if self._curr: 9645 return self._parse_as_command(start) 9646 9647 return self.expression( 9648 exp.Copy(this=this, kind=kind, credentials=credentials, files=files, params=params) 9649 ) 9650 9651 def _parse_normalize(self) -> exp.Normalize: 9652 return self.expression( 9653 exp.Normalize( 9654 this=self._parse_bitwise(), form=self._match(TokenType.COMMA) and self._parse_var() 9655 ) 9656 ) 9657 9658 def _parse_ceil_floor(self, expr_type: type[TCeilFloor]) -> TCeilFloor: 9659 args = self._parse_csv(lambda: self._parse_lambda()) 9660 9661 this = seq_get(args, 0) 9662 decimals = seq_get(args, 1) 9663 9664 return expr_type( 9665 this=this, 9666 decimals=decimals, 9667 to=self._parse_var() if self._match_text_seq("TO") else None, 9668 ) 9669 9670 def _parse_star_ops(self) -> exp.Expr | None: 9671 star_token = self._prev 9672 9673 if self._match_text_seq("COLUMNS", "(", advance=False): 9674 this = self._parse_function() 9675 if isinstance(this, exp.Columns): 9676 this.set("unpack", True) 9677 return this 9678 9679 ilike = self._parse_string() if self._match(TokenType.ILIKE) else None 9680 9681 return self.expression( 9682 exp.Star( 9683 ilike=ilike, 9684 except_=self._parse_star_op("EXCEPT", "EXCLUDE"), 9685 replace=self._parse_star_op("REPLACE"), 9686 rename=self._parse_star_op("RENAME"), 9687 ) 9688 ).update_positions(star_token) 9689 9690 def _parse_grant_privilege(self) -> exp.GrantPrivilege | None: 9691 privilege_parts = [] 9692 9693 # Keep consuming consecutive keywords until comma (end of this privilege) or ON 9694 # (end of privilege list) or L_PAREN (start of column list) are met 9695 while self._curr and not self._match_set(self.PRIVILEGE_FOLLOW_TOKENS, advance=False): 9696 privilege_parts.append(self._curr.text.upper()) 9697 self._advance() 9698 9699 this = exp.var(" ".join(privilege_parts)) 9700 expressions = ( 9701 self._parse_wrapped_csv(self._parse_column) 9702 if self._match(TokenType.L_PAREN, advance=False) 9703 else None 9704 ) 9705 9706 return self.expression(exp.GrantPrivilege(this=this, expressions=expressions)) 9707 9708 def _parse_grant_principal(self) -> exp.GrantPrincipal | None: 9709 kind = self._match_texts(("ROLE", "GROUP")) and self._prev.text.upper() 9710 principal = self._parse_id_var() 9711 9712 if not principal: 9713 return None 9714 9715 return self.expression(exp.GrantPrincipal(this=principal, kind=kind)) 9716 9717 def _parse_grant_revoke_common( 9718 self, 9719 ) -> tuple[list | None, str | None, exp.Expr | None]: 9720 privileges = self._parse_csv(self._parse_grant_privilege) 9721 9722 self._match(TokenType.ON) 9723 kind = self._prev.text.upper() if self._match_set(self.CREATABLES) else None 9724 9725 # Attempt to parse the securable e.g. MySQL allows names 9726 # such as "foo.*", "*.*" which are not easily parseable yet 9727 securable = self._try_parse(self._parse_table_parts) 9728 9729 return privileges, kind, securable 9730 9731 def _parse_grant(self) -> exp.Grant | exp.Command: 9732 start = self._prev 9733 9734 privileges, kind, securable = self._parse_grant_revoke_common() 9735 9736 if not securable or not self._match_text_seq("TO"): 9737 return self._parse_as_command(start) 9738 9739 principals = self._parse_csv(self._parse_grant_principal) 9740 9741 grant_option = self._match_text_seq("WITH", "GRANT", "OPTION") 9742 9743 if self._curr: 9744 return self._parse_as_command(start) 9745 9746 return self.expression( 9747 exp.Grant( 9748 privileges=privileges, 9749 kind=kind, 9750 securable=securable, 9751 principals=principals, 9752 grant_option=grant_option, 9753 ) 9754 ) 9755 9756 def _parse_revoke(self) -> exp.Revoke | exp.Command: 9757 start = self._prev 9758 9759 grant_option = self._match_text_seq("GRANT", "OPTION", "FOR") 9760 9761 privileges, kind, securable = self._parse_grant_revoke_common() 9762 9763 if not securable or not self._match_text_seq("FROM"): 9764 return self._parse_as_command(start) 9765 9766 principals = self._parse_csv(self._parse_grant_principal) 9767 9768 cascade = None 9769 if self._match_texts(("CASCADE", "RESTRICT")): 9770 cascade = self._prev.text.upper() 9771 9772 if self._curr: 9773 return self._parse_as_command(start) 9774 9775 return self.expression( 9776 exp.Revoke( 9777 privileges=privileges, 9778 kind=kind, 9779 securable=securable, 9780 principals=principals, 9781 grant_option=grant_option, 9782 cascade=cascade, 9783 ) 9784 ) 9785 9786 def _parse_overlay(self) -> exp.Overlay: 9787 def _parse_overlay_arg(text: str) -> exp.Expr | None: 9788 return ( 9789 self._parse_bitwise() 9790 if self._match(TokenType.COMMA) or self._match_text_seq(text) 9791 else None 9792 ) 9793 9794 return self.expression( 9795 exp.Overlay( 9796 this=self._parse_bitwise(), 9797 expression=_parse_overlay_arg("PLACING"), 9798 from_=_parse_overlay_arg("FROM"), 9799 for_=_parse_overlay_arg("FOR"), 9800 ) 9801 ) 9802 9803 def _parse_format_name(self) -> exp.Property: 9804 # Note: Although not specified in the docs, Snowflake does accept a string/identifier 9805 # for FILE_FORMAT = <format_name> 9806 return self.expression( 9807 exp.Property( 9808 this=exp.var("FORMAT_NAME"), value=self._parse_string() or self._parse_table_parts() 9809 ) 9810 ) 9811 9812 def _parse_max_min_by(self, expr_type: type[exp.AggFunc]) -> exp.AggFunc: 9813 args: list[exp.Expr] = [] 9814 9815 if self._match(TokenType.DISTINCT): 9816 args.append(self.expression(exp.Distinct(expressions=[self._parse_lambda()]))) 9817 self._match(TokenType.COMMA) 9818 9819 args.extend(self._parse_function_args()) 9820 9821 return self.expression( 9822 expr_type(this=seq_get(args, 0), expression=seq_get(args, 1), count=seq_get(args, 2)) 9823 ) 9824 9825 def _identifier_expression( 9826 self, token: Token | None = None, quoted: bool | None = None 9827 ) -> exp.Identifier: 9828 token = token or self._prev 9829 return self.expression(exp.Identifier(this=token.text, quoted=quoted), token) 9830 9831 def _build_pipe_cte( 9832 self, 9833 query: exp.Query, 9834 expressions: list[exp.Expr], 9835 alias_cte: exp.TableAlias | None = None, 9836 ) -> exp.Select: 9837 new_cte: str | exp.TableAlias | None 9838 if alias_cte: 9839 new_cte = alias_cte 9840 else: 9841 self._pipe_cte_counter += 1 9842 new_cte = f"__tmp{self._pipe_cte_counter}" 9843 9844 with_ = query.args.get("with_") 9845 ctes = with_.pop() if with_ else None 9846 9847 new_select = exp.select(*expressions, copy=False).from_(new_cte, copy=False) 9848 if ctes: 9849 new_select.set("with_", ctes) 9850 9851 return new_select.with_(new_cte, as_=query, copy=False) 9852 9853 def _parse_pipe_syntax_select(self, query: exp.Select) -> exp.Select: 9854 select = self._parse_select(consume_pipe=False) 9855 if not select: 9856 return query 9857 9858 return self._build_pipe_cte( 9859 query=query.select(*select.expressions, append=False), expressions=[exp.Star()] 9860 ) 9861 9862 def _parse_pipe_syntax_limit(self, query: exp.Select) -> exp.Select: 9863 limit = self._parse_limit() 9864 offset = self._parse_offset() 9865 if limit: 9866 curr_limit = query.args.get("limit", limit) 9867 if curr_limit.expression.to_py() >= limit.expression.to_py(): 9868 query.limit(limit, copy=False) 9869 if offset: 9870 curr_offset = query.args.get("offset") 9871 curr_offset = curr_offset.expression.to_py() if curr_offset else 0 9872 query.offset(exp.Literal.number(curr_offset + offset.expression.to_py()), copy=False) 9873 9874 return query 9875 9876 def _parse_pipe_syntax_aggregate_fields(self) -> exp.Expr | None: 9877 this = self._parse_disjunction() 9878 if self._match_text_seq("GROUP", "AND", advance=False): 9879 return this 9880 9881 this = self._parse_alias(this) 9882 9883 if self._match_set((TokenType.ASC, TokenType.DESC), advance=False): 9884 return self._parse_ordered(lambda: this) 9885 9886 return this 9887 9888 def _parse_pipe_syntax_aggregate_group_order_by( 9889 self, query: exp.Select, group_by_exists: bool = True 9890 ) -> exp.Select: 9891 expr = self._parse_csv(self._parse_pipe_syntax_aggregate_fields) 9892 aggregates_or_groups, orders = [], [] 9893 for element in expr: 9894 if isinstance(element, exp.Ordered): 9895 this = element.this 9896 if isinstance(this, exp.Alias): 9897 element.set("this", this.args["alias"]) 9898 orders.append(element) 9899 else: 9900 this = element 9901 aggregates_or_groups.append(this) 9902 9903 if group_by_exists: 9904 query.select( 9905 *aggregates_or_groups, *query.expressions, append=False, copy=False 9906 ).group_by( 9907 *[projection.args.get("alias", projection) for projection in aggregates_or_groups], 9908 copy=False, 9909 ) 9910 else: 9911 query.select(*aggregates_or_groups, append=False, copy=False) 9912 9913 if orders: 9914 return query.order_by(*orders, append=False, copy=False) 9915 9916 return query 9917 9918 def _parse_pipe_syntax_aggregate(self, query: exp.Select) -> exp.Select: 9919 self._match_text_seq("AGGREGATE") 9920 query = self._parse_pipe_syntax_aggregate_group_order_by(query, group_by_exists=False) 9921 9922 if self._match(TokenType.GROUP_BY) or ( 9923 self._match_text_seq("GROUP", "AND") and self._match(TokenType.ORDER_BY) 9924 ): 9925 query = self._parse_pipe_syntax_aggregate_group_order_by(query) 9926 9927 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 9928 9929 def _parse_pipe_syntax_set_operator(self, query: exp.Query) -> exp.Query | None: 9930 first_setop = self.parse_set_operation(this=query) 9931 if not first_setop: 9932 return None 9933 9934 def _parse_and_unwrap_query() -> exp.Expr | None: 9935 expr = self._parse_paren() 9936 return expr.assert_is(exp.Subquery).unnest() if expr else None 9937 9938 first_setop.this.pop() 9939 9940 setops = [ 9941 first_setop.expression.pop().assert_is(exp.Subquery).unnest(), 9942 *self._parse_csv(_parse_and_unwrap_query), 9943 ] 9944 9945 query = self._build_pipe_cte(query=query, expressions=[exp.Star()]) 9946 with_ = query.args.get("with_") 9947 ctes = with_.pop() if with_ else None 9948 9949 if isinstance(first_setop, exp.Union): 9950 query = query.union(*setops, copy=False, **first_setop.args) 9951 elif isinstance(first_setop, exp.Except): 9952 query = query.except_(*setops, copy=False, **first_setop.args) 9953 else: 9954 query = query.intersect(*setops, copy=False, **first_setop.args) 9955 9956 query.set("with_", ctes) 9957 9958 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 9959 9960 def _parse_pipe_syntax_join(self, query: exp.Query) -> exp.Query | None: 9961 join = self._parse_join() 9962 if not join: 9963 return None 9964 9965 if isinstance(query, exp.Select): 9966 return query.join(join, copy=False) 9967 9968 return query 9969 9970 def _parse_pipe_syntax_pivot(self, query: exp.Select) -> exp.Select: 9971 pivots = self._parse_pivots() 9972 if not pivots: 9973 return query 9974 9975 from_ = query.args.get("from_") 9976 if from_: 9977 from_.this.set("pivots", pivots) 9978 9979 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 9980 9981 def _parse_pipe_syntax_extend(self, query: exp.Select) -> exp.Select: 9982 self._match_text_seq("EXTEND") 9983 query.select(*[exp.Star(), *self._parse_expressions()], append=False, copy=False) 9984 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 9985 9986 def _parse_pipe_syntax_tablesample(self, query: exp.Select) -> exp.Select: 9987 sample = self._parse_table_sample() 9988 9989 with_ = query.args.get("with_") 9990 if with_: 9991 with_.expressions[-1].this.set("sample", sample) 9992 else: 9993 query.set("sample", sample) 9994 9995 return query 9996 9997 def _parse_pipe_syntax_query(self, query: exp.Query) -> exp.Query | None: 9998 if isinstance(query, exp.Subquery): 9999 query = exp.select("*").from_(query, copy=False) 10000 10001 if not query.args.get("from_"): 10002 query = exp.select("*").from_(query.subquery(copy=False), copy=False) 10003 10004 while self._match(TokenType.PIPE_GT): 10005 start_index = self._index 10006 start_text = self._curr.text.upper() 10007 parser = self.PIPE_SYNTAX_TRANSFORM_PARSERS.get(start_text) 10008 if not parser: 10009 # The set operators (UNION, etc) and the JOIN operator have a few common starting 10010 # keywords, making it tricky to disambiguate them without lookahead. The approach 10011 # here is to try and parse a set operation and if that fails, then try to parse a 10012 # join operator. If that fails as well, then the operator is not supported. 10013 parsed_query = self._parse_pipe_syntax_set_operator(query) 10014 parsed_query = parsed_query or self._parse_pipe_syntax_join(query) 10015 if not parsed_query: 10016 self._retreat(start_index) 10017 self.raise_error(f"Unsupported pipe syntax operator: '{start_text}'.") 10018 break 10019 query = parsed_query 10020 else: 10021 query = parser(self, query) 10022 10023 return query 10024 10025 def _parse_declareitem(self) -> exp.DeclareItem | None: 10026 self._match_texts(("VAR", "VARIABLE")) 10027 10028 vars = self._parse_csv(self._parse_id_var) 10029 if not vars: 10030 return None 10031 10032 self._match(TokenType.ALIAS) 10033 kind = self._parse_schema() if self._match(TokenType.TABLE) else self._parse_types() 10034 default = ( 10035 self._match(TokenType.DEFAULT) or self._match(TokenType.EQ) 10036 ) and self._parse_bitwise() 10037 10038 return self.expression(exp.DeclareItem(this=vars, kind=kind, default=default)) 10039 10040 def _parse_declare(self) -> exp.Declare | exp.Command: 10041 start = self._prev 10042 replace = self._match_text_seq("OR", "REPLACE") 10043 expressions = self._try_parse(lambda: self._parse_csv(self._parse_declareitem)) 10044 10045 if not expressions or self._curr: 10046 return self._parse_as_command(start) 10047 10048 return self.expression(exp.Declare(expressions=expressions, replace=replace)) 10049 10050 def build_cast(self, strict: bool, **kwargs) -> exp.Expr: 10051 exp_class = exp.Cast if strict else exp.TryCast 10052 10053 if exp_class == exp.TryCast: 10054 kwargs["requires_string"] = self.dialect.TRY_CAST_REQUIRES_STRING 10055 10056 return self.expression(exp_class(**kwargs)) 10057 10058 def _parse_json_value(self) -> exp.JSONValue: 10059 this = self._parse_bitwise() 10060 self._match(TokenType.COMMA) 10061 path = self._parse_bitwise() 10062 10063 returning = self._match(TokenType.RETURNING) and self._parse_type() 10064 10065 return self.expression( 10066 exp.JSONValue( 10067 this=this, 10068 path=self.dialect.to_json_path(path), 10069 returning=returning, 10070 on_condition=self._parse_on_condition(), 10071 ) 10072 ) 10073 10074 def _parse_group_concat(self) -> exp.Expr | None: 10075 def concat_exprs(node: exp.Expr | None, exprs: list[exp.Expr]) -> exp.Expr: 10076 if isinstance(node, exp.Distinct) and len(node.expressions) > 1: 10077 concat_exprs = [ 10078 self.expression( 10079 exp.Concat( 10080 expressions=node.expressions, 10081 safe=True, 10082 coalesce=self.dialect.CONCAT_COALESCE, 10083 ) 10084 ) 10085 ] 10086 node.set("expressions", concat_exprs) 10087 return node 10088 if len(exprs) == 1: 10089 return exprs[0] 10090 return self.expression( 10091 exp.Concat(expressions=args, safe=True, coalesce=self.dialect.CONCAT_COALESCE) 10092 ) 10093 10094 args = self._parse_csv(self._parse_lambda) 10095 10096 if args: 10097 order = args[-1] if isinstance(args[-1], exp.Order) else None 10098 10099 if order: 10100 # Order By is the last (or only) expression in the list and has consumed the 'expr' before it, 10101 # remove 'expr' from exp.Order and add it back to args 10102 args[-1] = order.this 10103 order.set("this", concat_exprs(order.this, args)) 10104 10105 this = order or concat_exprs(args[0], args) 10106 else: 10107 this = None 10108 10109 separator = self._parse_field() if self._match(TokenType.SEPARATOR) else None 10110 10111 return self.expression(exp.GroupConcat(this=this, separator=separator)) 10112 10113 def _parse_initcap(self) -> exp.Initcap: 10114 expr = exp.Initcap.from_arg_list(self._parse_function_args()) 10115 10116 # attach dialect's default delimiters 10117 if expr.args.get("expression") is None: 10118 expr.set("expression", exp.Literal.string(self.dialect.INITCAP_DEFAULT_DELIMITER_CHARS)) 10119 10120 return expr 10121 10122 def _parse_operator(self, this: exp.Expr | None) -> exp.Expr | None: 10123 while True: 10124 if not self._match(TokenType.L_PAREN): 10125 break 10126 10127 op = "" 10128 while self._curr and not self._match(TokenType.R_PAREN): 10129 op += self._curr.text 10130 self._advance() 10131 10132 comments = self._prev_comments 10133 this = self.expression( 10134 exp.Operator(this=this, operator=op, expression=self._parse_bitwise()), 10135 comments=comments, 10136 ) 10137 10138 if not self._match(TokenType.OPERATOR): 10139 break 10140 10141 return this
45def build_var_map(args: BuilderArgs) -> exp.StarMap | exp.VarMap: 46 if len(args) == 1 and args[0].is_star: 47 return exp.StarMap(this=args[0]) 48 49 keys: list[ExpOrStr] = [] 50 values: list[ExpOrStr] = [] 51 for i in range(0, len(args), 2): 52 keys.append(args[i]) 53 values.append(args[i + 1]) 54 55 return exp.VarMap(keys=exp.array(*keys, copy=False), values=exp.array(*values, copy=False))
63def binary_range_parser( 64 expr_type: Type[exp.Expr], reverse_args: bool = False 65) -> t.Callable[[Parser, exp.Expr | None], exp.Expr | None]: 66 def _parse_binary_range(self: Parser, this: exp.Expr | None) -> exp.Expr | None: 67 expression = self._parse_bitwise() 68 if reverse_args: 69 this, expression = expression, this 70 return self._parse_escape(self.expression(expr_type(this=this, expression=expression))) 71 72 return _parse_binary_range
75def build_logarithm(args: BuilderArgs, dialect: Dialect) -> exp.Func: 76 # Default argument order is base, expression 77 this = seq_get(args, 0) 78 expression = seq_get(args, 1) 79 80 if expression: 81 if not dialect.LOG_BASE_FIRST: 82 this, expression = expression, this 83 return exp.Log(this=this, expression=expression) 84 85 return (exp.Ln if dialect.parser_class.LOG_DEFAULTS_TO_LN else exp.Log)(this=this)
105def build_extract_json_with_path( 106 expr_type: Type[E], 107) -> t.Callable[[BuilderArgs, Dialect], E]: 108 def _builder(args: BuilderArgs, dialect: Dialect) -> E: 109 expression = expr_type( 110 this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1)) 111 ) 112 if len(args) > 2 and expr_type is exp.JSONExtract: 113 expression.set("expressions", args[2:]) 114 if expr_type is exp.JSONExtractScalar: 115 expression.set("scalar_only", dialect.JSON_EXTRACT_SCALAR_SCALAR_ONLY) 116 117 return expression 118 119 return _builder
122def build_mod(args: BuilderArgs) -> exp.Mod: 123 this = seq_get(args, 0) 124 expression = seq_get(args, 1) 125 126 # Wrap the operands if they are binary nodes, e.g. MOD(a + 1, 7) -> (a + 1) % 7 127 this = exp.Paren(this=this) if isinstance(this, exp.Binary) else this 128 expression = exp.Paren(this=expression) if isinstance(expression, exp.Binary) else expression 129 130 return exp.Mod(this=this, expression=expression)
142def build_array_constructor( 143 exp_class: Type[E], args: list[t.Any], bracket_kind: TokenType, dialect: Dialect 144) -> exp.Expr: 145 array_exp = exp_class(expressions=args) 146 147 if exp_class == exp.Array and dialect.HAS_DISTINCT_ARRAY_CONSTRUCTORS: 148 array_exp.set("bracket_notation", bracket_kind == TokenType.L_BRACKET) 149 150 return array_exp
153def build_convert_timezone( 154 args: BuilderArgs, default_source_tz: str | None = None 155) -> exp.ConvertTimezone | exp.Anonymous: 156 if len(args) == 2: 157 source_tz = exp.Literal.string(default_source_tz) if default_source_tz else None 158 return exp.ConvertTimezone( 159 source_tz=source_tz, target_tz=seq_get(args, 0), timestamp=seq_get(args, 1) 160 ) 161 162 return exp.ConvertTimezone.from_arg_list(args)
165def build_trim(args: BuilderArgs, is_left: bool = True, reverse_args: bool = False) -> exp.Trim: 166 this, expression = seq_get(args, 0), seq_get(args, 1) 167 168 if expression and reverse_args: 169 this, expression = expression, this 170 171 return exp.Trim(this=this, expression=expression, position="LEADING" if is_left else "TRAILING")
188def build_array_append(args: BuilderArgs, dialect: Dialect) -> exp.ArrayAppend: 189 """ 190 Builds ArrayAppend with NULL propagation semantics based on the dialect configuration. 191 192 Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. 193 Others (DuckDB, PostgreSQL) create a new single-element array instead. 194 195 Args: 196 args: Function arguments [array, element] 197 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 198 199 Returns: 200 ArrayAppend expression with appropriate null_propagation flag 201 """ 202 return exp.ArrayAppend( 203 this=seq_get(args, 0), 204 expression=seq_get(args, 1), 205 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 206 )
Builds ArrayAppend with NULL propagation semantics based on the dialect configuration.
Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. Others (DuckDB, PostgreSQL) create a new single-element array instead.
Arguments:
- args: Function arguments [array, element]
- dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
Returns:
ArrayAppend expression with appropriate null_propagation flag
209def build_array_prepend(args: BuilderArgs, dialect: Dialect) -> exp.ArrayPrepend: 210 """ 211 Builds ArrayPrepend with NULL propagation semantics based on the dialect configuration. 212 213 Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. 214 Others (DuckDB, PostgreSQL) create a new single-element array instead. 215 216 Args: 217 args: Function arguments [array, element] 218 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 219 220 Returns: 221 ArrayPrepend expression with appropriate null_propagation flag 222 """ 223 return exp.ArrayPrepend( 224 this=seq_get(args, 0), 225 expression=seq_get(args, 1), 226 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 227 )
Builds ArrayPrepend with NULL propagation semantics based on the dialect configuration.
Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. Others (DuckDB, PostgreSQL) create a new single-element array instead.
Arguments:
- args: Function arguments [array, element]
- dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
Returns:
ArrayPrepend expression with appropriate null_propagation flag
230def build_array_concat(args: BuilderArgs, dialect: Dialect) -> exp.ArrayConcat: 231 """ 232 Builds ArrayConcat with NULL propagation semantics based on the dialect configuration. 233 234 Some dialects (Redshift, Snowflake) return NULL when any input array is NULL. 235 Others (DuckDB, PostgreSQL) skip NULL arrays and continue concatenation. 236 237 Args: 238 args: Function arguments [array1, array2, ...] (variadic) 239 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 240 241 Returns: 242 ArrayConcat expression with appropriate null_propagation flag 243 """ 244 return exp.ArrayConcat( 245 this=seq_get(args, 0), 246 expressions=args[1:], 247 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 248 )
Builds ArrayConcat with NULL propagation semantics based on the dialect configuration.
Some dialects (Redshift, Snowflake) return NULL when any input array is NULL. Others (DuckDB, PostgreSQL) skip NULL arrays and continue concatenation.
Arguments:
- args: Function arguments [array1, array2, ...] (variadic)
- dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
Returns:
ArrayConcat expression with appropriate null_propagation flag
251def build_array_remove(args: BuilderArgs, dialect: Dialect) -> exp.ArrayRemove: 252 """ 253 Builds ArrayRemove with NULL propagation semantics based on the dialect configuration. 254 255 Some dialects (Snowflake) return NULL when the removal value is NULL. 256 Others (DuckDB) may return empty array due to NULL comparison semantics. 257 258 Args: 259 args: Function arguments [array, value_to_remove] 260 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 261 262 Returns: 263 ArrayRemove expression with appropriate null_propagation flag 264 """ 265 return exp.ArrayRemove( 266 this=seq_get(args, 0), 267 expression=seq_get(args, 1), 268 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 269 )
Builds ArrayRemove with NULL propagation semantics based on the dialect configuration.
Some dialects (Snowflake) return NULL when the removal value is NULL. Others (DuckDB) may return empty array due to NULL comparison semantics.
Arguments:
- args: Function arguments [array, value_to_remove]
- dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
Returns:
ArrayRemove expression with appropriate null_propagation flag
290class Parser: 291 """ 292 Parser consumes a list of tokens produced by the Tokenizer and produces a parsed syntax tree. 293 294 Args: 295 error_level: The desired error level. 296 Default: ErrorLevel.IMMEDIATE 297 error_message_context: The amount of context to capture from a query string when displaying 298 the error message (in number of characters). 299 Default: 100 300 max_errors: Maximum number of error messages to include in a raised ParseError. 301 This is only relevant if error_level is ErrorLevel.RAISE. 302 Default: 3 303 max_nodes: Maximum number of AST nodes to prevent memory exhaustion. 304 Set to -1 (default) to disable the check. 305 """ 306 307 __slots__ = ( 308 "error_level", 309 "error_message_context", 310 "max_errors", 311 "max_nodes", 312 "dialect", 313 "sql", 314 "errors", 315 "_tokens", 316 "_index", 317 "_curr", 318 "_next", 319 "_prev", 320 "_prev_comments", 321 "_pipe_cte_counter", 322 "_chunks", 323 "_chunk_index", 324 "_tokens_size", 325 "_node_count", 326 ) 327 328 FUNCTIONS: t.ClassVar[dict[str, t.Callable]] = { 329 **{name: func.from_arg_list for name, func in exp.FUNCTION_BY_NAME.items()}, 330 **dict.fromkeys(("COALESCE", "IFNULL", "NVL"), build_coalesce), 331 "ARRAY": lambda args, dialect: exp.Array(expressions=args), 332 "ARRAYAGG": lambda args, dialect: exp.ArrayAgg( 333 this=seq_get(args, 0), nulls_excluded=dialect.ARRAY_AGG_INCLUDES_NULLS is None or None 334 ), 335 "ARRAY_AGG": lambda args, dialect: exp.ArrayAgg( 336 this=seq_get(args, 0), nulls_excluded=dialect.ARRAY_AGG_INCLUDES_NULLS is None or None 337 ), 338 "ARRAY_APPEND": build_array_append, 339 "ARRAY_CAT": build_array_concat, 340 "ARRAY_CONCAT": build_array_concat, 341 "ARRAY_INTERSECT": lambda args: exp.ArrayIntersect(expressions=args), 342 "ARRAY_INTERSECTION": lambda args: exp.ArrayIntersect(expressions=args), 343 "ARRAY_PREPEND": build_array_prepend, 344 "ARRAY_REMOVE": build_array_remove, 345 "COUNT": lambda args: exp.Count(this=seq_get(args, 0), expressions=args[1:], big_int=True), 346 "CONCAT": lambda args, dialect: exp.Concat( 347 expressions=args, 348 safe=not dialect.STRICT_STRING_CONCAT, 349 coalesce=dialect.CONCAT_COALESCE, 350 ), 351 "CONCAT_WS": lambda args, dialect: exp.ConcatWs( 352 expressions=args, 353 safe=not dialect.STRICT_STRING_CONCAT, 354 coalesce=dialect.CONCAT_WS_COALESCE, 355 ), 356 "CONVERT_TIMEZONE": build_convert_timezone, 357 "DATE_TO_DATE_STR": lambda args: exp.Cast( 358 this=seq_get(args, 0), 359 to=exp.DataType(this=exp.DType.TEXT), 360 ), 361 "GENERATE_DATE_ARRAY": lambda args: exp.GenerateDateArray( 362 start=seq_get(args, 0), 363 end=seq_get(args, 1), 364 step=seq_get(args, 2) or exp.Interval(this=exp.Literal.string(1), unit=exp.var("DAY")), 365 ), 366 "GENERATE_UUID": lambda args, dialect: exp.Uuid( 367 is_string=dialect.UUID_IS_STRING_TYPE or None 368 ), 369 "GLOB": lambda args: exp.Glob(this=seq_get(args, 1), expression=seq_get(args, 0)), 370 "GREATEST": lambda args, dialect: exp.Greatest( 371 this=seq_get(args, 0), 372 expressions=args[1:], 373 ignore_nulls=dialect.LEAST_GREATEST_IGNORES_NULLS, 374 ), 375 "LEAST": lambda args, dialect: exp.Least( 376 this=seq_get(args, 0), 377 expressions=args[1:], 378 ignore_nulls=dialect.LEAST_GREATEST_IGNORES_NULLS, 379 ), 380 "HEX": build_hex, 381 "JSON_EXTRACT": build_extract_json_with_path(exp.JSONExtract), 382 "JSON_EXTRACT_SCALAR": build_extract_json_with_path(exp.JSONExtractScalar), 383 "JSON_EXTRACT_PATH_TEXT": build_extract_json_with_path(exp.JSONExtractScalar), 384 "JSON_KEYS": lambda args, dialect: exp.JSONKeys( 385 this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1)) 386 ), 387 "LIKE": build_like, 388 "LOG": build_logarithm, 389 "LOG2": lambda args: exp.Log(this=exp.Literal.number(2), expression=seq_get(args, 0)), 390 "LOG10": lambda args: exp.Log(this=exp.Literal.number(10), expression=seq_get(args, 0)), 391 "LOWER": build_lower, 392 "LPAD": lambda args: build_pad(args), 393 "LEFTPAD": lambda args: build_pad(args), 394 "LTRIM": lambda args: build_trim(args), 395 "MOD": build_mod, 396 "RIGHTPAD": lambda args: build_pad(args, is_left=False), 397 "RPAD": lambda args: build_pad(args, is_left=False), 398 "RTRIM": lambda args: build_trim(args, is_left=False), 399 "SCOPE_RESOLUTION": lambda args: ( 400 exp.ScopeResolution(expression=seq_get(args, 0)) 401 if len(args) != 2 402 else exp.ScopeResolution(this=seq_get(args, 0), expression=seq_get(args, 1)) 403 ), 404 "STRPOS": exp.StrPosition.from_arg_list, 405 "CHARINDEX": lambda args: build_locate_strposition(args), 406 "INSTR": exp.StrPosition.from_arg_list, 407 "LOCATE": lambda args: build_locate_strposition(args), 408 "TIME_TO_TIME_STR": lambda args: exp.Cast( 409 this=seq_get(args, 0), 410 to=exp.DataType(this=exp.DType.TEXT), 411 ), 412 "TO_HEX": build_hex, 413 "TS_OR_DS_TO_DATE_STR": lambda args: exp.Substring( 414 this=exp.Cast( 415 this=seq_get(args, 0), 416 to=exp.DataType(this=exp.DType.TEXT), 417 ), 418 start=exp.Literal.number(1), 419 length=exp.Literal.number(10), 420 ), 421 "UNNEST": lambda args: exp.Unnest(expressions=ensure_list(seq_get(args, 0))), 422 "UPPER": build_upper, 423 "UUID": lambda args, dialect: exp.Uuid(is_string=dialect.UUID_IS_STRING_TYPE or None), 424 "UUID_STRING": lambda args, dialect: exp.Uuid( 425 this=seq_get(args, 0), 426 name=seq_get(args, 1), 427 is_string=dialect.UUID_IS_STRING_TYPE or None, 428 ), 429 "VAR_MAP": build_var_map, 430 } 431 432 NO_PAREN_FUNCTIONS: t.ClassVar[dict] = { 433 TokenType.CURRENT_DATE: exp.CurrentDate, 434 TokenType.CURRENT_DATETIME: exp.CurrentDate, 435 TokenType.CURRENT_TIME: exp.CurrentTime, 436 TokenType.CURRENT_TIMESTAMP: exp.CurrentTimestamp, 437 TokenType.CURRENT_USER: exp.CurrentUser, 438 TokenType.CURRENT_ROLE: exp.CurrentRole, 439 } 440 441 STRUCT_TYPE_TOKENS: t.ClassVar = { 442 TokenType.NESTED, 443 TokenType.OBJECT, 444 TokenType.STRUCT, 445 TokenType.UNION, 446 } 447 448 NESTED_TYPE_TOKENS: t.ClassVar = { 449 TokenType.ARRAY, 450 TokenType.LIST, 451 TokenType.LOWCARDINALITY, 452 TokenType.MAP, 453 TokenType.NULLABLE, 454 TokenType.RANGE, 455 *STRUCT_TYPE_TOKENS, 456 } 457 458 ENUM_TYPE_TOKENS: t.ClassVar = { 459 TokenType.DYNAMIC, 460 TokenType.ENUM, 461 TokenType.ENUM8, 462 TokenType.ENUM16, 463 } 464 465 AGGREGATE_TYPE_TOKENS: t.ClassVar = { 466 TokenType.AGGREGATEFUNCTION, 467 TokenType.SIMPLEAGGREGATEFUNCTION, 468 } 469 470 TYPE_TOKENS: t.ClassVar = { 471 TokenType.BIT, 472 TokenType.BOOLEAN, 473 TokenType.TINYINT, 474 TokenType.UTINYINT, 475 TokenType.SMALLINT, 476 TokenType.USMALLINT, 477 TokenType.INT, 478 TokenType.UINT, 479 TokenType.BIGINT, 480 TokenType.UBIGINT, 481 TokenType.BIGNUM, 482 TokenType.INT128, 483 TokenType.UINT128, 484 TokenType.INT256, 485 TokenType.UINT256, 486 TokenType.MEDIUMINT, 487 TokenType.UMEDIUMINT, 488 TokenType.FIXEDSTRING, 489 TokenType.FLOAT, 490 TokenType.DOUBLE, 491 TokenType.UDOUBLE, 492 TokenType.CHAR, 493 TokenType.NCHAR, 494 TokenType.VARCHAR, 495 TokenType.NVARCHAR, 496 TokenType.BPCHAR, 497 TokenType.TEXT, 498 TokenType.MEDIUMTEXT, 499 TokenType.LONGTEXT, 500 TokenType.BLOB, 501 TokenType.MEDIUMBLOB, 502 TokenType.LONGBLOB, 503 TokenType.BINARY, 504 TokenType.VARBINARY, 505 TokenType.JSON, 506 TokenType.JSONB, 507 TokenType.INTERVAL, 508 TokenType.TINYBLOB, 509 TokenType.TINYTEXT, 510 TokenType.TIME, 511 TokenType.TIMETZ, 512 TokenType.TIME_NS, 513 TokenType.TIMESTAMP, 514 TokenType.TIMESTAMP_S, 515 TokenType.TIMESTAMP_MS, 516 TokenType.TIMESTAMP_NS, 517 TokenType.TIMESTAMPTZ, 518 TokenType.TIMESTAMPLTZ, 519 TokenType.TIMESTAMPNTZ, 520 TokenType.DATETIME, 521 TokenType.DATETIME2, 522 TokenType.DATETIME64, 523 TokenType.SMALLDATETIME, 524 TokenType.DATE, 525 TokenType.DATE32, 526 TokenType.INT4RANGE, 527 TokenType.INT4MULTIRANGE, 528 TokenType.INT8RANGE, 529 TokenType.INT8MULTIRANGE, 530 TokenType.NUMRANGE, 531 TokenType.NUMMULTIRANGE, 532 TokenType.TSRANGE, 533 TokenType.TSMULTIRANGE, 534 TokenType.TSTZRANGE, 535 TokenType.TSTZMULTIRANGE, 536 TokenType.DATERANGE, 537 TokenType.DATEMULTIRANGE, 538 TokenType.DECIMAL, 539 TokenType.DECIMAL32, 540 TokenType.DECIMAL64, 541 TokenType.DECIMAL128, 542 TokenType.DECIMAL256, 543 TokenType.DECFLOAT, 544 TokenType.UDECIMAL, 545 TokenType.BIGDECIMAL, 546 TokenType.UUID, 547 TokenType.GEOGRAPHY, 548 TokenType.GEOGRAPHYPOINT, 549 TokenType.GEOMETRY, 550 TokenType.POINT, 551 TokenType.RING, 552 TokenType.LINESTRING, 553 TokenType.MULTILINESTRING, 554 TokenType.POLYGON, 555 TokenType.MULTIPOLYGON, 556 TokenType.HLLSKETCH, 557 TokenType.HSTORE, 558 TokenType.PSEUDO_TYPE, 559 TokenType.SUPER, 560 TokenType.SERIAL, 561 TokenType.SMALLSERIAL, 562 TokenType.BIGSERIAL, 563 TokenType.XML, 564 TokenType.YEAR, 565 TokenType.USERDEFINED, 566 TokenType.MONEY, 567 TokenType.SMALLMONEY, 568 TokenType.ROWVERSION, 569 TokenType.IMAGE, 570 TokenType.VARIANT, 571 TokenType.VECTOR, 572 TokenType.VOID, 573 TokenType.OBJECT, 574 TokenType.OBJECT_IDENTIFIER, 575 TokenType.INET, 576 TokenType.IPADDRESS, 577 TokenType.IPPREFIX, 578 TokenType.IPV4, 579 TokenType.IPV6, 580 TokenType.UNKNOWN, 581 TokenType.NOTHING, 582 TokenType.NULL, 583 TokenType.NAME, 584 TokenType.TDIGEST, 585 TokenType.DYNAMIC, 586 *ENUM_TYPE_TOKENS, 587 *NESTED_TYPE_TOKENS, 588 *AGGREGATE_TYPE_TOKENS, 589 } 590 591 SIGNED_TO_UNSIGNED_TYPE_TOKEN: t.ClassVar = { 592 TokenType.BIGINT: TokenType.UBIGINT, 593 TokenType.INT: TokenType.UINT, 594 TokenType.MEDIUMINT: TokenType.UMEDIUMINT, 595 TokenType.SMALLINT: TokenType.USMALLINT, 596 TokenType.TINYINT: TokenType.UTINYINT, 597 TokenType.DECIMAL: TokenType.UDECIMAL, 598 TokenType.DOUBLE: TokenType.UDOUBLE, 599 } 600 601 SUBQUERY_PREDICATES: t.ClassVar = { 602 TokenType.ANY: exp.Any, 603 TokenType.ALL: exp.All, 604 TokenType.EXISTS: exp.Exists, 605 TokenType.SOME: exp.Any, 606 } 607 608 SUBQUERY_TOKENS: t.ClassVar = { 609 TokenType.SELECT, 610 TokenType.WITH, 611 TokenType.FROM, 612 } 613 614 RESERVED_TOKENS: t.ClassVar = { 615 *Tokenizer.SINGLE_TOKENS.values(), 616 TokenType.SELECT, 617 } - {TokenType.IDENTIFIER} 618 619 DB_CREATABLES: t.ClassVar = { 620 TokenType.DATABASE, 621 TokenType.DICTIONARY, 622 TokenType.FILE_FORMAT, 623 TokenType.MODEL, 624 TokenType.NAMESPACE, 625 TokenType.SCHEMA, 626 TokenType.SEMANTIC_VIEW, 627 TokenType.SEQUENCE, 628 TokenType.SINK, 629 TokenType.SOURCE, 630 TokenType.STAGE, 631 TokenType.STORAGE_INTEGRATION, 632 TokenType.STREAMLIT, 633 TokenType.TABLE, 634 TokenType.TAG, 635 TokenType.VIEW, 636 TokenType.WAREHOUSE, 637 } 638 639 CREATABLES: t.ClassVar = { 640 TokenType.COLUMN, 641 TokenType.CONSTRAINT, 642 TokenType.FOREIGN_KEY, 643 TokenType.FUNCTION, 644 TokenType.INDEX, 645 TokenType.PROCEDURE, 646 TokenType.TRIGGER, 647 TokenType.TYPE, 648 *DB_CREATABLES, 649 } 650 651 TRIGGER_EVENTS: t.ClassVar = { 652 TokenType.INSERT, 653 TokenType.UPDATE, 654 TokenType.DELETE, 655 TokenType.TRUNCATE, 656 } 657 658 ALTERABLES: t.ClassVar = { 659 TokenType.INDEX, 660 TokenType.TABLE, 661 TokenType.VIEW, 662 TokenType.SESSION, 663 } 664 665 # Tokens that can represent identifiers 666 ID_VAR_TOKENS: t.ClassVar[set] = { 667 TokenType.ALL, 668 TokenType.ANALYZE, 669 TokenType.ATTACH, 670 TokenType.VAR, 671 TokenType.ANTI, 672 TokenType.APPLY, 673 TokenType.ASC, 674 TokenType.ASOF, 675 TokenType.AUTO_INCREMENT, 676 TokenType.BEGIN, 677 TokenType.BPCHAR, 678 TokenType.CACHE, 679 TokenType.CASE, 680 TokenType.COLLATE, 681 TokenType.COMMAND, 682 TokenType.COMMENT, 683 TokenType.COMMIT, 684 TokenType.CONSTRAINT, 685 TokenType.COPY, 686 TokenType.CUBE, 687 TokenType.CURRENT_SCHEMA, 688 TokenType.DEFAULT, 689 TokenType.DELETE, 690 TokenType.DESC, 691 TokenType.DESCRIBE, 692 TokenType.DETACH, 693 TokenType.DICTIONARY, 694 TokenType.DIV, 695 TokenType.END, 696 TokenType.EXECUTE, 697 TokenType.EXPORT, 698 TokenType.ESCAPE, 699 TokenType.FALSE, 700 TokenType.FIRST, 701 TokenType.FILE, 702 TokenType.FILTER, 703 TokenType.FINAL, 704 TokenType.FORMAT, 705 TokenType.FULL, 706 TokenType.GET, 707 TokenType.IDENTIFIER, 708 TokenType.INOUT, 709 TokenType.IS, 710 TokenType.ISNULL, 711 TokenType.INTERVAL, 712 TokenType.KEEP, 713 TokenType.KILL, 714 TokenType.LEFT, 715 TokenType.LIMIT, 716 TokenType.LOAD, 717 TokenType.LOCK, 718 TokenType.MATCH, 719 TokenType.MERGE, 720 TokenType.NATURAL, 721 TokenType.NEXT, 722 TokenType.OFFSET, 723 TokenType.OPERATOR, 724 TokenType.ORDINALITY, 725 TokenType.OVER, 726 TokenType.OVERLAPS, 727 TokenType.OVERWRITE, 728 TokenType.PARTITION, 729 TokenType.PERCENT, 730 TokenType.PIVOT, 731 TokenType.PRAGMA, 732 TokenType.PUT, 733 TokenType.RANGE, 734 TokenType.RECURSIVE, 735 TokenType.REFERENCES, 736 TokenType.REFRESH, 737 TokenType.RENAME, 738 TokenType.REPLACE, 739 TokenType.RIGHT, 740 TokenType.ROLLUP, 741 TokenType.ROW, 742 TokenType.ROWS, 743 TokenType.SEMI, 744 TokenType.SET, 745 TokenType.SETTINGS, 746 TokenType.SHOW, 747 TokenType.STREAM, 748 TokenType.STREAMLIT, 749 TokenType.TEMPORARY, 750 TokenType.TOP, 751 TokenType.TRUE, 752 TokenType.TRUNCATE, 753 TokenType.UNIQUE, 754 TokenType.UNNEST, 755 TokenType.UNPIVOT, 756 TokenType.UPDATE, 757 TokenType.USE, 758 TokenType.VOLATILE, 759 TokenType.WINDOW, 760 TokenType.CURRENT_CATALOG, 761 TokenType.LOCALTIME, 762 TokenType.LOCALTIMESTAMP, 763 TokenType.SESSION_USER, 764 TokenType.STRAIGHT_JOIN, 765 *ALTERABLES, 766 *CREATABLES, 767 *SUBQUERY_PREDICATES, 768 *TYPE_TOKENS, 769 *NO_PAREN_FUNCTIONS, 770 } - {TokenType.UNION} 771 772 TABLE_ALIAS_TOKENS: t.ClassVar[set] = ID_VAR_TOKENS - { 773 TokenType.ANTI, 774 TokenType.ASOF, 775 TokenType.FULL, 776 TokenType.LEFT, 777 TokenType.LOCK, 778 TokenType.NATURAL, 779 TokenType.RIGHT, 780 TokenType.SEMI, 781 TokenType.WINDOW, 782 } 783 784 ALIAS_TOKENS: t.ClassVar = ID_VAR_TOKENS 785 786 COLON_PLACEHOLDER_TOKENS: t.ClassVar = ID_VAR_TOKENS 787 788 ARRAY_CONSTRUCTORS: t.ClassVar = { 789 "ARRAY": exp.Array, 790 "LIST": exp.List, 791 } 792 793 COMMENT_TABLE_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - {TokenType.IS} 794 795 UPDATE_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - {TokenType.SET} 796 797 TRIM_TYPES: t.ClassVar = {"LEADING", "TRAILING", "BOTH"} 798 799 # Tokens that indicate a simple column reference 800 IDENTIFIER_TOKENS: t.ClassVar[frozenset] = frozenset({TokenType.VAR, TokenType.IDENTIFIER}) 801 802 BRACKETS: t.ClassVar[frozenset] = frozenset({TokenType.L_BRACKET, TokenType.L_BRACE}) 803 804 # Postfix tokens that prevent the bare column fast path 805 COLUMN_POSTFIX_TOKENS: t.ClassVar[frozenset] = frozenset( 806 { 807 TokenType.L_PAREN, 808 TokenType.L_BRACKET, 809 TokenType.L_BRACE, 810 TokenType.COLON, 811 TokenType.JOIN_MARKER, 812 } 813 ) 814 815 TABLE_POSTFIX_TOKENS: t.ClassVar[frozenset] = frozenset( 816 { 817 TokenType.L_PAREN, 818 TokenType.L_BRACKET, 819 TokenType.L_BRACE, 820 TokenType.PIVOT, 821 TokenType.UNPIVOT, 822 TokenType.TABLE_SAMPLE, 823 } 824 ) 825 826 FUNC_TOKENS: t.ClassVar = { 827 TokenType.COLLATE, 828 TokenType.COMMAND, 829 TokenType.CURRENT_DATE, 830 TokenType.CURRENT_DATETIME, 831 TokenType.CURRENT_SCHEMA, 832 TokenType.CURRENT_TIMESTAMP, 833 TokenType.CURRENT_TIME, 834 TokenType.CURRENT_USER, 835 TokenType.CURRENT_CATALOG, 836 TokenType.FILTER, 837 TokenType.FIRST, 838 TokenType.FORMAT, 839 TokenType.GET, 840 TokenType.GLOB, 841 TokenType.IDENTIFIER, 842 TokenType.INDEX, 843 TokenType.ISNULL, 844 TokenType.ILIKE, 845 TokenType.INSERT, 846 TokenType.LIKE, 847 TokenType.LOCALTIME, 848 TokenType.LOCALTIMESTAMP, 849 TokenType.MERGE, 850 TokenType.NEXT, 851 TokenType.OFFSET, 852 TokenType.PRIMARY_KEY, 853 TokenType.RANGE, 854 TokenType.REPLACE, 855 TokenType.RLIKE, 856 TokenType.ROW, 857 TokenType.SESSION_USER, 858 TokenType.UNNEST, 859 TokenType.VAR, 860 TokenType.LEFT, 861 TokenType.RIGHT, 862 TokenType.SEQUENCE, 863 TokenType.DATE, 864 TokenType.DATETIME, 865 TokenType.TABLE, 866 TokenType.TIMESTAMP, 867 TokenType.TIMESTAMPTZ, 868 TokenType.TRUNCATE, 869 TokenType.UTC_DATE, 870 TokenType.UTC_TIME, 871 TokenType.UTC_TIMESTAMP, 872 TokenType.WINDOW, 873 TokenType.XOR, 874 *TYPE_TOKENS, 875 *SUBQUERY_PREDICATES, 876 } 877 878 CONJUNCTION: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 879 TokenType.AND: exp.And, 880 } 881 882 ASSIGNMENT: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 883 TokenType.COLON_EQ: exp.PropertyEQ, 884 } 885 886 DISJUNCTION: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 887 TokenType.OR: exp.Or, 888 } 889 890 EQUALITY: t.ClassVar = { 891 TokenType.EQ: exp.EQ, 892 TokenType.NEQ: exp.NEQ, 893 TokenType.NULLSAFE_EQ: exp.NullSafeEQ, 894 } 895 896 COMPARISON: t.ClassVar = { 897 TokenType.GT: exp.GT, 898 TokenType.GTE: exp.GTE, 899 TokenType.LT: exp.LT, 900 TokenType.LTE: exp.LTE, 901 } 902 903 BITWISE: t.ClassVar = { 904 TokenType.AMP: exp.BitwiseAnd, 905 TokenType.CARET: exp.BitwiseXor, 906 TokenType.PIPE: exp.BitwiseOr, 907 } 908 909 TERM: t.ClassVar = { 910 TokenType.DASH: exp.Sub, 911 TokenType.PLUS: exp.Add, 912 TokenType.MOD: exp.Mod, 913 TokenType.COLLATE: exp.Collate, 914 } 915 916 FACTOR: t.ClassVar = { 917 TokenType.DIV: exp.IntDiv, 918 TokenType.LR_ARROW: exp.Distance, 919 TokenType.LLRR_ARROW: exp.DistanceNd, 920 TokenType.SLASH: exp.Div, 921 TokenType.STAR: exp.Mul, 922 } 923 924 EXPONENT: t.ClassVar[dict[TokenType, type[exp.Expr]]] = {} 925 926 TIMES: t.ClassVar = { 927 TokenType.TIME, 928 TokenType.TIMETZ, 929 } 930 931 TIMESTAMPS: t.ClassVar = { 932 TokenType.TIMESTAMP, 933 TokenType.TIMESTAMPNTZ, 934 TokenType.TIMESTAMPTZ, 935 TokenType.TIMESTAMPLTZ, 936 *TIMES, 937 } 938 939 SET_OPERATIONS: t.ClassVar = { 940 TokenType.UNION, 941 TokenType.INTERSECT, 942 TokenType.EXCEPT, 943 } 944 945 JOIN_METHODS: t.ClassVar = { 946 TokenType.ASOF, 947 TokenType.NATURAL, 948 TokenType.POSITIONAL, 949 } 950 951 JOIN_SIDES: t.ClassVar = { 952 TokenType.LEFT, 953 TokenType.RIGHT, 954 TokenType.FULL, 955 } 956 957 JOIN_KINDS: t.ClassVar = { 958 TokenType.ANTI, 959 TokenType.CROSS, 960 TokenType.INNER, 961 TokenType.OUTER, 962 TokenType.SEMI, 963 TokenType.STRAIGHT_JOIN, 964 } 965 966 JOIN_HINTS: t.ClassVar[set[str]] = set() 967 968 # Tokens that unambiguously end a table reference on the fast path 969 TABLE_TERMINATORS: t.ClassVar[frozenset] = frozenset( 970 { 971 TokenType.COMMA, 972 TokenType.GROUP_BY, 973 TokenType.HAVING, 974 TokenType.JOIN, 975 TokenType.LIMIT, 976 TokenType.ON, 977 TokenType.ORDER_BY, 978 TokenType.R_PAREN, 979 TokenType.SEMICOLON, 980 TokenType.SENTINEL, 981 TokenType.WHERE, 982 *SET_OPERATIONS, 983 *JOIN_KINDS, 984 *JOIN_METHODS, 985 *JOIN_SIDES, 986 } 987 ) 988 989 LAMBDAS: t.ClassVar = { 990 TokenType.ARROW: lambda self, expressions: self.expression( 991 exp.Lambda( 992 this=self._replace_lambda( 993 self._parse_disjunction(), 994 expressions, 995 ), 996 expressions=expressions, 997 ) 998 ), 999 TokenType.FARROW: lambda self, expressions: self.expression( 1000 exp.Kwarg(this=exp.var(expressions[0].name), expression=self._parse_disjunction()) 1001 ), 1002 } 1003 1004 # Whether lambda args include type annotations, e.g. TRANSFORM(arr, x INT -> x + 1) in Snowflake 1005 TYPED_LAMBDA_ARGS: t.ClassVar[bool] = False 1006 1007 LAMBDA_ARG_TERMINATORS: t.ClassVar[frozenset] = frozenset({TokenType.COMMA, TokenType.R_PAREN}) 1008 1009 COLUMN_OPERATORS: t.ClassVar = { 1010 TokenType.DOT: None, 1011 TokenType.DOTCOLON: lambda self, this, to: self.expression(exp.JSONCast(this=this, to=to)), 1012 TokenType.DCOLON: lambda self, this, to: self.build_cast( 1013 strict=self.STRICT_CAST, this=this, to=to 1014 ), 1015 TokenType.ARROW: lambda self, this, path: self.expression( 1016 exp.JSONExtract( 1017 this=this, 1018 expression=self.dialect.to_json_path(path), 1019 only_json_types=self.JSON_ARROWS_REQUIRE_JSON_TYPE, 1020 ) 1021 ), 1022 TokenType.DARROW: lambda self, this, path: self.expression( 1023 exp.JSONExtractScalar( 1024 this=this, 1025 expression=self.dialect.to_json_path(path), 1026 only_json_types=self.JSON_ARROWS_REQUIRE_JSON_TYPE, 1027 scalar_only=self.dialect.JSON_EXTRACT_SCALAR_SCALAR_ONLY, 1028 ) 1029 ), 1030 TokenType.HASH_ARROW: lambda self, this, path: self.expression( 1031 exp.JSONBExtract(this=this, expression=path) 1032 ), 1033 TokenType.DHASH_ARROW: lambda self, this, path: self.expression( 1034 exp.JSONBExtractScalar(this=this, expression=path) 1035 ), 1036 TokenType.PLACEHOLDER: lambda self, this, key: self.expression( 1037 exp.JSONBContains(this=this, expression=key) 1038 ), 1039 } 1040 1041 CAST_COLUMN_OPERATORS: t.ClassVar = { 1042 TokenType.DOTCOLON, 1043 TokenType.DCOLON, 1044 } 1045 1046 EXPRESSION_PARSERS: t.ClassVar = { 1047 exp.Cluster: lambda self: self._parse_sort(exp.Cluster, TokenType.CLUSTER_BY), 1048 exp.Column: lambda self: self._parse_column(), 1049 exp.ColumnDef: lambda self: self._parse_column_def(self._parse_column()), 1050 exp.Condition: lambda self: self._parse_disjunction(), 1051 exp.DataType: lambda self: self._parse_types(allow_identifiers=False, schema=True), 1052 exp.Expr: lambda self: self._parse_expression(), 1053 exp.From: lambda self: self._parse_from(joins=True), 1054 exp.GrantPrincipal: lambda self: self._parse_grant_principal(), 1055 exp.GrantPrivilege: lambda self: self._parse_grant_privilege(), 1056 exp.Group: lambda self: self._parse_group(), 1057 exp.Having: lambda self: self._parse_having(), 1058 exp.Hint: lambda self: self._parse_hint_body(), 1059 exp.Identifier: lambda self: self._parse_id_var(), 1060 exp.Join: lambda self: self._parse_join(), 1061 exp.Lambda: lambda self: self._parse_lambda(), 1062 exp.Lateral: lambda self: self._parse_lateral(), 1063 exp.Limit: lambda self: self._parse_limit(), 1064 exp.Offset: lambda self: self._parse_offset(), 1065 exp.Order: lambda self: self._parse_order(), 1066 exp.Ordered: lambda self: self._parse_ordered(), 1067 exp.Properties: lambda self: self._parse_properties(), 1068 exp.PartitionedByProperty: lambda self: self._parse_partitioned_by(), 1069 exp.Qualify: lambda self: self._parse_qualify(), 1070 exp.Returning: lambda self: self._parse_returning(), 1071 exp.Select: lambda self: self._parse_select(), 1072 exp.Sort: lambda self: self._parse_sort(exp.Sort, TokenType.SORT_BY), 1073 exp.Table: lambda self: self._parse_table_parts(), 1074 exp.TableAlias: lambda self: self._parse_table_alias(), 1075 exp.Tuple: lambda self: self._parse_value(values=False), 1076 exp.Whens: lambda self: self._parse_when_matched(), 1077 exp.Where: lambda self: self._parse_where(), 1078 exp.Window: lambda self: self._parse_named_window(), 1079 exp.With: lambda self: self._parse_with(), 1080 } 1081 1082 STATEMENT_PARSERS: t.ClassVar = { 1083 TokenType.ALTER: lambda self: self._parse_alter(), 1084 TokenType.ANALYZE: lambda self: self._parse_analyze(), 1085 TokenType.BEGIN: lambda self: self._parse_transaction(), 1086 TokenType.CACHE: lambda self: self._parse_cache(), 1087 TokenType.COMMENT: lambda self: self._parse_comment(), 1088 TokenType.COMMIT: lambda self: self._parse_commit_or_rollback(), 1089 TokenType.COPY: lambda self: self._parse_copy(), 1090 TokenType.CREATE: lambda self: self._parse_create(), 1091 TokenType.DELETE: lambda self: self._parse_delete(), 1092 TokenType.DESC: lambda self: self._parse_describe(), 1093 TokenType.DESCRIBE: lambda self: self._parse_describe(), 1094 TokenType.DROP: lambda self: self._parse_drop(), 1095 TokenType.GRANT: lambda self: self._parse_grant(), 1096 TokenType.REVOKE: lambda self: self._parse_revoke(), 1097 TokenType.INSERT: lambda self: self._parse_insert(), 1098 TokenType.KILL: lambda self: self._parse_kill(), 1099 TokenType.LOAD: lambda self: self._parse_load(), 1100 TokenType.MERGE: lambda self: self._parse_merge(), 1101 TokenType.PIVOT: lambda self: self._parse_simplified_pivot(), 1102 TokenType.PRAGMA: lambda self: self.expression(exp.Pragma(this=self._parse_expression())), 1103 TokenType.REFRESH: lambda self: self._parse_refresh(), 1104 TokenType.ROLLBACK: lambda self: self._parse_commit_or_rollback(), 1105 TokenType.SET: lambda self: self._parse_set(), 1106 TokenType.TRUNCATE: lambda self: self._parse_truncate_table(), 1107 TokenType.UNCACHE: lambda self: self._parse_uncache(), 1108 TokenType.UNPIVOT: lambda self: self._parse_simplified_pivot(is_unpivot=True), 1109 TokenType.UPDATE: lambda self: self._parse_update(), 1110 TokenType.USE: lambda self: self._parse_use(), 1111 TokenType.SEMICOLON: lambda self: exp.Semicolon(), 1112 } 1113 1114 UNARY_PARSERS: t.ClassVar = { 1115 TokenType.PLUS: lambda self: self._parse_unary(), # Unary + is handled as a no-op 1116 TokenType.NOT: lambda self: self.expression(exp.Not(this=self._parse_equality())), 1117 TokenType.TILDE: lambda self: self.expression(exp.BitwiseNot(this=self._parse_unary())), 1118 TokenType.DASH: lambda self: self.expression(exp.Neg(this=self._parse_unary())), 1119 TokenType.PIPE_SLASH: lambda self: self.expression(exp.Sqrt(this=self._parse_unary())), 1120 TokenType.DPIPE_SLASH: lambda self: self.expression(exp.Cbrt(this=self._parse_unary())), 1121 } 1122 1123 STRING_PARSERS: t.ClassVar = { 1124 TokenType.HEREDOC_STRING: lambda self, token: self.expression( 1125 exp.RawString(this=token.text), token 1126 ), 1127 TokenType.NATIONAL_STRING: lambda self, token: self.expression( 1128 exp.National(this=token.text), token 1129 ), 1130 TokenType.RAW_STRING: lambda self, token: self.expression( 1131 exp.RawString(this=token.text), token 1132 ), 1133 TokenType.STRING: lambda self, token: self.expression( 1134 exp.Literal(this=token.text, is_string=True), token 1135 ), 1136 TokenType.UNICODE_STRING: lambda self, token: self.expression( 1137 exp.UnicodeString( 1138 this=token.text, escape=self._match_text_seq("UESCAPE") and self._parse_string() 1139 ), 1140 token, 1141 ), 1142 } 1143 1144 NUMERIC_PARSERS: t.ClassVar = { 1145 TokenType.BIT_STRING: lambda self, token: self.expression( 1146 exp.BitString(this=token.text), token 1147 ), 1148 TokenType.BYTE_STRING: lambda self, token: self.expression( 1149 exp.ByteString( 1150 this=token.text, is_bytes=self.dialect.BYTE_STRING_IS_BYTES_TYPE or None 1151 ), 1152 token, 1153 ), 1154 TokenType.HEX_STRING: lambda self, token: self.expression( 1155 exp.HexString( 1156 this=token.text, is_integer=self.dialect.HEX_STRING_IS_INTEGER_TYPE or None 1157 ), 1158 token, 1159 ), 1160 TokenType.NUMBER: lambda self, token: self.expression( 1161 exp.Literal(this=token.text, is_string=False), token 1162 ), 1163 } 1164 1165 PRIMARY_PARSERS: t.ClassVar = { 1166 **STRING_PARSERS, 1167 **NUMERIC_PARSERS, 1168 TokenType.INTRODUCER: lambda self, token: self._parse_introducer(token), 1169 TokenType.NULL: lambda self, _: self.expression(exp.Null()), 1170 TokenType.TRUE: lambda self, _: self.expression(exp.Boolean(this=True)), 1171 TokenType.FALSE: lambda self, _: self.expression(exp.Boolean(this=False)), 1172 TokenType.SESSION_PARAMETER: lambda self, _: self._parse_session_parameter(), 1173 TokenType.STAR: lambda self, _: self._parse_star_ops(), 1174 } 1175 1176 PLACEHOLDER_PARSERS: t.ClassVar = { 1177 TokenType.PLACEHOLDER: lambda self: self.expression(exp.Placeholder()), 1178 TokenType.PARAMETER: lambda self: self._parse_parameter(), 1179 TokenType.COLON: lambda self: ( 1180 self.expression(exp.Placeholder(this=self._prev.text)) 1181 if self._match_set(self.COLON_PLACEHOLDER_TOKENS) 1182 else None 1183 ), 1184 } 1185 1186 RANGE_PARSERS: t.ClassVar = { 1187 TokenType.AT_GT: binary_range_parser(exp.ArrayContainsAll), 1188 TokenType.BETWEEN: lambda self, this: self._parse_between(this), 1189 TokenType.GLOB: binary_range_parser(exp.Glob), 1190 TokenType.ILIKE: binary_range_parser(exp.ILike), 1191 TokenType.IN: lambda self, this: self._parse_in(this), 1192 TokenType.IRLIKE: binary_range_parser(exp.RegexpILike), 1193 TokenType.IS: lambda self, this: self._parse_is(this), 1194 TokenType.LIKE: binary_range_parser(exp.Like), 1195 TokenType.LT_AT: binary_range_parser(exp.ArrayContainedBy), 1196 TokenType.OVERLAPS: binary_range_parser(exp.Overlaps), 1197 TokenType.RLIKE: binary_range_parser(exp.RegexpLike), 1198 TokenType.SIMILAR_TO: binary_range_parser(exp.SimilarTo), 1199 TokenType.FOR: lambda self, this: self._parse_comprehension(this), 1200 TokenType.QMARK_AMP: binary_range_parser(exp.JSONBContainsAllTopKeys), 1201 TokenType.QMARK_PIPE: binary_range_parser(exp.JSONBContainsAnyTopKeys), 1202 TokenType.HASH_DASH: binary_range_parser(exp.JSONBDeleteAtPath), 1203 TokenType.AT_QMARK: binary_range_parser(exp.JSONBPathExists), 1204 TokenType.ADJACENT: binary_range_parser(exp.Adjacent), 1205 TokenType.OPERATOR: lambda self, this: self._parse_operator(this), 1206 TokenType.AMP_LT: binary_range_parser(exp.ExtendsLeft), 1207 TokenType.AMP_GT: binary_range_parser(exp.ExtendsRight), 1208 } 1209 1210 PIPE_SYNTAX_TRANSFORM_PARSERS: t.ClassVar = { 1211 "AGGREGATE": lambda self, query: self._parse_pipe_syntax_aggregate(query), 1212 "AS": lambda self, query: self._build_pipe_cte( 1213 query, [exp.Star()], self._parse_table_alias() 1214 ), 1215 "DISTINCT": lambda self, query: self._advance() or query.distinct(copy=False), 1216 "EXTEND": lambda self, query: self._parse_pipe_syntax_extend(query), 1217 "LIMIT": lambda self, query: self._parse_pipe_syntax_limit(query), 1218 "ORDER BY": lambda self, query: query.order_by( 1219 self._parse_order(), append=False, copy=False 1220 ), 1221 "PIVOT": lambda self, query: self._parse_pipe_syntax_pivot(query), 1222 "SELECT": lambda self, query: self._parse_pipe_syntax_select(query), 1223 "TABLESAMPLE": lambda self, query: self._parse_pipe_syntax_tablesample(query), 1224 "UNPIVOT": lambda self, query: self._parse_pipe_syntax_pivot(query), 1225 "WHERE": lambda self, query: query.where(self._parse_where(), copy=False), 1226 } 1227 1228 PROPERTY_PARSERS: t.ClassVar[dict[str, t.Callable]] = { 1229 "ALLOWED_VALUES": lambda self: self.expression( 1230 exp.AllowedValuesProperty(expressions=self._parse_csv(self._parse_primary)) 1231 ), 1232 "ALGORITHM": lambda self: self._parse_property_assignment(exp.AlgorithmProperty), 1233 "AUTO": lambda self: self._parse_auto_property(), 1234 "AUTO_INCREMENT": lambda self: self._parse_property_assignment(exp.AutoIncrementProperty), 1235 "BACKUP": lambda self: self.expression( 1236 exp.BackupProperty(this=self._parse_var(any_token=True)) 1237 ), 1238 "BLOCKCOMPRESSION": lambda self: self._parse_blockcompression(), 1239 "CALLED": lambda self: self._parse_called_on_null_input_property(), 1240 "CHARSET": lambda self, **kwargs: self._parse_character_set(**kwargs), 1241 "CHARACTER SET": lambda self, **kwargs: self._parse_character_set(**kwargs), 1242 "CHECKSUM": lambda self: self._parse_checksum(), 1243 "CLUSTER BY": lambda self: self._parse_cluster_property(), 1244 "CLUSTERED": lambda self: self._parse_clustered_by(), 1245 "COLLATE": lambda self, **kwargs: self._parse_property_assignment( 1246 exp.CollateProperty, **kwargs 1247 ), 1248 "COMMENT": lambda self: self._parse_property_assignment(exp.SchemaCommentProperty), 1249 "CONTAINS": lambda self: self._parse_contains_property(), 1250 "COPY": lambda self: self._parse_copy_property(), 1251 "DATABLOCKSIZE": lambda self, **kwargs: self._parse_datablocksize(**kwargs), 1252 "DATA_DELETION": lambda self: self._parse_data_deletion_property(), 1253 "DEFINER": lambda self: self._parse_definer(), 1254 "DETERMINISTIC": lambda self: self.expression( 1255 exp.StabilityProperty(this=exp.Literal.string("IMMUTABLE")) 1256 ), 1257 "DISTRIBUTED": lambda self: self._parse_distributed_property(), 1258 "DUPLICATE": lambda self: self._parse_composite_key_property(exp.DuplicateKeyProperty), 1259 "DYNAMIC": lambda self: self.expression(exp.DynamicProperty()), 1260 "DISTKEY": lambda self: self._parse_distkey(), 1261 "DISTSTYLE": lambda self: self._parse_property_assignment(exp.DistStyleProperty), 1262 "EMPTY": lambda self: self.expression(exp.EmptyProperty()), 1263 "ENGINE": lambda self: self._parse_property_assignment(exp.EngineProperty), 1264 "ENVIRONMENT": lambda self: self.expression( 1265 exp.EnviromentProperty(expressions=self._parse_wrapped_csv(self._parse_assignment)) 1266 ), 1267 "HANDLER": lambda self: self._parse_property_assignment(exp.HandlerProperty), 1268 "EXECUTE": lambda self: self._parse_property_assignment(exp.ExecuteAsProperty), 1269 "EXTERNAL": lambda self: self.expression(exp.ExternalProperty()), 1270 "FALLBACK": lambda self, **kwargs: self._parse_fallback(**kwargs), 1271 "FORMAT": lambda self: self._parse_property_assignment(exp.FileFormatProperty), 1272 "FREESPACE": lambda self: self._parse_freespace(), 1273 "GLOBAL": lambda self: self.expression(exp.GlobalProperty()), 1274 "HEAP": lambda self: self.expression(exp.HeapProperty()), 1275 "ICEBERG": lambda self: self.expression(exp.IcebergProperty()), 1276 "IMMUTABLE": lambda self: self.expression( 1277 exp.StabilityProperty(this=exp.Literal.string("IMMUTABLE")) 1278 ), 1279 "INHERITS": lambda self: self.expression( 1280 exp.InheritsProperty(expressions=self._parse_wrapped_csv(self._parse_table)) 1281 ), 1282 "INPUT": lambda self: self.expression(exp.InputModelProperty(this=self._parse_schema())), 1283 "JOURNAL": lambda self, **kwargs: self._parse_journal(**kwargs), 1284 "LANGUAGE": lambda self: self._parse_property_assignment(exp.LanguageProperty), 1285 "LAYOUT": lambda self: self._parse_dict_property(this="LAYOUT"), 1286 "LIFETIME": lambda self: self._parse_dict_range(this="LIFETIME"), 1287 "LIKE": lambda self: self._parse_create_like(), 1288 "LOCATION": lambda self: self._parse_property_assignment(exp.LocationProperty), 1289 "LOCK": lambda self: self._parse_locking(), 1290 "LOCKING": lambda self: self._parse_locking(), 1291 "LOG": lambda self, **kwargs: self._parse_log(**kwargs), 1292 "MATERIALIZED": lambda self: self.expression(exp.MaterializedProperty()), 1293 "MERGEBLOCKRATIO": lambda self, **kwargs: self._parse_mergeblockratio(**kwargs), 1294 "MODIFIES": lambda self: self._parse_modifies_property(), 1295 "MULTISET": lambda self: self.expression(exp.SetProperty(multi=True)), 1296 "NO": lambda self: self._parse_no_property(), 1297 "ON": lambda self: self._parse_on_property(), 1298 "ORDER BY": lambda self: self._parse_order(skip_order_token=True), 1299 "OUTPUT": lambda self: self.expression(exp.OutputModelProperty(this=self._parse_schema())), 1300 "PARTITION": lambda self: self._parse_partitioned_of(), 1301 "PARTITION BY": lambda self: self._parse_partitioned_by(), 1302 "PARTITIONED BY": lambda self: self._parse_partitioned_by(), 1303 "PARTITIONED_BY": lambda self: self._parse_partitioned_by(), 1304 "PRIMARY KEY": lambda self: self._parse_primary_key(in_props=True), 1305 "RANGE": lambda self: self._parse_dict_range(this="RANGE"), 1306 "READS": lambda self: self._parse_reads_property(), 1307 "REMOTE": lambda self: self._parse_remote_with_connection(), 1308 "RETURNS": lambda self: self._parse_returns(), 1309 "STRICT": lambda self: self.expression(exp.StrictProperty()), 1310 "STREAMING": lambda self: self.expression(exp.StreamingTableProperty()), 1311 "ROW": lambda self: self._parse_row(), 1312 "ROW_FORMAT": lambda self: self._parse_property_assignment(exp.RowFormatProperty), 1313 "SAMPLE": lambda self: self.expression( 1314 exp.SampleProperty(this=self._match_text_seq("BY") and self._parse_bitwise()) 1315 ), 1316 "SECURE": lambda self: self.expression(exp.SecureProperty()), 1317 "SECURITY": lambda self: self._parse_sql_security(), 1318 "SQL SECURITY": lambda self: self._parse_sql_security(), 1319 "SET": lambda self: self.expression(exp.SetProperty(multi=False)), 1320 "SETTINGS": lambda self: self._parse_settings_property(), 1321 "SHARING": lambda self: self._parse_property_assignment(exp.SharingProperty), 1322 "SORTKEY": lambda self: self._parse_sortkey(), 1323 "SOURCE": lambda self: self._parse_dict_property(this="SOURCE"), 1324 "STABLE": lambda self: self.expression( 1325 exp.StabilityProperty(this=exp.Literal.string("STABLE")) 1326 ), 1327 "STORED": lambda self: self._parse_stored(), 1328 "SYSTEM_VERSIONING": lambda self: self._parse_system_versioning_property(), 1329 "TBLPROPERTIES": lambda self: self._parse_wrapped_properties(), 1330 "TEMP": lambda self: self.expression(exp.TemporaryProperty()), 1331 "TEMPORARY": lambda self: self.expression(exp.TemporaryProperty()), 1332 "TO": lambda self: self._parse_to_table(), 1333 "TRANSIENT": lambda self: self.expression(exp.TransientProperty()), 1334 "TRANSFORM": lambda self: self.expression( 1335 exp.TransformModelProperty(expressions=self._parse_wrapped_csv(self._parse_expression)) 1336 ), 1337 "TTL": lambda self: self._parse_ttl(), 1338 "USING": lambda self: self._parse_property_assignment(exp.FileFormatProperty), 1339 "UNLOGGED": lambda self: self.expression(exp.UnloggedProperty()), 1340 "VOLATILE": lambda self: self._parse_volatile_property(), 1341 "WITH": lambda self: self._parse_with_property(), 1342 } 1343 1344 CONSTRAINT_PARSERS: t.ClassVar = { 1345 "AUTOINCREMENT": lambda self: self._parse_auto_increment(), 1346 "AUTO_INCREMENT": lambda self: self._parse_auto_increment(), 1347 "CASESPECIFIC": lambda self: self.expression(exp.CaseSpecificColumnConstraint(not_=False)), 1348 "CHARACTER SET": lambda self: self.expression( 1349 exp.CharacterSetColumnConstraint(this=self._parse_var_or_string()) 1350 ), 1351 "CHECK": lambda self: self._parse_check_constraint(), 1352 "COLLATE": lambda self: self.expression( 1353 exp.CollateColumnConstraint(this=self._parse_identifier() or self._parse_column()) 1354 ), 1355 "COMMENT": lambda self: self.expression( 1356 exp.CommentColumnConstraint(this=self._parse_string()) 1357 ), 1358 "COMPRESS": lambda self: self._parse_compress(), 1359 "CLUSTERED": lambda self: self.expression( 1360 exp.ClusteredColumnConstraint(this=self._parse_wrapped_csv(self._parse_ordered)) 1361 ), 1362 "NONCLUSTERED": lambda self: self.expression( 1363 exp.NonClusteredColumnConstraint(this=self._parse_wrapped_csv(self._parse_ordered)) 1364 ), 1365 "DEFAULT": lambda self: self.expression( 1366 exp.DefaultColumnConstraint(this=self._parse_bitwise()) 1367 ), 1368 "ENCODE": lambda self: self.expression(exp.EncodeColumnConstraint(this=self._parse_var())), 1369 "EPHEMERAL": lambda self: self.expression( 1370 exp.EphemeralColumnConstraint(this=self._parse_bitwise()) 1371 ), 1372 "EXCLUDE": lambda self: self.expression( 1373 exp.ExcludeColumnConstraint(this=self._parse_index_params()) 1374 ), 1375 "FOREIGN KEY": lambda self: self._parse_foreign_key(), 1376 "FORMAT": lambda self: self.expression( 1377 exp.DateFormatColumnConstraint(this=self._parse_var_or_string()) 1378 ), 1379 "GENERATED": lambda self: self._parse_generated_as_identity(), 1380 "IDENTITY": lambda self: self._parse_auto_increment(), 1381 "INLINE": lambda self: self._parse_inline(), 1382 "LIKE": lambda self: self._parse_create_like(), 1383 "NOT": lambda self: self._parse_not_constraint(), 1384 "NULL": lambda self: self.expression(exp.NotNullColumnConstraint(allow_null=True)), 1385 "ON": lambda self: ( 1386 ( 1387 self._match(TokenType.UPDATE) 1388 and self.expression(exp.OnUpdateColumnConstraint(this=self._parse_function())) 1389 ) 1390 or self.expression(exp.OnProperty(this=self._parse_id_var())) 1391 ), 1392 "PATH": lambda self: self.expression(exp.PathColumnConstraint(this=self._parse_string())), 1393 "PERIOD": lambda self: self._parse_period_for_system_time(), 1394 "PRIMARY KEY": lambda self: self._parse_primary_key(), 1395 "REFERENCES": lambda self: self._parse_references(match=False), 1396 "TITLE": lambda self: self.expression( 1397 exp.TitleColumnConstraint(this=self._parse_var_or_string()) 1398 ), 1399 "TTL": lambda self: self.expression(exp.MergeTreeTTL(expressions=[self._parse_bitwise()])), 1400 "UNIQUE": lambda self: self._parse_unique(), 1401 "UPPERCASE": lambda self: self.expression(exp.UppercaseColumnConstraint()), 1402 "WITH": lambda self: self.expression( 1403 exp.Properties(expressions=self._parse_wrapped_properties()) 1404 ), 1405 "BUCKET": lambda self: self._parse_partitioned_by_bucket_or_truncate(), 1406 "TRUNCATE": lambda self: self._parse_partitioned_by_bucket_or_truncate(), 1407 } 1408 1409 def _parse_partitioned_by_bucket_or_truncate(self) -> exp.Expr | None: 1410 if not self._match(TokenType.L_PAREN, advance=False): 1411 # Partitioning by bucket or truncate follows the syntax: 1412 # PARTITION BY (BUCKET(..) | TRUNCATE(..)) 1413 # If we don't have parenthesis after each keyword, we should instead parse this as an identifier 1414 self._retreat(self._index - 1) 1415 return None 1416 1417 klass = ( 1418 exp.PartitionedByBucket 1419 if self._prev.text.upper() == "BUCKET" 1420 else exp.PartitionByTruncate 1421 ) 1422 1423 args = self._parse_wrapped_csv(lambda: self._parse_primary() or self._parse_column()) 1424 this, expression = seq_get(args, 0), seq_get(args, 1) 1425 1426 if isinstance(this, exp.Literal): 1427 # Check for Iceberg partition transforms (bucket / truncate) and ensure their arguments are in the right order 1428 # - For Hive, it's `bucket(<num buckets>, <col name>)` or `truncate(<num_chars>, <col_name>)` 1429 # - For Trino, it's reversed - `bucket(<col name>, <num buckets>)` or `truncate(<col_name>, <num_chars>)` 1430 # Both variants are canonicalized in the latter i.e `bucket(<col name>, <num buckets>)` 1431 # 1432 # Hive ref: https://docs.aws.amazon.com/athena/latest/ug/querying-iceberg-creating-tables.html#querying-iceberg-partitioning 1433 # Trino ref: https://docs.aws.amazon.com/athena/latest/ug/create-table-as.html#ctas-table-properties 1434 this, expression = expression, this 1435 1436 return self.expression(klass(this=this, expression=expression)) 1437 1438 ALTER_PARSERS: t.ClassVar = { 1439 "ADD": lambda self: self._parse_alter_table_add(), 1440 "AS": lambda self: self._parse_select(), 1441 "ALTER": lambda self: self._parse_alter_table_alter(), 1442 "CLUSTER BY": lambda self: self._parse_cluster_property(), 1443 "DELETE": lambda self: self.expression(exp.Delete(where=self._parse_where())), 1444 "DROP": lambda self: self._parse_alter_table_drop(), 1445 "RENAME": lambda self: self._parse_alter_table_rename(), 1446 "SET": lambda self: self._parse_alter_table_set(), 1447 "SWAP": lambda self: self.expression( 1448 exp.SwapTable(this=self._match(TokenType.WITH) and self._parse_table(schema=True)) 1449 ), 1450 } 1451 1452 ALTER_ALTER_PARSERS: t.ClassVar = { 1453 "DISTKEY": lambda self: self._parse_alter_diststyle(), 1454 "DISTSTYLE": lambda self: self._parse_alter_diststyle(), 1455 "SORTKEY": lambda self: self._parse_alter_sortkey(), 1456 "COMPOUND": lambda self: self._parse_alter_sortkey(compound=True), 1457 } 1458 1459 SCHEMA_UNNAMED_CONSTRAINTS: t.ClassVar = { 1460 "CHECK", 1461 "EXCLUDE", 1462 "FOREIGN KEY", 1463 "LIKE", 1464 "PERIOD", 1465 "PRIMARY KEY", 1466 "UNIQUE", 1467 "BUCKET", 1468 "TRUNCATE", 1469 } 1470 1471 NO_PAREN_FUNCTION_PARSERS: t.ClassVar = { 1472 "ANY": lambda self: self.expression(exp.Any(this=self._parse_bitwise())), 1473 "CASE": lambda self: self._parse_case(), 1474 "CONNECT_BY_ROOT": lambda self: self.expression( 1475 exp.ConnectByRoot(this=self._parse_column()) 1476 ), 1477 "IF": lambda self: self._parse_if(), 1478 } 1479 1480 INVALID_FUNC_NAME_TOKENS: t.ClassVar = { 1481 TokenType.IDENTIFIER, 1482 TokenType.STRING, 1483 } 1484 1485 FUNCTIONS_WITH_ALIASED_ARGS: t.ClassVar = {"STRUCT"} 1486 1487 KEY_VALUE_DEFINITIONS: t.ClassVar = (exp.Alias, exp.EQ, exp.PropertyEQ, exp.Slice) 1488 1489 FUNCTION_PARSERS: t.ClassVar[dict[str, t.Callable]] = { 1490 **{ 1491 name: lambda self: self._parse_max_min_by(exp.ArgMax) for name in exp.ArgMax.sql_names() 1492 }, 1493 **{ 1494 name: lambda self: self._parse_max_min_by(exp.ArgMin) for name in exp.ArgMin.sql_names() 1495 }, 1496 "CAST": lambda self: self._parse_cast(self.STRICT_CAST), 1497 "CEIL": lambda self: self._parse_ceil_floor(exp.Ceil), 1498 "CONVERT": lambda self: self._parse_convert(self.STRICT_CAST), 1499 "CHAR": lambda self: self._parse_char(), 1500 "CHR": lambda self: self._parse_char(), 1501 "DECODE": lambda self: self._parse_decode(), 1502 "EXTRACT": lambda self: self._parse_extract(), 1503 "FLOOR": lambda self: self._parse_ceil_floor(exp.Floor), 1504 "GAP_FILL": lambda self: self._parse_gap_fill(), 1505 "INITCAP": lambda self: self._parse_initcap(), 1506 "JSON_OBJECT": lambda self: self._parse_json_object(), 1507 "JSON_OBJECTAGG": lambda self: self._parse_json_object(agg=True), 1508 "JSON_TABLE": lambda self: self._parse_json_table(), 1509 "MATCH": lambda self: self._parse_match_against(), 1510 "NORMALIZE": lambda self: self._parse_normalize(), 1511 "OPENJSON": lambda self: self._parse_open_json(), 1512 "OVERLAY": lambda self: self._parse_overlay(), 1513 "POSITION": lambda self: self._parse_position(), 1514 "SAFE_CAST": lambda self: self._parse_cast(False, safe=True), 1515 "STRING_AGG": lambda self: self._parse_string_agg(), 1516 "SUBSTRING": lambda self: self._parse_substring(), 1517 "TRIM": lambda self: self._parse_trim(), 1518 "TRY_CAST": lambda self: self._parse_cast(False, safe=True), 1519 "TRY_CONVERT": lambda self: self._parse_convert(False, safe=True), 1520 "XMLELEMENT": lambda self: self._parse_xml_element(), 1521 "XMLTABLE": lambda self: self._parse_xml_table(), 1522 } 1523 1524 QUERY_MODIFIER_PARSERS: t.ClassVar = { 1525 TokenType.MATCH_RECOGNIZE: lambda self: ("match", self._parse_match_recognize()), 1526 TokenType.PREWHERE: lambda self: ("prewhere", self._parse_prewhere()), 1527 TokenType.WHERE: lambda self: ("where", self._parse_where()), 1528 TokenType.GROUP_BY: lambda self: ("group", self._parse_group()), 1529 TokenType.HAVING: lambda self: ("having", self._parse_having()), 1530 TokenType.QUALIFY: lambda self: ("qualify", self._parse_qualify()), 1531 TokenType.WINDOW: lambda self: ("windows", self._parse_window_clause()), 1532 TokenType.ORDER_BY: lambda self: ("order", self._parse_order()), 1533 TokenType.LIMIT: lambda self: ("limit", self._parse_limit()), 1534 TokenType.FETCH: lambda self: ("limit", self._parse_limit()), 1535 TokenType.OFFSET: lambda self: ("offset", self._parse_offset()), 1536 TokenType.FOR: lambda self: ("locks", self._parse_locks()), 1537 TokenType.LOCK: lambda self: ("locks", self._parse_locks()), 1538 TokenType.TABLE_SAMPLE: lambda self: ("sample", self._parse_table_sample(as_modifier=True)), 1539 TokenType.USING: lambda self: ("sample", self._parse_table_sample(as_modifier=True)), 1540 TokenType.CLUSTER_BY: lambda self: ( 1541 "cluster", 1542 self._parse_cluster(), 1543 ), 1544 TokenType.DISTRIBUTE_BY: lambda self: ( 1545 "distribute", 1546 self._parse_sort(exp.Distribute, TokenType.DISTRIBUTE_BY), 1547 ), 1548 TokenType.SORT_BY: lambda self: ("sort", self._parse_sort(exp.Sort, TokenType.SORT_BY)), 1549 TokenType.CONNECT_BY: lambda self: ("connect", self._parse_connect(skip_start_token=True)), 1550 TokenType.START_WITH: lambda self: ("connect", self._parse_connect()), 1551 } 1552 QUERY_MODIFIER_TOKENS: t.ClassVar = set(QUERY_MODIFIER_PARSERS) 1553 1554 SET_PARSERS: t.ClassVar = { 1555 "GLOBAL": lambda self: self._parse_set_item_assignment("GLOBAL"), 1556 "LOCAL": lambda self: self._parse_set_item_assignment("LOCAL"), 1557 "SESSION": lambda self: self._parse_set_item_assignment("SESSION"), 1558 "TRANSACTION": lambda self: self._parse_set_transaction(), 1559 } 1560 1561 SHOW_PARSERS: t.ClassVar[dict[str, t.Callable]] = {} 1562 1563 TYPE_LITERAL_PARSERS: t.ClassVar = { 1564 exp.DType.JSON: lambda self, this, _: self.expression(exp.ParseJSON(this=this)), 1565 } 1566 1567 TYPE_CONVERTERS: t.ClassVar[dict[exp.DType, t.Callable[[exp.DataType], exp.DataType]]] = {} 1568 1569 DDL_SELECT_TOKENS: t.ClassVar = {TokenType.SELECT, TokenType.WITH, TokenType.L_PAREN} 1570 1571 PRE_VOLATILE_TOKENS: t.ClassVar = {TokenType.CREATE, TokenType.REPLACE, TokenType.UNIQUE} 1572 1573 TRANSACTION_KIND: t.ClassVar = {"DEFERRED", "IMMEDIATE", "EXCLUSIVE"} 1574 TRANSACTION_CHARACTERISTICS: t.ClassVar[OPTIONS_TYPE] = { 1575 "ISOLATION": ( 1576 ("LEVEL", "REPEATABLE", "READ"), 1577 ("LEVEL", "READ", "COMMITTED"), 1578 ("LEVEL", "READ", "UNCOMITTED"), 1579 ("LEVEL", "SERIALIZABLE"), 1580 ), 1581 "READ": ("WRITE", "ONLY"), 1582 } 1583 1584 CONFLICT_ACTIONS: t.ClassVar[OPTIONS_TYPE] = { 1585 **dict.fromkeys(("ABORT", "FAIL", "IGNORE", "REPLACE", "ROLLBACK", "UPDATE"), tuple()), 1586 "DO": ("NOTHING", "UPDATE"), 1587 } 1588 1589 TRIGGER_TIMING: t.ClassVar[OPTIONS_TYPE] = { 1590 "INSTEAD": (("OF",),), 1591 "BEFORE": tuple(), 1592 "AFTER": tuple(), 1593 } 1594 1595 TRIGGER_DEFERRABLE: t.ClassVar[OPTIONS_TYPE] = { 1596 "NOT": (("DEFERRABLE",),), 1597 "DEFERRABLE": tuple(), 1598 } 1599 1600 CREATE_SEQUENCE: t.ClassVar[OPTIONS_TYPE] = { 1601 "SCALE": ("EXTEND", "NOEXTEND"), 1602 "SHARD": ("EXTEND", "NOEXTEND"), 1603 "NO": ("CYCLE", "CACHE", "MAXVALUE", "MINVALUE"), 1604 **dict.fromkeys( 1605 ( 1606 "SESSION", 1607 "GLOBAL", 1608 "KEEP", 1609 "NOKEEP", 1610 "ORDER", 1611 "NOORDER", 1612 "NOCACHE", 1613 "CYCLE", 1614 "NOCYCLE", 1615 "NOMINVALUE", 1616 "NOMAXVALUE", 1617 "NOSCALE", 1618 "NOSHARD", 1619 ), 1620 tuple(), 1621 ), 1622 } 1623 1624 ISOLATED_LOADING_OPTIONS: t.ClassVar[OPTIONS_TYPE] = {"FOR": ("ALL", "INSERT", "NONE")} 1625 1626 USABLES: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys( 1627 ("ROLE", "WAREHOUSE", "DATABASE", "SCHEMA", "CATALOG"), tuple() 1628 ) 1629 1630 CAST_ACTIONS: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys(("RENAME", "ADD"), ("FIELDS",)) 1631 1632 SCHEMA_BINDING_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1633 "TYPE": ("EVOLUTION",), 1634 **dict.fromkeys(("BINDING", "COMPENSATION", "EVOLUTION"), tuple()), 1635 } 1636 1637 PROCEDURE_OPTIONS: t.ClassVar[OPTIONS_TYPE] = {} 1638 1639 EXECUTE_AS_OPTIONS: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys( 1640 ("CALLER", "SELF", "OWNER"), tuple() 1641 ) 1642 1643 KEY_CONSTRAINT_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1644 "NOT": ("ENFORCED",), 1645 "MATCH": ( 1646 "FULL", 1647 "PARTIAL", 1648 "SIMPLE", 1649 ), 1650 "INITIALLY": ("DEFERRED", "IMMEDIATE"), 1651 "USING": ( 1652 "BTREE", 1653 "HASH", 1654 ), 1655 **dict.fromkeys(("DEFERRABLE", "NORELY", "RELY"), tuple()), 1656 } 1657 1658 WINDOW_EXCLUDE_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1659 "NO": ("OTHERS",), 1660 "CURRENT": ("ROW",), 1661 **dict.fromkeys(("GROUP", "TIES"), tuple()), 1662 } 1663 1664 INSERT_ALTERNATIVES: t.ClassVar = {"ABORT", "FAIL", "IGNORE", "REPLACE", "ROLLBACK"} 1665 1666 CLONE_KEYWORDS: t.ClassVar = {"CLONE", "COPY"} 1667 HISTORICAL_DATA_PREFIX: t.ClassVar = {"AT", "BEFORE", "END"} 1668 HISTORICAL_DATA_KIND: t.ClassVar = {"OFFSET", "STATEMENT", "STREAM", "TIMESTAMP", "VERSION"} 1669 1670 OPCLASS_FOLLOW_KEYWORDS: t.ClassVar = {"ASC", "DESC", "NULLS", "WITH"} 1671 1672 OPTYPE_FOLLOW_TOKENS: t.ClassVar = {TokenType.COMMA, TokenType.R_PAREN} 1673 1674 TABLE_INDEX_HINT_TOKENS: t.ClassVar = {TokenType.FORCE, TokenType.IGNORE, TokenType.USE} 1675 1676 VIEW_ATTRIBUTES: t.ClassVar = {"ENCRYPTION", "SCHEMABINDING", "VIEW_METADATA"} 1677 1678 WINDOW_ALIAS_TOKENS: t.ClassVar = ID_VAR_TOKENS - {TokenType.RANGE, TokenType.ROWS} 1679 WINDOW_BEFORE_PAREN_TOKENS: t.ClassVar = {TokenType.OVER} 1680 WINDOW_SIDES: t.ClassVar = {"FOLLOWING", "PRECEDING"} 1681 1682 JSON_KEY_VALUE_SEPARATOR_TOKENS: t.ClassVar = {TokenType.COLON, TokenType.COMMA, TokenType.IS} 1683 1684 FETCH_TOKENS: t.ClassVar = ID_VAR_TOKENS - {TokenType.ROW, TokenType.ROWS, TokenType.PERCENT} 1685 1686 ADD_CONSTRAINT_TOKENS: t.ClassVar = { 1687 TokenType.CONSTRAINT, 1688 TokenType.FOREIGN_KEY, 1689 TokenType.INDEX, 1690 TokenType.KEY, 1691 TokenType.PRIMARY_KEY, 1692 TokenType.UNIQUE, 1693 } 1694 1695 DISTINCT_TOKENS: t.ClassVar = {TokenType.DISTINCT} 1696 1697 UNNEST_OFFSET_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - SET_OPERATIONS 1698 1699 SELECT_START_TOKENS: t.ClassVar = {TokenType.L_PAREN, TokenType.WITH, TokenType.SELECT} 1700 1701 COPY_INTO_VARLEN_OPTIONS: t.ClassVar = { 1702 "FILE_FORMAT", 1703 "COPY_OPTIONS", 1704 "FORMAT_OPTIONS", 1705 "CREDENTIAL", 1706 } 1707 1708 IS_JSON_PREDICATE_KIND: t.ClassVar = {"VALUE", "SCALAR", "ARRAY", "OBJECT"} 1709 1710 ODBC_DATETIME_LITERALS: t.ClassVar[dict[str, type[exp.Expr]]] = {} 1711 1712 ON_CONDITION_TOKENS: t.ClassVar = {"ERROR", "NULL", "TRUE", "FALSE", "EMPTY"} 1713 1714 PRIVILEGE_FOLLOW_TOKENS: t.ClassVar = {TokenType.ON, TokenType.COMMA, TokenType.L_PAREN} 1715 1716 # The style options for the DESCRIBE statement 1717 DESCRIBE_STYLES: t.ClassVar = {"ANALYZE", "EXTENDED", "FORMATTED", "HISTORY"} 1718 1719 SET_ASSIGNMENT_DELIMITERS: t.ClassVar = {"=", ":=", "TO"} 1720 1721 # The style options for the ANALYZE statement 1722 ANALYZE_STYLES: t.ClassVar = { 1723 "BUFFER_USAGE_LIMIT", 1724 "FULL", 1725 "LOCAL", 1726 "NO_WRITE_TO_BINLOG", 1727 "SAMPLE", 1728 "SKIP_LOCKED", 1729 "VERBOSE", 1730 } 1731 1732 ANALYZE_EXPRESSION_PARSERS: t.ClassVar = { 1733 "ALL": lambda self: self._parse_analyze_columns(), 1734 "COMPUTE": lambda self: self._parse_analyze_statistics(), 1735 "DELETE": lambda self: self._parse_analyze_delete(), 1736 "DROP": lambda self: self._parse_analyze_histogram(), 1737 "ESTIMATE": lambda self: self._parse_analyze_statistics(), 1738 "LIST": lambda self: self._parse_analyze_list(), 1739 "PREDICATE": lambda self: self._parse_analyze_columns(), 1740 "UPDATE": lambda self: self._parse_analyze_histogram(), 1741 "VALIDATE": lambda self: self._parse_analyze_validate(), 1742 } 1743 1744 PARTITION_KEYWORDS: t.ClassVar = {"PARTITION", "SUBPARTITION"} 1745 1746 AMBIGUOUS_ALIAS_TOKENS: t.ClassVar = (TokenType.LIMIT, TokenType.OFFSET) 1747 1748 OPERATION_MODIFIERS: t.ClassVar[set[str]] = set() 1749 1750 RECURSIVE_CTE_SEARCH_KIND: t.ClassVar = {"BREADTH", "DEPTH", "CYCLE"} 1751 1752 SECURITY_PROPERTY_KEYWORDS: t.ClassVar = {"DEFINER", "INVOKER", "NONE"} 1753 1754 MODIFIABLES: t.ClassVar = (exp.Query, exp.Table, exp.TableFromRows, exp.Values) 1755 1756 STRICT_CAST: t.ClassVar = True 1757 1758 PREFIXED_PIVOT_COLUMNS: t.ClassVar = False 1759 IDENTIFY_PIVOT_STRINGS: t.ClassVar = False 1760 # Controls when an aggregation's name is included in a pivoted column's name: 1761 # "agg_name_if_aliased" - only for aggregations that carry an explicit alias 1762 # "agg_name_if_aliased_or_multiple" - if aliased, or whenever there are multiple aggregations 1763 # "agg_name_if_multiple" - only when there are multiple aggregations (a lone agg is value-only) 1764 PIVOT_COLUMN_NAMING: t.ClassVar[str] = "agg_name_if_aliased" 1765 1766 LOG_DEFAULTS_TO_LN: t.ClassVar = False 1767 1768 # Whether the table sample clause expects CSV syntax 1769 TABLESAMPLE_CSV: t.ClassVar = False 1770 1771 # The default method used for table sampling 1772 DEFAULT_SAMPLING_METHOD: t.ClassVar[str | None] = None 1773 1774 # Whether the SET command needs a delimiter (e.g. "=") for assignments 1775 SET_REQUIRES_ASSIGNMENT_DELIMITER: t.ClassVar = True 1776 1777 # Whether the TRIM function expects the characters to trim as its first argument 1778 TRIM_PATTERN_FIRST: t.ClassVar = False 1779 1780 # Whether string aliases are supported `SELECT COUNT(*) 'count'` 1781 STRING_ALIASES: t.ClassVar = False 1782 1783 # Whether query modifiers such as LIMIT are attached to the UNION node (vs its right operand) 1784 MODIFIERS_ATTACHED_TO_SET_OP: t.ClassVar = True 1785 SET_OP_MODIFIERS: t.ClassVar = {"order", "limit", "offset"} 1786 1787 # Whether to parse IF statements that aren't followed by a left parenthesis as commands 1788 NO_PAREN_IF_COMMANDS: t.ClassVar = True 1789 1790 # Whether the -> and ->> operators expect documents of type JSON (e.g. Postgres) 1791 JSON_ARROWS_REQUIRE_JSON_TYPE: t.ClassVar = False 1792 1793 # Whether the `:` operator is used to extract a value from a VARIANT column 1794 COLON_IS_VARIANT_EXTRACT: t.ClassVar = False 1795 1796 # Whether a chain of colon extractions (x:y:z) is a single extraction with a merged 1797 # path (x:y.z, e.g. Snowflake) or each colon extracts from the previous result (e.g. Databricks) 1798 COLON_CHAIN_IS_SINGLE_EXTRACT: t.ClassVar = True 1799 1800 # Whether or not a VALUES keyword needs to be followed by '(' to form a VALUES clause. 1801 # If this is True and '(' is not found, the keyword will be treated as an identifier 1802 VALUES_FOLLOWED_BY_PAREN: t.ClassVar = True 1803 1804 # Whether implicit unnesting is supported, e.g. SELECT 1 FROM y.z AS z, z.a (Redshift) 1805 SUPPORTS_IMPLICIT_UNNEST: t.ClassVar = False 1806 1807 # Whether or not interval spans are supported, INTERVAL 1 YEAR TO MONTHS 1808 INTERVAL_SPANS: t.ClassVar = True 1809 1810 # Whether a PARTITION clause can follow a table reference 1811 SUPPORTS_PARTITION_SELECTION: t.ClassVar = False 1812 1813 # Whether the `name AS expr` schema/column constraint requires parentheses around `expr` 1814 WRAPPED_TRANSFORM_COLUMN_CONSTRAINT: t.ClassVar = True 1815 1816 # Whether the 'AS' keyword is optional in the CTE definition syntax 1817 OPTIONAL_ALIAS_TOKEN_CTE: t.ClassVar = True 1818 1819 # Whether renaming a column with an ALTER statement requires the presence of the COLUMN keyword 1820 ALTER_RENAME_REQUIRES_COLUMN: t.ClassVar = True 1821 1822 # Whether Alter statements are allowed to contain Partition specifications 1823 ALTER_TABLE_PARTITIONS: t.ClassVar = False 1824 1825 # Whether all join types have the same precedence, i.e., they "naturally" produce a left-deep tree. 1826 # In standard SQL, joins that use the JOIN keyword take higher precedence than comma-joins. That is 1827 # to say, JOIN operators happen before comma operators. This is not the case in some dialects, such 1828 # as BigQuery, where all joins have the same precedence. 1829 JOINS_HAVE_EQUAL_PRECEDENCE: t.ClassVar = False 1830 1831 # Whether TIMESTAMP <literal> can produce a zone-aware timestamp 1832 ZONE_AWARE_TIMESTAMP_CONSTRUCTOR: t.ClassVar = False 1833 1834 # Whether map literals support arbitrary expressions as keys. 1835 # When True, allows complex keys like arrays or literals: {[1, 2]: 3}, {1: 2} (e.g. DuckDB). 1836 # When False, keys are typically restricted to identifiers. 1837 MAP_KEYS_ARE_ARBITRARY_EXPRESSIONS: t.ClassVar = False 1838 1839 # Whether JSON_EXTRACT requires a JSON expression as the first argument, e.g this 1840 # is true for Snowflake but not for BigQuery which can also process strings 1841 JSON_EXTRACT_REQUIRES_JSON_EXPRESSION: t.ClassVar = False 1842 1843 # Dialects like Databricks support JOINS without join criteria 1844 # Adding an ON TRUE, makes transpilation semantically correct for other dialects 1845 ADD_JOIN_ON_TRUE: t.ClassVar = False 1846 1847 # Whether INTERVAL spans with literal format '\d+ hh:[mm:[ss[.ff]]]' 1848 # can omit the span unit `DAY TO MINUTE` or `DAY TO SECOND` 1849 SUPPORTS_OMITTED_INTERVAL_SPAN_UNIT: t.ClassVar = False 1850 1851 # Whether adjacent string literals like 'foo' 'bar' require a whitespace or comment between them 1852 # to be considered valid syntactically. Such expressions evaluate to the strings' concatenation. 1853 ADJACENT_STRINGS_CANNOT_BE_CONNECTED: t.ClassVar = False 1854 1855 SHOW_TRIE: t.ClassVar[dict] = new_trie(key.split(" ") for key in SHOW_PARSERS) 1856 SET_TRIE: t.ClassVar[dict] = new_trie(key.split(" ") for key in SET_PARSERS) 1857 1858 def __init__( 1859 self, 1860 error_level: ErrorLevel | None = None, 1861 error_message_context: int = 100, 1862 max_errors: int = 3, 1863 max_nodes: int = -1, 1864 dialect: DialectType = None, 1865 ): 1866 self.error_level: ErrorLevel = error_level or ErrorLevel.IMMEDIATE 1867 self.error_message_context: int = error_message_context 1868 self.max_errors: int = max_errors 1869 self.max_nodes: int = max_nodes 1870 self.dialect: t.Any = _resolve_dialect(dialect) 1871 self.sql: str = "" 1872 self.errors: list[ParseError] = [] 1873 self._tokens: list[Token] = [] 1874 self._tokens_size: i64 = 0 1875 self._index: i64 = 0 1876 self._curr: Token = SENTINEL_NONE 1877 self._next: Token = SENTINEL_NONE 1878 self._prev: Token = SENTINEL_NONE 1879 self._prev_comments: list[str] = [] 1880 self._pipe_cte_counter: int = 0 1881 self._chunks: list[list[Token]] = [] 1882 self._chunk_index: i64 = 0 1883 self._node_count: int = 0 1884 1885 def reset(self) -> None: 1886 self.sql = "" 1887 self.errors = [] 1888 self._tokens = [] 1889 self._tokens_size = 0 1890 self._index = 0 1891 self._curr = SENTINEL_NONE 1892 self._next = SENTINEL_NONE 1893 self._prev = SENTINEL_NONE 1894 self._prev_comments = [] 1895 self._pipe_cte_counter = 0 1896 self._chunks = [] 1897 self._chunk_index = 0 1898 self._node_count = 0 1899 1900 def _advance(self, times: i64 = 1) -> None: 1901 index = self._index + times 1902 self._index = index 1903 tokens = self._tokens 1904 size = self._tokens_size 1905 self._curr = tokens[index] if index < size else SENTINEL_NONE 1906 self._next = tokens[index + 1] if index + 1 < size else SENTINEL_NONE 1907 1908 if index > 0: 1909 prev = tokens[index - 1] 1910 self._prev = prev 1911 self._prev_comments = prev.comments 1912 else: 1913 self._prev = SENTINEL_NONE 1914 self._prev_comments = [] 1915 1916 def _advance_chunk(self) -> None: 1917 self._index = -1 1918 self._tokens = self._chunks[self._chunk_index] 1919 self._tokens_size = i64(len(self._tokens)) 1920 self._chunk_index += 1 1921 self._advance() 1922 1923 def _retreat(self, index: i64) -> None: 1924 if index != self._index: 1925 self._advance(index - self._index) 1926 1927 def _add_comments(self, expression: exp.Expr | None) -> None: 1928 if expression and self._prev_comments: 1929 expression.add_comments(self._prev_comments) 1930 self._prev_comments = [] 1931 1932 def _match( 1933 self, token_type: TokenType, advance: bool = True, expression: exp.Expr | None = None 1934 ) -> bool: 1935 if self._curr.token_type == token_type: 1936 if advance: 1937 self._advance() 1938 self._add_comments(expression) 1939 return True 1940 return False 1941 1942 def _match_set(self, types: t.Collection[TokenType], advance: bool = True) -> bool: 1943 if self._curr.token_type in types: 1944 if advance: 1945 self._advance() 1946 return True 1947 return False 1948 1949 def _match_pair( 1950 self, token_type_a: TokenType, token_type_b: TokenType, advance: bool = True 1951 ) -> bool: 1952 if self._curr.token_type == token_type_a and self._next.token_type == token_type_b: 1953 if advance: 1954 self._advance(2) 1955 return True 1956 return False 1957 1958 def _match_texts(self, texts: t.Collection[str], advance: bool = True) -> bool: 1959 if self._curr.token_type != TokenType.STRING and self._curr.text.upper() in texts: 1960 if advance: 1961 self._advance() 1962 return True 1963 return False 1964 1965 def _match_text_seq(self, *texts: str, advance: bool = True) -> bool: 1966 index = self._index 1967 string_type = TokenType.STRING 1968 for text in texts: 1969 if self._curr.token_type != string_type and self._curr.text.upper() == text: 1970 self._advance() 1971 else: 1972 self._retreat(index) 1973 return False 1974 1975 if not advance: 1976 self._retreat(index) 1977 1978 return True 1979 1980 def _is_connected(self) -> bool: 1981 prev = self._prev 1982 curr = self._curr 1983 return bool(prev and curr and prev.end + 1 == curr.start) 1984 1985 def _find_sql(self, start: Token, end: Token) -> str: 1986 return self.sql[start.start : end.end + 1] 1987 1988 def raise_error(self, message: str, token: Token = SENTINEL_NONE) -> None: 1989 token = token or self._curr or self._prev or Token.string("") 1990 formatted_sql, start_context, highlight, end_context = highlight_sql( 1991 sql=self.sql, 1992 positions=[(token.start, token.end)], 1993 context_length=self.error_message_context, 1994 ) 1995 formatted_message = f"{message}. Line {token.line}, Col: {token.col}.\n {formatted_sql}" 1996 1997 error = ParseError.new( 1998 formatted_message, 1999 description=message, 2000 line=token.line, 2001 col=token.col, 2002 start_context=start_context, 2003 highlight=highlight, 2004 end_context=end_context, 2005 ) 2006 2007 if self.error_level == ErrorLevel.IMMEDIATE: 2008 raise error 2009 2010 self.errors.append(error) 2011 2012 def validate_expression(self, expression: E, args: list | None = None) -> E: 2013 if self.max_nodes > -1: 2014 self._node_count += 1 2015 if self._node_count > self.max_nodes: 2016 self.raise_error(f"Maximum number of AST nodes ({self.max_nodes}) exceeded") 2017 if self.error_level != ErrorLevel.IGNORE: 2018 for error_message in expression.error_messages(args): 2019 self.raise_error(error_message) 2020 return expression 2021 2022 def _try_parse(self, parse_method: t.Callable[[], T], retreat: bool = False) -> T | None: 2023 index = self._index 2024 error_level = self.error_level 2025 this: T | None = None 2026 2027 self.error_level = ErrorLevel.IMMEDIATE 2028 try: 2029 this = parse_method() 2030 except ParseError: 2031 this = None 2032 finally: 2033 if not this or retreat: 2034 self._retreat(index) 2035 self.error_level = error_level 2036 2037 return this 2038 2039 def parse(self, raw_tokens: list[Token], sql: str) -> list[exp.Expr | None]: 2040 """ 2041 Parses a list of tokens and returns a list of syntax trees, one tree 2042 per parsed SQL statement. 2043 2044 Args: 2045 raw_tokens: The list of tokens. 2046 sql: The original SQL string. 2047 2048 Returns: 2049 The list of the produced syntax trees. 2050 """ 2051 return self._parse( 2052 parse_method=self.__class__._parse_statement, raw_tokens=raw_tokens, sql=sql 2053 ) 2054 2055 def parse_into( 2056 self, 2057 expression_types: exp.IntoType, 2058 raw_tokens: list[Token], 2059 sql: str | None = None, 2060 ) -> list[exp.Expr | None]: 2061 """ 2062 Parses a list of tokens into a given Expr type. If a collection of Expr 2063 types is given instead, this method will try to parse the token list into each one 2064 of them, stopping at the first for which the parsing succeeds. 2065 2066 Args: 2067 expression_types: The expression type(s) to try and parse the token list into. 2068 raw_tokens: The list of tokens. 2069 sql: The original SQL string, used to produce helpful debug messages. 2070 2071 Returns: 2072 The target Expr. 2073 """ 2074 errors = [] 2075 for expression_type in ensure_list(expression_types): 2076 parser = self.EXPRESSION_PARSERS.get(t.cast(type[exp.Expr], expression_type)) 2077 if not parser: 2078 raise TypeError(f"No parser registered for {expression_type}") 2079 2080 try: 2081 return self._parse(parser, raw_tokens, sql) 2082 except ParseError as e: 2083 e.errors[0]["into_expression"] = expression_type 2084 errors.append(e) 2085 2086 raise ParseError( 2087 f"Failed to parse '{sql or raw_tokens}' into {expression_types}", 2088 errors=merge_errors(errors), 2089 ) from errors[-1] 2090 2091 def check_errors(self) -> None: 2092 """Logs or raises any found errors, depending on the chosen error level setting.""" 2093 if self.error_level == ErrorLevel.WARN: 2094 for error in self.errors: 2095 logger.error(str(error)) 2096 elif self.error_level == ErrorLevel.RAISE and self.errors: 2097 raise ParseError( 2098 concat_messages(self.errors, self.max_errors), 2099 errors=merge_errors(self.errors), 2100 ) 2101 2102 def expression( 2103 self, 2104 instance: E, 2105 token: Token | None = None, 2106 comments: list[str] | None = None, 2107 ) -> E: 2108 if token: 2109 instance.update_positions(token) 2110 instance.add_comments(comments) if comments else self._add_comments(instance) 2111 if not instance.is_primitive: 2112 instance = self.validate_expression(instance) 2113 return instance 2114 2115 def _parse_batch_statements( 2116 self, 2117 parse_method: t.Callable[[Parser], exp.Expr | None], 2118 sep_first_statement: bool = True, 2119 ) -> list[exp.Expr | None]: 2120 expressions = [] 2121 2122 # Chunkification binds if/while statements with the first statement of the body 2123 if sep_first_statement: 2124 self._match(TokenType.BEGIN) 2125 expressions.append(parse_method(self)) 2126 2127 chunks_length = len(self._chunks) 2128 while self._chunk_index < chunks_length: 2129 self._advance_chunk() 2130 2131 if self._match(TokenType.ELSE, advance=False): 2132 return expressions 2133 2134 if expressions and not self._next and self._match(TokenType.END): 2135 expressions.append(exp.EndStatement()) 2136 continue 2137 2138 expressions.append(parse_method(self)) 2139 2140 if self._index < self._tokens_size: 2141 self.raise_error("Invalid expression / Unexpected token") 2142 2143 self.check_errors() 2144 2145 return expressions 2146 2147 def _parse( 2148 self, 2149 parse_method: t.Callable[[Parser], exp.Expr | None], 2150 raw_tokens: list[Token], 2151 sql: str | None = None, 2152 ) -> list[exp.Expr | None]: 2153 self.reset() 2154 self.sql = sql or "" 2155 2156 total = len(raw_tokens) 2157 chunks: list[list[Token]] = [[]] 2158 2159 for i, token in enumerate(raw_tokens): 2160 if token.token_type == TokenType.SEMICOLON: 2161 if token.comments: 2162 chunks.append([token]) 2163 2164 if i < total - 1: 2165 chunks.append([]) 2166 else: 2167 chunks[-1].append(token) 2168 2169 self._chunks = chunks 2170 2171 return self._parse_batch_statements(parse_method=parse_method, sep_first_statement=False) 2172 2173 def _warn_unsupported(self) -> None: 2174 if self._tokens_size <= 1: 2175 return 2176 2177 # We use _find_sql because self.sql may comprise multiple chunks, and we're only 2178 # interested in emitting a warning for the one being currently processed. 2179 sql = self._find_sql(self._tokens[0], self._tokens[-1])[: self.error_message_context] 2180 2181 logger.warning( 2182 f"'{sql}' contains unsupported syntax. Falling back to parsing as a 'Command'." 2183 ) 2184 2185 def _parse_command(self) -> exp.Command: 2186 self._warn_unsupported() 2187 comments = self._prev_comments 2188 return self.expression( 2189 exp.Command(this=self._prev.text.upper(), expression=self._parse_string()), 2190 comments=comments, 2191 ) 2192 2193 def _parse_comment(self, allow_exists: bool = True) -> exp.Expr: 2194 start = self._prev 2195 exists = self._parse_exists() if allow_exists else None 2196 2197 self._match(TokenType.ON) 2198 2199 materialized = self._match_text_seq("MATERIALIZED") 2200 kind = self._match_set(self.CREATABLES) and self._prev 2201 if not kind: 2202 return self._parse_as_command(start) 2203 2204 if kind.token_type in (TokenType.FUNCTION, TokenType.PROCEDURE): 2205 this = self._parse_user_defined_function(kind=kind.token_type) 2206 elif kind.token_type == TokenType.TABLE: 2207 this = self._parse_table(alias_tokens=self.COMMENT_TABLE_ALIAS_TOKENS) 2208 elif kind.token_type == TokenType.COLUMN: 2209 this = self._parse_column() 2210 else: 2211 this = self._parse_table_parts(schema=True) 2212 2213 self._match(TokenType.IS) 2214 2215 return self.expression( 2216 exp.Comment( 2217 this=this, 2218 kind=kind.text, 2219 expression=self._parse_string(), 2220 exists=exists, 2221 materialized=materialized, 2222 ) 2223 ) 2224 2225 def _parse_to_table( 2226 self, 2227 ) -> exp.ToTableProperty: 2228 table = self._parse_table_parts(schema=True) 2229 return self.expression(exp.ToTableProperty(this=table)) 2230 2231 # https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#mergetree-table-ttl 2232 def _parse_ttl(self) -> exp.Expr: 2233 def _parse_ttl_action() -> exp.Expr | None: 2234 this = self._parse_bitwise() 2235 2236 if self._match_text_seq("DELETE"): 2237 return self.expression(exp.MergeTreeTTLAction(this=this, delete=True)) 2238 if self._match_text_seq("RECOMPRESS"): 2239 return self.expression( 2240 exp.MergeTreeTTLAction(this=this, recompress=self._parse_bitwise()) 2241 ) 2242 if self._match_text_seq("TO", "DISK"): 2243 return self.expression( 2244 exp.MergeTreeTTLAction(this=this, to_disk=self._parse_string()) 2245 ) 2246 if self._match_text_seq("TO", "VOLUME"): 2247 return self.expression( 2248 exp.MergeTreeTTLAction(this=this, to_volume=self._parse_string()) 2249 ) 2250 2251 return this 2252 2253 expressions = self._parse_csv(_parse_ttl_action) 2254 where = self._parse_where() 2255 group = self._parse_group() 2256 2257 aggregates = None 2258 if group and self._match(TokenType.SET): 2259 aggregates = self._parse_csv(self._parse_set_item) 2260 2261 return self.expression( 2262 exp.MergeTreeTTL( 2263 expressions=expressions, where=where, group=group, aggregates=aggregates 2264 ) 2265 ) 2266 2267 def _parse_condition(self) -> exp.Expr | None: 2268 return self._parse_wrapped(parse_method=self._parse_expression, optional=True) 2269 2270 def _parse_block(self) -> exp.Block: 2271 return self.expression( 2272 exp.Block( 2273 expressions=self._parse_batch_statements( 2274 parse_method=lambda self: self._parse_statement() 2275 ) 2276 ) 2277 ) 2278 2279 def _parse_whileblock(self) -> exp.WhileBlock: 2280 return self.expression( 2281 exp.WhileBlock(this=self._parse_condition(), body=self._parse_block()) 2282 ) 2283 2284 def _parse_statement(self) -> exp.Expr | None: 2285 if not self._curr: 2286 return None 2287 2288 if self._match_set(self.STATEMENT_PARSERS): 2289 comments = self._prev_comments 2290 stmt = self.STATEMENT_PARSERS[self._prev.token_type](self) 2291 stmt.add_comments(comments, prepend=True) 2292 return stmt 2293 2294 if self._match_set(self.dialect.tokenizer_class.COMMANDS): 2295 return self._parse_command() 2296 2297 if self._match_text_seq("WHILE"): 2298 return self._parse_whileblock() 2299 2300 expression = self._parse_expression() 2301 expression = self._parse_set_operations(expression) if expression else self._parse_select() 2302 2303 if isinstance(expression, exp.Subquery) and self._match(TokenType.PIPE_GT, advance=False): 2304 expression = self._parse_pipe_syntax_query(expression) 2305 2306 return self._parse_query_modifiers(expression) 2307 2308 def _parse_drop(self, exists: bool = False) -> exp.Drop | exp.Command: 2309 start = self._prev 2310 temporary = self._match(TokenType.TEMPORARY) 2311 materialized = self._match_text_seq("MATERIALIZED") 2312 iceberg = self._match_text_seq("ICEBERG") 2313 2314 kind = self._match_set(self.CREATABLES) and self._prev.text.upper() 2315 if not kind or (iceberg and kind and kind != "TABLE"): 2316 return self._parse_as_command(start) 2317 2318 concurrently = self._match_text_seq("CONCURRENTLY") 2319 if_exists = exists or self._parse_exists() 2320 2321 if kind == "COLUMN": 2322 this = self._parse_column() 2323 else: 2324 this = self._parse_table_parts(schema=True, is_db_reference=kind == "SCHEMA") 2325 2326 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 2327 2328 if self._match(TokenType.L_PAREN, advance=False): 2329 expressions = self._parse_wrapped_csv(self._parse_types) 2330 else: 2331 expressions = None 2332 2333 cascade_or_restrict = self._match_texts(("CASCADE", "RESTRICT")) and self._prev.text.upper() 2334 2335 return self.expression( 2336 exp.Drop( 2337 exists=if_exists, 2338 this=this, 2339 expressions=expressions, 2340 kind=self.dialect.CREATABLE_KIND_MAPPING.get(kind) or kind, 2341 temporary=temporary, 2342 materialized=materialized, 2343 cascade=cascade_or_restrict == "CASCADE", 2344 restrict=cascade_or_restrict == "RESTRICT", 2345 constraints=self._match_text_seq("CONSTRAINTS"), 2346 purge=self._match_text_seq("PURGE"), 2347 cluster=cluster, 2348 concurrently=concurrently, 2349 sync=self._match_text_seq("SYNC"), 2350 iceberg=iceberg, 2351 ) 2352 ) 2353 2354 def _parse_exists(self, not_: bool = False) -> bool | None: 2355 return ( 2356 self._match_text_seq("IF") 2357 and (not not_ or self._match(TokenType.NOT)) 2358 and self._match(TokenType.EXISTS) 2359 ) 2360 2361 def _parse_create(self) -> exp.Create | exp.Command: 2362 # Note: this can't be None because we've matched a statement parser 2363 start = self._prev 2364 2365 replace = ( 2366 start.token_type == TokenType.REPLACE 2367 or self._match_pair(TokenType.OR, TokenType.REPLACE) 2368 or self._match_pair(TokenType.OR, TokenType.ALTER) 2369 ) 2370 refresh = self._match_pair(TokenType.OR, TokenType.REFRESH) 2371 2372 unique = self._match(TokenType.UNIQUE) 2373 2374 if self._match_text_seq("CLUSTERED", "COLUMNSTORE"): 2375 clustered = True 2376 elif self._match_text_seq("NONCLUSTERED", "COLUMNSTORE") or self._match_text_seq( 2377 "COLUMNSTORE" 2378 ): 2379 clustered = False 2380 else: 2381 clustered = None 2382 2383 if self._match_pair(TokenType.TABLE, TokenType.FUNCTION, advance=False): 2384 self._advance() 2385 2386 properties = None 2387 create_token = self._match_set(self.CREATABLES) and self._prev 2388 2389 if not create_token: 2390 # exp.Properties.Location.POST_CREATE 2391 properties = self._parse_properties() 2392 create_token = self._match_set(self.CREATABLES) and self._prev 2393 2394 if not properties or not create_token: 2395 return self._parse_as_command(start) 2396 2397 create_token_type = t.cast(Token, create_token).token_type 2398 2399 concurrently = self._match_text_seq("CONCURRENTLY") 2400 exists = self._parse_exists(not_=True) 2401 this = None 2402 expression: exp.Expr | None = None 2403 indexes = None 2404 no_schema_binding = None 2405 begin = None 2406 clone = None 2407 2408 def extend_props(temp_props: exp.Properties | None) -> None: 2409 nonlocal properties 2410 if properties and temp_props: 2411 properties.expressions.extend(temp_props.expressions) 2412 elif temp_props: 2413 properties = temp_props 2414 2415 if create_token_type in (TokenType.FUNCTION, TokenType.PROCEDURE): 2416 this = self._parse_user_defined_function(kind=create_token_type) 2417 2418 # exp.Properties.Location.POST_SCHEMA ("schema" here is the UDF's type signature) 2419 extend_props(self._parse_properties()) 2420 2421 expression = self._parse_heredoc() if self._match(TokenType.ALIAS) else None 2422 2423 if ( 2424 not expression 2425 and create_token_type == TokenType.FUNCTION 2426 and isinstance(this, exp.UserDefinedFunction) 2427 and this.args.get("wrapped") 2428 ): 2429 pre_table_index = self._index 2430 is_table = self._match(TokenType.TABLE) 2431 2432 expression = self._parse_expression() 2433 overload_mode = bool( 2434 expression 2435 and self._curr.token_type == TokenType.COMMA 2436 and self._next.token_type == TokenType.L_PAREN 2437 ) 2438 if not overload_mode: 2439 self._retreat(pre_table_index) 2440 is_table = False 2441 expression = None 2442 else: 2443 is_table = False 2444 overload_mode = False 2445 2446 extend_props(self._parse_function_properties()) 2447 2448 if not expression: 2449 if self._match(TokenType.COMMAND): 2450 expression = self._parse_as_command(self._prev) 2451 else: 2452 begin = self._match(TokenType.BEGIN) 2453 return_ = self._match_text_seq("RETURN") 2454 2455 if self._match(TokenType.STRING, advance=False): 2456 # Takes care of BigQuery's JavaScript UDF definitions that end in an OPTIONS property 2457 # # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_function_statement 2458 expression = self._parse_string() 2459 extend_props(self._parse_properties()) 2460 else: 2461 expression = ( 2462 self._parse_user_defined_function_expression() 2463 if create_token_type == TokenType.FUNCTION 2464 else self._parse_block() 2465 ) 2466 2467 if return_: 2468 expression = self.expression(exp.Return(this=expression)) 2469 2470 if overload_mode and expression: 2471 expression = self._parse_macro_overloads( 2472 t.cast(exp.UserDefinedFunction, this), expression, is_table 2473 ) 2474 elif create_token_type == TokenType.INDEX: 2475 # Postgres allows anonymous indexes, eg. CREATE INDEX IF NOT EXISTS ON t(c) 2476 if not self._match(TokenType.ON): 2477 index = self._parse_id_var() 2478 anonymous = False 2479 else: 2480 index = None 2481 anonymous = True 2482 2483 this = self._parse_index(index=index, anonymous=anonymous) 2484 elif ( 2485 create_token_type == TokenType.CONSTRAINT and self._match(TokenType.TRIGGER) 2486 ) or create_token_type == TokenType.TRIGGER: 2487 if is_constraint := (create_token_type == TokenType.CONSTRAINT): 2488 create_token = self._prev 2489 2490 trigger_name = self._parse_id_var() 2491 if not trigger_name: 2492 return self._parse_as_command(start) 2493 2494 timing_var = self._parse_var_from_options(self.TRIGGER_TIMING, raise_unmatched=False) 2495 timing = timing_var.this if timing_var else None 2496 if not timing: 2497 return self._parse_as_command(start) 2498 2499 events = self._parse_trigger_events() 2500 if not self._match(TokenType.ON): 2501 self.raise_error("Expected ON in trigger definition") 2502 2503 table = self._parse_table_parts() 2504 referenced_table = self._parse_table_parts() if self._match(TokenType.FROM) else None 2505 deferrable, initially = self._parse_trigger_deferrable() 2506 referencing = self._parse_trigger_referencing() 2507 for_each = self._parse_trigger_for_each() 2508 when = self._match_text_seq("WHEN") and self._parse_wrapped( 2509 self._parse_disjunction, optional=True 2510 ) 2511 execute = self._parse_trigger_execute() 2512 2513 if execute is None: 2514 return self._parse_as_command(start) 2515 2516 trigger_props = self.expression( 2517 exp.TriggerProperties( 2518 table=table, 2519 timing=timing, 2520 events=events, 2521 execute=execute, 2522 constraint=is_constraint, 2523 referenced_table=referenced_table, 2524 deferrable=deferrable, 2525 initially=initially, 2526 referencing=referencing, 2527 for_each=for_each, 2528 when=when, 2529 ) 2530 ) 2531 2532 this = trigger_name 2533 extend_props(exp.Properties(expressions=[trigger_props] if trigger_props else [])) 2534 elif create_token_type == TokenType.TYPE: 2535 this = self._parse_table_parts(schema=True) 2536 if not this or not self._match(TokenType.ALIAS): 2537 return self._parse_as_command(start) 2538 2539 if self._match(TokenType.ENUM): 2540 expression = exp.DataType( 2541 this=exp.DType.ENUM, 2542 expressions=self._parse_wrapped_csv(self._parse_string), 2543 ) 2544 elif self._match(TokenType.L_PAREN, advance=False): 2545 expression = self._parse_schema() 2546 else: 2547 return self._parse_as_command(start) 2548 elif create_token_type in self.DB_CREATABLES: 2549 table_parts = self._parse_table_parts( 2550 schema=True, is_db_reference=create_token_type == TokenType.SCHEMA 2551 ) 2552 2553 # exp.Properties.Location.POST_NAME 2554 self._match(TokenType.COMMA) 2555 extend_props(self._parse_properties(before=True)) 2556 2557 this = self._parse_schema(this=table_parts) 2558 2559 # exp.Properties.Location.POST_SCHEMA and POST_WITH 2560 extend_props(self._parse_properties()) 2561 2562 has_alias = self._match(TokenType.ALIAS) 2563 if not self._match_set(self.DDL_SELECT_TOKENS, advance=False): 2564 # exp.Properties.Location.POST_ALIAS 2565 extend_props(self._parse_properties()) 2566 2567 if create_token_type == TokenType.SEQUENCE: 2568 expression = self._parse_types() 2569 props = self._parse_properties() 2570 if props: 2571 sequence_props = exp.SequenceProperties() 2572 options = [] 2573 for prop in props: 2574 if isinstance(prop, exp.SequenceProperties): 2575 for arg, value in prop.args.items(): 2576 if arg == "options": 2577 options.extend(value) 2578 else: 2579 sequence_props.set(arg, value) 2580 prop.pop() 2581 2582 if options: 2583 sequence_props.set("options", options) 2584 2585 props.append("expressions", sequence_props) 2586 extend_props(props) 2587 else: 2588 expression = self._parse_ddl_select() 2589 2590 # Some dialects also support using a table as an alias instead of a SELECT. 2591 # Here we fallback to this as an alternative. 2592 if not expression and has_alias: 2593 expression = self._try_parse(self._parse_table_parts) 2594 2595 if create_token_type == TokenType.TABLE: 2596 # exp.Properties.Location.POST_EXPRESSION 2597 extend_props(self._parse_properties()) 2598 2599 indexes = [] 2600 while True: 2601 index = self._parse_index() 2602 2603 # exp.Properties.Location.POST_INDEX 2604 extend_props(self._parse_properties()) 2605 if not index: 2606 break 2607 else: 2608 self._match(TokenType.COMMA) 2609 indexes.append(index) 2610 elif create_token_type == TokenType.VIEW: 2611 if self._match_text_seq("WITH", "NO", "SCHEMA", "BINDING"): 2612 no_schema_binding = True 2613 elif create_token_type in (TokenType.SINK, TokenType.SOURCE): 2614 extend_props(self._parse_properties()) 2615 2616 shallow = self._match_text_seq("SHALLOW") 2617 2618 if self._match_texts(self.CLONE_KEYWORDS): 2619 copy = self._prev.text.lower() == "copy" 2620 clone = self.expression( 2621 exp.Clone(this=self._parse_table(schema=True), shallow=shallow, copy=copy) 2622 ) 2623 2624 if self._curr and not self._match_set((TokenType.R_PAREN, TokenType.COMMA), advance=False): 2625 return self._parse_as_command(start) 2626 2627 create_kind_text = create_token.text.upper() 2628 return self.expression( 2629 exp.Create( 2630 this=this, 2631 kind=self.dialect.CREATABLE_KIND_MAPPING.get(create_kind_text) or create_kind_text, 2632 replace=replace, 2633 refresh=refresh, 2634 unique=unique, 2635 expression=expression, 2636 exists=exists, 2637 properties=properties, 2638 indexes=indexes, 2639 no_schema_binding=no_schema_binding, 2640 begin=begin, 2641 clone=clone, 2642 concurrently=concurrently, 2643 clustered=clustered, 2644 ) 2645 ) 2646 2647 def _parse_sequence_properties(self) -> exp.SequenceProperties | None: 2648 seq = exp.SequenceProperties() 2649 2650 options = [] 2651 index = self._index 2652 2653 while self._curr: 2654 self._match(TokenType.COMMA) 2655 if self._match_text_seq("INCREMENT"): 2656 self._match_text_seq("BY") 2657 self._match_text_seq("=") 2658 seq.set("increment", self._parse_term()) 2659 elif self._match_text_seq("MINVALUE"): 2660 seq.set("minvalue", self._parse_term()) 2661 elif self._match_text_seq("MAXVALUE"): 2662 seq.set("maxvalue", self._parse_term()) 2663 elif self._match(TokenType.START_WITH) or self._match_text_seq("START"): 2664 self._match_text_seq("=") 2665 seq.set("start", self._parse_term()) 2666 elif self._match_text_seq("CACHE"): 2667 # T-SQL allows empty CACHE which is initialized dynamically 2668 seq.set("cache", self._parse_number() or True) 2669 elif self._match_text_seq("OWNED", "BY"): 2670 # "OWNED BY NONE" is the default 2671 seq.set("owned", None if self._match_text_seq("NONE") else self._parse_column()) 2672 else: 2673 opt = self._parse_var_from_options(self.CREATE_SEQUENCE, raise_unmatched=False) 2674 if opt: 2675 options.append(opt) 2676 else: 2677 break 2678 2679 seq.set("options", options if options else None) 2680 return None if self._index == index else seq 2681 2682 def _parse_trigger_events(self) -> list[exp.TriggerEvent]: 2683 events = [] 2684 2685 while True: 2686 event_type = self._match_set(self.TRIGGER_EVENTS) and self._prev.text.upper() 2687 2688 if not event_type: 2689 self.raise_error("Expected trigger event (INSERT, UPDATE, DELETE, TRUNCATE)") 2690 2691 columns = ( 2692 self._parse_csv(self._parse_column) 2693 if event_type == "UPDATE" and self._match_text_seq("OF") 2694 else None 2695 ) 2696 2697 events.append(self.expression(exp.TriggerEvent(this=event_type, columns=columns))) 2698 2699 if not self._match(TokenType.OR): 2700 break 2701 2702 return events 2703 2704 def _parse_trigger_deferrable( 2705 self, 2706 ) -> tuple[str | None, str | None]: 2707 deferrable_var = self._parse_var_from_options( 2708 self.TRIGGER_DEFERRABLE, raise_unmatched=False 2709 ) 2710 deferrable = deferrable_var.this if deferrable_var else None 2711 2712 initially = None 2713 if deferrable and self._match_text_seq("INITIALLY"): 2714 initially = ( 2715 self._prev.text.upper() if self._match_texts(("IMMEDIATE", "DEFERRED")) else None 2716 ) 2717 2718 return deferrable, initially 2719 2720 def _parse_trigger_referencing_clause(self, keyword: str) -> exp.Expr | None: 2721 if not self._match_text_seq(keyword): 2722 return None 2723 if not self._match_text_seq("TABLE"): 2724 self.raise_error(f"Expected TABLE after {keyword} in REFERENCING clause") 2725 self._match_text_seq("AS") 2726 return self._parse_id_var() 2727 2728 def _parse_trigger_referencing(self) -> exp.TriggerReferencing | None: 2729 if not self._match_text_seq("REFERENCING"): 2730 return None 2731 2732 old_alias = None 2733 new_alias = None 2734 2735 while True: 2736 if alias := self._parse_trigger_referencing_clause("OLD"): 2737 if old_alias is not None: 2738 self.raise_error("Duplicate OLD clause in REFERENCING") 2739 old_alias = alias 2740 elif alias := self._parse_trigger_referencing_clause("NEW"): 2741 if new_alias is not None: 2742 self.raise_error("Duplicate NEW clause in REFERENCING") 2743 new_alias = alias 2744 else: 2745 break 2746 2747 if old_alias is None and new_alias is None: 2748 self.raise_error("REFERENCING clause requires at least OLD TABLE or NEW TABLE") 2749 2750 return self.expression(exp.TriggerReferencing(old=old_alias, new=new_alias)) 2751 2752 def _parse_trigger_for_each(self) -> str | None: 2753 if not self._match_text_seq("FOR", "EACH"): 2754 return None 2755 2756 return self._prev.text.upper() if self._match_texts(("ROW", "STATEMENT")) else None 2757 2758 def _parse_trigger_execute(self) -> exp.TriggerExecute | None: 2759 if not self._match(TokenType.EXECUTE): 2760 return None 2761 2762 if not self._match_set((TokenType.FUNCTION, TokenType.PROCEDURE)): 2763 self.raise_error("Expected FUNCTION or PROCEDURE after EXECUTE") 2764 2765 func_call = self._parse_column() 2766 return self.expression(exp.TriggerExecute(this=func_call)) 2767 2768 def _parse_property_before(self) -> exp.Expr | list[exp.Expr] | None: 2769 # only used for teradata currently 2770 self._match(TokenType.COMMA) 2771 2772 kwargs = { 2773 "no": self._match_text_seq("NO"), 2774 "dual": self._match_text_seq("DUAL"), 2775 "before": self._match_text_seq("BEFORE"), 2776 "default": self._match_text_seq("DEFAULT"), 2777 "local": (self._match_text_seq("LOCAL") and "LOCAL") 2778 or (self._match_text_seq("NOT", "LOCAL") and "NOT LOCAL"), 2779 "after": self._match_text_seq("AFTER"), 2780 "minimum": self._match_texts(("MIN", "MINIMUM")), 2781 "maximum": self._match_texts(("MAX", "MAXIMUM")), 2782 } 2783 2784 if self._match_texts(self.PROPERTY_PARSERS): 2785 parser = self.PROPERTY_PARSERS[self._prev.text.upper()] 2786 try: 2787 return parser(self, **{k: v for k, v in kwargs.items() if v}) 2788 except TypeError: 2789 self.raise_error(f"Cannot parse property '{self._prev.text}'") 2790 2791 return None 2792 2793 def _parse_wrapped_properties(self) -> list[exp.Expr | list[exp.Expr]]: 2794 return self._parse_wrapped_csv(self._parse_property) 2795 2796 def _parse_property(self) -> exp.Expr | list[exp.Expr] | None: 2797 if self._match_texts(self.PROPERTY_PARSERS): 2798 return self.PROPERTY_PARSERS[self._prev.text.upper()](self) 2799 2800 if self._match(TokenType.DEFAULT) and self._match_texts(self.PROPERTY_PARSERS): 2801 return self.PROPERTY_PARSERS[self._prev.text.upper()](self, default=True) 2802 2803 if self._match_text_seq("COMPOUND", "SORTKEY"): 2804 return self._parse_sortkey(compound=True) 2805 2806 if self._match_text_seq("PARAMETER", "STYLE", "PANDAS"): 2807 return self.expression(exp.ParameterStyleProperty(this="PANDAS")) 2808 2809 index = self._index 2810 2811 seq_props = self._parse_sequence_properties() 2812 if seq_props: 2813 return seq_props 2814 2815 self._retreat(index) 2816 return self._parse_key_value_property() 2817 2818 def _parse_key_value_property( 2819 self, parse_value: t.Callable[[], exp.Expr | None] | None = None 2820 ) -> exp.Property | None: 2821 index = self._index 2822 key = self._parse_column() 2823 2824 if not self._match(TokenType.EQ): 2825 self._retreat(index) 2826 return None 2827 2828 # Transform the key to exp.Dot if it's dotted identifiers wrapped in exp.Column or to exp.Var otherwise 2829 if isinstance(key, exp.Column): 2830 key = key.to_dot() if len(key.parts) > 1 else exp.var(key.name) 2831 2832 value = ( 2833 parse_value() 2834 if parse_value 2835 else self._parse_bitwise() or self._parse_var(any_token=True) 2836 ) 2837 2838 # Transform the value to exp.Var if it was parsed as exp.Column(exp.Identifier()) 2839 if isinstance(value, exp.Column): 2840 value = exp.var(value.name) 2841 2842 return self.expression(exp.Property(this=key, value=value)) 2843 2844 def _parse_stored(self) -> exp.FileFormatProperty | exp.StorageHandlerProperty: 2845 if self._match_text_seq("BY"): 2846 return self.expression(exp.StorageHandlerProperty(this=self._parse_var_or_string())) 2847 2848 self._match(TokenType.ALIAS) 2849 input_format = self._parse_string() if self._match_text_seq("INPUTFORMAT") else None 2850 output_format = self._parse_string() if self._match_text_seq("OUTPUTFORMAT") else None 2851 2852 return self.expression( 2853 exp.FileFormatProperty( 2854 this=( 2855 self.expression( 2856 exp.InputOutputFormat( 2857 input_format=input_format, output_format=output_format 2858 ) 2859 ) 2860 if input_format or output_format 2861 else self._parse_var_or_string() or self._parse_number() or self._parse_id_var() 2862 ), 2863 hive_format=True, 2864 ) 2865 ) 2866 2867 def _parse_unquoted_field(self) -> exp.Expr | None: 2868 field = self._parse_field() 2869 if isinstance(field, exp.Identifier) and not field.quoted: 2870 field = exp.var(field) 2871 2872 return field 2873 2874 def _parse_property_assignment(self, exp_class: type[E], **kwargs: t.Any) -> E: 2875 self._match(TokenType.EQ) 2876 self._match(TokenType.ALIAS) 2877 2878 return self.expression(exp_class(this=self._parse_unquoted_field(), **kwargs)) 2879 2880 def _parse_properties(self, before: bool | None = None) -> exp.Properties | None: 2881 properties = [] 2882 while True: 2883 if before: 2884 prop = self._parse_property_before() 2885 else: 2886 prop = self._parse_property() 2887 if not prop: 2888 break 2889 for p in ensure_list(prop): 2890 properties.append(p) 2891 2892 if properties: 2893 return self.expression(exp.Properties(expressions=properties)) 2894 2895 return None 2896 2897 def _parse_fallback(self, no: bool = False) -> exp.FallbackProperty: 2898 return self.expression( 2899 exp.FallbackProperty(no=no, protection=self._match_text_seq("PROTECTION")) 2900 ) 2901 2902 def _parse_sql_security(self) -> exp.SqlSecurityProperty: 2903 return self.expression( 2904 exp.SqlSecurityProperty( 2905 this=self._match_texts(self.SECURITY_PROPERTY_KEYWORDS) and self._prev.text.upper() 2906 ) 2907 ) 2908 2909 def _parse_settings_property(self) -> exp.SettingsProperty: 2910 return self.expression( 2911 exp.SettingsProperty(expressions=self._parse_csv(self._parse_assignment)) 2912 ) 2913 2914 def _parse_called_on_null_input_property(self) -> exp.CalledOnNullInputProperty | None: 2915 if not self._match_text_seq("ON", "NULL", "INPUT"): 2916 self._retreat(self._index - 1) 2917 return None 2918 2919 return self.expression(exp.CalledOnNullInputProperty()) 2920 2921 def _parse_volatile_property(self) -> exp.VolatileProperty | exp.StabilityProperty: 2922 if self._index >= 2: 2923 pre_volatile_token = self._tokens[self._index - 2] 2924 else: 2925 pre_volatile_token = None 2926 2927 if pre_volatile_token and pre_volatile_token.token_type in self.PRE_VOLATILE_TOKENS: 2928 return exp.VolatileProperty() 2929 2930 return self.expression(exp.StabilityProperty(this=exp.Literal.string("VOLATILE"))) 2931 2932 def _parse_retention_period(self) -> exp.Var: 2933 # Parse TSQL's HISTORY_RETENTION_PERIOD: {INFINITE | <number> DAY | DAYS | MONTH ...} 2934 number = self._parse_number() 2935 number_str = f"{number} " if number else "" 2936 unit = self._parse_var(any_token=True) 2937 return exp.var(f"{number_str}{unit}") 2938 2939 def _parse_system_versioning_property( 2940 self, with_: bool = False 2941 ) -> exp.WithSystemVersioningProperty: 2942 self._match(TokenType.EQ) 2943 prop = self.expression(exp.WithSystemVersioningProperty(on=True, with_=with_)) 2944 2945 if self._match_text_seq("OFF"): 2946 prop.set("on", False) 2947 return prop 2948 2949 self._match(TokenType.ON) 2950 if self._match(TokenType.L_PAREN): 2951 while self._curr and not self._match(TokenType.R_PAREN): 2952 if self._match_text_seq("HISTORY_TABLE", "="): 2953 prop.set("this", self._parse_table_parts()) 2954 elif self._match_text_seq("DATA_CONSISTENCY_CHECK", "="): 2955 prop.set("data_consistency", self._advance_any() and self._prev.text.upper()) 2956 elif self._match_text_seq("HISTORY_RETENTION_PERIOD", "="): 2957 prop.set("retention_period", self._parse_retention_period()) 2958 2959 self._match(TokenType.COMMA) 2960 2961 return prop 2962 2963 def _parse_data_deletion_property(self) -> exp.DataDeletionProperty: 2964 self._match(TokenType.EQ) 2965 on = self._match_text_seq("ON") or not self._match_text_seq("OFF") 2966 prop = self.expression(exp.DataDeletionProperty(on=on)) 2967 2968 if self._match(TokenType.L_PAREN): 2969 while self._curr and not self._match(TokenType.R_PAREN): 2970 if self._match_text_seq("FILTER_COLUMN", "="): 2971 prop.set("filter_column", self._parse_column()) 2972 elif self._match_text_seq("RETENTION_PERIOD", "="): 2973 prop.set("retention_period", self._parse_retention_period()) 2974 2975 self._match(TokenType.COMMA) 2976 2977 return prop 2978 2979 def _parse_distributed_property(self) -> exp.DistributedByProperty: 2980 kind = "HASH" 2981 expressions: list[exp.Expr] | None = None 2982 if self._match_text_seq("BY", "HASH"): 2983 expressions = self._parse_wrapped_csv(self._parse_id_var) 2984 elif self._match_text_seq("BY", "RANDOM"): 2985 kind = "RANDOM" 2986 2987 # If the BUCKETS keyword is not present, the number of buckets is AUTO 2988 buckets: exp.Expr | None = None 2989 if self._match_text_seq("BUCKETS") and not self._match_text_seq("AUTO"): 2990 buckets = self._parse_number() 2991 2992 return self.expression( 2993 exp.DistributedByProperty( 2994 expressions=expressions, kind=kind, buckets=buckets, order=self._parse_order() 2995 ) 2996 ) 2997 2998 def _parse_composite_key_property(self, expr_type: type[E]) -> E: 2999 self._match_text_seq("KEY") 3000 expressions = self._parse_wrapped_id_vars() 3001 return self.expression(expr_type(expressions=expressions)) 3002 3003 def _parse_with_property(self) -> exp.Expr | None | list[exp.Expr]: 3004 if self._match_text_seq("(", "SYSTEM_VERSIONING"): 3005 prop = self._parse_system_versioning_property(with_=True) 3006 self._match_r_paren() 3007 return prop 3008 3009 if self._match(TokenType.L_PAREN, advance=False): 3010 result: list[exp.Expr] = [] 3011 for i in self._parse_wrapped_properties(): 3012 result.extend(i) if isinstance(i, list) else result.append(i) 3013 return result 3014 3015 if self._match_text_seq("JOURNAL"): 3016 return self._parse_withjournaltable() 3017 3018 if self._match_texts(self.VIEW_ATTRIBUTES): 3019 return self.expression(exp.ViewAttributeProperty(this=self._prev.text.upper())) 3020 3021 if self._match_text_seq("DATA"): 3022 return self._parse_withdata(no=False) 3023 elif self._match_text_seq("NO", "DATA"): 3024 return self._parse_withdata(no=True) 3025 3026 if self._match(TokenType.SERDE_PROPERTIES, advance=False): 3027 return self._parse_serde_properties(with_=True) 3028 3029 if self._match(TokenType.SCHEMA): 3030 return self.expression( 3031 exp.WithSchemaBindingProperty( 3032 this=self._parse_var_from_options(self.SCHEMA_BINDING_OPTIONS) 3033 ) 3034 ) 3035 3036 if self._match_texts(self.PROCEDURE_OPTIONS, advance=False): 3037 return self.expression( 3038 exp.WithProcedureOptions(expressions=self._parse_csv(self._parse_procedure_option)) 3039 ) 3040 3041 if not self._next: 3042 return None 3043 3044 return self._parse_withisolatedloading() 3045 3046 def _parse_procedure_option(self) -> exp.Expr | None: 3047 if self._match_text_seq("EXECUTE", "AS"): 3048 return self.expression( 3049 exp.ExecuteAsProperty( 3050 this=self._parse_var_from_options( 3051 self.EXECUTE_AS_OPTIONS, raise_unmatched=False 3052 ) 3053 or self._parse_string() 3054 ) 3055 ) 3056 3057 return self._parse_var_from_options(self.PROCEDURE_OPTIONS) 3058 3059 # https://dev.mysql.com/doc/refman/8.0/en/create-view.html 3060 def _parse_definer(self) -> exp.DefinerProperty | None: 3061 self._match(TokenType.EQ) 3062 3063 user = self._parse_id_var() 3064 self._match(TokenType.PARAMETER) 3065 host = self._parse_id_var() or (self._match(TokenType.MOD) and self._prev.text) 3066 3067 if not user or not host: 3068 return None 3069 3070 return exp.DefinerProperty(this=f"{user}@{host}") 3071 3072 def _parse_withjournaltable(self) -> exp.WithJournalTableProperty: 3073 self._match(TokenType.TABLE) 3074 self._match(TokenType.EQ) 3075 return self.expression(exp.WithJournalTableProperty(this=self._parse_table_parts())) 3076 3077 def _parse_log(self, no: bool = False) -> exp.LogProperty: 3078 return self.expression(exp.LogProperty(no=no)) 3079 3080 def _parse_journal(self, **kwargs) -> exp.JournalProperty: 3081 return self.expression(exp.JournalProperty(**kwargs)) 3082 3083 def _parse_checksum(self) -> exp.ChecksumProperty: 3084 self._match(TokenType.EQ) 3085 3086 on = None 3087 if self._match(TokenType.ON): 3088 on = True 3089 elif self._match_text_seq("OFF"): 3090 on = False 3091 3092 return self.expression(exp.ChecksumProperty(on=on, default=self._match(TokenType.DEFAULT))) 3093 3094 def _parse_cluster(self) -> exp.Cluster: 3095 self._match(TokenType.CLUSTER_BY) 3096 return self.expression( 3097 exp.Cluster( 3098 expressions=self._parse_csv(self._parse_column), 3099 ) 3100 ) 3101 3102 def _parse_cluster_property(self) -> exp.ClusterProperty: 3103 return self.expression( 3104 exp.ClusterProperty( 3105 expressions=self._parse_wrapped_csv(self._parse_column), 3106 ) 3107 ) 3108 3109 def _parse_clustered_by(self) -> exp.ClusteredByProperty: 3110 self._match_text_seq("BY") 3111 3112 self._match_l_paren() 3113 expressions = self._parse_csv(self._parse_column) 3114 self._match_r_paren() 3115 3116 if self._match_text_seq("SORTED", "BY"): 3117 self._match_l_paren() 3118 sorted_by = self._parse_csv(self._parse_ordered) 3119 self._match_r_paren() 3120 else: 3121 sorted_by = None 3122 3123 self._match(TokenType.INTO) 3124 buckets = self._parse_number() 3125 self._match_text_seq("BUCKETS") 3126 3127 return self.expression( 3128 exp.ClusteredByProperty(expressions=expressions, sorted_by=sorted_by, buckets=buckets) 3129 ) 3130 3131 def _parse_copy_property(self) -> exp.CopyGrantsProperty | None: 3132 if not self._match_text_seq("GRANTS"): 3133 self._retreat(self._index - 1) 3134 return None 3135 3136 return self.expression(exp.CopyGrantsProperty()) 3137 3138 def _parse_freespace(self) -> exp.FreespaceProperty: 3139 self._match(TokenType.EQ) 3140 return self.expression( 3141 exp.FreespaceProperty(this=self._parse_number(), percent=self._match(TokenType.PERCENT)) 3142 ) 3143 3144 def _parse_mergeblockratio( 3145 self, no: bool = False, default: bool = False 3146 ) -> exp.MergeBlockRatioProperty: 3147 if self._match(TokenType.EQ): 3148 return self.expression( 3149 exp.MergeBlockRatioProperty( 3150 this=self._parse_number(), percent=self._match(TokenType.PERCENT) 3151 ) 3152 ) 3153 3154 return self.expression(exp.MergeBlockRatioProperty(no=no, default=default)) 3155 3156 def _parse_datablocksize( 3157 self, 3158 default: bool | None = None, 3159 minimum: bool | None = None, 3160 maximum: bool | None = None, 3161 ) -> exp.DataBlocksizeProperty: 3162 self._match(TokenType.EQ) 3163 size = self._parse_number() 3164 3165 units = None 3166 if self._match_texts(("BYTES", "KBYTES", "KILOBYTES")): 3167 units = self._prev.text 3168 3169 return self.expression( 3170 exp.DataBlocksizeProperty( 3171 size=size, units=units, default=default, minimum=minimum, maximum=maximum 3172 ) 3173 ) 3174 3175 def _parse_blockcompression(self) -> exp.BlockCompressionProperty: 3176 self._match(TokenType.EQ) 3177 always = self._match_text_seq("ALWAYS") 3178 manual = self._match_text_seq("MANUAL") 3179 never = self._match_text_seq("NEVER") 3180 default = self._match_text_seq("DEFAULT") 3181 3182 autotemp = None 3183 if self._match_text_seq("AUTOTEMP"): 3184 autotemp = self._parse_schema() 3185 3186 return self.expression( 3187 exp.BlockCompressionProperty( 3188 always=always, manual=manual, never=never, default=default, autotemp=autotemp 3189 ) 3190 ) 3191 3192 def _parse_withisolatedloading(self) -> exp.IsolatedLoadingProperty | None: 3193 index = self._index 3194 no = self._match_text_seq("NO") 3195 concurrent = self._match_text_seq("CONCURRENT") 3196 3197 if not self._match_text_seq("ISOLATED", "LOADING"): 3198 self._retreat(index) 3199 return None 3200 3201 target = self._parse_var_from_options(self.ISOLATED_LOADING_OPTIONS, raise_unmatched=False) 3202 return self.expression( 3203 exp.IsolatedLoadingProperty(no=no, concurrent=concurrent, target=target) 3204 ) 3205 3206 def _parse_locking(self) -> exp.LockingProperty: 3207 if self._match(TokenType.TABLE): 3208 kind = "TABLE" 3209 elif self._match(TokenType.VIEW): 3210 kind = "VIEW" 3211 elif self._match(TokenType.ROW): 3212 kind = "ROW" 3213 elif self._match_text_seq("DATABASE"): 3214 kind = "DATABASE" 3215 else: 3216 kind = None 3217 3218 if kind in ("DATABASE", "TABLE", "VIEW"): 3219 this = self._parse_table_parts() 3220 else: 3221 this = None 3222 3223 if self._match(TokenType.FOR): 3224 for_or_in = "FOR" 3225 elif self._match(TokenType.IN): 3226 for_or_in = "IN" 3227 else: 3228 for_or_in = None 3229 3230 if self._match_text_seq("ACCESS"): 3231 lock_type = "ACCESS" 3232 elif self._match_texts(("EXCL", "EXCLUSIVE")): 3233 lock_type = "EXCLUSIVE" 3234 elif self._match_text_seq("SHARE"): 3235 lock_type = "SHARE" 3236 elif self._match_text_seq("READ"): 3237 lock_type = "READ" 3238 elif self._match_text_seq("WRITE"): 3239 lock_type = "WRITE" 3240 elif self._match_text_seq("CHECKSUM"): 3241 lock_type = "CHECKSUM" 3242 else: 3243 lock_type = None 3244 3245 override = self._match_text_seq("OVERRIDE") 3246 3247 return self.expression( 3248 exp.LockingProperty( 3249 this=this, kind=kind, for_or_in=for_or_in, lock_type=lock_type, override=override 3250 ) 3251 ) 3252 3253 def _parse_partition_by(self) -> list[exp.Expr]: 3254 if self._match(TokenType.PARTITION_BY): 3255 return self._parse_csv(self._parse_disjunction) 3256 return [] 3257 3258 def _parse_partition_bound_spec(self) -> exp.PartitionBoundSpec: 3259 def _parse_partition_bound_expr() -> exp.Expr | None: 3260 if self._match_text_seq("MINVALUE"): 3261 return exp.var("MINVALUE") 3262 if self._match_text_seq("MAXVALUE"): 3263 return exp.var("MAXVALUE") 3264 return self._parse_bitwise() 3265 3266 this: exp.Expr | list[exp.Expr] | None = None 3267 expression = None 3268 from_expressions = None 3269 to_expressions = None 3270 3271 if self._match(TokenType.IN): 3272 this = self._parse_wrapped_csv(self._parse_bitwise) 3273 elif self._match(TokenType.FROM): 3274 from_expressions = self._parse_wrapped_csv(_parse_partition_bound_expr) 3275 self._match_text_seq("TO") 3276 to_expressions = self._parse_wrapped_csv(_parse_partition_bound_expr) 3277 elif self._match_text_seq("WITH", "(", "MODULUS"): 3278 this = self._parse_number() 3279 self._match_text_seq(",", "REMAINDER") 3280 expression = self._parse_number() 3281 self._match_r_paren() 3282 else: 3283 self.raise_error("Failed to parse partition bound spec.") 3284 3285 return self.expression( 3286 exp.PartitionBoundSpec( 3287 this=this, 3288 expression=expression, 3289 from_expressions=from_expressions, 3290 to_expressions=to_expressions, 3291 ) 3292 ) 3293 3294 # https://www.postgresql.org/docs/current/sql-createtable.html 3295 def _parse_partitioned_of(self) -> exp.PartitionedOfProperty | None: 3296 if not self._match_text_seq("OF"): 3297 self._retreat(self._index - 1) 3298 return None 3299 3300 this = self._parse_table(schema=True) 3301 3302 if self._match(TokenType.DEFAULT): 3303 expression: exp.Var | exp.PartitionBoundSpec = exp.var("DEFAULT") 3304 elif self._match_text_seq("FOR", "VALUES"): 3305 expression = self._parse_partition_bound_spec() 3306 else: 3307 self.raise_error("Expecting either DEFAULT or FOR VALUES clause.") 3308 3309 return self.expression(exp.PartitionedOfProperty(this=this, expression=expression)) 3310 3311 def _parse_partitioned_by(self) -> exp.PartitionedByProperty: 3312 self._match(TokenType.EQ) 3313 return self.expression( 3314 exp.PartitionedByProperty( 3315 this=self._parse_schema() or self._parse_bracket(self._parse_field()) 3316 ) 3317 ) 3318 3319 def _parse_withdata(self, no: bool = False) -> exp.WithDataProperty: 3320 if self._match_text_seq("AND", "STATISTICS"): 3321 statistics = True 3322 elif self._match_text_seq("AND", "NO", "STATISTICS"): 3323 statistics = False 3324 else: 3325 statistics = None 3326 3327 return self.expression(exp.WithDataProperty(no=no, statistics=statistics)) 3328 3329 def _parse_contains_property(self) -> exp.SqlReadWriteProperty | None: 3330 if self._match_text_seq("SQL"): 3331 return self.expression(exp.SqlReadWriteProperty(this="CONTAINS SQL")) 3332 return None 3333 3334 def _parse_modifies_property(self) -> exp.SqlReadWriteProperty | None: 3335 if self._match_text_seq("SQL", "DATA"): 3336 return self.expression(exp.SqlReadWriteProperty(this="MODIFIES SQL DATA")) 3337 return None 3338 3339 def _parse_no_property(self) -> exp.Expr | None: 3340 if self._match_text_seq("PRIMARY", "INDEX"): 3341 return exp.NoPrimaryIndexProperty() 3342 if self._match_text_seq("SQL"): 3343 return self.expression(exp.SqlReadWriteProperty(this="NO SQL")) 3344 return None 3345 3346 def _parse_on_property(self) -> exp.Expr | None: 3347 if self._match_text_seq("COMMIT", "PRESERVE", "ROWS"): 3348 return exp.OnCommitProperty() 3349 if self._match_text_seq("COMMIT", "DELETE", "ROWS"): 3350 return exp.OnCommitProperty(delete=True) 3351 return self.expression(exp.OnProperty(this=self._parse_schema(self._parse_id_var()))) 3352 3353 def _parse_reads_property(self) -> exp.SqlReadWriteProperty | None: 3354 if self._match_text_seq("SQL", "DATA"): 3355 return self.expression(exp.SqlReadWriteProperty(this="READS SQL DATA")) 3356 return None 3357 3358 def _parse_distkey(self) -> exp.DistKeyProperty: 3359 return self.expression(exp.DistKeyProperty(this=self._parse_wrapped(self._parse_id_var))) 3360 3361 def _parse_create_like(self) -> exp.LikeProperty | None: 3362 table = self._parse_table(schema=True) 3363 3364 options = [] 3365 while self._match_texts(("INCLUDING", "EXCLUDING")): 3366 this = self._prev.text.upper() 3367 3368 id_var = self._parse_id_var() 3369 if not id_var: 3370 return None 3371 3372 options.append( 3373 self.expression(exp.Property(this=this, value=exp.var(id_var.this.upper()))) 3374 ) 3375 3376 return self.expression(exp.LikeProperty(this=table, expressions=options)) 3377 3378 def _parse_sortkey(self, compound: bool = False) -> exp.SortKeyProperty: 3379 return self.expression( 3380 exp.SortKeyProperty(this=self._parse_wrapped_id_vars(), compound=compound) 3381 ) 3382 3383 def _parse_character_set(self, default: bool = False) -> exp.CharacterSetProperty: 3384 self._match(TokenType.EQ) 3385 return self.expression( 3386 exp.CharacterSetProperty(this=self._parse_var_or_string(), default=default) 3387 ) 3388 3389 def _parse_remote_with_connection(self) -> exp.RemoteWithConnectionModelProperty: 3390 self._match_text_seq("WITH", "CONNECTION") 3391 return self.expression( 3392 exp.RemoteWithConnectionModelProperty(this=self._parse_table_parts()) 3393 ) 3394 3395 def _parse_returns(self) -> exp.ReturnsProperty: 3396 value: exp.Expr | None 3397 null = None 3398 is_table = self._match(TokenType.TABLE) 3399 3400 if is_table: 3401 if self._match(TokenType.LT): 3402 value = self.expression( 3403 exp.Schema(this="TABLE", expressions=self._parse_csv(self._parse_struct_types)) 3404 ) 3405 if not self._match(TokenType.GT): 3406 self.raise_error("Expecting >") 3407 else: 3408 value = self._parse_schema(exp.var("TABLE")) 3409 elif self._match_text_seq("NULL", "ON", "NULL", "INPUT"): 3410 null = True 3411 value = None 3412 else: 3413 value = self._parse_types() 3414 3415 return self.expression(exp.ReturnsProperty(this=value, is_table=is_table, null=null)) 3416 3417 def _parse_describe(self) -> exp.Describe: 3418 kind = self._prev.text if self._match_set(self.CREATABLES) else None 3419 style: str | None = ( 3420 self._prev.text.upper() if self._match_texts(self.DESCRIBE_STYLES) else None 3421 ) 3422 if self._match(TokenType.DOT): 3423 style = None 3424 self._retreat(self._index - 2) 3425 3426 format = self._parse_property() if self._match(TokenType.FORMAT, advance=False) else None 3427 3428 if self._match_set(self.STATEMENT_PARSERS, advance=False): 3429 this = self._parse_statement() 3430 else: 3431 this = self._parse_table(schema=True) 3432 3433 properties = self._parse_properties() 3434 expressions = properties.expressions if properties else None 3435 partition = self._parse_partition() 3436 return self.expression( 3437 exp.Describe( 3438 this=this, 3439 style=style, 3440 kind=kind, 3441 expressions=expressions, 3442 partition=partition, 3443 format=format, 3444 as_json=self._match_text_seq("AS", "JSON"), 3445 ) 3446 ) 3447 3448 def _parse_multitable_inserts(self, comments: list[str] | None) -> exp.MultitableInserts: 3449 kind = self._prev.text.upper() 3450 expressions = [] 3451 3452 def parse_conditional_insert() -> exp.ConditionalInsert | None: 3453 if self._match(TokenType.WHEN): 3454 expression = self._parse_disjunction() 3455 self._match(TokenType.THEN) 3456 else: 3457 expression = None 3458 3459 else_ = self._match(TokenType.ELSE) 3460 3461 if not self._match(TokenType.INTO): 3462 return None 3463 3464 return self.expression( 3465 exp.ConditionalInsert( 3466 this=self.expression( 3467 exp.Insert( 3468 this=self._parse_table(schema=True), 3469 expression=self._parse_derived_table_values(), 3470 ) 3471 ), 3472 expression=expression, 3473 else_=else_, 3474 ) 3475 ) 3476 3477 expression = parse_conditional_insert() 3478 while expression is not None: 3479 expressions.append(expression) 3480 expression = parse_conditional_insert() 3481 3482 return self.expression( 3483 exp.MultitableInserts(kind=kind, expressions=expressions, source=self._parse_table()), 3484 comments=comments, 3485 ) 3486 3487 def _parse_insert(self) -> exp.Insert | exp.MultitableInserts: 3488 comments: list[str] = [] 3489 hint = self._parse_hint() 3490 overwrite = self._match(TokenType.OVERWRITE) 3491 ignore = self._match(TokenType.IGNORE) 3492 local = self._match_text_seq("LOCAL") 3493 alternative = None 3494 is_function = None 3495 3496 if self._match_text_seq("DIRECTORY"): 3497 this: exp.Expr | None = self.expression( 3498 exp.Directory( 3499 this=self._parse_var_or_string(), 3500 local=local, 3501 row_format=self._parse_row_format(match_row=True), 3502 ) 3503 ) 3504 else: 3505 if self._match_set((TokenType.FIRST, TokenType.ALL)): 3506 comments += ensure_list(self._prev_comments) 3507 return self._parse_multitable_inserts(comments) 3508 3509 if self._match(TokenType.OR): 3510 alternative = self._match_texts(self.INSERT_ALTERNATIVES) and self._prev.text 3511 3512 self._match(TokenType.INTO) 3513 comments += ensure_list(self._prev_comments) 3514 self._match(TokenType.TABLE) 3515 is_function = self._match(TokenType.FUNCTION) 3516 3517 this = self._parse_function() if is_function else self._parse_insert_table() 3518 3519 returning = self._parse_returning() # TSQL allows RETURNING before source 3520 3521 return self.expression( 3522 exp.Insert( 3523 hint=hint, 3524 is_function=is_function, 3525 this=this, 3526 stored=self._match_text_seq("STORED") and self._parse_stored(), 3527 by_name=self._match_text_seq("BY", "NAME"), 3528 exists=self._parse_exists(), 3529 where=self._match_pair(TokenType.REPLACE, TokenType.WHERE) 3530 and self._parse_disjunction(), 3531 partition=self._match(TokenType.PARTITION_BY) and self._parse_partitioned_by(), 3532 settings=self._match_text_seq("SETTINGS") and self._parse_settings_property(), 3533 default=self._match_text_seq("DEFAULT", "VALUES"), 3534 expression=self._parse_derived_table_values() or self._parse_ddl_select(), 3535 conflict=self._parse_on_conflict(), 3536 returning=returning or self._parse_returning(), 3537 overwrite=overwrite, 3538 alternative=alternative, 3539 ignore=ignore, 3540 source=self._match(TokenType.TABLE) and self._parse_table(), 3541 ), 3542 comments=comments, 3543 ) 3544 3545 def _parse_insert_table(self) -> exp.Expr | None: 3546 this = self._parse_table(schema=True, parse_partition=True) 3547 if isinstance(this, exp.Table) and self._match(TokenType.ALIAS, advance=False): 3548 this.set("alias", self._parse_table_alias()) 3549 return this 3550 3551 def _parse_kill(self) -> exp.Kill: 3552 kind = exp.var(self._prev.text) if self._match_texts(("CONNECTION", "QUERY")) else None 3553 3554 return self.expression(exp.Kill(this=self._parse_primary(), kind=kind)) 3555 3556 def _parse_on_conflict(self) -> exp.OnConflict | None: 3557 conflict = self._match_text_seq("ON", "CONFLICT") 3558 duplicate = self._match_text_seq("ON", "DUPLICATE", "KEY") 3559 3560 if not conflict and not duplicate: 3561 return None 3562 3563 conflict_keys = None 3564 constraint = None 3565 3566 if conflict: 3567 if self._match_text_seq("ON", "CONSTRAINT"): 3568 constraint = self._parse_id_var() 3569 elif self._match(TokenType.L_PAREN): 3570 conflict_keys = self._parse_csv(self._parse_indexed_column) 3571 self._match_r_paren() 3572 3573 index_predicate = self._parse_where() 3574 3575 action = self._parse_var_from_options(self.CONFLICT_ACTIONS) 3576 if self._prev.token_type == TokenType.UPDATE: 3577 self._match(TokenType.SET) 3578 expressions = self._parse_csv(self._parse_equality) 3579 else: 3580 expressions = None 3581 3582 return self.expression( 3583 exp.OnConflict( 3584 duplicate=duplicate, 3585 expressions=expressions, 3586 action=action, 3587 conflict_keys=conflict_keys, 3588 index_predicate=index_predicate, 3589 constraint=constraint, 3590 where=self._parse_where(), 3591 ) 3592 ) 3593 3594 def _parse_returning(self) -> exp.Returning | None: 3595 if not self._match(TokenType.RETURNING): 3596 return None 3597 return self.expression( 3598 exp.Returning( 3599 expressions=self._parse_csv(self._parse_expression), 3600 into=self._match(TokenType.INTO) and self._parse_table_part(), 3601 ) 3602 ) 3603 3604 def _parse_row(self) -> exp.RowFormatSerdeProperty | exp.RowFormatDelimitedProperty | None: 3605 if not self._match(TokenType.FORMAT): 3606 return None 3607 return self._parse_row_format() 3608 3609 def _parse_serde_properties(self, with_: bool = False) -> exp.SerdeProperties | None: 3610 index = self._index 3611 with_ = with_ or self._match_text_seq("WITH") 3612 3613 if not self._match(TokenType.SERDE_PROPERTIES): 3614 self._retreat(index) 3615 return None 3616 return self.expression( 3617 exp.SerdeProperties(expressions=self._parse_wrapped_properties(), with_=with_) 3618 ) 3619 3620 def _parse_row_format( 3621 self, match_row: bool = False 3622 ) -> exp.RowFormatSerdeProperty | exp.RowFormatDelimitedProperty | None: 3623 if match_row and not self._match_pair(TokenType.ROW, TokenType.FORMAT): 3624 return None 3625 3626 if self._match_text_seq("SERDE"): 3627 this = self._parse_string() 3628 3629 serde_properties = self._parse_serde_properties() 3630 3631 return self.expression( 3632 exp.RowFormatSerdeProperty(this=this, serde_properties=serde_properties) 3633 ) 3634 3635 self._match_text_seq("DELIMITED") 3636 3637 kwargs = {} 3638 3639 if self._match_text_seq("FIELDS", "TERMINATED", "BY"): 3640 kwargs["fields"] = self._parse_string() 3641 if self._match_text_seq("ESCAPED", "BY"): 3642 kwargs["escaped"] = self._parse_string() 3643 if self._match_text_seq("COLLECTION", "ITEMS", "TERMINATED", "BY"): 3644 kwargs["collection_items"] = self._parse_string() 3645 if self._match_text_seq("MAP", "KEYS", "TERMINATED", "BY"): 3646 kwargs["map_keys"] = self._parse_string() 3647 if self._match_text_seq("LINES", "TERMINATED", "BY"): 3648 kwargs["lines"] = self._parse_string() 3649 if self._match_text_seq("NULL", "DEFINED", "AS"): 3650 kwargs["null"] = self._parse_string() 3651 3652 return self.expression(exp.RowFormatDelimitedProperty(**kwargs)) # type: ignore 3653 3654 def _parse_load(self) -> exp.LoadData | exp.Command: 3655 if self._match_text_seq("DATA"): 3656 local = self._match_text_seq("LOCAL") 3657 self._match_text_seq("INPATH") 3658 inpath = self._parse_string() 3659 overwrite = self._match(TokenType.OVERWRITE) 3660 temp: bool | None = None 3661 if self._match(TokenType.INTO): 3662 temp = self._match(TokenType.TEMPORARY) 3663 self._match(TokenType.TABLE) 3664 3665 return self.expression( 3666 exp.LoadData( 3667 this=self._parse_table(schema=True), 3668 local=local, 3669 overwrite=overwrite, 3670 temp=temp, 3671 inpath=inpath, 3672 files=self._match_text_seq("FROM", "FILES") 3673 and exp.Properties(expressions=self._parse_wrapped_properties()), 3674 partition=self._parse_partition(), 3675 input_format=self._match_text_seq("INPUTFORMAT") and self._parse_string(), 3676 serde=self._match_text_seq("SERDE") and self._parse_string(), 3677 ) 3678 ) 3679 return self._parse_as_command(self._prev) 3680 3681 def _parse_delete(self) -> exp.Delete: 3682 hint = self._parse_hint() 3683 3684 # This handles MySQL's "Multiple-Table Syntax" 3685 # https://dev.mysql.com/doc/refman/8.0/en/delete.html 3686 tables = None 3687 if not self._match(TokenType.FROM, advance=False): 3688 tables = self._parse_csv(self._parse_table) or None 3689 3690 returning = self._parse_returning() 3691 3692 return self.expression( 3693 exp.Delete( 3694 hint=hint, 3695 tables=tables, 3696 this=self._match(TokenType.FROM) and self._parse_table(joins=True), 3697 using=self._match(TokenType.USING) 3698 and self._parse_csv(lambda: self._parse_table(joins=True)), 3699 cluster=self._match(TokenType.ON) and self._parse_on_property(), 3700 where=self._parse_where(), 3701 returning=returning or self._parse_returning(), 3702 order=self._parse_order(), 3703 limit=self._parse_limit(), 3704 ) 3705 ) 3706 3707 def _parse_update(self) -> exp.Update: 3708 hint = self._parse_hint() 3709 kwargs: dict[str, object] = { 3710 "hint": hint, 3711 "this": self._parse_table(joins=True, alias_tokens=self.UPDATE_ALIAS_TOKENS), 3712 } 3713 while self._curr: 3714 if self._match(TokenType.SET): 3715 kwargs["expressions"] = self._parse_csv(self._parse_equality) 3716 elif self._match(TokenType.RETURNING, advance=False): 3717 kwargs["returning"] = self._parse_returning() 3718 elif self._match(TokenType.FROM, advance=False): 3719 from_ = self._parse_from(joins=True) 3720 table = from_.this if from_ else None 3721 if isinstance(table, exp.Subquery) and self._match(TokenType.JOIN, advance=False): 3722 table.set("joins", list(self._parse_joins()) or None) 3723 3724 kwargs["from_"] = from_ 3725 elif self._match(TokenType.WHERE, advance=False): 3726 kwargs["where"] = self._parse_where() 3727 elif self._match(TokenType.ORDER_BY, advance=False): 3728 kwargs["order"] = self._parse_order() 3729 elif self._match(TokenType.LIMIT, advance=False): 3730 kwargs["limit"] = self._parse_limit() 3731 else: 3732 break 3733 3734 return self.expression(exp.Update(**kwargs)) 3735 3736 def _parse_use(self) -> exp.Use: 3737 return self.expression( 3738 exp.Use( 3739 kind=self._parse_var_from_options(self.USABLES, raise_unmatched=False), 3740 this=self._parse_table(schema=False), 3741 ) 3742 ) 3743 3744 def _parse_uncache(self) -> exp.Uncache: 3745 if not self._match(TokenType.TABLE): 3746 self.raise_error("Expecting TABLE after UNCACHE") 3747 3748 return self.expression( 3749 exp.Uncache(exists=self._parse_exists(), this=self._parse_table(schema=True)) 3750 ) 3751 3752 def _parse_cache(self) -> exp.Cache: 3753 lazy = self._match_text_seq("LAZY") 3754 self._match(TokenType.TABLE) 3755 table = self._parse_table(schema=True) 3756 3757 options = [] 3758 if self._match_text_seq("OPTIONS"): 3759 self._match_l_paren() 3760 k = self._parse_string() 3761 self._match(TokenType.EQ) 3762 v = self._parse_string() 3763 options = [k, v] 3764 self._match_r_paren() 3765 3766 self._match(TokenType.ALIAS) 3767 return self.expression( 3768 exp.Cache( 3769 this=table, lazy=lazy, options=options, expression=self._parse_select(nested=True) 3770 ) 3771 ) 3772 3773 def _parse_partition(self) -> exp.Partition | None: 3774 if not self._match_texts(self.PARTITION_KEYWORDS): 3775 return None 3776 3777 return self.expression( 3778 exp.Partition( 3779 subpartition=self._prev.text.upper() == "SUBPARTITION", 3780 expressions=self._parse_wrapped_csv(self._parse_disjunction), 3781 ) 3782 ) 3783 3784 def _parse_value(self, values: bool = True) -> exp.Tuple | None: 3785 def _parse_value_expression() -> exp.Expr | None: 3786 if self.dialect.SUPPORTS_VALUES_DEFAULT and self._match(TokenType.DEFAULT): 3787 return exp.var(self._prev.text.upper()) 3788 return self._parse_expression() 3789 3790 if self._match(TokenType.L_PAREN): 3791 expressions = self._parse_csv(_parse_value_expression) 3792 self._match_r_paren() 3793 return self.expression(exp.Tuple(expressions=expressions)) 3794 3795 # In some dialects we can have VALUES 1, 2 which results in 1 column & 2 rows. 3796 expression = self._parse_expression() 3797 if expression: 3798 return self.expression(exp.Tuple(expressions=[expression])) 3799 return None 3800 3801 def _parse_projections( 3802 self, 3803 ) -> tuple[list[exp.Expr], list[exp.Expr] | None]: 3804 return self._parse_expressions(), None 3805 3806 def _parse_wrapped_select(self, table: bool = False) -> exp.Expr | None: 3807 if self._match_set((TokenType.PIVOT, TokenType.UNPIVOT)): 3808 this: exp.Expr | None = self._parse_simplified_pivot( 3809 is_unpivot=self._prev.token_type == TokenType.UNPIVOT 3810 ) 3811 elif self._match(TokenType.FROM): 3812 from_ = self._parse_from(joins=True, skip_from_token=True, consume_pipe=True) 3813 # Support parentheses for duckdb FROM-first syntax 3814 select = self._parse_select(from_=from_) 3815 if select: 3816 if not select.args.get("from_"): 3817 select.set("from_", from_) 3818 this = select 3819 else: 3820 this = exp.select("*").from_(t.cast(exp.From, from_)) 3821 this = self._parse_query_modifiers(self._parse_set_operations(this)) 3822 else: 3823 this = ( 3824 self._parse_table(consume_pipe=True) 3825 if table 3826 else self._parse_select(nested=True, parse_set_operation=False) 3827 ) 3828 3829 # Transform exp.Values into a exp.Table to pass through parse_query_modifiers 3830 # in case a modifier (e.g. join) is following 3831 if table and isinstance(this, exp.Values) and this.alias: 3832 alias = this.args["alias"].pop() 3833 this = exp.Table(this=this, alias=alias) 3834 3835 this = self._parse_query_modifiers(self._parse_set_operations(this)) 3836 3837 return this 3838 3839 def _parse_select( 3840 self, 3841 nested: bool = False, 3842 table: bool = False, 3843 parse_subquery_alias: bool = True, 3844 parse_set_operation: bool = True, 3845 consume_pipe: bool = True, 3846 from_: exp.From | None = None, 3847 ) -> exp.Expr | None: 3848 query = self._parse_select_query( 3849 nested=nested, 3850 table=table, 3851 parse_subquery_alias=parse_subquery_alias, 3852 parse_set_operation=parse_set_operation, 3853 ) 3854 3855 if consume_pipe and self._match(TokenType.PIPE_GT, advance=False): 3856 if not query and from_: 3857 query = exp.select("*").from_(from_) 3858 if isinstance(query, exp.Query): 3859 query = self._parse_pipe_syntax_query(query) 3860 query = query.subquery(copy=False) if query and table else query 3861 3862 return query 3863 3864 def _parse_select_query( 3865 self, 3866 nested: bool = False, 3867 table: bool = False, 3868 parse_subquery_alias: bool = True, 3869 parse_set_operation: bool = True, 3870 ) -> exp.Expr | None: 3871 cte = self._parse_with() 3872 3873 if cte: 3874 this = self._parse_statement() 3875 3876 if not this: 3877 self.raise_error("Failed to parse any statement following CTE") 3878 return cte 3879 3880 while isinstance(this, exp.Subquery) and this.is_wrapper: 3881 this = this.this 3882 3883 assert this is not None 3884 if "with_" in this.arg_types: 3885 if inner_cte := this.args.get("with_"): 3886 cte.set("expressions", cte.expressions + inner_cte.expressions) 3887 if inner_cte.args.get("recursive"): 3888 cte.set("recursive", True) 3889 this.set("with_", cte) 3890 else: 3891 self.raise_error(f"{this.key} does not support CTE") 3892 this = cte 3893 3894 return this 3895 3896 # duckdb supports leading with FROM x 3897 from_ = ( 3898 self._parse_from(joins=True, consume_pipe=True) 3899 if self._match(TokenType.FROM, advance=False) 3900 else None 3901 ) 3902 3903 if self._match(TokenType.SELECT): 3904 comments = self._prev_comments 3905 3906 hint = self._parse_hint() 3907 3908 if self._next and not self._next.token_type == TokenType.DOT: 3909 all_ = self._match(TokenType.ALL) 3910 matched_distinct = self._match_set(self.DISTINCT_TOKENS) 3911 else: 3912 all_, matched_distinct = None, False 3913 3914 kind = ( 3915 self._prev.text.upper() 3916 if self._match(TokenType.ALIAS) and self._match_texts(("STRUCT", "VALUE")) 3917 else None 3918 ) 3919 3920 distinct: exp.Expr | None = ( 3921 self.expression( 3922 exp.Distinct( 3923 on=self._parse_value(values=False) if self._match(TokenType.ON) else None 3924 ) 3925 ) 3926 if matched_distinct 3927 else None 3928 ) 3929 3930 operation_modifiers = [] 3931 while self._curr and self._match_texts(self.OPERATION_MODIFIERS): 3932 operation_modifiers.append(exp.var(self._prev.text.upper())) 3933 3934 limit = self._parse_limit(top=True) 3935 3936 # Some dialects (e.g. Redshift, T-SQL) allow SELECT TOP N DISTINCT ... 3937 if limit and not matched_distinct and not all_: 3938 matched_distinct = self._match_set(self.DISTINCT_TOKENS) 3939 if matched_distinct: 3940 distinct = self.expression( 3941 exp.Distinct( 3942 on=self._parse_value(values=False) 3943 if self._match(TokenType.ON) 3944 else None 3945 ) 3946 ) 3947 else: 3948 all_ = self._match(TokenType.ALL) 3949 3950 if all_ and distinct: 3951 self.raise_error("Cannot specify both ALL and DISTINCT after SELECT") 3952 3953 projections, exclude = self._parse_projections() 3954 3955 this = self.expression( 3956 exp.Select( 3957 kind=kind, 3958 hint=hint, 3959 distinct=distinct, 3960 expressions=projections, 3961 limit=limit, 3962 exclude=exclude, 3963 operation_modifiers=operation_modifiers or None, 3964 ) 3965 ) 3966 this.comments = comments 3967 3968 into = self._parse_into() 3969 if into: 3970 this.set("into", into) 3971 3972 if not from_: 3973 from_ = self._parse_from() 3974 3975 if from_: 3976 this.set("from_", from_) 3977 3978 this = self._parse_query_modifiers(this) 3979 elif (table or nested) and self._match(TokenType.L_PAREN): 3980 comments = self._prev_comments 3981 this = self._parse_wrapped_select(table=table) 3982 3983 if this: 3984 this.add_comments(comments, prepend=True) 3985 3986 # We return early here so that the UNION isn't attached to the subquery by the 3987 # following call to _parse_set_operations, but instead becomes the parent node 3988 self._match_r_paren() 3989 return self._parse_subquery(this, parse_alias=parse_subquery_alias) 3990 elif self._match(TokenType.VALUES, advance=False): 3991 this = self._parse_derived_table_values() 3992 elif from_: 3993 this = exp.select("*").from_(from_.this, copy=False) 3994 this = self._parse_query_modifiers(this) 3995 elif self._match(TokenType.SUMMARIZE): 3996 table = self._match(TokenType.TABLE) 3997 this = self._parse_select() or self._parse_string() or self._parse_table() 3998 return self.expression(exp.Summarize(this=this, table=table)) 3999 elif self._match(TokenType.DESCRIBE): 4000 this = self._parse_describe() 4001 else: 4002 this = None 4003 4004 return self._parse_set_operations(this) if parse_set_operation else this 4005 4006 def _parse_recursive_with_search(self) -> exp.RecursiveWithSearch | None: 4007 self._match_text_seq("SEARCH") 4008 4009 kind = self._match_texts(self.RECURSIVE_CTE_SEARCH_KIND) and self._prev.text.upper() 4010 4011 if not kind: 4012 return None 4013 4014 self._match_text_seq("FIRST", "BY") 4015 4016 return self.expression( 4017 exp.RecursiveWithSearch( 4018 kind=kind, 4019 this=self._parse_id_var(), 4020 expression=self._match_text_seq("SET") and self._parse_id_var(), 4021 using=self._match_text_seq("USING") and self._parse_id_var(), 4022 ) 4023 ) 4024 4025 def _parse_with(self, skip_with_token: bool = False) -> exp.With | None: 4026 if not skip_with_token and not self._match(TokenType.WITH): 4027 return None 4028 4029 comments = self._prev_comments 4030 recursive = self._match(TokenType.RECURSIVE) 4031 4032 last_comments = None 4033 expressions = [] 4034 while True: 4035 cte = self._parse_cte() 4036 if isinstance(cte, exp.CTE): 4037 expressions.append(cte) 4038 if last_comments: 4039 cte.add_comments(last_comments) 4040 4041 if not self._match(TokenType.COMMA) and not self._match(TokenType.WITH): 4042 break 4043 else: 4044 self._match(TokenType.WITH) 4045 4046 last_comments = self._prev_comments 4047 4048 return self.expression( 4049 exp.With( 4050 expressions=expressions, 4051 recursive=recursive or None, 4052 search=self._parse_recursive_with_search(), 4053 ), 4054 comments=comments, 4055 ) 4056 4057 def _parse_cte(self) -> exp.CTE | None: 4058 index = self._index 4059 4060 alias = self._parse_table_alias(self.ID_VAR_TOKENS) 4061 if not alias or not alias.this: 4062 self.raise_error("Expected CTE to have alias") 4063 4064 key_expressions = ( 4065 self._parse_wrapped_id_vars() if self._match_text_seq("USING", "KEY") else None 4066 ) 4067 4068 if not self._match(TokenType.ALIAS) and not self.OPTIONAL_ALIAS_TOKEN_CTE: 4069 self._retreat(index) 4070 return None 4071 4072 comments = self._prev_comments 4073 4074 if self._match_text_seq("NOT", "MATERIALIZED"): 4075 materialized = False 4076 elif self._match_text_seq("MATERIALIZED"): 4077 materialized = True 4078 else: 4079 materialized = None 4080 4081 cte = self.expression( 4082 exp.CTE( 4083 this=self._parse_wrapped(self._parse_statement), 4084 alias=alias, 4085 materialized=materialized, 4086 key_expressions=key_expressions, 4087 ), 4088 comments=comments, 4089 ) 4090 4091 values = cte.this 4092 if isinstance(values, exp.Values): 4093 if values.alias: 4094 cte.set("this", exp.select("*").from_(values)) 4095 else: 4096 cte.set("this", exp.select("*").from_(exp.alias_(values, "_values", table=True))) 4097 4098 return cte 4099 4100 def _parse_table_alias( 4101 self, alias_tokens: t.Collection[TokenType] | None = None 4102 ) -> exp.TableAlias | None: 4103 # In some dialects, LIMIT and OFFSET can act as both identifiers and keywords (clauses) 4104 # so this section tries to parse the clause version and if it fails, it treats the token 4105 # as an identifier (alias) 4106 if self._can_parse_limit_or_offset(): 4107 return None 4108 4109 any_token = self._match(TokenType.ALIAS) 4110 alias = ( 4111 self._parse_id_var(any_token=any_token, tokens=alias_tokens or self.TABLE_ALIAS_TOKENS) 4112 or self._parse_string_as_identifier() 4113 ) 4114 4115 index = self._index 4116 if self._match(TokenType.L_PAREN): 4117 columns = self._parse_csv(self._parse_function_parameter) 4118 self._match_r_paren() if columns else self._retreat(index) 4119 else: 4120 columns = None 4121 4122 if not alias and not columns: 4123 return None 4124 4125 table_alias = self.expression(exp.TableAlias(this=alias, columns=columns)) 4126 4127 # We bubble up comments from the Identifier to the TableAlias 4128 if isinstance(alias, exp.Identifier): 4129 table_alias.add_comments(alias.pop_comments()) 4130 4131 return table_alias 4132 4133 def _parse_subquery( 4134 self, this: exp.Expr | None, parse_alias: bool = True 4135 ) -> exp.Subquery | None: 4136 if not this: 4137 return None 4138 4139 return self.expression( 4140 exp.Subquery( 4141 this=this, 4142 pivots=self._parse_pivots(), 4143 alias=self._parse_table_alias() if parse_alias else None, 4144 sample=self._parse_table_sample(), 4145 ) 4146 ) 4147 4148 def _implicit_unnests_to_explicit(self, this: E) -> E: 4149 from sqlglot.optimizer.normalize_identifiers import normalize_identifiers as _norm 4150 4151 refs = {_norm(this.args["from_"].this.copy(), dialect=self.dialect).alias_or_name} 4152 for i, join in enumerate(this.args.get("joins") or []): 4153 table = join.this 4154 normalized_table = table.copy() 4155 normalized_table.meta["maybe_column"] = True 4156 normalized_table = _norm(normalized_table, dialect=self.dialect) 4157 4158 if isinstance(table, exp.Table) and not join.args.get("on"): 4159 if len(normalized_table.parts) > 1 and normalized_table.parts[0].name in refs: 4160 table_as_column = table.to_column() 4161 unnest = exp.Unnest(expressions=[table_as_column]) 4162 4163 # Table.to_column creates a parent Alias node that we want to convert to 4164 # a TableAlias and attach to the Unnest, so it matches the parser's output 4165 if isinstance(table.args.get("alias"), exp.TableAlias): 4166 table_as_column.replace(table_as_column.this) 4167 exp.alias_(unnest, None, table=[table.args["alias"].this], copy=False) 4168 4169 table.replace(unnest) 4170 4171 refs.add(normalized_table.alias_or_name) 4172 4173 return this 4174 4175 @t.overload 4176 def _parse_query_modifiers(self, this: E) -> E: ... 4177 4178 @t.overload 4179 def _parse_query_modifiers(self, this: None) -> None: ... 4180 4181 def _parse_query_modifiers(self, this): 4182 if isinstance(this, self.MODIFIABLES): 4183 for join in self._parse_joins(): 4184 this.append("joins", join) 4185 for lateral in iter(self._parse_lateral, None): 4186 this.append("laterals", lateral) 4187 4188 while True: 4189 if self._match_set(self.QUERY_MODIFIER_PARSERS, advance=False): 4190 modifier_token = self._curr 4191 parser = self.QUERY_MODIFIER_PARSERS[modifier_token.token_type] 4192 key, expression = parser(self) 4193 4194 if expression: 4195 if this.args.get(key): 4196 self.raise_error( 4197 f"Found multiple '{modifier_token.text.upper()}' clauses", 4198 token=modifier_token, 4199 ) 4200 4201 this.set(key, expression) 4202 if key == "limit": 4203 offset = expression.args.get("offset") 4204 expression.set("offset", None) 4205 4206 if offset: 4207 offset = exp.Offset(expression=offset) 4208 this.set("offset", offset) 4209 4210 limit_by_expressions = expression.expressions 4211 expression.set("expressions", None) 4212 offset.set("expressions", limit_by_expressions) 4213 continue 4214 break 4215 4216 if self.SUPPORTS_IMPLICIT_UNNEST and this and this.args.get("from_"): 4217 this = self._implicit_unnests_to_explicit(this) 4218 4219 return this 4220 4221 def _parse_hint_fallback_to_string(self) -> exp.Hint | None: 4222 start = self._curr 4223 while self._curr: 4224 self._advance() 4225 4226 end = self._tokens[self._index - 1] 4227 return exp.Hint(expressions=[self._find_sql(start, end)]) 4228 4229 def _parse_hint_function_call(self) -> exp.Expr | None: 4230 return self._parse_function_call() 4231 4232 def _parse_hint_body(self) -> exp.Hint | None: 4233 start_index = self._index 4234 should_fallback_to_string = False 4235 4236 hints = [] 4237 try: 4238 for hint in iter( 4239 lambda: self._parse_csv( 4240 lambda: self._parse_hint_function_call() or self._parse_var(upper=True), 4241 ), 4242 [], 4243 ): 4244 hints.extend(hint) 4245 except ParseError: 4246 should_fallback_to_string = True 4247 4248 if should_fallback_to_string or self._curr: 4249 self._retreat(start_index) 4250 return self._parse_hint_fallback_to_string() 4251 4252 return self.expression(exp.Hint(expressions=hints)) 4253 4254 def _parse_hint(self) -> exp.Hint | None: 4255 if self._match(TokenType.HINT) and self._prev_comments: 4256 return exp.maybe_parse(self._prev_comments[0], into=exp.Hint, dialect=self.dialect) 4257 4258 return None 4259 4260 def _parse_into(self) -> exp.Into | None: 4261 if not self._match(TokenType.INTO): 4262 return None 4263 4264 temp = self._match(TokenType.TEMPORARY) 4265 unlogged = self._match_text_seq("UNLOGGED") 4266 self._match(TokenType.TABLE) 4267 4268 return self.expression( 4269 exp.Into(this=self._parse_table(schema=True), temporary=temp, unlogged=unlogged) 4270 ) 4271 4272 def _parse_from( 4273 self, 4274 joins: bool = False, 4275 skip_from_token: bool = False, 4276 consume_pipe: bool = False, 4277 ) -> exp.From | None: 4278 if not skip_from_token and not self._match(TokenType.FROM): 4279 return None 4280 4281 comments = self._prev_comments 4282 return self.expression( 4283 exp.From(this=self._parse_table(joins=joins, consume_pipe=consume_pipe)), 4284 comments=comments, 4285 ) 4286 4287 def _parse_match_recognize_measure(self) -> exp.MatchRecognizeMeasure: 4288 return self.expression( 4289 exp.MatchRecognizeMeasure( 4290 window_frame=self._match_texts(("FINAL", "RUNNING")) and self._prev.text.upper(), 4291 this=self._parse_expression(), 4292 ) 4293 ) 4294 4295 def _parse_match_recognize(self) -> exp.MatchRecognize | None: 4296 if not self._match(TokenType.MATCH_RECOGNIZE): 4297 return None 4298 4299 self._match_l_paren() 4300 4301 partition = self._parse_partition_by() 4302 order = self._parse_order() 4303 4304 measures = ( 4305 self._parse_csv(self._parse_match_recognize_measure) 4306 if self._match_text_seq("MEASURES") 4307 else None 4308 ) 4309 4310 if self._match_text_seq("ONE", "ROW", "PER", "MATCH"): 4311 rows = exp.var("ONE ROW PER MATCH") 4312 elif self._match_text_seq("ALL", "ROWS", "PER", "MATCH"): 4313 text = "ALL ROWS PER MATCH" 4314 if self._match_text_seq("SHOW", "EMPTY", "MATCHES"): 4315 text += " SHOW EMPTY MATCHES" 4316 elif self._match_text_seq("OMIT", "EMPTY", "MATCHES"): 4317 text += " OMIT EMPTY MATCHES" 4318 elif self._match_text_seq("WITH", "UNMATCHED", "ROWS"): 4319 text += " WITH UNMATCHED ROWS" 4320 rows = exp.var(text) 4321 else: 4322 rows = None 4323 4324 if self._match_text_seq("AFTER", "MATCH", "SKIP"): 4325 text = "AFTER MATCH SKIP" 4326 if self._match_text_seq("PAST", "LAST", "ROW"): 4327 text += " PAST LAST ROW" 4328 elif self._match_text_seq("TO", "NEXT", "ROW"): 4329 text += " TO NEXT ROW" 4330 elif self._match_text_seq("TO", "FIRST"): 4331 text += f" TO FIRST {self._advance_any().text}" # type: ignore 4332 elif self._match_text_seq("TO", "LAST"): 4333 text += f" TO LAST {self._advance_any().text}" # type: ignore 4334 after = exp.var(text) 4335 else: 4336 after = None 4337 4338 if self._match_text_seq("PATTERN"): 4339 self._match_l_paren() 4340 4341 if not self._curr: 4342 self.raise_error("Expecting )", self._curr) 4343 4344 paren = 1 4345 start = self._curr 4346 4347 while self._curr and paren > 0: 4348 if self._curr.token_type == TokenType.L_PAREN: 4349 paren += 1 4350 if self._curr.token_type == TokenType.R_PAREN: 4351 paren -= 1 4352 4353 end = self._prev 4354 self._advance() 4355 4356 if paren > 0: 4357 self.raise_error("Expecting )", self._curr) 4358 4359 pattern = exp.var(self._find_sql(start, end)) 4360 else: 4361 pattern = None 4362 4363 define = ( 4364 self._parse_csv(self._parse_name_as_expression) 4365 if self._match_text_seq("DEFINE") 4366 else None 4367 ) 4368 4369 self._match_r_paren() 4370 4371 return self.expression( 4372 exp.MatchRecognize( 4373 partition_by=partition, 4374 order=order, 4375 measures=measures, 4376 rows=rows, 4377 after=after, 4378 pattern=pattern, 4379 define=define, 4380 alias=self._parse_table_alias(), 4381 ) 4382 ) 4383 4384 def _parse_lateral(self) -> exp.Lateral | None: 4385 cross_apply: bool | None = None 4386 if self._match_pair(TokenType.CROSS, TokenType.APPLY): 4387 cross_apply = True 4388 elif self._match_pair(TokenType.OUTER, TokenType.APPLY): 4389 cross_apply = False 4390 4391 if cross_apply is not None: 4392 this = self._parse_select(table=True) 4393 view = None 4394 outer = None 4395 elif self._match(TokenType.LATERAL): 4396 this = self._parse_select(table=True) 4397 view = self._match(TokenType.VIEW) 4398 outer = self._match(TokenType.OUTER) 4399 else: 4400 return None 4401 4402 if not this: 4403 this = ( 4404 self._parse_unnest() 4405 or self._parse_function() 4406 or self._parse_id_var(any_token=False) 4407 ) 4408 4409 while self._match(TokenType.DOT): 4410 this = exp.Dot( 4411 this=this, 4412 expression=self._parse_function() or self._parse_id_var(any_token=False), 4413 ) 4414 4415 ordinality: bool | None = None 4416 4417 if view: 4418 table = self._parse_id_var(any_token=False) 4419 columns = self._parse_csv(self._parse_id_var) if self._match(TokenType.ALIAS) else [] 4420 table_alias: exp.TableAlias | None = self.expression( 4421 exp.TableAlias(this=table, columns=columns) 4422 ) 4423 elif isinstance(this, (exp.Subquery, exp.Unnest)) and this.alias: 4424 # We move the alias from the lateral's child node to the lateral itself 4425 table_alias = this.args["alias"].pop() 4426 else: 4427 ordinality = self._match_pair(TokenType.WITH, TokenType.ORDINALITY) 4428 table_alias = self._parse_table_alias() 4429 4430 return self.expression( 4431 exp.Lateral( 4432 this=this, 4433 view=view, 4434 outer=outer, 4435 alias=table_alias, 4436 cross_apply=cross_apply, 4437 ordinality=ordinality, 4438 ) 4439 ) 4440 4441 def _parse_stream(self) -> exp.Stream | None: 4442 index = self._index 4443 if self._match(TokenType.STREAM): 4444 if this := self._try_parse(self._parse_table): 4445 return self.expression(exp.Stream(this=this)) 4446 self._retreat(index) 4447 return None 4448 4449 def _parse_join_parts( 4450 self, 4451 ) -> tuple[Token | None, Token | None, Token | None]: 4452 return ( 4453 self._prev if self._match_set(self.JOIN_METHODS) else None, 4454 self._prev if self._match_set(self.JOIN_SIDES) else None, 4455 self._prev if self._match_set(self.JOIN_KINDS) else None, 4456 ) 4457 4458 def _parse_using_identifiers(self) -> list[exp.Expr]: 4459 def _parse_column_as_identifier() -> exp.Expr | None: 4460 this = self._parse_column() 4461 if isinstance(this, exp.Column): 4462 return this.this 4463 return this 4464 4465 return self._parse_wrapped_csv(_parse_column_as_identifier, optional=True) 4466 4467 def _parse_join( 4468 self, 4469 skip_join_token: bool = False, 4470 parse_bracket: bool = False, 4471 alias_tokens: t.Collection[TokenType] | None = None, 4472 ) -> exp.Join | None: 4473 if self._match(TokenType.COMMA): 4474 table = self._try_parse(lambda: self._parse_table(alias_tokens=alias_tokens)) 4475 cross_join = self.expression(exp.Join(this=table)) if table else None 4476 4477 if cross_join and self.JOINS_HAVE_EQUAL_PRECEDENCE: 4478 cross_join.set("kind", "CROSS") 4479 4480 return cross_join 4481 4482 index = self._index 4483 method, side, kind = self._parse_join_parts() 4484 directed = self._match_text_seq("DIRECTED") 4485 hint = self._prev.text if self._match_texts(self.JOIN_HINTS) else None 4486 join = self._match(TokenType.JOIN) or (kind and kind.token_type == TokenType.STRAIGHT_JOIN) 4487 join_comments = self._prev_comments 4488 4489 if not skip_join_token and not join: 4490 self._retreat(index) 4491 kind = None 4492 method = None 4493 side = None 4494 4495 outer_apply = self._match_pair(TokenType.OUTER, TokenType.APPLY, False) 4496 cross_apply = self._match_pair(TokenType.CROSS, TokenType.APPLY, False) 4497 4498 if not skip_join_token and not join and not outer_apply and not cross_apply: 4499 return None 4500 4501 kwargs: dict[str, t.Any] = { 4502 "this": self._parse_table(parse_bracket=parse_bracket, alias_tokens=alias_tokens) 4503 } 4504 if kind and kind.token_type == TokenType.ARRAY and self._match(TokenType.COMMA): 4505 kwargs["expressions"] = self._parse_csv( 4506 lambda: self._parse_table(parse_bracket=parse_bracket, alias_tokens=alias_tokens) 4507 ) 4508 4509 if method: 4510 kwargs["method"] = method.text.upper() 4511 if side: 4512 kwargs["side"] = side.text.upper() 4513 if kind: 4514 kwargs["kind"] = kind.text.upper() 4515 if hint: 4516 kwargs["hint"] = hint 4517 4518 if self._match(TokenType.MATCH_CONDITION): 4519 kwargs["match_condition"] = self._parse_wrapped(self._parse_comparison) 4520 4521 if self._match(TokenType.ON): 4522 kwargs["on"] = self._parse_disjunction() 4523 elif self._match(TokenType.USING): 4524 kwargs["using"] = self._parse_using_identifiers() 4525 elif ( 4526 not method 4527 and not (outer_apply or cross_apply) 4528 and not isinstance(kwargs["this"], exp.Unnest) 4529 and not (kind and kind.token_type in (TokenType.CROSS, TokenType.ARRAY)) 4530 ): 4531 index = self._index 4532 joins: list | None = list(self._parse_joins(alias_tokens=alias_tokens)) 4533 4534 if joins and self._match(TokenType.ON): 4535 kwargs["on"] = self._parse_disjunction() 4536 elif joins and self._match(TokenType.USING): 4537 kwargs["using"] = self._parse_using_identifiers() 4538 else: 4539 joins = None 4540 self._retreat(index) 4541 4542 kwargs["this"].set("joins", joins if joins else None) 4543 4544 kwargs["pivots"] = self._parse_pivots() 4545 4546 comments = [c for token in (method, side, kind) if token for c in token.comments] 4547 comments = (join_comments or []) + comments 4548 4549 if ( 4550 self.ADD_JOIN_ON_TRUE 4551 and not kwargs.get("on") 4552 and not kwargs.get("using") 4553 and not kwargs.get("method") 4554 and kwargs.get("kind") in (None, "INNER", "OUTER") 4555 ): 4556 kwargs["on"] = exp.true() 4557 4558 if directed: 4559 kwargs["directed"] = directed 4560 4561 return self.expression(exp.Join(**kwargs), comments=comments) 4562 4563 def _parse_opclass(self) -> exp.Expr | None: 4564 this = self._parse_disjunction() 4565 4566 if self._match_texts(self.OPCLASS_FOLLOW_KEYWORDS, advance=False): 4567 return this 4568 4569 if not self._match_set(self.OPTYPE_FOLLOW_TOKENS, advance=False): 4570 return self.expression(exp.Opclass(this=this, expression=self._parse_table_parts())) 4571 4572 return this 4573 4574 def _parse_index_params(self) -> exp.IndexParameters: 4575 using = self._parse_var(any_token=True) if self._match(TokenType.USING) else None 4576 4577 if self._match(TokenType.L_PAREN, advance=False): 4578 columns = self._parse_wrapped_csv(self._parse_with_operator) 4579 else: 4580 columns = None 4581 4582 include = self._parse_wrapped_id_vars() if self._match_text_seq("INCLUDE") else None 4583 partition_by = self._parse_partition_by() 4584 with_storage = self._match(TokenType.WITH) and self._parse_wrapped_properties() 4585 tablespace = ( 4586 self._parse_var(any_token=True) 4587 if self._match_text_seq("USING", "INDEX", "TABLESPACE") 4588 else None 4589 ) 4590 where = self._parse_where() 4591 4592 on = self._parse_field() if self._match(TokenType.ON) else None 4593 4594 return self.expression( 4595 exp.IndexParameters( 4596 using=using, 4597 columns=columns, 4598 include=include, 4599 partition_by=partition_by, 4600 where=where, 4601 with_storage=with_storage, 4602 tablespace=tablespace, 4603 on=on, 4604 ) 4605 ) 4606 4607 def _parse_index( 4608 self, index: exp.Expr | None = None, anonymous: bool = False 4609 ) -> exp.Index | None: 4610 if index or anonymous: 4611 unique = None 4612 primary = None 4613 amp = None 4614 4615 self._match(TokenType.ON) 4616 self._match(TokenType.TABLE) # hive 4617 table = self._parse_table_parts(schema=True) 4618 else: 4619 unique = self._match(TokenType.UNIQUE) 4620 primary = self._match_text_seq("PRIMARY") 4621 amp = self._match_text_seq("AMP") 4622 4623 if not self._match(TokenType.INDEX): 4624 return None 4625 4626 index = self._parse_id_var() 4627 table = None 4628 4629 params = self._parse_index_params() 4630 4631 return self.expression( 4632 exp.Index( 4633 this=index, table=table, unique=unique, primary=primary, amp=amp, params=params 4634 ) 4635 ) 4636 4637 def _parse_table_hints(self) -> list[exp.Expr] | None: 4638 hints: list[exp.Expr] = [] 4639 if self._match_pair(TokenType.WITH, TokenType.L_PAREN): 4640 # https://learn.microsoft.com/en-us/sql/t-sql/queries/hints-transact-sql-table?view=sql-server-ver16 4641 hints.append( 4642 self.expression( 4643 exp.WithTableHint( 4644 expressions=self._parse_csv( 4645 lambda: self._parse_function() or self._parse_var(any_token=True) 4646 ) 4647 ) 4648 ) 4649 ) 4650 self._match_r_paren() 4651 else: 4652 # https://dev.mysql.com/doc/refman/8.0/en/index-hints.html 4653 while self._match_set(self.TABLE_INDEX_HINT_TOKENS): 4654 hint = exp.IndexTableHint(this=self._prev.text.upper()) 4655 4656 self._match_set((TokenType.INDEX, TokenType.KEY)) 4657 if self._match(TokenType.FOR): 4658 hint.set("target", self._advance_any() and self._prev.text.upper()) 4659 4660 hint.set("expressions", self._parse_wrapped_id_vars()) 4661 hints.append(hint) 4662 4663 return hints or None 4664 4665 def _parse_table_part(self, schema: bool = False) -> exp.Expr | None: 4666 return ( 4667 (not schema and self._parse_function(optional_parens=False)) 4668 or self._parse_id_var(any_token=False) 4669 or self._parse_string_as_identifier() 4670 or self._parse_placeholder() 4671 ) 4672 4673 def _parse_table_parts_fast(self) -> exp.Table | None: 4674 index = self._index 4675 parts: list[exp.Identifier] | None = None 4676 all_comments: list[str] | None = None 4677 4678 while self._match_set(self.IDENTIFIER_TOKENS): 4679 token = self._prev 4680 comments = self._prev_comments 4681 4682 has_dot = self._match(TokenType.DOT) 4683 curr_tt = self._curr.token_type 4684 4685 if not has_dot: 4686 if curr_tt in self.TABLE_POSTFIX_TOKENS: 4687 self._retreat(index) 4688 return None 4689 elif curr_tt not in self.IDENTIFIER_TOKENS: 4690 self._retreat(index) 4691 return None 4692 4693 if parts is None: 4694 parts = [] 4695 4696 if comments: 4697 if all_comments is None: 4698 all_comments = [] 4699 all_comments.extend(comments) 4700 self._prev_comments = [] 4701 4702 parts.append( 4703 self.expression( 4704 exp.Identifier( 4705 this=token.text, quoted=token.token_type == TokenType.IDENTIFIER 4706 ), 4707 token, 4708 ) 4709 ) 4710 4711 if not has_dot: 4712 break 4713 4714 if parts is None: 4715 return None 4716 4717 n = len(parts) 4718 4719 if n == 1: 4720 table: exp.Table = exp.Table(this=parts[0]) 4721 elif n == 2: 4722 table = exp.Table(this=parts[1], db=parts[0]) 4723 elif n >= 3: 4724 this: exp.Identifier | exp.Dot = parts[2] 4725 for i in range(3, n): 4726 this = exp.Dot(this=this, expression=parts[i]) 4727 4728 table = exp.Table(this=this, db=parts[1], catalog=parts[0]) 4729 4730 if table is None: 4731 self._retreat(index) 4732 elif all_comments: 4733 table.add_comments(all_comments) 4734 return table 4735 4736 def _parse_table_parts( 4737 self, 4738 schema: bool = False, 4739 is_db_reference: bool = False, 4740 wildcard: bool = False, 4741 fast: bool = False, 4742 ) -> exp.Table | exp.Dot | None: 4743 if fast: 4744 return self._parse_table_parts_fast() 4745 4746 catalog: exp.Expr | str | None = None 4747 db: exp.Expr | str | None = None 4748 table: exp.Expr | str | None = self._parse_table_part(schema=schema) 4749 4750 while self._match(TokenType.DOT): 4751 if catalog: 4752 # This allows nesting the table in arbitrarily many dot expressions if needed 4753 table = self.expression( 4754 exp.Dot(this=table, expression=self._parse_table_part(schema=schema)) 4755 ) 4756 else: 4757 catalog = db 4758 db = table 4759 # "" used for tsql FROM a..b case 4760 table = self._parse_table_part(schema=schema) or "" 4761 4762 if ( 4763 wildcard 4764 and self._is_connected() 4765 and (isinstance(table, exp.Identifier) or not table) 4766 and self._match(TokenType.STAR) 4767 ): 4768 if isinstance(table, exp.Identifier): 4769 table.args["this"] += "*" 4770 else: 4771 table = exp.Identifier(this="*") 4772 4773 if is_db_reference: 4774 catalog = db 4775 db = table 4776 table = None 4777 4778 if not table and not is_db_reference: 4779 self.raise_error(f"Expected table name but got {self._curr}") 4780 if not db and is_db_reference: 4781 self.raise_error(f"Expected database name but got {self._curr}") 4782 4783 table = self.expression(exp.Table(this=table, db=db, catalog=catalog)) 4784 4785 # Bubble up comments from identifier parts to the Table 4786 comments = [] 4787 for part in table.parts: 4788 if part_comments := part.pop_comments(): 4789 comments.extend(part_comments) 4790 if comments: 4791 table.add_comments(comments) 4792 4793 changes = self._parse_changes() 4794 if changes: 4795 table.set("changes", changes) 4796 4797 at_before = self._parse_historical_data() 4798 if at_before: 4799 table.set("when", at_before) 4800 4801 pivots = self._parse_pivots() 4802 if pivots: 4803 table.set("pivots", pivots) 4804 4805 return table 4806 4807 def _parse_table( 4808 self, 4809 schema: bool = False, 4810 joins: bool = False, 4811 alias_tokens: t.Collection[TokenType] | None = None, 4812 parse_bracket: bool = False, 4813 is_db_reference: bool = False, 4814 parse_partition: bool = False, 4815 consume_pipe: bool = False, 4816 ) -> exp.Expr | None: 4817 if not schema and not is_db_reference and not consume_pipe and not joins: 4818 index = self._index 4819 table = self._parse_table_parts(fast=True) 4820 4821 if table is not None: 4822 curr_tt = self._curr.token_type 4823 next_tt = self._next.token_type 4824 4825 fast_terminators = self.TABLE_TERMINATORS 4826 4827 # only return the table if we're sure there are no other operators 4828 # MATCH_CONDITION is a special case because it accepts any alias before it like LIMIT 4829 if curr_tt in fast_terminators and next_tt != TokenType.MATCH_CONDITION: 4830 return table 4831 4832 postfix_tokens = self.TABLE_POSTFIX_TOKENS 4833 4834 if curr_tt not in postfix_tokens and next_tt not in postfix_tokens: 4835 if alias := self._parse_table_alias( 4836 alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS 4837 ): 4838 table.set("alias", alias) 4839 4840 if self._curr.token_type in fast_terminators: 4841 return table 4842 4843 self._retreat(index) 4844 4845 if stream := self._parse_stream(): 4846 return stream 4847 4848 if lateral := self._parse_lateral(): 4849 return lateral 4850 4851 if unnest := self._parse_unnest(): 4852 return unnest 4853 4854 if values := self._parse_derived_table_values(): 4855 return values 4856 4857 if subquery := self._parse_select(table=True, consume_pipe=consume_pipe): 4858 if not subquery.args.get("pivots"): 4859 subquery.set("pivots", self._parse_pivots()) 4860 if joins: 4861 for join in self._parse_joins(): 4862 subquery.append("joins", join) 4863 return subquery 4864 4865 bracket = parse_bracket and self._parse_bracket(None) 4866 bracket = self.expression(exp.Table(this=bracket)) if bracket else None 4867 4868 rows_from_tables = ( 4869 self._parse_wrapped_csv(self._parse_table) 4870 if self._match_text_seq("ROWS", "FROM") 4871 else None 4872 ) 4873 rows_from = ( 4874 self.expression(exp.Table(rows_from=rows_from_tables)) if rows_from_tables else None 4875 ) 4876 4877 only = self._match(TokenType.ONLY) 4878 4879 this = t.cast( 4880 exp.Expr, 4881 bracket 4882 or rows_from 4883 or self._parse_bracket( 4884 self._parse_table_parts(schema=schema, is_db_reference=is_db_reference) 4885 ), 4886 ) 4887 4888 if only: 4889 this.set("only", only) 4890 4891 # Postgres supports a wildcard (table) suffix operator, which is a no-op in this context 4892 self._match(TokenType.STAR) 4893 4894 parse_partition = parse_partition or self.SUPPORTS_PARTITION_SELECTION 4895 if parse_partition and self._match(TokenType.PARTITION, advance=False): 4896 this.set("partition", self._parse_partition()) 4897 4898 if schema: 4899 return self._parse_schema(this=this) 4900 4901 if self.dialect.ALIAS_POST_VERSION: 4902 this.set("version", self._parse_version()) 4903 4904 if self.dialect.ALIAS_POST_TABLESAMPLE: 4905 this.set("sample", self._parse_table_sample()) 4906 4907 alias = self._parse_table_alias(alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS) 4908 if alias: 4909 this.set("alias", alias) 4910 4911 if self._match(TokenType.INDEXED_BY): 4912 this.set("indexed", self._parse_table_parts()) 4913 elif self._match_text_seq("NOT", "INDEXED"): 4914 this.set("indexed", False) 4915 4916 if isinstance(this, exp.Table) and self._match_text_seq("AT"): 4917 return self.expression( 4918 exp.AtIndex(this=this.to_column(copy=False), expression=self._parse_id_var()) 4919 ) 4920 4921 this.set("hints", self._parse_table_hints()) 4922 4923 if not this.args.get("pivots"): 4924 this.set("pivots", self._parse_pivots()) 4925 4926 if not self.dialect.ALIAS_POST_TABLESAMPLE: 4927 this.set("sample", self._parse_table_sample()) 4928 4929 if not self.dialect.ALIAS_POST_VERSION: 4930 this.set("version", self._parse_version()) 4931 4932 if joins: 4933 for join in self._parse_joins(alias_tokens=alias_tokens): 4934 this.append("joins", join) 4935 4936 if self._match_pair(TokenType.WITH, TokenType.ORDINALITY): 4937 this.set("ordinality", True) 4938 this.set("alias", self._parse_table_alias()) 4939 4940 return this 4941 4942 def _parse_version(self) -> exp.Version | None: 4943 if self._match(TokenType.TIMESTAMP_SNAPSHOT): 4944 this = "TIMESTAMP" 4945 elif self._match(TokenType.VERSION_SNAPSHOT): 4946 this = "VERSION" 4947 else: 4948 return None 4949 4950 if self._match_set((TokenType.FROM, TokenType.BETWEEN)): 4951 kind = self._prev.text.upper() 4952 start = self._parse_bitwise() 4953 self._match_texts(("TO", "AND")) 4954 end = self._parse_bitwise() 4955 expression: exp.Expr | None = self.expression(exp.Tuple(expressions=[start, end])) 4956 elif self._match_text_seq("CONTAINED", "IN"): 4957 kind = "CONTAINED IN" 4958 expression = self.expression( 4959 exp.Tuple(expressions=self._parse_wrapped_csv(self._parse_bitwise)) 4960 ) 4961 elif self._match(TokenType.ALL): 4962 kind = "ALL" 4963 expression = None 4964 else: 4965 self._match_text_seq("AS", "OF") 4966 kind = "AS OF" 4967 expression = self._parse_type() 4968 4969 return self.expression(exp.Version(this=this, expression=expression, kind=kind)) 4970 4971 def _parse_historical_data(self) -> exp.HistoricalData | None: 4972 # https://docs.snowflake.com/en/sql-reference/constructs/at-before 4973 index = self._index 4974 historical_data = None 4975 if self._match_texts(self.HISTORICAL_DATA_PREFIX): 4976 this = self._prev.text.upper() 4977 kind = ( 4978 self._match(TokenType.L_PAREN) 4979 and self._match_texts(self.HISTORICAL_DATA_KIND) 4980 and self._prev.text.upper() 4981 ) 4982 expression = self._match(TokenType.FARROW) and self._parse_bitwise() 4983 4984 if expression: 4985 self._match_r_paren() 4986 historical_data = self.expression( 4987 exp.HistoricalData(this=this, kind=kind, expression=expression) 4988 ) 4989 else: 4990 self._retreat(index) 4991 4992 return historical_data 4993 4994 def _parse_changes(self) -> exp.Changes | None: 4995 if not self._match_text_seq("CHANGES", "(", "INFORMATION", "=>"): 4996 return None 4997 4998 information = self._parse_var(any_token=True) 4999 self._match_r_paren() 5000 5001 return self.expression( 5002 exp.Changes( 5003 information=information, 5004 at_before=self._parse_historical_data(), 5005 end=self._parse_historical_data(), 5006 ) 5007 ) 5008 5009 def _parse_unnest(self, with_alias: bool = True) -> exp.Unnest | None: 5010 if not self._match_pair(TokenType.UNNEST, TokenType.L_PAREN, advance=False): 5011 return None 5012 5013 self._advance() 5014 5015 expressions = self._parse_wrapped_csv(self._parse_equality) 5016 offset: bool | exp.Expr = self._match_pair(TokenType.WITH, TokenType.ORDINALITY) 5017 5018 alias = self._parse_table_alias() if with_alias else None 5019 5020 if alias: 5021 if self.dialect.UNNEST_COLUMN_ONLY: 5022 if alias.args.get("columns"): 5023 self.raise_error("Unexpected extra column alias in unnest.") 5024 5025 alias.set("columns", [alias.this]) 5026 alias.set("this", None) 5027 5028 columns = alias.args.get("columns") or [] 5029 if offset and len(expressions) < len(columns): 5030 offset = columns.pop() 5031 5032 if not offset and self._match_pair(TokenType.WITH, TokenType.OFFSET): 5033 self._match(TokenType.ALIAS) 5034 offset = self._parse_id_var( 5035 any_token=False, tokens=self.UNNEST_OFFSET_ALIAS_TOKENS 5036 ) or exp.to_identifier("offset") 5037 5038 return self.expression(exp.Unnest(expressions=expressions, alias=alias, offset=offset)) 5039 5040 def _parse_derived_table_values(self) -> exp.Values | None: 5041 is_derived = self._match_pair(TokenType.L_PAREN, TokenType.VALUES) 5042 if not is_derived and not ( 5043 # ClickHouse's `FORMAT Values` is equivalent to `VALUES` 5044 self._match_text_seq("VALUES") or self._match_text_seq("FORMAT", "VALUES") 5045 ): 5046 return None 5047 5048 expressions = self._parse_csv(self._parse_value) 5049 alias = self._parse_table_alias() 5050 5051 if is_derived: 5052 self._match_r_paren() 5053 5054 return self.expression( 5055 exp.Values(expressions=expressions, alias=alias or self._parse_table_alias()) 5056 ) 5057 5058 def _parse_table_sample(self, as_modifier: bool = False) -> exp.TableSample | None: 5059 if not self._match(TokenType.TABLE_SAMPLE) and not ( 5060 as_modifier and self._match_text_seq("USING", "SAMPLE") 5061 ): 5062 return None 5063 5064 bucket_numerator = None 5065 bucket_denominator = None 5066 bucket_field = None 5067 percent = None 5068 size = None 5069 seed = None 5070 5071 method = self._parse_var(tokens=(TokenType.ROW,), upper=True) 5072 matched_l_paren = self._match(TokenType.L_PAREN) 5073 5074 if self.TABLESAMPLE_CSV: 5075 num = None 5076 expressions = self._parse_csv(self._parse_primary) 5077 else: 5078 expressions = None 5079 num = ( 5080 self._parse_factor() 5081 if self._match(TokenType.NUMBER, advance=False) 5082 else self._parse_primary() or self._parse_placeholder() 5083 ) 5084 5085 if self._match_text_seq("BUCKET"): 5086 bucket_numerator = self._parse_number() 5087 self._match_text_seq("OUT", "OF") 5088 bucket_denominator = bucket_denominator = self._parse_number() 5089 self._match(TokenType.ON) 5090 bucket_field = self._parse_field() 5091 elif self._match_set((TokenType.PERCENT, TokenType.MOD)): 5092 percent = num 5093 elif self._match(TokenType.ROWS) or not self.dialect.TABLESAMPLE_SIZE_IS_PERCENT: 5094 size = num 5095 else: 5096 percent = num 5097 5098 if matched_l_paren: 5099 self._match_r_paren() 5100 5101 if self._match(TokenType.L_PAREN): 5102 method = self._parse_var(upper=True) 5103 seed = self._match(TokenType.COMMA) and self._parse_number() 5104 self._match_r_paren() 5105 elif self._match_texts(("SEED", "REPEATABLE")): 5106 seed = self._parse_wrapped(self._parse_number) 5107 5108 if not method and self.DEFAULT_SAMPLING_METHOD: 5109 method = exp.var(self.DEFAULT_SAMPLING_METHOD) 5110 5111 return self.expression( 5112 exp.TableSample( 5113 expressions=expressions, 5114 method=method, 5115 bucket_numerator=bucket_numerator, 5116 bucket_denominator=bucket_denominator, 5117 bucket_field=bucket_field, 5118 percent=percent, 5119 size=size, 5120 seed=seed, 5121 ) 5122 ) 5123 5124 def _parse_pivots(self) -> list[exp.Pivot] | None: 5125 if self._curr.token_type not in (TokenType.PIVOT, TokenType.UNPIVOT): 5126 return None 5127 return list(iter(self._parse_pivot, None)) or None 5128 5129 def _parse_joins( 5130 self, alias_tokens: t.Collection[TokenType] | None = None 5131 ) -> t.Iterator[exp.Join]: 5132 return iter(lambda: self._parse_join(alias_tokens=alias_tokens), None) 5133 5134 def _parse_unpivot_columns(self) -> exp.UnpivotColumns | None: 5135 if not self._match(TokenType.INTO): 5136 return None 5137 5138 return self.expression( 5139 exp.UnpivotColumns( 5140 this=self._match_text_seq("NAME") and self._parse_column(), 5141 expressions=self._match_text_seq("VALUE") and self._parse_csv(self._parse_column), 5142 ) 5143 ) 5144 5145 # https://duckdb.org/docs/sql/statements/pivot 5146 def _parse_simplified_pivot(self, is_unpivot: bool | None = None) -> exp.Pivot: 5147 def _parse_on() -> exp.Expr | None: 5148 this = self._parse_bitwise() 5149 5150 if self._match(TokenType.IN): 5151 # PIVOT ... ON col IN (row_val1, row_val2) 5152 return self._parse_in(this) 5153 if self._match(TokenType.ALIAS, advance=False): 5154 # UNPIVOT ... ON (col1, col2, col3) AS row_val 5155 return self._parse_alias(this) 5156 5157 return this 5158 5159 this = self._parse_table() 5160 expressions = self._match(TokenType.ON) and self._parse_csv(_parse_on) 5161 into = self._parse_unpivot_columns() 5162 using = self._match(TokenType.USING) and self._parse_csv( 5163 lambda: self._parse_alias(self._parse_column()) 5164 ) 5165 group = self._parse_group() 5166 5167 return self.expression( 5168 exp.Pivot( 5169 this=this, 5170 expressions=expressions, 5171 using=using, 5172 group=group, 5173 unpivot=is_unpivot, 5174 into=into, 5175 ) 5176 ) 5177 5178 def _parse_pivot_in(self) -> exp.In: 5179 def _parse_aliased_expression() -> exp.Expr | None: 5180 this = self._parse_select_or_expression() 5181 5182 self._match(TokenType.ALIAS) 5183 alias = self._parse_bitwise() 5184 if alias: 5185 if isinstance(alias, exp.Column) and not alias.db: 5186 alias = alias.this 5187 return self.expression(exp.PivotAlias(this=this, alias=alias)) 5188 5189 return this 5190 5191 value = self._parse_column() 5192 5193 if not self._match(TokenType.IN): 5194 self.raise_error("Expecting IN") 5195 5196 if self._match(TokenType.L_PAREN): 5197 if self._match(TokenType.ANY): 5198 exprs: list[exp.Expr] = ensure_list(exp.PivotAny(this=self._parse_order())) 5199 else: 5200 exprs = self._parse_csv(_parse_aliased_expression) 5201 self._match_r_paren() 5202 return self.expression(exp.In(this=value, expressions=exprs)) 5203 5204 return self.expression(exp.In(this=value, field=self._parse_id_var())) 5205 5206 def _parse_pivot_aggregation(self) -> exp.Expr | None: 5207 func = self._parse_function() 5208 if not func: 5209 if self._prev.token_type == TokenType.COMMA: 5210 return None 5211 self.raise_error("Expecting an aggregation function in PIVOT") 5212 5213 return self._parse_alias(func) 5214 5215 def _parse_pivot(self) -> exp.Pivot | None: 5216 index = self._index 5217 include_nulls = None 5218 5219 if self._match(TokenType.PIVOT): 5220 unpivot = False 5221 elif self._match(TokenType.UNPIVOT): 5222 unpivot = True 5223 5224 # https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-qry-select-unpivot.html#syntax 5225 if self._match_text_seq("INCLUDE", "NULLS"): 5226 include_nulls = True 5227 elif self._match_text_seq("EXCLUDE", "NULLS"): 5228 include_nulls = False 5229 else: 5230 return None 5231 5232 expressions = [] 5233 5234 if not self._match(TokenType.L_PAREN): 5235 self._retreat(index) 5236 return None 5237 5238 if unpivot: 5239 expressions = self._parse_csv(self._parse_column) 5240 else: 5241 expressions = self._parse_csv(self._parse_pivot_aggregation) 5242 5243 if not expressions: 5244 self.raise_error("Failed to parse PIVOT's aggregation list") 5245 5246 if not self._match(TokenType.FOR): 5247 self.raise_error("Expecting FOR") 5248 5249 fields = [] 5250 while True: 5251 field = self._try_parse(self._parse_pivot_in) 5252 if not field: 5253 break 5254 fields.append(field) 5255 5256 default_on_null = self._match_text_seq("DEFAULT", "ON", "NULL") and self._parse_wrapped( 5257 self._parse_bitwise 5258 ) 5259 5260 group = self._parse_group() 5261 5262 self._match_r_paren() 5263 5264 pivot = self.expression( 5265 exp.Pivot( 5266 expressions=expressions, 5267 fields=fields, 5268 unpivot=unpivot, 5269 include_nulls=include_nulls, 5270 default_on_null=default_on_null, 5271 group=group, 5272 ) 5273 ) 5274 5275 if unpivot: 5276 pivot.set("expressions", [_unpivot_target(e) for e in pivot.expressions]) 5277 for pivot_field in pivot.fields: 5278 if isinstance(pivot_field, exp.In): 5279 pivot_field.set("this", _unpivot_target(pivot_field.this)) 5280 5281 if not self._match_set((TokenType.PIVOT, TokenType.UNPIVOT), advance=False): 5282 pivot.set("alias", self._parse_table_alias()) 5283 5284 if not unpivot: 5285 names = self._pivot_column_names(t.cast(list[exp.Expr], expressions)) 5286 5287 columns: list[exp.Expr] = [] 5288 all_fields = [] 5289 for pivot_field in pivot.fields: 5290 pivot_field_expressions = pivot_field.expressions 5291 5292 # The `PivotAny` expression corresponds to `ANY ORDER BY <column>`; we can't infer in this case. 5293 if isinstance(seq_get(pivot_field_expressions, 0), exp.PivotAny): 5294 continue 5295 5296 all_fields.append( 5297 [ 5298 fld.sql() if self.IDENTIFY_PIVOT_STRINGS else fld.alias_or_name 5299 for fld in pivot_field_expressions 5300 ] 5301 ) 5302 5303 if all_fields: 5304 if names: 5305 all_fields.append(names) 5306 5307 # Generate all possible combinations of the pivot columns 5308 # e.g PIVOT(sum(...) as total FOR year IN (2000, 2010) FOR country IN ('NL', 'US')) 5309 # generates the product between [[2000, 2010], ['NL', 'US'], ['total']] 5310 for fld_parts_tuple in itertools.product(*all_fields): 5311 fld_parts = list(fld_parts_tuple) 5312 5313 if names and self.PREFIXED_PIVOT_COLUMNS: 5314 # Move the "name" to the front of the list 5315 fld_parts.insert(0, fld_parts.pop(-1)) 5316 5317 columns.append(exp.to_identifier("_".join(fld_parts))) 5318 5319 pivot.set("columns", columns) 5320 pivot.set("identify_pivot_strings", self.IDENTIFY_PIVOT_STRINGS) 5321 pivot.set("prefixed_pivot_columns", self.PREFIXED_PIVOT_COLUMNS) 5322 pivot.set("pivot_column_naming", self.PIVOT_COLUMN_NAMING) 5323 5324 return pivot 5325 5326 def _pivot_column_names(self, aggregations: list[exp.Expr]) -> list[str]: 5327 return [agg.alias for agg in aggregations if agg.alias] 5328 5329 def _parse_prewhere(self, skip_where_token: bool = False) -> exp.PreWhere | None: 5330 if not skip_where_token and not self._match(TokenType.PREWHERE): 5331 return None 5332 5333 comments = self._prev_comments 5334 return self.expression( 5335 exp.PreWhere(this=self._parse_disjunction()), 5336 comments=comments, 5337 ) 5338 5339 def _parse_where(self, skip_where_token: bool = False) -> exp.Where | None: 5340 if not skip_where_token and not self._match(TokenType.WHERE): 5341 return None 5342 5343 comments = self._prev_comments 5344 return self.expression( 5345 exp.Where(this=self._parse_disjunction()), 5346 comments=comments, 5347 ) 5348 5349 def _parse_group(self, skip_group_by_token: bool = False) -> exp.Group | None: 5350 if not skip_group_by_token and not self._match(TokenType.GROUP_BY): 5351 return None 5352 comments = self._prev_comments 5353 5354 elements: dict[str, t.Any] = defaultdict(list) 5355 5356 if self._match(TokenType.ALL): 5357 elements["all"] = True 5358 elif self._match(TokenType.DISTINCT): 5359 elements["all"] = False 5360 5361 if self._match_set(self.QUERY_MODIFIER_TOKENS, advance=False): 5362 return self.expression(exp.Group(**elements), comments=comments) # type: ignore 5363 5364 while True: 5365 index = self._index 5366 5367 elements["expressions"].extend( 5368 self._parse_csv( 5369 lambda: ( 5370 None 5371 if self._match_set((TokenType.CUBE, TokenType.ROLLUP), advance=False) 5372 else self._parse_disjunction() 5373 ) 5374 ) 5375 ) 5376 5377 before_with_index = self._index 5378 with_prefix = self._match(TokenType.WITH) 5379 5380 if cube_or_rollup := self._parse_cube_or_rollup(with_prefix=with_prefix): 5381 key = "rollup" if isinstance(cube_or_rollup, exp.Rollup) else "cube" 5382 elements[key].append(cube_or_rollup) 5383 elif grouping_sets := self._parse_grouping_sets(): 5384 elements["grouping_sets"].append(grouping_sets) 5385 elif self._match_text_seq("TOTALS"): 5386 elements["totals"] = True # type: ignore 5387 5388 if before_with_index <= self._index <= before_with_index + 1: 5389 self._retreat(before_with_index) 5390 break 5391 5392 if index == self._index: 5393 break 5394 5395 return self.expression(exp.Group(**elements), comments=comments) # type: ignore 5396 5397 def _parse_cube_or_rollup(self, with_prefix: bool = False) -> exp.Cube | exp.Rollup | None: 5398 if self._match(TokenType.CUBE): 5399 kind: type[exp.Cube | exp.Rollup] = exp.Cube 5400 elif self._match(TokenType.ROLLUP): 5401 kind = exp.Rollup 5402 else: 5403 return None 5404 5405 return self.expression( 5406 kind(expressions=[] if with_prefix else self._parse_wrapped_csv(self._parse_bitwise)) 5407 ) 5408 5409 def _parse_grouping_sets(self) -> exp.GroupingSets | None: 5410 if self._match(TokenType.GROUPING_SETS): 5411 return self.expression( 5412 exp.GroupingSets(expressions=self._parse_wrapped_csv(self._parse_grouping_set)) 5413 ) 5414 return None 5415 5416 def _parse_grouping_set(self) -> exp.Expr | None: 5417 return self._parse_grouping_sets() or self._parse_cube_or_rollup() or self._parse_bitwise() 5418 5419 def _parse_having(self, skip_having_token: bool = False) -> exp.Having | None: 5420 if not skip_having_token and not self._match(TokenType.HAVING): 5421 return None 5422 comments = self._prev_comments 5423 return self.expression( 5424 exp.Having(this=self._parse_disjunction()), 5425 comments=comments, 5426 ) 5427 5428 def _parse_qualify(self) -> exp.Qualify | None: 5429 if not self._match(TokenType.QUALIFY): 5430 return None 5431 return self.expression(exp.Qualify(this=self._parse_disjunction())) 5432 5433 def _parse_connect_with_prior(self) -> exp.Expr | None: 5434 self.NO_PAREN_FUNCTION_PARSERS["PRIOR"] = lambda self: self.expression( 5435 exp.Prior(this=self._parse_bitwise()) 5436 ) 5437 connect = self._parse_disjunction() 5438 self.NO_PAREN_FUNCTION_PARSERS.pop("PRIOR") 5439 return connect 5440 5441 def _parse_connect(self, skip_start_token: bool = False) -> exp.Connect | None: 5442 if skip_start_token: 5443 start = None 5444 elif self._match(TokenType.START_WITH): 5445 start = self._parse_disjunction() 5446 else: 5447 return None 5448 5449 self._match(TokenType.CONNECT_BY) 5450 nocycle = self._match_text_seq("NOCYCLE") 5451 connect = self._parse_connect_with_prior() 5452 5453 if not start and self._match(TokenType.START_WITH): 5454 start = self._parse_disjunction() 5455 5456 return self.expression(exp.Connect(start=start, connect=connect, nocycle=nocycle)) 5457 5458 def _parse_name_as_expression(self) -> exp.Expr | None: 5459 this = self._parse_id_var(any_token=True) 5460 if self._match(TokenType.ALIAS): 5461 this = self.expression(exp.Alias(alias=this, this=self._parse_disjunction())) 5462 return this 5463 5464 def _parse_interpolate(self) -> list[exp.Expr] | None: 5465 if self._match_text_seq("INTERPOLATE"): 5466 return self._parse_wrapped_csv(self._parse_name_as_expression) 5467 return None 5468 5469 def _parse_order( 5470 self, this: exp.Expr | None = None, skip_order_token: bool = False 5471 ) -> exp.Expr | None: 5472 siblings = None 5473 if not skip_order_token and not self._match(TokenType.ORDER_BY): 5474 if not self._match(TokenType.ORDER_SIBLINGS_BY): 5475 return this 5476 5477 siblings = True 5478 5479 comments = self._prev_comments 5480 return self.expression( 5481 exp.Order( 5482 this=this, 5483 expressions=self._parse_csv(self._parse_ordered), 5484 siblings=siblings, 5485 ), 5486 comments=comments, 5487 ) 5488 5489 def _parse_sort(self, exp_class: type[E], token: TokenType) -> E | None: 5490 if not self._match(token): 5491 return None 5492 return self.expression(exp_class(expressions=self._parse_csv(self._parse_ordered))) 5493 5494 def _parse_ordered( 5495 self, parse_method: t.Callable[[], exp.Expr | None] | None = None 5496 ) -> exp.Ordered | None: 5497 this = parse_method() if parse_method else self._parse_disjunction() 5498 if not this: 5499 return None 5500 5501 if this.name.upper() == "ALL" and self.dialect.SUPPORTS_ORDER_BY_ALL: 5502 this = exp.var("ALL") 5503 5504 asc = self._match(TokenType.ASC) 5505 desc: bool | None = True if self._match(TokenType.DESC) else (False if asc else None) 5506 5507 is_nulls_first = self._match_text_seq("NULLS", "FIRST") 5508 is_nulls_last = self._match_text_seq("NULLS", "LAST") 5509 5510 nulls_first = is_nulls_first or False 5511 explicitly_null_ordered = is_nulls_first or is_nulls_last 5512 5513 if ( 5514 not explicitly_null_ordered 5515 and ( 5516 (not desc and self.dialect.NULL_ORDERING == "nulls_are_small") 5517 or (desc and self.dialect.NULL_ORDERING != "nulls_are_small") 5518 ) 5519 and self.dialect.NULL_ORDERING != "nulls_are_last" 5520 ): 5521 nulls_first = True 5522 5523 if self._match_text_seq("WITH", "FILL"): 5524 with_fill = self.expression( 5525 exp.WithFill( 5526 from_=self._match(TokenType.FROM) and self._parse_bitwise(), 5527 to=self._match_text_seq("TO") and self._parse_bitwise(), 5528 step=self._match_text_seq("STEP") and self._parse_bitwise(), 5529 interpolate=self._parse_interpolate(), 5530 ) 5531 ) 5532 else: 5533 with_fill = None 5534 5535 return self.expression( 5536 exp.Ordered(this=this, desc=desc, nulls_first=nulls_first, with_fill=with_fill) 5537 ) 5538 5539 def _parse_limit_options(self) -> exp.LimitOptions | None: 5540 percent = self._match_set((TokenType.PERCENT, TokenType.MOD)) 5541 rows = self._match_set((TokenType.ROW, TokenType.ROWS)) 5542 self._match_text_seq("ONLY") 5543 with_ties = self._match_text_seq("WITH", "TIES") 5544 5545 if not (percent or rows or with_ties): 5546 return None 5547 5548 return self.expression(exp.LimitOptions(percent=percent, rows=rows, with_ties=with_ties)) 5549 5550 def _parse_limit( 5551 self, 5552 this: exp.Expr | None = None, 5553 top: bool = False, 5554 skip_limit_token: bool = False, 5555 ) -> exp.Expr | None: 5556 if skip_limit_token or self._match(TokenType.TOP if top else TokenType.LIMIT): 5557 comments = self._prev_comments 5558 if top: 5559 limit_paren = self._match(TokenType.L_PAREN) 5560 expression = ( 5561 self._parse_term() or self._parse_select() 5562 if limit_paren 5563 else self._parse_number() 5564 ) 5565 5566 if limit_paren: 5567 self._match_r_paren() 5568 5569 else: 5570 if self.dialect.SUPPORTS_LIMIT_ALL and self._match(TokenType.ALL): 5571 return this 5572 5573 # Parsing LIMIT x% (i.e x PERCENT) as a term leads to an error, since 5574 # we try to build an exp.Mod expr. For that matter, we backtrack and instead 5575 # consume the factor plus parse the percentage separately 5576 index = self._index 5577 expression = self._try_parse(self._parse_term) 5578 if isinstance(expression, exp.Mod): 5579 self._retreat(index) 5580 expression = self._parse_factor() 5581 elif not expression: 5582 expression = self._parse_factor() 5583 limit_options = self._parse_limit_options() 5584 5585 if self._match(TokenType.COMMA): 5586 offset = expression 5587 expression = self._parse_term() 5588 else: 5589 offset = None 5590 5591 limit_exp = self.expression( 5592 exp.Limit( 5593 this=this, 5594 expression=expression, 5595 offset=offset, 5596 limit_options=limit_options, 5597 expressions=self._parse_limit_by(), 5598 ), 5599 comments=comments, 5600 ) 5601 5602 return limit_exp 5603 5604 if self._match(TokenType.FETCH): 5605 direction = ( 5606 self._prev.text.upper() 5607 if self._match_set((TokenType.FIRST, TokenType.NEXT)) 5608 else "FIRST" 5609 ) 5610 5611 count = self._parse_field(tokens=self.FETCH_TOKENS) 5612 5613 return self.expression( 5614 exp.Fetch( 5615 direction=direction, count=count, limit_options=self._parse_limit_options() 5616 ) 5617 ) 5618 5619 return this 5620 5621 def _parse_offset(self, this: exp.Expr | None = None) -> exp.Expr | None: 5622 if not self._match(TokenType.OFFSET): 5623 return this 5624 5625 count = self._parse_term() 5626 self._match_set((TokenType.ROW, TokenType.ROWS)) 5627 5628 return self.expression( 5629 exp.Offset(this=this, expression=count, expressions=self._parse_limit_by()) 5630 ) 5631 5632 def _can_parse_limit_or_offset(self) -> bool: 5633 if not self._match_set(self.AMBIGUOUS_ALIAS_TOKENS, advance=False): 5634 return False 5635 5636 index = self._index 5637 result = bool( 5638 self._try_parse(self._parse_limit, retreat=True) 5639 or self._try_parse(self._parse_offset, retreat=True) 5640 ) 5641 self._retreat(index) 5642 5643 # MATCH_CONDITION (...) is a special construct that should not be consumed by limit/offset 5644 if self._next.token_type == TokenType.MATCH_CONDITION: 5645 result = False 5646 5647 return result 5648 5649 def _can_parse_named_window(self) -> bool: 5650 # `WINDOW` is in ID_VAR_TOKENS so it could be mistakenly consumed as an implicit alias. 5651 # Refuse only when the following tokens look like a named-window clause: `WINDOW <id> AS (`. 5652 if not self._match(TokenType.WINDOW, advance=False): 5653 return False 5654 5655 name = self._tokens[self._index + 1] if self._index + 1 < len(self._tokens) else None 5656 if name is None or name.token_type not in self.ID_VAR_TOKENS: 5657 return False 5658 5659 alias_tok = self._tokens[self._index + 2] if self._index + 2 < len(self._tokens) else None 5660 if alias_tok is None or alias_tok.token_type != TokenType.ALIAS: 5661 return False 5662 5663 body = self._tokens[self._index + 3] if self._index + 3 < len(self._tokens) else None 5664 return body is not None and body.token_type == TokenType.L_PAREN 5665 5666 def _parse_limit_by(self) -> list[exp.Expr] | None: 5667 return self._parse_csv(self._parse_bitwise) if self._match_text_seq("BY") else None 5668 5669 def _parse_locks(self) -> list[exp.Lock]: 5670 locks = [] 5671 while True: 5672 update, key = None, None 5673 if self._match_text_seq("FOR", "UPDATE"): 5674 update = True 5675 elif self._match_text_seq("FOR", "SHARE") or self._match_text_seq( 5676 "LOCK", "IN", "SHARE", "MODE" 5677 ): 5678 update = False 5679 elif self._match_text_seq("FOR", "KEY", "SHARE"): 5680 update, key = False, True 5681 elif self._match_text_seq("FOR", "NO", "KEY", "UPDATE"): 5682 update, key = True, True 5683 else: 5684 break 5685 5686 expressions = None 5687 if self._match_text_seq("OF"): 5688 expressions = self._parse_csv(lambda: self._parse_table(schema=True)) 5689 5690 wait: bool | exp.Expr | None = None 5691 if self._match_text_seq("NOWAIT"): 5692 wait = True 5693 elif self._match_text_seq("WAIT"): 5694 wait = self._parse_primary() 5695 elif self._match_text_seq("SKIP", "LOCKED"): 5696 wait = False 5697 5698 locks.append( 5699 self.expression( 5700 exp.Lock(update=update, expressions=expressions, wait=wait, key=key) 5701 ) 5702 ) 5703 5704 return locks 5705 5706 def parse_set_operation( 5707 self, this: exp.Expr | None, consume_pipe: bool = False 5708 ) -> exp.Expr | None: 5709 start = self._index 5710 _, side_token, kind_token = self._parse_join_parts() 5711 5712 side = side_token.text if side_token else None 5713 kind = kind_token.text if kind_token else None 5714 5715 if not self._match_set(self.SET_OPERATIONS): 5716 self._retreat(start) 5717 return None 5718 5719 token_type = self._prev.token_type 5720 5721 if token_type == TokenType.UNION: 5722 operation: type[exp.SetOperation] = exp.Union 5723 elif token_type == TokenType.EXCEPT: 5724 operation = exp.Except 5725 else: 5726 operation = exp.Intersect 5727 5728 comments = self._prev.comments 5729 5730 if self._match(TokenType.DISTINCT): 5731 distinct: bool | None = True 5732 elif self._match(TokenType.ALL): 5733 distinct = False 5734 else: 5735 distinct = self.dialect.SET_OP_DISTINCT_BY_DEFAULT[operation] 5736 if distinct is None: 5737 self.raise_error(f"Expected DISTINCT or ALL for {operation.__name__}") 5738 5739 by_name = ( 5740 self._match_text_seq("BY", "NAME") 5741 or self._match_text_seq("STRICT", "CORRESPONDING") 5742 or None 5743 ) 5744 if self._match_text_seq("CORRESPONDING"): 5745 by_name = True 5746 if not side and not kind: 5747 kind = "INNER" 5748 5749 on_column_list = None 5750 if by_name and self._match_texts(("ON", "BY")): 5751 on_column_list = self._parse_wrapped_csv(self._parse_column) 5752 5753 expression = self._parse_select( 5754 nested=True, parse_set_operation=False, consume_pipe=consume_pipe 5755 ) 5756 5757 return self.expression( 5758 operation( 5759 this=this, 5760 distinct=distinct, 5761 by_name=by_name, 5762 expression=expression, 5763 side=side, 5764 kind=kind, 5765 on=on_column_list, 5766 ), 5767 comments=comments, 5768 ) 5769 5770 def _parse_set_operations(self, this: exp.Expr | None) -> exp.Expr | None: 5771 while this: 5772 setop = self.parse_set_operation(this) 5773 if not setop: 5774 break 5775 this = setop 5776 5777 if isinstance(this, exp.SetOperation) and self.MODIFIERS_ATTACHED_TO_SET_OP: 5778 expression = this.expression 5779 5780 if expression: 5781 for arg in self.SET_OP_MODIFIERS: 5782 expr = expression.args.get(arg) 5783 if expr: 5784 this.set(arg, expr.pop()) 5785 5786 return this 5787 5788 def _parse_expression(self) -> exp.Expr | None: 5789 return self._parse_alias(self._parse_assignment()) 5790 5791 def _parse_assignment(self) -> exp.Expr | None: 5792 this = self._parse_disjunction() 5793 if not this and self._next.token_type in self.ASSIGNMENT: 5794 # This allows us to parse <non-identifier token> := <expr> 5795 this = exp.column( 5796 t.cast(str, self._advance_any(ignore_reserved=True) and self._prev.text) 5797 ) 5798 5799 while self._match_set(self.ASSIGNMENT): 5800 if isinstance(this, exp.Column) and len(this.parts) == 1: 5801 this = this.this 5802 5803 comments = self._prev_comments 5804 this = self.expression( 5805 self.ASSIGNMENT[self._prev.token_type]( 5806 this=this, expression=self._parse_assignment() 5807 ), 5808 comments=comments, 5809 ) 5810 5811 return this 5812 5813 def _parse_disjunction(self) -> exp.Expr | None: 5814 this = self._parse_conjunction() 5815 while self._match_set(self.DISJUNCTION): 5816 comments = self._prev_comments 5817 this = self.expression( 5818 self.DISJUNCTION[self._prev.token_type]( 5819 this=this, expression=self._parse_conjunction() 5820 ), 5821 comments=comments, 5822 ) 5823 return this 5824 5825 def _parse_conjunction(self) -> exp.Expr | None: 5826 this = self._parse_equality() 5827 while self._match_set(self.CONJUNCTION): 5828 comments = self._prev_comments 5829 this = self.expression( 5830 self.CONJUNCTION[self._prev.token_type]( 5831 this=this, expression=self._parse_equality() 5832 ), 5833 comments=comments, 5834 ) 5835 return this 5836 5837 def _parse_equality(self) -> exp.Expr | None: 5838 this = self._parse_comparison() 5839 while self._match_set(self.EQUALITY): 5840 comments = self._prev_comments 5841 this = self.expression( 5842 self.EQUALITY[self._prev.token_type]( 5843 this=this, expression=self._parse_comparison() 5844 ), 5845 comments=comments, 5846 ) 5847 return this 5848 5849 def _parse_comparison(self) -> exp.Expr | None: 5850 this = self._parse_range() 5851 while self._match_set(self.COMPARISON): 5852 comments = self._prev_comments 5853 this = self.expression( 5854 self.COMPARISON[self._prev.token_type](this=this, expression=self._parse_range()), 5855 comments=comments, 5856 ) 5857 return this 5858 5859 def _parse_range(self, this: exp.Expr | None = None) -> exp.Expr | None: 5860 this = this or self._parse_bitwise() 5861 negate = self._match(TokenType.NOT) 5862 5863 if self._match_set(self.RANGE_PARSERS): 5864 expression = self.RANGE_PARSERS[self._prev.token_type](self, this) 5865 if not expression: 5866 return this 5867 5868 this = expression 5869 elif self._match(TokenType.ISNULL) or (negate and self._match(TokenType.NULL)): 5870 this = self.expression(exp.Is(this=this, expression=exp.Null())) 5871 5872 # Postgres supports ISNULL and NOTNULL for conditions. 5873 # https://blog.andreiavram.ro/postgresql-null-composite-type/ 5874 if self._match(TokenType.NOTNULL): 5875 this = self.expression(exp.Is(this=this, expression=exp.Null())) 5876 this = self.expression(exp.Not(this=this)) 5877 5878 if negate: 5879 this = self._negate_range(this) 5880 5881 if self._match(TokenType.IS): 5882 this = self._parse_is(this) 5883 5884 return this 5885 5886 def _negate_range(self, this: exp.Expr | None = None) -> exp.Expr | None: 5887 if not this: 5888 return this 5889 5890 expression = this.this if isinstance(this, exp.Escape) else this 5891 if isinstance(expression, (exp.Like, exp.ILike)): 5892 expression.set("negate", True) 5893 return this 5894 5895 return self.expression(exp.Not(this=this)) 5896 5897 def _parse_is(self, this: exp.Expr | None) -> exp.Expr | None: 5898 index = self._index - 1 5899 negate = self._match(TokenType.NOT) 5900 5901 if self._match_text_seq("DISTINCT", "FROM"): 5902 klass = exp.NullSafeEQ if negate else exp.NullSafeNEQ 5903 return self.expression(klass(this=this, expression=self._parse_bitwise())) 5904 5905 if self._match(TokenType.JSON): 5906 kind = self._match_texts(self.IS_JSON_PREDICATE_KIND) and self._prev.text.upper() 5907 5908 if self._match_text_seq("WITH"): 5909 _with = True 5910 elif self._match_text_seq("WITHOUT"): 5911 _with = False 5912 else: 5913 _with = None 5914 5915 unique = self._match(TokenType.UNIQUE) 5916 self._match_text_seq("KEYS") 5917 expression: exp.Expr | None = self.expression( 5918 exp.JSON(this=kind, with_=_with, unique=unique) 5919 ) 5920 else: 5921 expression = self._parse_null() or self._parse_bitwise() 5922 if not expression: 5923 self._retreat(index) 5924 return None 5925 5926 this = self.expression(exp.Is(this=this, expression=expression)) 5927 this = self.expression(exp.Not(this=this)) if negate else this 5928 return self._parse_column_ops(this) 5929 5930 def _parse_in(self, this: exp.Expr | None, alias: bool = False) -> exp.In: 5931 unnest = self._parse_unnest(with_alias=False) 5932 if unnest: 5933 this = self.expression(exp.In(this=this, unnest=unnest)) 5934 elif self._match_set((TokenType.L_PAREN, TokenType.L_BRACKET)): 5935 matched_l_paren = self._prev.token_type == TokenType.L_PAREN 5936 expressions = self._parse_csv(lambda: self._parse_select_or_expression(alias=alias)) 5937 5938 if len(expressions) == 1 and isinstance(query := expressions[0], exp.Query): 5939 this = self.expression( 5940 exp.In(this=this, query=self._parse_query_modifiers(query).subquery(copy=False)) 5941 ) 5942 else: 5943 this = self.expression(exp.In(this=this, expressions=expressions)) 5944 5945 if matched_l_paren: 5946 self._match_r_paren(this) 5947 elif not self._match(TokenType.R_BRACKET, expression=this): 5948 self.raise_error("Expecting ]") 5949 else: 5950 this = self.expression(exp.In(this=this, field=self._parse_column())) 5951 5952 return this 5953 5954 def _parse_between(self, this: exp.Expr | None) -> exp.Between: 5955 symmetric = None 5956 if self._match_text_seq("SYMMETRIC"): 5957 symmetric = True 5958 elif self._match_text_seq("ASYMMETRIC"): 5959 symmetric = False 5960 5961 low = self._parse_bitwise() 5962 self._match(TokenType.AND) 5963 high = self._parse_bitwise() 5964 5965 return self.expression(exp.Between(this=this, low=low, high=high, symmetric=symmetric)) 5966 5967 def _parse_escape(self, this: exp.Expr | None) -> exp.Expr | None: 5968 if not self._match(TokenType.ESCAPE): 5969 return this 5970 return self.expression( 5971 exp.Escape(this=this, expression=self._parse_string() or self._parse_null()) 5972 ) 5973 5974 def _parse_interval_span(self, this: exp.Expr) -> exp.Interval: 5975 # handle day-time format interval span with omitted units: 5976 # INTERVAL '<number days> hh[:][mm[:ss[.ff]]]' <maybe `unit TO unit`> 5977 interval_span_units_omitted = None 5978 if ( 5979 this 5980 and this.is_string 5981 and self.SUPPORTS_OMITTED_INTERVAL_SPAN_UNIT 5982 and exp.INTERVAL_DAY_TIME_RE.match(this.name) 5983 ): 5984 index = self._index 5985 5986 # Var "TO" Var 5987 first_unit = self._parse_var(any_token=True, upper=True) 5988 second_unit = None 5989 if first_unit and self._match_text_seq("TO"): 5990 second_unit = self._parse_var(any_token=True, upper=True) 5991 5992 interval_span_units_omitted = not (first_unit and second_unit) 5993 5994 self._retreat(index) 5995 5996 if interval_span_units_omitted: 5997 unit = None 5998 else: 5999 unit = self._parse_function() 6000 if not unit and ( 6001 self._curr.token_type == TokenType.VAR 6002 or self._curr.text.upper() in self.dialect.VALID_INTERVAL_UNITS 6003 ): 6004 unit = self._parse_var(any_token=True, upper=True) 6005 6006 # Most dialects support, e.g., the form INTERVAL '5' day, thus we try to parse 6007 # each INTERVAL expression into this canonical form so it's easy to transpile 6008 if this and this.is_number: 6009 this = exp.Literal.string(this.to_py()) 6010 elif this and this.is_string: 6011 parts = exp.INTERVAL_STRING_RE.findall(this.name) 6012 if parts and unit: 6013 # Unconsume the eagerly-parsed unit, since the real unit was part of the string 6014 unit = None 6015 self._retreat(self._index - 1) 6016 6017 if len(parts) == 1: 6018 this = exp.Literal.string(parts[0][0]) 6019 unit = self.expression(exp.Var(this=parts[0][1].upper())) 6020 6021 if self.INTERVAL_SPANS and self._match_text_seq("TO"): 6022 unit = self.expression( 6023 exp.IntervalSpan( 6024 this=unit, 6025 expression=self._parse_function() 6026 or self._parse_var(any_token=True, upper=True), 6027 ) 6028 ) 6029 6030 return self.expression(exp.Interval(this=this, unit=unit)) 6031 6032 def _parse_interval(self, require_interval: bool = True) -> exp.Add | exp.Interval | None: 6033 index = self._index 6034 6035 if not self._match(TokenType.INTERVAL) and require_interval: 6036 return None 6037 6038 if self._match(TokenType.STRING, advance=False): 6039 this = self._parse_primary() 6040 else: 6041 this = self._parse_term() 6042 6043 if not this or ( 6044 isinstance(this, exp.Column) 6045 and not this.table 6046 and not this.this.quoted 6047 and self._curr 6048 and self._curr.text.upper() not in self.dialect.VALID_INTERVAL_UNITS 6049 ): 6050 self._retreat(index) 6051 return None 6052 6053 interval = self._parse_interval_span(this) 6054 6055 index = self._index 6056 self._match(TokenType.PLUS) 6057 6058 # Convert INTERVAL 'val_1' unit_1 [+] ... [+] 'val_n' unit_n into a sum of intervals 6059 if self._match_set((TokenType.STRING, TokenType.NUMBER), advance=False): 6060 return self.expression(exp.Add(this=interval, expression=self._parse_interval(False))) 6061 6062 self._retreat(index) 6063 return interval 6064 6065 def _parse_bitwise(self) -> exp.Expr | None: 6066 this = self._parse_term() 6067 6068 while True: 6069 if self._match_set(self.BITWISE): 6070 this = self.expression( 6071 self.BITWISE[self._prev.token_type](this=this, expression=self._parse_term()) 6072 ) 6073 elif self.dialect.DPIPE_IS_STRING_CONCAT and self._match(TokenType.DPIPE): 6074 this = self.expression( 6075 exp.DPipe( 6076 this=this, 6077 expression=self._parse_term(), 6078 safe=not self.dialect.STRICT_STRING_CONCAT, 6079 ) 6080 ) 6081 elif self._match(TokenType.DQMARK): 6082 this = self.expression( 6083 exp.Coalesce(this=this, expressions=ensure_list(self._parse_term())) 6084 ) 6085 elif self._match_pair(TokenType.LT, TokenType.LT): 6086 this = self.expression( 6087 exp.BitwiseLeftShift(this=this, expression=self._parse_term()) 6088 ) 6089 elif self._match_pair(TokenType.GT, TokenType.GT): 6090 this = self.expression( 6091 exp.BitwiseRightShift(this=this, expression=self._parse_term()) 6092 ) 6093 else: 6094 break 6095 6096 return this 6097 6098 def _parse_term(self) -> exp.Expr | None: 6099 this = self._parse_factor() 6100 6101 while self._match_set(self.TERM): 6102 klass = self.TERM[self._prev.token_type] 6103 comments = self._prev_comments 6104 expression = self._parse_factor() 6105 6106 this = self.expression(klass(this=this, expression=expression), comments=comments) 6107 6108 if isinstance(this, exp.Collate): 6109 expr = this.expression 6110 6111 # Preserve collations such as pg_catalog."default" (Postgres) as columns, otherwise 6112 # fallback to Identifier / Var 6113 if isinstance(expr, exp.Column) and len(expr.parts) == 1: 6114 ident = expr.this 6115 if isinstance(ident, exp.Identifier): 6116 this.set("expression", ident if ident.quoted else exp.var(ident.name)) 6117 6118 return this 6119 6120 def _parse_factor(self) -> exp.Expr | None: 6121 parse_method = self._parse_exponent if self.EXPONENT else self._parse_unary 6122 this = self._parse_at_time_zone(parse_method()) 6123 6124 while self._match_set(self.FACTOR): 6125 klass = self.FACTOR[self._prev.token_type] 6126 comments = self._prev_comments 6127 expression = parse_method() 6128 6129 if not expression and klass is exp.IntDiv and self._prev.text.isalpha(): 6130 self._retreat(self._index - 1) 6131 return this 6132 6133 this = self.expression(klass(this=this, expression=expression), comments=comments) 6134 6135 if isinstance(this, exp.Div): 6136 this.set("typed", self.dialect.TYPED_DIVISION) 6137 this.set("safe", self.dialect.SAFE_DIVISION) 6138 6139 return this 6140 6141 def _parse_exponent(self) -> exp.Expr | None: 6142 this = self._parse_unary() 6143 while self._match_set(self.EXPONENT): 6144 comments = self._prev_comments 6145 this = self.expression( 6146 self.EXPONENT[self._prev.token_type](this=this, expression=self._parse_unary()), 6147 comments=comments, 6148 ) 6149 return this 6150 6151 def _parse_unary(self) -> exp.Expr | None: 6152 if self._match_set(self.UNARY_PARSERS): 6153 return self.UNARY_PARSERS[self._prev.token_type](self) 6154 return self._parse_type() 6155 6156 def _parse_type( 6157 self, parse_interval: bool = True, fallback_to_identifier: bool = False 6158 ) -> exp.Expr | None: 6159 if not fallback_to_identifier and (atom := self._parse_atom()) is not None: 6160 return atom 6161 6162 if interval := parse_interval and self._parse_interval(): 6163 return self._parse_column_ops(interval) 6164 6165 index = self._index 6166 data_type = self._parse_types(check_func=True, allow_identifiers=False) 6167 6168 # parse_types() returns a Cast if we parsed BQ's inline constructor <type>(<values>) e.g. 6169 # STRUCT<a INT, b STRING>(1, 'foo'), which is canonicalized to CAST(<values> AS <type>) 6170 if isinstance(data_type, exp.Cast): 6171 # This constructor can contain ops directly after it, for instance struct unnesting: 6172 # STRUCT<a INT, b STRING>(1, 'foo').* --> CAST(STRUCT(1, 'foo') AS STRUCT<a iNT, b STRING).* 6173 return self._parse_column_ops(data_type) 6174 6175 if data_type: 6176 index2 = self._index 6177 this = self._parse_primary() 6178 6179 if isinstance(this, exp.Literal): 6180 literal = this.name 6181 this = self._parse_column_ops(this) 6182 6183 parser = self.TYPE_LITERAL_PARSERS.get(data_type.this) 6184 if parser: 6185 return parser(self, this, data_type) 6186 6187 if ( 6188 self.ZONE_AWARE_TIMESTAMP_CONSTRUCTOR 6189 and data_type.is_type(exp.DType.TIMESTAMP) 6190 and TIME_ZONE_RE.search(literal) 6191 ): 6192 data_type = exp.DType.TIMESTAMPTZ.into_expr() 6193 6194 return self.expression(exp.Cast(this=this, to=data_type)) 6195 6196 # The expressions arg gets set by the parser when we have something like DECIMAL(38, 0) 6197 # in the input SQL. In that case, we'll produce these tokens: DECIMAL ( 38 , 0 ) 6198 # 6199 # If the index difference here is greater than 1, that means the parser itself must have 6200 # consumed additional tokens such as the DECIMAL scale and precision in the above example. 6201 # 6202 # If it's not greater than 1, then it must be 1, because we've consumed at least the type 6203 # keyword, meaning that the expressions arg of the DataType must have gotten set by a 6204 # callable in the TYPE_CONVERTERS mapping. For example, Snowflake converts DECIMAL to 6205 # DECIMAL(38, 0)) in order to facilitate the data type's transpilation. 6206 # 6207 # In these cases, we don't really want to return the converted type, but instead retreat 6208 # and try to parse a Column or Identifier in the section below. 6209 if data_type.expressions and index2 - index > 1: 6210 self._retreat(index2) 6211 return self._parse_column_ops(data_type) 6212 6213 self._retreat(index) 6214 6215 if fallback_to_identifier: 6216 return self._parse_id_var() 6217 6218 return self._parse_column() 6219 6220 def _parse_type_size(self) -> exp.DataTypeParam | None: 6221 this = self._parse_type() 6222 if not this: 6223 return None 6224 6225 if isinstance(this, exp.Column) and not this.table: 6226 this = exp.var(this.name.upper()) 6227 6228 return self.expression( 6229 exp.DataTypeParam(this=this, expression=self._parse_var(any_token=True)) 6230 ) 6231 6232 def _parse_user_defined_type(self, identifier: exp.Identifier) -> exp.Expr | None: 6233 type_name = identifier.name 6234 6235 while self._match(TokenType.DOT): 6236 type_name = f"{type_name}.{self._advance_any() and self._prev.text}" 6237 6238 return exp.DataType.from_str(type_name, dialect=self.dialect, udt=True) 6239 6240 def _parse_types( 6241 self, 6242 check_func: bool = False, 6243 schema: bool = False, 6244 allow_identifiers: bool = True, 6245 with_collation: bool = False, 6246 ) -> exp.Expr | None: 6247 index = self._index 6248 this: exp.Expr | None = None 6249 6250 if self._match_set(self.TYPE_TOKENS): 6251 type_token = self._prev.token_type 6252 else: 6253 type_token = None 6254 identifier = allow_identifiers and self._parse_id_var( 6255 any_token=False, tokens=(TokenType.VAR,) 6256 ) 6257 if isinstance(identifier, exp.Identifier): 6258 try: 6259 tokens = self.dialect.tokenize(identifier.name) 6260 except TokenError: 6261 tokens = None 6262 6263 if tokens and (type_token := tokens[0].token_type) in self.TYPE_TOKENS: 6264 if len(tokens) > 1: 6265 return exp.DataType.from_str(identifier.name, dialect=self.dialect) 6266 elif self.dialect.SUPPORTS_USER_DEFINED_TYPES: 6267 this = self._parse_user_defined_type(identifier) 6268 else: 6269 self._retreat(self._index - 1) 6270 return None 6271 else: 6272 return None 6273 6274 if type_token == TokenType.PSEUDO_TYPE: 6275 return self.expression(exp.PseudoType(this=self._prev.text.upper())) 6276 6277 if type_token == TokenType.OBJECT_IDENTIFIER: 6278 return self.expression(exp.ObjectIdentifier(this=self._prev.text.upper())) 6279 6280 # https://materialize.com/docs/sql/types/map/ 6281 if type_token == TokenType.MAP and self._match(TokenType.L_BRACKET): 6282 key_type = self._parse_types( 6283 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6284 ) 6285 if not self._match(TokenType.FARROW): 6286 self._retreat(index) 6287 return None 6288 6289 value_type = self._parse_types( 6290 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6291 ) 6292 if not self._match(TokenType.R_BRACKET): 6293 self._retreat(index) 6294 return None 6295 6296 return exp.DataType( 6297 this=exp.DType.MAP, 6298 expressions=[key_type, value_type], 6299 nested=True, 6300 ) 6301 6302 nested = type_token in self.NESTED_TYPE_TOKENS 6303 is_struct = type_token in self.STRUCT_TYPE_TOKENS 6304 is_aggregate = type_token in self.AGGREGATE_TYPE_TOKENS 6305 expressions = None 6306 maybe_func = False 6307 6308 if self._match(TokenType.L_PAREN): 6309 if is_struct: 6310 expressions = self._parse_csv(lambda: self._parse_struct_types(type_required=True)) 6311 elif nested: 6312 expressions = self._parse_csv( 6313 lambda: self._parse_types( 6314 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6315 ) 6316 ) 6317 if type_token == TokenType.NULLABLE and len(expressions) == 1: 6318 this = expressions[0] 6319 this.set("nullable", True) 6320 self._match_r_paren() 6321 return this 6322 elif type_token in self.ENUM_TYPE_TOKENS: 6323 expressions = self._parse_csv(self._parse_equality) 6324 elif type_token == TokenType.JSON: 6325 # ClickHouse JSON type supports arguments: JSON(col Type, SKIP col, param=value) 6326 # https://clickhouse.com/docs/sql-reference/data-types/newjson 6327 expressions = self._parse_csv(self._parse_json_type_arg) 6328 elif is_aggregate: 6329 func_or_ident = self._parse_function(anonymous=True) or self._parse_id_var( 6330 any_token=False, tokens=(TokenType.VAR, TokenType.ANY) 6331 ) 6332 if not func_or_ident: 6333 return None 6334 expressions = [func_or_ident] 6335 if self._match(TokenType.COMMA): 6336 expressions.extend( 6337 self._parse_csv( 6338 lambda: self._parse_types( 6339 check_func=check_func, 6340 schema=schema, 6341 allow_identifiers=allow_identifiers, 6342 ) 6343 ) 6344 ) 6345 else: 6346 expressions = self._parse_csv(self._parse_type_size) 6347 6348 # https://docs.snowflake.com/en/sql-reference/data-types-vector 6349 if type_token == TokenType.VECTOR and len(expressions) == 2: 6350 expressions = self._parse_vector_expressions(expressions) 6351 6352 if not self._match(TokenType.R_PAREN): 6353 self._retreat(index) 6354 return None 6355 6356 maybe_func = True 6357 6358 values: list[exp.Expr] | None = None 6359 6360 if nested and self._match(TokenType.LT): 6361 if is_struct: 6362 expressions = self._parse_csv(lambda: self._parse_struct_types(type_required=True)) 6363 else: 6364 expressions = self._parse_csv( 6365 lambda: self._parse_types( 6366 check_func=check_func, 6367 schema=schema, 6368 allow_identifiers=allow_identifiers, 6369 with_collation=True, 6370 ) 6371 ) 6372 6373 if not self._match(TokenType.GT): 6374 self.raise_error("Expecting >") 6375 6376 if self._match_set((TokenType.L_BRACKET, TokenType.L_PAREN)): 6377 values = self._parse_csv(self._parse_disjunction) 6378 if not values and is_struct: 6379 values = None 6380 self._retreat(self._index - 1) 6381 else: 6382 self._match_set((TokenType.R_BRACKET, TokenType.R_PAREN)) 6383 6384 if type_token in self.TIMESTAMPS: 6385 if self._match_text_seq("WITH", "TIME", "ZONE"): 6386 maybe_func = False 6387 tz_type = exp.DType.TIMETZ if type_token in self.TIMES else exp.DType.TIMESTAMPTZ 6388 this = exp.DataType(this=tz_type, expressions=expressions) 6389 elif self._match_text_seq("WITH", "LOCAL", "TIME", "ZONE"): 6390 maybe_func = False 6391 this = exp.DataType(this=exp.DType.TIMESTAMPLTZ, expressions=expressions) 6392 elif self._match_text_seq("WITHOUT", "TIME", "ZONE"): 6393 maybe_func = False 6394 elif type_token == TokenType.INTERVAL: 6395 if self._curr.text.upper() in self.dialect.VALID_INTERVAL_UNITS: 6396 unit = self._parse_var(upper=True) 6397 if self._match_text_seq("TO"): 6398 unit = exp.IntervalSpan(this=unit, expression=self._parse_var(upper=True)) 6399 6400 this = self.expression(exp.DataType(this=self.expression(exp.Interval(unit=unit)))) 6401 else: 6402 this = self.expression(exp.DataType(this=exp.DType.INTERVAL)) 6403 elif type_token == TokenType.VOID: 6404 this = exp.DataType(this=exp.DType.NULL) 6405 6406 if maybe_func and check_func: 6407 index2 = self._index 6408 peek = self._parse_string() 6409 6410 if not peek: 6411 self._retreat(index) 6412 return None 6413 6414 self._retreat(index2) 6415 6416 if not this: 6417 assert type_token is not None 6418 if self._match_text_seq("UNSIGNED"): 6419 unsigned_type_token = self.SIGNED_TO_UNSIGNED_TYPE_TOKEN.get(type_token) 6420 if not unsigned_type_token: 6421 self.raise_error(f"Cannot convert {type_token.name} to unsigned.") 6422 6423 type_token = unsigned_type_token or type_token 6424 6425 # NULLABLE without parentheses can be a column (Presto/Trino) 6426 if type_token == TokenType.NULLABLE and not expressions: 6427 self._retreat(index) 6428 return None 6429 6430 this = exp.DataType( 6431 this=exp.DType[type_token.name], 6432 expressions=expressions, 6433 nested=nested, 6434 ) 6435 6436 # Empty arrays/structs are allowed 6437 if values is not None: 6438 cls = exp.Struct if is_struct else exp.Array 6439 this = exp.cast(cls(expressions=values), this, copy=False) 6440 6441 elif expressions: 6442 this.set("expressions", expressions) 6443 6444 # https://materialize.com/docs/sql/types/list/#type-name 6445 while self._match(TokenType.LIST): 6446 this = exp.DataType(this=exp.DType.LIST, expressions=[this], nested=True) 6447 6448 index = self._index 6449 6450 # Postgres supports the INT ARRAY[3] syntax as a synonym for INT[3] 6451 matched_array = self._match(TokenType.ARRAY) 6452 6453 while self._curr: 6454 datatype_token = self._prev.token_type 6455 matched_l_bracket = self._match(TokenType.L_BRACKET) 6456 6457 if (not matched_l_bracket and not matched_array) or ( 6458 datatype_token == TokenType.ARRAY and self._match(TokenType.R_BRACKET) 6459 ): 6460 # Postgres allows casting empty arrays such as ARRAY[]::INT[], 6461 # not to be confused with the fixed size array parsing 6462 break 6463 6464 matched_array = False 6465 values = self._parse_csv(self._parse_disjunction) or None 6466 if ( 6467 values 6468 and not schema 6469 and ( 6470 not self.dialect.SUPPORTS_FIXED_SIZE_ARRAYS 6471 or datatype_token == TokenType.ARRAY 6472 or not self._match(TokenType.R_BRACKET, advance=False) 6473 ) 6474 ): 6475 # Retreating here means that we should not parse the following values as part of the data type, e.g. in DuckDB 6476 # ARRAY[1] should retreat and instead be parsed into exp.Array in contrast to INT[x][y] which denotes a fixed-size array data type 6477 self._retreat(index) 6478 break 6479 6480 this = exp.DataType( 6481 this=exp.DType.ARRAY, expressions=[this], values=values, nested=True 6482 ) 6483 self._match(TokenType.R_BRACKET) 6484 6485 if self.TYPE_CONVERTERS and isinstance(this.this, exp.DType): 6486 converter = self.TYPE_CONVERTERS.get(this.this) 6487 if converter: 6488 this = converter(t.cast(exp.DataType, this)) 6489 6490 if with_collation and isinstance(this, exp.DataType) and self._match(TokenType.COLLATE): 6491 this.set("collate", self._parse_identifier() or self._parse_column()) 6492 6493 return this 6494 6495 def _parse_json_type_arg(self) -> exp.Expr | None: 6496 """Parse a single argument to ClickHouse's JSON type.""" 6497 6498 # SKIP col or SKIP REGEXP 'pattern' 6499 if self._match_text_seq("SKIP"): 6500 regexp = self._match(TokenType.RLIKE) 6501 arg = self._parse_column() 6502 if isinstance(arg, exp.Column): 6503 arg = arg.to_dot() 6504 return self.expression(exp.SkipJSONColumn(regexp=regexp, expression=arg)) 6505 6506 param_or_col = self._parse_column() 6507 if not isinstance(param_or_col, exp.Column): 6508 return None 6509 6510 # Parameter: name=value (e.g., max_dynamic_paths=2) 6511 if len(param_or_col.parts) == 1 and self._match(TokenType.EQ): 6512 param = param_or_col.name 6513 value = self._parse_primary() 6514 return self.expression(exp.EQ(this=exp.var(param), expression=value)) 6515 6516 # Column type hint: col_name Type 6517 col = param_or_col.to_dot() 6518 kind = self._parse_types(check_func=False, allow_identifiers=False) 6519 return self.expression(exp.ColumnDef(this=col, kind=kind)) 6520 6521 def _parse_vector_expressions(self, expressions: list[exp.Expr]) -> list[exp.Expr]: 6522 return [exp.DataType.from_str(expressions[0].name, dialect=self.dialect), *expressions[1:]] 6523 6524 def _parse_struct_types(self, type_required: bool = False) -> exp.Expr | None: 6525 index = self._index 6526 6527 if ( 6528 self._curr 6529 and self._next 6530 and self._curr.token_type in self.TYPE_TOKENS 6531 and self._next.token_type in self.TYPE_TOKENS 6532 ): 6533 # Takes care of special cases like `STRUCT<list ARRAY<...>>` where the identifier is also a 6534 # type token. Without this, the list will be parsed as a type and we'll eventually crash 6535 this = self._parse_id_var() 6536 else: 6537 this = ( 6538 self._parse_type(parse_interval=False, fallback_to_identifier=True) 6539 or self._parse_id_var() 6540 ) 6541 6542 self._match(TokenType.COLON) 6543 6544 if ( 6545 type_required 6546 and not isinstance(this, exp.DataType) 6547 and not self._match_set(self.TYPE_TOKENS, advance=False) 6548 ): 6549 self._retreat(index) 6550 return self._parse_types() 6551 6552 return self._parse_column_def(this) 6553 6554 def _parse_at_time_zone(self, this: exp.Expr | None) -> exp.Expr | None: 6555 if not self._match_text_seq("AT", "TIME", "ZONE"): 6556 return this 6557 return self._parse_at_time_zone( 6558 self.expression(exp.AtTimeZone(this=this, zone=self._parse_unary())) 6559 ) 6560 6561 def _parse_atom(self) -> exp.Expr | None: 6562 if ( 6563 self._curr.token_type in self.IDENTIFIER_TOKENS 6564 and (column := self._parse_column()) is not None 6565 ): 6566 return column 6567 6568 token = self._curr 6569 token_type = token.token_type 6570 6571 if not (primary_parser := self.PRIMARY_PARSERS.get(token_type)): 6572 return None 6573 6574 next_type = self._next.token_type 6575 6576 if ( 6577 next_type in self.COLUMN_OPERATORS 6578 or next_type in self.COLUMN_POSTFIX_TOKENS 6579 or (token_type == TokenType.STRING and next_type == TokenType.STRING) 6580 ): 6581 return None 6582 6583 self._advance() 6584 return primary_parser(self, token) 6585 6586 def _parse_column(self) -> exp.Expr | None: 6587 column: exp.Expr | None = self._parse_column_parts_fast() 6588 if column is None: 6589 this = self._parse_column_reference() 6590 if not this: 6591 this = self._parse_bracket(this) 6592 column = self._parse_column_ops(this) if this else this 6593 6594 if column: 6595 if self.dialect.SUPPORTS_COLUMN_JOIN_MARKS: 6596 column.set("join_mark", self._match(TokenType.JOIN_MARKER)) 6597 if self.COLON_IS_VARIANT_EXTRACT: 6598 column = self._parse_colon_as_variant_extract(column) 6599 6600 return column 6601 6602 def _parse_column_parts_fast(self) -> exp.Column | exp.Dot | None: 6603 """Fast path for simple column and dot references (a, a.b, ...). 6604 6605 Greedily consumes VAR/IDENTIFIER tokens separated by DOTs, then checks 6606 that nothing complex follows. If it does, retreats and returns None so 6607 the slow path can handle it. For >4 parts, wraps in exp.Dot nodes. 6608 """ 6609 index = self._index 6610 parts: list[exp.Identifier] | None = None 6611 all_comments: list[str] | None = None 6612 6613 while self._match_set(self.IDENTIFIER_TOKENS): 6614 token = self._prev 6615 comments = self._prev_comments 6616 6617 if parts is None and token.text.upper() in self.NO_PAREN_FUNCTION_PARSERS: 6618 self._retreat(index) 6619 return None 6620 6621 has_dot = self._match(TokenType.DOT) 6622 curr_tt = self._curr.token_type 6623 6624 if not has_dot: 6625 if curr_tt in self.COLUMN_OPERATORS or curr_tt in self.COLUMN_POSTFIX_TOKENS: 6626 self._retreat(index) 6627 return None 6628 elif curr_tt not in self.IDENTIFIER_TOKENS: 6629 self._retreat(index) 6630 return None 6631 6632 if parts is None: 6633 parts = [] 6634 6635 if comments: 6636 if all_comments is None: 6637 all_comments = [] 6638 all_comments.extend(comments) 6639 self._prev_comments = [] 6640 6641 parts.append( 6642 self.expression( 6643 exp.Identifier( 6644 this=token.text, quoted=token.token_type == TokenType.IDENTIFIER 6645 ), 6646 token, 6647 ) 6648 ) 6649 6650 if not has_dot: 6651 break 6652 6653 if parts is None: 6654 return None 6655 6656 n = len(parts) 6657 6658 if n == 1: 6659 column: exp.Column | exp.Dot = exp.Column(this=parts[0]) 6660 elif n == 2: 6661 column = exp.Column(this=parts[1], table=parts[0]) 6662 elif n == 3: 6663 column = exp.Column(this=parts[2], table=parts[1], db=parts[0]) 6664 else: 6665 column = exp.Column(this=parts[3], table=parts[2], db=parts[1], catalog=parts[0]) 6666 6667 for i in range(4, n): 6668 column = exp.Dot(this=column, expression=parts[i]) 6669 6670 if all_comments: 6671 column.add_comments(all_comments) 6672 6673 return column 6674 6675 def _parse_column_reference(self) -> exp.Expr | None: 6676 this = self._parse_field() 6677 if ( 6678 not this 6679 and self._match(TokenType.VALUES, advance=False) 6680 and self.VALUES_FOLLOWED_BY_PAREN 6681 and (not self._next or self._next.token_type != TokenType.L_PAREN) 6682 ): 6683 this = self._parse_id_var() 6684 6685 if isinstance(this, exp.Identifier): 6686 # We bubble up comments from the Identifier to the Column 6687 this = self.expression(exp.Column(this=this), comments=this.pop_comments()) 6688 6689 return this 6690 6691 def _build_json_extract( 6692 self, 6693 this: exp.Expr | None, 6694 path_parts: list[exp.JSONPathPart], 6695 ) -> tuple[exp.Expr | None, list[exp.JSONPathPart]]: 6696 if len(path_parts) > 1: 6697 this = self.expression( 6698 exp.JSONExtract( 6699 this=this, 6700 expression=exp.JSONPath(expressions=path_parts), 6701 variant_extract=True, 6702 requires_json=self.JSON_EXTRACT_REQUIRES_JSON_EXPRESSION, 6703 ) 6704 ) 6705 path_parts = [exp.JSONPathRoot()] 6706 6707 return this, path_parts 6708 6709 def _parse_colon_as_variant_extract(self, this: exp.Expr | None) -> exp.Expr | None: 6710 path_parts: list[exp.JSONPathPart] = [exp.JSONPathRoot()] 6711 6712 while self._match(TokenType.COLON): 6713 if not self.COLON_CHAIN_IS_SINGLE_EXTRACT: 6714 this, path_parts = self._build_json_extract(this, path_parts) 6715 6716 key = self._parse_id_var(any_token=True, tokens=(TokenType.SELECT,)) 6717 6718 if key: 6719 quoted = isinstance(key, exp.Identifier) and key.quoted 6720 path_parts.append(exp.JSONPathKey(this=key.name, quoted=quoted)) 6721 6722 while True: 6723 if self._match(TokenType.DOT): 6724 next_key = self._parse_id_var(any_token=True, tokens=(TokenType.SELECT,)) 6725 6726 if next_key: 6727 quoted = isinstance(next_key, exp.Identifier) and next_key.quoted 6728 path_parts.append(exp.JSONPathKey(this=next_key.name, quoted=quoted)) 6729 elif self._match(TokenType.L_BRACKET): 6730 bracket_expr = self._parse_bracket_key_value() 6731 6732 if not self._match(TokenType.R_BRACKET): 6733 self.raise_error("Expected ]") 6734 6735 if bracket_expr: 6736 if bracket_expr.is_string: 6737 path_parts.append(exp.JSONPathKey(this=bracket_expr.name, quoted=True)) 6738 elif bracket_expr.is_star: 6739 path_parts.append(exp.JSONPathSubscript(this=exp.JSONPathWildcard())) 6740 elif bracket_expr.is_number: 6741 path_parts.append(exp.JSONPathSubscript(this=bracket_expr.to_py())) 6742 else: 6743 this, path_parts = self._build_json_extract(this, path_parts) 6744 6745 this = self.expression( 6746 exp.Bracket( 6747 this=this, expressions=[bracket_expr], json_access=True 6748 ), 6749 ) 6750 6751 elif self._match(TokenType.DCOLON): 6752 this, path_parts = self._build_json_extract(this, path_parts) 6753 6754 cast_type = self._parse_types() 6755 if cast_type: 6756 this = self.expression(exp.Cast(this=this, to=cast_type)) 6757 else: 6758 self.raise_error("Expected type after '::'") 6759 else: 6760 break 6761 6762 this, _ = self._build_json_extract(this, path_parts) 6763 6764 return this 6765 6766 def _parse_dcolon(self) -> exp.Expr | None: 6767 return self._parse_types() 6768 6769 def _parse_column_ops(self, this: exp.Expr | None) -> exp.Expr | None: 6770 while self._curr.token_type in self.BRACKETS: 6771 this = self._parse_bracket(this) 6772 6773 column_operators = self.COLUMN_OPERATORS 6774 cast_column_operators = self.CAST_COLUMN_OPERATORS 6775 while self._curr: 6776 op_token = self._curr.token_type 6777 6778 if op_token not in column_operators: 6779 break 6780 op = column_operators[op_token] 6781 self._advance() 6782 6783 if op_token in cast_column_operators: 6784 field = self._parse_dcolon() 6785 if not field: 6786 self.raise_error("Expected type") 6787 elif op and self._curr: 6788 field = self._parse_column_reference() or self._parse_bitwise() 6789 if isinstance(field, exp.Column) and self._match(TokenType.DOT, advance=False): 6790 field = self._parse_column_ops(field) 6791 else: 6792 field = self._parse_field(any_token=True, anonymous_func=True) 6793 6794 # Function calls can be qualified, e.g., x.y.FOO() 6795 # This converts the final AST to a series of Dots leading to the function call 6796 # https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-reference#function_call_rules 6797 if isinstance(field, (exp.Func, exp.Window)) and this: 6798 this = this.transform( 6799 lambda n: n.to_dot(include_dots=False) if isinstance(n, exp.Column) else n 6800 ) 6801 6802 if op: 6803 this = op(self, this, field) 6804 elif isinstance(this, exp.Column) and not this.args.get("catalog"): 6805 this = self.expression( 6806 exp.Column( 6807 this=field, 6808 table=this.this, 6809 db=this.args.get("table"), 6810 catalog=this.args.get("db"), 6811 ), 6812 comments=this.comments, 6813 ) 6814 elif isinstance(field, exp.Window): 6815 # Move the exp.Dot's to the window's function 6816 window_func = self.expression(exp.Dot(this=this, expression=field.this)) 6817 field.set("this", window_func) 6818 this = field 6819 else: 6820 this = self.expression(exp.Dot(this=this, expression=field)) 6821 6822 if field and field.comments: 6823 t.cast(exp.Expr, this).add_comments(field.pop_comments()) 6824 6825 this = self._parse_bracket(this) 6826 6827 return this 6828 6829 def _parse_paren(self) -> exp.Expr | None: 6830 if not self._match(TokenType.L_PAREN): 6831 return None 6832 6833 comments = self._prev_comments 6834 query = self._parse_select() 6835 6836 if query: 6837 expressions = [query] 6838 else: 6839 expressions = self._parse_expressions() 6840 6841 this = seq_get(expressions, 0) 6842 6843 if not this and self._match(TokenType.R_PAREN, advance=False): 6844 this = self.expression(exp.Tuple()) 6845 elif isinstance(this, exp.UNWRAPPED_QUERIES): 6846 this = self._parse_subquery(this=this, parse_alias=False) 6847 elif isinstance(this, (exp.Subquery, exp.Values)): 6848 this = self._parse_subquery( 6849 this=self._parse_query_modifiers(self._parse_set_operations(this)), 6850 parse_alias=False, 6851 ) 6852 elif len(expressions) > 1 or self._prev.token_type == TokenType.COMMA: 6853 this = self.expression(exp.Tuple(expressions=expressions)) 6854 else: 6855 this = self.expression(exp.Paren(this=this)) 6856 6857 if this: 6858 this.add_comments(comments) 6859 6860 self._match_r_paren(expression=this) 6861 6862 if isinstance(this, exp.Paren) and isinstance(this.this, exp.AggFunc): 6863 return self._parse_window(this) 6864 6865 return this 6866 6867 def _parse_primary(self) -> exp.Expr | None: 6868 if self._match_set(self.PRIMARY_PARSERS): 6869 token_type = self._prev.token_type 6870 primary = self.PRIMARY_PARSERS[token_type](self, self._prev) 6871 6872 if token_type == TokenType.STRING: 6873 expressions = [primary] 6874 while self._match(TokenType.STRING, advance=False): 6875 if self._is_connected() and self.ADJACENT_STRINGS_CANNOT_BE_CONNECTED: 6876 self.raise_error( 6877 "Adjacent string literals need to be separated by whitespace or comments" 6878 ) 6879 6880 self._advance() 6881 expressions.append(exp.Literal.string(self._prev.text)) 6882 6883 if len(expressions) > 1: 6884 return self.expression( 6885 exp.Concat(expressions=expressions, coalesce=self.dialect.CONCAT_COALESCE) 6886 ) 6887 6888 return primary 6889 6890 if self._match_pair(TokenType.DOT, TokenType.NUMBER): 6891 return exp.Literal.number(f"0.{self._prev.text}") 6892 6893 return self._parse_paren() 6894 6895 def _parse_field( 6896 self, 6897 any_token: bool = False, 6898 tokens: t.Collection[TokenType] | None = None, 6899 anonymous_func: bool = False, 6900 ) -> exp.Expr | None: 6901 if anonymous_func: 6902 field = ( 6903 self._parse_function(anonymous=anonymous_func, any_token=any_token) 6904 or self._parse_primary() 6905 ) 6906 else: 6907 field = self._parse_primary() or self._parse_function( 6908 anonymous=anonymous_func, any_token=any_token 6909 ) 6910 return field or self._parse_id_var(any_token=any_token, tokens=tokens) 6911 6912 def _parse_function( 6913 self, 6914 functions: dict[str, t.Callable] | None = None, 6915 anonymous: bool = False, 6916 optional_parens: bool = True, 6917 any_token: bool = False, 6918 ) -> exp.Expr | None: 6919 # This allows us to also parse {fn <function>} syntax (Snowflake, MySQL support this) 6920 # See: https://community.snowflake.com/s/article/SQL-Escape-Sequences 6921 fn_syntax = False 6922 if ( 6923 self._match(TokenType.L_BRACE, advance=False) 6924 and self._next 6925 and self._next.text.upper() == "FN" 6926 ): 6927 self._advance(2) 6928 fn_syntax = True 6929 6930 func = self._parse_function_call( 6931 functions=functions, 6932 anonymous=anonymous, 6933 optional_parens=optional_parens, 6934 any_token=any_token, 6935 ) 6936 6937 if fn_syntax: 6938 self._match(TokenType.R_BRACE) 6939 6940 return func 6941 6942 def _parse_function_args(self, alias: bool = False) -> list[exp.Expr]: 6943 return self._parse_csv(lambda: self._parse_lambda(alias=alias)) 6944 6945 def _parse_function_call( 6946 self, 6947 functions: dict[str, t.Callable] | None = None, 6948 anonymous: bool = False, 6949 optional_parens: bool = True, 6950 any_token: bool = False, 6951 ) -> exp.Expr | None: 6952 if not self._curr: 6953 return None 6954 6955 comments = self._curr.comments 6956 prev = self._prev 6957 token = self._curr 6958 token_type = self._curr.token_type 6959 this: str | exp.Expr = self._curr.text 6960 upper = self._curr.text.upper() 6961 6962 after_dot = prev.token_type == TokenType.DOT 6963 parser = self.NO_PAREN_FUNCTION_PARSERS.get(upper) 6964 if ( 6965 optional_parens 6966 and parser 6967 and token_type not in self.INVALID_FUNC_NAME_TOKENS 6968 and not after_dot 6969 ): 6970 self._advance() 6971 return self._parse_window(parser(self)) 6972 6973 if self._next.token_type != TokenType.L_PAREN: 6974 if optional_parens and token_type in self.NO_PAREN_FUNCTIONS and not after_dot: 6975 self._advance() 6976 return self.expression(self.NO_PAREN_FUNCTIONS[token_type]()) 6977 6978 return None 6979 6980 if any_token: 6981 if token_type in self.RESERVED_TOKENS: 6982 return None 6983 elif token_type not in self.FUNC_TOKENS: 6984 return None 6985 6986 self._advance(2) 6987 6988 parser = self.FUNCTION_PARSERS.get(upper) 6989 if parser and not anonymous: 6990 result = parser(self) 6991 else: 6992 subquery_predicate = self.SUBQUERY_PREDICATES.get(token_type) 6993 6994 if subquery_predicate: 6995 expr = None 6996 if self._curr.token_type in self.SUBQUERY_TOKENS: 6997 expr = self._parse_select() 6998 self._match_r_paren() 6999 elif prev and prev.token_type in (TokenType.LIKE, TokenType.ILIKE): 7000 # Backtrack one token since we've consumed the L_PAREN here. Instead, we'd like 7001 # to parse "LIKE [ANY | ALL] (...)" as a whole into an exp.Tuple or exp.Paren 7002 self._advance(-1) 7003 expr = self._parse_bitwise() 7004 7005 if expr: 7006 return self.expression(subquery_predicate(this=expr), comments=comments) 7007 7008 if functions is None: 7009 functions = self.FUNCTIONS 7010 7011 function = functions.get(upper) 7012 known_function = function and not anonymous 7013 7014 alias = not known_function or upper in self.FUNCTIONS_WITH_ALIASED_ARGS 7015 args = self._parse_function_args(alias) 7016 7017 post_func_comments = self._curr.comments if self._curr else None 7018 if known_function and post_func_comments: 7019 # If the user-inputted comment "/* sqlglot.anonymous */" is following the function 7020 # call we'll construct it as exp.Anonymous, even if it's "known" 7021 if any( 7022 comment.lstrip().startswith(exp.SQLGLOT_ANONYMOUS) 7023 for comment in post_func_comments 7024 ): 7025 known_function = False 7026 7027 if alias and known_function: 7028 args = self._kv_to_prop_eq(args) 7029 7030 if known_function: 7031 func_builder = t.cast(t.Callable, function) 7032 7033 # mypyc compiled functions don't have __code__, so we use 7034 # try/except to check if func_builder accepts 'dialect'. 7035 try: 7036 func = func_builder(args) 7037 except TypeError: 7038 func = func_builder(args, dialect=self.dialect) 7039 7040 func = self.validate_expression(func, args) 7041 if self.dialect.PRESERVE_ORIGINAL_NAMES: 7042 func.meta["name"] = this 7043 7044 result = func 7045 else: 7046 if token_type == TokenType.IDENTIFIER: 7047 this = exp.Identifier(this=this, quoted=True).update_positions(token) 7048 7049 result = self.expression(exp.Anonymous(this=this, expressions=args)) 7050 7051 result = result.update_positions(token) 7052 7053 if isinstance(result, exp.Expr): 7054 result.add_comments(comments) 7055 7056 if parser: 7057 self._match(TokenType.R_PAREN, expression=result) 7058 else: 7059 self._match_r_paren(result) 7060 return self._parse_window(result) 7061 7062 def _to_prop_eq(self, expression: exp.Expr, index: int) -> exp.Expr: 7063 return expression 7064 7065 def _kv_to_prop_eq( 7066 self, expressions: list[exp.Expr], parse_map: bool = False 7067 ) -> list[exp.Expr]: 7068 transformed = [] 7069 7070 for index, e in enumerate(expressions): 7071 if isinstance(e, self.KEY_VALUE_DEFINITIONS): 7072 if isinstance(e, exp.Alias): 7073 e = self.expression(exp.PropertyEQ(this=e.args.get("alias"), expression=e.this)) 7074 7075 if not isinstance(e, exp.PropertyEQ): 7076 e = self.expression( 7077 exp.PropertyEQ( 7078 this=e.this if parse_map else exp.to_identifier(e.this.name), 7079 expression=e.expression, 7080 ) 7081 ) 7082 7083 if isinstance(e.this, exp.Column): 7084 e.this.replace(e.this.this) 7085 else: 7086 e = self._to_prop_eq(e, index) 7087 7088 transformed.append(e) 7089 7090 return transformed 7091 7092 def _parse_function_properties(self) -> exp.Properties | None: 7093 # Skip the generic `key = value` fallback in _parse_property since this 7094 # runs post-AS where a function body like `name = expr` can be misread 7095 # as a property. 7096 properties = [] 7097 while True: 7098 if self._match_texts(self.PROPERTY_PARSERS): 7099 prop = self.PROPERTY_PARSERS[self._prev.text.upper()](self) 7100 elif self._match(TokenType.DEFAULT) and self._match_texts(self.PROPERTY_PARSERS): 7101 prop = self.PROPERTY_PARSERS[self._prev.text.upper()](self, default=True) 7102 else: 7103 break 7104 for p in ensure_list(prop): 7105 properties.append(p) 7106 7107 return self.expression(exp.Properties(expressions=properties)) if properties else None 7108 7109 def _parse_user_defined_function_expression(self) -> exp.Expr | None: 7110 return self._parse_statement() 7111 7112 def _parse_function_parameter(self) -> exp.Expr | None: 7113 return self._parse_column_def(this=self._parse_id_var(), computed_column=False) 7114 7115 def _parse_user_defined_function(self, kind: TokenType | None = None) -> exp.Expr | None: 7116 this = self._parse_table_parts(schema=True) 7117 7118 if not self._match(TokenType.L_PAREN): 7119 return this 7120 7121 expressions = self._parse_csv(self._parse_function_parameter) 7122 self._match_r_paren() 7123 return self.expression( 7124 exp.UserDefinedFunction(this=this, expressions=expressions, wrapped=True) 7125 ) 7126 7127 def _parse_macro_overloads( 7128 self, 7129 this: exp.UserDefinedFunction, 7130 first_body: exp.Expr, 7131 first_is_table: bool = False, 7132 ) -> exp.MacroOverloads: 7133 overloads = [ 7134 self.expression( 7135 exp.MacroOverload( 7136 this=first_body, 7137 expressions=this.expressions or None, 7138 is_table=first_is_table, 7139 ) 7140 ) 7141 ] 7142 this.set("expressions", None) 7143 this.set("wrapped", False) 7144 7145 while self._match(TokenType.COMMA): 7146 if not self._match(TokenType.L_PAREN): 7147 break 7148 7149 params = self._parse_csv(self._parse_function_parameter) 7150 self._match_r_paren() 7151 7152 if not self._match(TokenType.ALIAS): 7153 break 7154 7155 is_table = self._match(TokenType.TABLE) 7156 body = self._parse_expression() 7157 macro = exp.MacroOverload(this=body, expressions=params, is_table=is_table) 7158 overloads.append(self.expression(macro)) 7159 7160 return self.expression(exp.MacroOverloads(expressions=overloads)) 7161 7162 def _parse_introducer(self, token: Token) -> exp.Introducer | exp.Identifier: 7163 literal = self._parse_primary() 7164 if literal: 7165 return self.expression(exp.Introducer(this=token.text, expression=literal), token) 7166 7167 return self._identifier_expression(token) 7168 7169 def _parse_session_parameter(self) -> exp.SessionParameter: 7170 kind = None 7171 this = self._parse_id_var() or self._parse_primary() 7172 7173 if this and self._match(TokenType.DOT): 7174 kind = this.name 7175 this = self._parse_var() or self._parse_primary() 7176 7177 return self.expression(exp.SessionParameter(this=this, kind=kind)) 7178 7179 def _parse_lambda_arg(self) -> exp.Expr | None: 7180 return self._parse_id_var() 7181 7182 def _parse_lambda(self, alias: bool = False) -> exp.Expr | None: 7183 next_token_type = self._next.token_type 7184 7185 # Fast path: simple atom (column, literal, null, bool) followed by , or ) 7186 if ( 7187 next_token_type in self.LAMBDA_ARG_TERMINATORS 7188 and (atom := self._parse_atom()) is not None 7189 ): 7190 return atom 7191 7192 index = self._index 7193 7194 if self._match(TokenType.L_PAREN): 7195 expressions = t.cast( 7196 list[t.Optional[exp.Expr]], self._parse_csv(self._parse_lambda_arg) 7197 ) 7198 7199 if not self._match(TokenType.R_PAREN): 7200 self._retreat(index) 7201 elif self._match_set(self.LAMBDAS): 7202 return self.LAMBDAS[self._prev.token_type](self, expressions) 7203 else: 7204 self._retreat(index) 7205 elif self.TYPED_LAMBDA_ARGS or next_token_type in self.LAMBDAS: 7206 expressions = [self._parse_lambda_arg()] 7207 7208 if self._match_set(self.LAMBDAS): 7209 return self.LAMBDAS[self._prev.token_type](self, expressions) 7210 7211 self._retreat(index) 7212 7213 this: exp.Expr | None 7214 7215 if self._match(TokenType.DISTINCT): 7216 this = self.expression( 7217 exp.Distinct(expressions=self._parse_csv(self._parse_disjunction)) 7218 ) 7219 else: 7220 self._match(TokenType.ALL) # ALL is the default/no-op aggregate modifier (SQL-92) 7221 this = self._parse_select_or_expression(alias=alias) 7222 7223 return self._parse_limit( 7224 self._parse_respect_or_ignore_nulls( 7225 self._parse_order(self._parse_having_max(self._parse_respect_or_ignore_nulls(this))) 7226 ) 7227 ) 7228 7229 def _parse_schema(self, this: exp.Expr | None = None) -> exp.Expr | None: 7230 index = self._index 7231 if not self._match(TokenType.L_PAREN): 7232 return this 7233 7234 # Disambiguate between schema and subquery/CTE, e.g. in INSERT INTO table (<expr>), 7235 # expr can be of both types 7236 if self._match_set(self.SELECT_START_TOKENS): 7237 self._retreat(index) 7238 return this 7239 args = self._parse_csv(lambda: self._parse_constraint() or self._parse_field_def()) 7240 self._match_r_paren() 7241 return self.expression(exp.Schema(this=this, expressions=args)) 7242 7243 def _parse_field_def(self) -> exp.Expr | None: 7244 return self._parse_column_def(self._parse_field(any_token=True)) 7245 7246 def _parse_column_def( 7247 self, this: exp.Expr | None, computed_column: bool = True 7248 ) -> exp.Expr | None: 7249 # column defs are not really columns, they're identifiers 7250 if isinstance(this, exp.Column): 7251 this = this.this 7252 7253 if not computed_column: 7254 self._match(TokenType.ALIAS) 7255 7256 kind = self._parse_types(schema=True) 7257 7258 if self._match_text_seq("FOR", "ORDINALITY"): 7259 return self.expression(exp.ColumnDef(this=this, ordinality=True)) 7260 7261 constraints: list[exp.Expr] = [] 7262 7263 if (not kind and self._match(TokenType.ALIAS)) or self._match_texts( 7264 ("ALIAS", "MATERIALIZED") 7265 ): 7266 persisted = self._prev.text.upper() == "MATERIALIZED" 7267 constraint_kind = exp.ComputedColumnConstraint( 7268 this=self._parse_disjunction(), 7269 persisted=persisted or self._match_text_seq("PERSISTED"), 7270 data_type=exp.Var(this="AUTO") 7271 if self._match_text_seq("AUTO") 7272 else self._parse_types(), 7273 not_null=self._match_pair(TokenType.NOT, TokenType.NULL), 7274 ) 7275 constraints.append(self.expression(exp.ColumnConstraint(kind=constraint_kind))) 7276 elif not kind and self._match_set({TokenType.IN, TokenType.OUT}, advance=False): 7277 in_out_constraint = self.expression( 7278 exp.InOutColumnConstraint( 7279 input_=self._match(TokenType.IN), output=self._match(TokenType.OUT) 7280 ) 7281 ) 7282 constraints.append(in_out_constraint) 7283 kind = self._parse_types() 7284 elif ( 7285 kind 7286 and self._match(TokenType.ALIAS, advance=False) 7287 and ( 7288 not self.WRAPPED_TRANSFORM_COLUMN_CONSTRAINT 7289 or self._next.token_type == TokenType.L_PAREN 7290 ) 7291 ): 7292 self._advance() 7293 constraints.append( 7294 self.expression( 7295 exp.ColumnConstraint( 7296 kind=exp.ComputedColumnConstraint( 7297 this=self._parse_disjunction(), 7298 persisted=self._match_texts(("STORED", "VIRTUAL")) 7299 and self._prev.text.upper() == "STORED", 7300 ) 7301 ) 7302 ) 7303 ) 7304 7305 while True: 7306 constraint = self._parse_column_constraint() 7307 if not constraint: 7308 break 7309 constraints.append(constraint) 7310 7311 if not kind and not constraints: 7312 return this 7313 7314 position = None 7315 if self._match_texts(("FIRST", "AFTER")): 7316 pos = self._prev.text 7317 position = self.expression(exp.ColumnPosition(this=self._parse_column(), position=pos)) 7318 7319 return self.expression( 7320 exp.ColumnDef(this=this, kind=kind, constraints=constraints, position=position) 7321 ) 7322 7323 def _parse_auto_increment( 7324 self, 7325 ) -> exp.GeneratedAsIdentityColumnConstraint | exp.AutoIncrementColumnConstraint: 7326 start = None 7327 increment = None 7328 order = None 7329 7330 if self._match(TokenType.L_PAREN, advance=False): 7331 args = self._parse_wrapped_csv(self._parse_bitwise) 7332 start = seq_get(args, 0) 7333 increment = seq_get(args, 1) 7334 elif self._match_text_seq("START"): 7335 start = self._parse_bitwise() 7336 self._match_text_seq("INCREMENT") 7337 increment = self._parse_bitwise() 7338 if self._match_text_seq("ORDER"): 7339 order = True 7340 elif self._match_text_seq("NOORDER"): 7341 order = False 7342 7343 if start and increment: 7344 return exp.GeneratedAsIdentityColumnConstraint( 7345 start=start, increment=increment, this=False, order=order 7346 ) 7347 7348 return exp.AutoIncrementColumnConstraint() 7349 7350 def _parse_check_constraint(self) -> exp.CheckColumnConstraint | None: 7351 if not self._match(TokenType.L_PAREN, advance=False): 7352 return None 7353 7354 return self.expression( 7355 exp.CheckColumnConstraint( 7356 this=self._parse_wrapped(self._parse_assignment), 7357 enforced=self._match_text_seq("ENFORCED"), 7358 ) 7359 ) 7360 7361 def _parse_auto_property(self) -> exp.AutoRefreshProperty | None: 7362 if not self._match_text_seq("REFRESH"): 7363 self._retreat(self._index - 1) 7364 return None 7365 return self.expression(exp.AutoRefreshProperty(this=self._parse_var(upper=True))) 7366 7367 def _parse_compress(self) -> exp.CompressColumnConstraint: 7368 if self._match(TokenType.L_PAREN, advance=False): 7369 return self.expression( 7370 exp.CompressColumnConstraint(this=self._parse_wrapped_csv(self._parse_bitwise)) 7371 ) 7372 7373 return self.expression(exp.CompressColumnConstraint(this=self._parse_bitwise())) 7374 7375 def _parse_generated_as_identity( 7376 self, 7377 ) -> ( 7378 exp.GeneratedAsIdentityColumnConstraint 7379 | exp.ComputedColumnConstraint 7380 | exp.GeneratedAsRowColumnConstraint 7381 ): 7382 if self._match_text_seq("BY", "DEFAULT"): 7383 on_null = self._match_pair(TokenType.ON, TokenType.NULL) 7384 this = self.expression( 7385 exp.GeneratedAsIdentityColumnConstraint(this=False, on_null=on_null) 7386 ) 7387 else: 7388 self._match_text_seq("ALWAYS") 7389 this = self.expression(exp.GeneratedAsIdentityColumnConstraint(this=True)) 7390 7391 self._match(TokenType.ALIAS) 7392 7393 if self._match_text_seq("ROW"): 7394 start = self._match_text_seq("START") 7395 if not start: 7396 self._match(TokenType.END) 7397 hidden = self._match_text_seq("HIDDEN") 7398 return self.expression(exp.GeneratedAsRowColumnConstraint(start=start, hidden=hidden)) 7399 7400 identity = self._match_text_seq("IDENTITY") 7401 7402 if self._match(TokenType.L_PAREN): 7403 if self._match(TokenType.START_WITH): 7404 this.set("start", self._parse_bitwise()) 7405 if self._match_text_seq("INCREMENT", "BY"): 7406 this.set("increment", self._parse_bitwise()) 7407 if self._match_text_seq("MINVALUE"): 7408 this.set("minvalue", self._parse_bitwise()) 7409 if self._match_text_seq("MAXVALUE"): 7410 this.set("maxvalue", self._parse_bitwise()) 7411 7412 if self._match_text_seq("CYCLE"): 7413 this.set("cycle", True) 7414 elif self._match_text_seq("NO", "CYCLE"): 7415 this.set("cycle", False) 7416 7417 if not identity: 7418 this.set("expression", self._parse_range()) 7419 elif not this.args.get("start") and self._match(TokenType.NUMBER, advance=False): 7420 args = self._parse_csv(self._parse_bitwise) 7421 this.set("start", seq_get(args, 0)) 7422 this.set("increment", seq_get(args, 1)) 7423 7424 self._match_r_paren() 7425 7426 return this 7427 7428 def _parse_inline(self) -> exp.InlineLengthColumnConstraint: 7429 self._match_text_seq("LENGTH") 7430 return self.expression(exp.InlineLengthColumnConstraint(this=self._parse_bitwise())) 7431 7432 def _parse_not_constraint(self) -> exp.Expr | None: 7433 if self._match_text_seq("NULL"): 7434 return self.expression(exp.NotNullColumnConstraint()) 7435 if self._match_text_seq("CASESPECIFIC"): 7436 return self.expression(exp.CaseSpecificColumnConstraint(not_=True)) 7437 if self._match_text_seq("FOR", "REPLICATION"): 7438 return self.expression(exp.NotForReplicationColumnConstraint()) 7439 7440 # Unconsume the `NOT` token 7441 self._retreat(self._index - 1) 7442 return None 7443 7444 def _parse_column_constraint(self) -> exp.Expr | None: 7445 this = self._parse_id_var() if self._match(TokenType.CONSTRAINT) else None 7446 7447 procedure_option_follows = ( 7448 self._match(TokenType.WITH, advance=False) 7449 and self._next 7450 and self._next.text.upper() in self.PROCEDURE_OPTIONS 7451 ) 7452 7453 if not procedure_option_follows and self._match_texts(self.CONSTRAINT_PARSERS): 7454 constraint = self.CONSTRAINT_PARSERS[self._prev.text.upper()](self) 7455 if not constraint: 7456 self._retreat(self._index - 1) 7457 return None 7458 7459 return self.expression(exp.ColumnConstraint(this=this, kind=constraint)) 7460 7461 return this 7462 7463 def _parse_constraint(self) -> exp.Expr | None: 7464 if not self._match(TokenType.CONSTRAINT): 7465 return self._parse_unnamed_constraint(constraints=self.SCHEMA_UNNAMED_CONSTRAINTS) 7466 7467 return self.expression( 7468 exp.Constraint(this=self._parse_id_var(), expressions=self._parse_unnamed_constraints()) 7469 ) 7470 7471 def _parse_unnamed_constraints(self) -> list[exp.Expr]: 7472 constraints = [] 7473 while True: 7474 constraint = self._parse_unnamed_constraint() or self._parse_function() 7475 if not constraint: 7476 break 7477 constraints.append(constraint) 7478 7479 return constraints 7480 7481 def _parse_unnamed_constraint( 7482 self, constraints: t.Collection[str] | None = None 7483 ) -> exp.Expr | None: 7484 index = self._index 7485 7486 if self._match(TokenType.IDENTIFIER, advance=False) or not self._match_texts( 7487 constraints or self.CONSTRAINT_PARSERS 7488 ): 7489 return None 7490 7491 constraint_key = self._prev.text.upper() 7492 if constraint_key not in self.CONSTRAINT_PARSERS: 7493 self.raise_error(f"No parser found for schema constraint {constraint_key}.") 7494 7495 result = self.CONSTRAINT_PARSERS[constraint_key](self) 7496 if not result: 7497 self._retreat(index) 7498 7499 return result 7500 7501 def _parse_unique_key(self) -> exp.Expr | None: 7502 if ( 7503 self._curr 7504 and self._curr.token_type != TokenType.IDENTIFIER 7505 and self._curr.text.upper() in self.CONSTRAINT_PARSERS 7506 ): 7507 return None 7508 return self._parse_id_var(any_token=False) 7509 7510 def _parse_unique(self) -> exp.UniqueColumnConstraint: 7511 self._match_texts(("KEY", "INDEX")) 7512 return self.expression( 7513 exp.UniqueColumnConstraint( 7514 nulls=self._match_text_seq("NULLS", "NOT", "DISTINCT"), 7515 this=self._parse_schema(self._parse_unique_key()), 7516 index_type=self._match(TokenType.USING) and self._advance_any() and self._prev.text, 7517 on_conflict=self._parse_on_conflict(), 7518 options=self._parse_key_constraint_options(), 7519 ) 7520 ) 7521 7522 def _parse_key_constraint_options(self) -> list[str]: 7523 options = [] 7524 while True: 7525 if not self._curr: 7526 break 7527 7528 if self._match(TokenType.ON): 7529 action = None 7530 on = self._advance_any() and self._prev.text 7531 7532 if self._match_text_seq("NO", "ACTION"): 7533 action = "NO ACTION" 7534 elif self._match_text_seq("CASCADE"): 7535 action = "CASCADE" 7536 elif self._match_text_seq("RESTRICT"): 7537 action = "RESTRICT" 7538 elif self._match_pair(TokenType.SET, TokenType.NULL): 7539 action = "SET NULL" 7540 elif self._match_pair(TokenType.SET, TokenType.DEFAULT): 7541 action = "SET DEFAULT" 7542 else: 7543 self.raise_error("Invalid key constraint") 7544 7545 options.append(f"ON {on} {action}") 7546 else: 7547 var = self._parse_var_from_options( 7548 self.KEY_CONSTRAINT_OPTIONS, raise_unmatched=False 7549 ) 7550 if not var: 7551 break 7552 options.append(var.name) 7553 7554 return options 7555 7556 def _parse_references(self, match: bool = True) -> exp.Reference | None: 7557 if match and not self._match(TokenType.REFERENCES): 7558 return None 7559 7560 expressions: list | None = None 7561 this = self._parse_table(schema=True) 7562 options = self._parse_key_constraint_options() 7563 return self.expression(exp.Reference(this=this, expressions=expressions, options=options)) 7564 7565 def _parse_foreign_key(self) -> exp.ForeignKey: 7566 expressions = ( 7567 self._parse_wrapped_id_vars() 7568 if not self._match(TokenType.REFERENCES, advance=False) 7569 else None 7570 ) 7571 reference = self._parse_references() 7572 on_options = {} 7573 7574 while self._match(TokenType.ON): 7575 if not self._match_set((TokenType.DELETE, TokenType.UPDATE)): 7576 self.raise_error("Expected DELETE or UPDATE") 7577 7578 kind = self._prev.text.lower() 7579 7580 if self._match_text_seq("NO", "ACTION"): 7581 action = "NO ACTION" 7582 elif self._match(TokenType.SET): 7583 self._match_set((TokenType.NULL, TokenType.DEFAULT)) 7584 action = "SET " + self._prev.text.upper() 7585 else: 7586 self._advance() 7587 action = self._prev.text.upper() 7588 7589 on_options[kind] = action 7590 7591 return self.expression( 7592 exp.ForeignKey( 7593 expressions=expressions, 7594 reference=reference, 7595 options=self._parse_key_constraint_options(), 7596 **on_options, 7597 ) 7598 ) 7599 7600 def _parse_primary_key_part(self) -> exp.Expr | None: 7601 return self._parse_field() 7602 7603 def _parse_period_for_system_time(self) -> exp.PeriodForSystemTimeConstraint | None: 7604 if not self._match(TokenType.TIMESTAMP_SNAPSHOT): 7605 self._retreat(self._index - 1) 7606 return None 7607 7608 id_vars = self._parse_wrapped_id_vars() 7609 return self.expression( 7610 exp.PeriodForSystemTimeConstraint( 7611 this=seq_get(id_vars, 0), expression=seq_get(id_vars, 1) 7612 ) 7613 ) 7614 7615 def _parse_primary_key( 7616 self, 7617 wrapped_optional: bool = False, 7618 in_props: bool = False, 7619 named_primary_key: bool = False, 7620 ) -> exp.PrimaryKeyColumnConstraint | exp.PrimaryKey: 7621 desc = ( 7622 self._prev.token_type == TokenType.DESC 7623 if self._match_set((TokenType.ASC, TokenType.DESC)) 7624 else None 7625 ) 7626 7627 this = None 7628 if ( 7629 named_primary_key 7630 and self._curr.text.upper() not in self.CONSTRAINT_PARSERS 7631 and self._next 7632 and self._next.token_type == TokenType.L_PAREN 7633 ): 7634 this = self._parse_id_var() 7635 7636 if not in_props and not self._match(TokenType.L_PAREN, advance=False): 7637 return self.expression( 7638 exp.PrimaryKeyColumnConstraint( 7639 desc=desc, options=self._parse_key_constraint_options() 7640 ) 7641 ) 7642 7643 expressions = self._parse_wrapped_csv( 7644 self._parse_primary_key_part, optional=wrapped_optional 7645 ) 7646 7647 return self.expression( 7648 exp.PrimaryKey( 7649 this=this, 7650 expressions=expressions, 7651 include=self._parse_index_params(), 7652 options=self._parse_key_constraint_options(), 7653 ) 7654 ) 7655 7656 def _parse_bracket_key_value(self, is_map: bool = False) -> exp.Expr | None: 7657 return self._parse_slice(self._parse_alias(self._parse_disjunction(), explicit=True)) 7658 7659 def _parse_odbc_datetime_literal(self) -> exp.Expr: 7660 """ 7661 Parses a datetime column in ODBC format. We parse the column into the corresponding 7662 types, for example `{d'yyyy-mm-dd'}` will be parsed as a `Date` column, exactly the 7663 same as we did for `DATE('yyyy-mm-dd')`. 7664 7665 Reference: 7666 https://learn.microsoft.com/en-us/sql/odbc/reference/develop-app/date-time-and-timestamp-literals 7667 """ 7668 self._match(TokenType.VAR) 7669 exp_class = self.ODBC_DATETIME_LITERALS[self._prev.text.lower()] 7670 expression = self.expression(exp_class(this=self._parse_string())) 7671 if not self._match(TokenType.R_BRACE): 7672 self.raise_error("Expected }") 7673 return expression 7674 7675 def _parse_bracket(self, this: exp.Expr | None = None) -> exp.Expr | None: 7676 if not self._match_set(self.BRACKETS): 7677 return this 7678 7679 if self.MAP_KEYS_ARE_ARBITRARY_EXPRESSIONS: 7680 map_token = seq_get(self._tokens, self._index - 2) 7681 parse_map = map_token is not None and map_token.text.upper() == "MAP" 7682 else: 7683 parse_map = False 7684 7685 bracket_kind = self._prev.token_type 7686 if ( 7687 bracket_kind == TokenType.L_BRACE 7688 and self._curr 7689 and self._curr.token_type == TokenType.VAR 7690 and self._curr.text.lower() in self.ODBC_DATETIME_LITERALS 7691 ): 7692 return self._parse_odbc_datetime_literal() 7693 7694 expressions = self._parse_csv( 7695 lambda: self._parse_bracket_key_value(is_map=bracket_kind == TokenType.L_BRACE) 7696 ) 7697 7698 if bracket_kind == TokenType.L_BRACKET and not self._match(TokenType.R_BRACKET): 7699 self.raise_error("Expected ]") 7700 elif bracket_kind == TokenType.L_BRACE and not self._match(TokenType.R_BRACE): 7701 self.raise_error("Expected }") 7702 7703 # https://duckdb.org/docs/sql/data_types/struct.html#creating-structs 7704 if bracket_kind == TokenType.L_BRACE: 7705 this = self.expression( 7706 exp.Struct( 7707 expressions=self._kv_to_prop_eq(expressions=expressions, parse_map=parse_map) 7708 ) 7709 ) 7710 elif not this: 7711 this = build_array_constructor( 7712 exp.Array, args=expressions, bracket_kind=bracket_kind, dialect=self.dialect 7713 ) 7714 else: 7715 constructor_type = self.ARRAY_CONSTRUCTORS.get(this.name.upper()) 7716 if constructor_type: 7717 return build_array_constructor( 7718 constructor_type, 7719 args=expressions, 7720 bracket_kind=bracket_kind, 7721 dialect=self.dialect, 7722 ) 7723 7724 expressions = apply_index_offset( 7725 this, expressions, -self.dialect.INDEX_OFFSET, dialect=self.dialect 7726 ) 7727 this = self.expression( 7728 exp.Bracket(this=this, expressions=expressions), comments=this.pop_comments() 7729 ) 7730 7731 self._add_comments(this) 7732 return self._parse_bracket(this) 7733 7734 def _parse_slice(self, this: exp.Expr | None) -> exp.Expr | None: 7735 if not self._match(TokenType.COLON): 7736 return this 7737 7738 if self._match_pair(TokenType.DASH, TokenType.COLON, advance=False): 7739 self._advance() 7740 end: exp.Expr | None = -exp.Literal.number("1") 7741 else: 7742 end = self._parse_assignment() 7743 step = self._parse_unary() if self._match(TokenType.COLON) else None 7744 return self.expression(exp.Slice(this=this, expression=end, step=step)) 7745 7746 def _parse_case(self) -> exp.Expr | None: 7747 if self._match(TokenType.DOT, advance=False): 7748 # Avoid raising on valid expressions like case.*, supported by, e.g., spark & snowflake 7749 self._retreat(self._index - 1) 7750 return None 7751 7752 ifs = [] 7753 default = None 7754 7755 comments = self._prev_comments 7756 expression = self._parse_disjunction() 7757 7758 while self._match(TokenType.WHEN): 7759 this = self._parse_disjunction() 7760 self._match(TokenType.THEN) 7761 then = self._parse_disjunction() 7762 ifs.append(self.expression(exp.If(this=this, true=then))) 7763 7764 if self._match(TokenType.ELSE): 7765 default = self._parse_disjunction() 7766 7767 if not self._match(TokenType.END): 7768 if isinstance(default, exp.Interval) and default.this.sql().upper() == "END": 7769 default = exp.column("interval") 7770 else: 7771 self.raise_error("Expected END after CASE", self._prev) 7772 7773 return self.expression( 7774 exp.Case(this=expression, ifs=ifs, default=default), comments=comments 7775 ) 7776 7777 def _parse_if(self) -> exp.Expr | None: 7778 if self._match(TokenType.L_PAREN): 7779 args = self._parse_csv( 7780 lambda: self._parse_alias(self._parse_assignment(), explicit=True) 7781 ) 7782 this = self.validate_expression(exp.If.from_arg_list(args), args) 7783 self._match_r_paren() 7784 else: 7785 index = self._index - 1 7786 7787 if self.NO_PAREN_IF_COMMANDS and index == 0: 7788 return self._parse_as_command(self._prev) 7789 7790 condition = self._parse_disjunction() 7791 7792 if not condition: 7793 self._retreat(index) 7794 return None 7795 7796 self._match(TokenType.THEN) 7797 true = self._parse_disjunction() 7798 false = self._parse_disjunction() if self._match(TokenType.ELSE) else None 7799 self._match(TokenType.END) 7800 this = self.expression(exp.If(this=condition, true=true, false=false)) 7801 7802 return this 7803 7804 def _parse_next_value_for(self) -> exp.Expr | None: 7805 if not self._match_text_seq("VALUE", "FOR"): 7806 self._retreat(self._index - 1) 7807 return None 7808 7809 return self.expression( 7810 exp.NextValueFor( 7811 this=self._parse_column(), 7812 order=self._match(TokenType.OVER) and self._parse_wrapped(self._parse_order), 7813 ) 7814 ) 7815 7816 def _parse_extract(self) -> exp.Extract | exp.Anonymous: 7817 this = self._parse_function() or self._parse_var_or_string(upper=True) 7818 7819 if self._match(TokenType.FROM): 7820 return self.expression(exp.Extract(this=this, expression=self._parse_bitwise())) 7821 7822 if not self._match(TokenType.COMMA): 7823 self.raise_error("Expected FROM or comma after EXTRACT", self._prev) 7824 7825 return self.expression(exp.Extract(this=this, expression=self._parse_bitwise())) 7826 7827 def _parse_gap_fill(self) -> exp.GapFill: 7828 self._match(TokenType.TABLE) 7829 this = self._parse_table() 7830 7831 self._match(TokenType.COMMA) 7832 args = [this, *self._parse_csv(self._parse_lambda)] 7833 7834 gap_fill = exp.GapFill.from_arg_list(args) 7835 return self.validate_expression(gap_fill, args) 7836 7837 def _parse_char(self) -> exp.Chr: 7838 return self.expression( 7839 exp.Chr( 7840 expressions=self._parse_csv(self._parse_assignment), 7841 charset=self._match(TokenType.USING) and self._parse_charset_name(), 7842 ) 7843 ) 7844 7845 def _parse_charset_name(self) -> exp.Expr | None: 7846 """ 7847 Parse a charset name after USING or CHARACTER SET. Dialects that need to preserve quoting 7848 for specific name shapes override this. 7849 """ 7850 return self._parse_var( 7851 tokens={TokenType.BINARY, TokenType.IDENTIFIER}, 7852 ) 7853 7854 def _parse_cast(self, strict: bool, safe: bool | None = None) -> exp.Expr: 7855 this = self._parse_assignment() 7856 7857 if not self._match(TokenType.ALIAS): 7858 if self._match(TokenType.COMMA): 7859 return self.expression(exp.CastToStrType(this=this, to=self._parse_string())) 7860 7861 self.raise_error("Expected AS after CAST") 7862 7863 fmt = None 7864 to = self._parse_types(with_collation=True) 7865 7866 default = None 7867 if self._match(TokenType.DEFAULT): 7868 default = self._parse_bitwise() 7869 self._match_text_seq("ON", "CONVERSION", "ERROR") 7870 7871 if self._match_set((TokenType.FORMAT, TokenType.COMMA)): 7872 fmt_string = self._parse_wrapped(self._parse_string, optional=True) 7873 fmt = self._parse_at_time_zone(fmt_string) 7874 7875 if not to: 7876 to = exp.DType.UNKNOWN.into_expr() 7877 if to.this in exp.DataType.TEMPORAL_TYPES: 7878 this = self.expression( 7879 (exp.StrToDate if to.this == exp.DType.DATE else exp.StrToTime)( 7880 this=this, 7881 format=exp.Literal.string( 7882 format_time( 7883 fmt_string.this if fmt_string else "", 7884 self.dialect.FORMAT_MAPPING or self.dialect.TIME_MAPPING, 7885 self.dialect.FORMAT_TRIE or self.dialect.TIME_TRIE, 7886 ) 7887 ), 7888 safe=safe, 7889 ) 7890 ) 7891 7892 if isinstance(fmt, exp.AtTimeZone) and isinstance(this, exp.StrToTime): 7893 this.set("zone", fmt.args["zone"]) 7894 return this 7895 elif not to: 7896 self.raise_error("Expected TYPE after CAST") 7897 elif isinstance(to, exp.Identifier): 7898 to = exp.DataType.from_str(to.name, dialect=self.dialect, udt=True) 7899 elif to.this == exp.DType.CHAR and self._match(TokenType.CHARACTER_SET): 7900 to = exp.DType.CHARACTER_SET.into_expr(kind=self._parse_var_or_string()) 7901 7902 return self.build_cast( 7903 strict=strict, 7904 this=this, 7905 to=to, 7906 format=fmt, 7907 safe=safe, 7908 action=self._parse_var_from_options(self.CAST_ACTIONS, raise_unmatched=False), 7909 default=default, 7910 ) 7911 7912 def _parse_string_agg(self) -> exp.GroupConcat: 7913 if self._match(TokenType.DISTINCT): 7914 args: list[exp.Expr | None] = [ 7915 self.expression(exp.Distinct(expressions=[self._parse_disjunction()])) 7916 ] 7917 if self._match(TokenType.COMMA): 7918 args.extend(self._parse_csv(self._parse_disjunction)) 7919 else: 7920 args = self._parse_csv(self._parse_disjunction) # type: ignore 7921 7922 if self._match_text_seq("ON", "OVERFLOW"): 7923 # trino: LISTAGG(expression [, separator] [ON OVERFLOW overflow_behavior]) 7924 if self._match_text_seq("ERROR"): 7925 on_overflow: exp.Expr | None = exp.var("ERROR") 7926 else: 7927 self._match_text_seq("TRUNCATE") 7928 on_overflow = self.expression( 7929 exp.OverflowTruncateBehavior( 7930 this=self._parse_string(), 7931 with_count=( 7932 self._match_text_seq("WITH", "COUNT") 7933 or not self._match_text_seq("WITHOUT", "COUNT") 7934 ), 7935 ) 7936 ) 7937 else: 7938 on_overflow = None 7939 7940 index = self._index 7941 if not self._match(TokenType.R_PAREN) and args: 7942 # postgres: STRING_AGG([DISTINCT] expression, separator [ORDER BY expression1 {ASC | DESC} [, ...]]) 7943 # bigquery: STRING_AGG([DISTINCT] expression [, separator] [ORDER BY key [{ASC | DESC}] [, ... ]] [LIMIT n]) 7944 # The order is parsed through `this` as a canonicalization for WITHIN GROUPs 7945 args[0] = self._parse_limit(this=self._parse_order(this=args[0])) 7946 return self.expression(exp.GroupConcat(this=args[0], separator=seq_get(args, 1))) 7947 7948 # Checks if we can parse an order clause: WITHIN GROUP (ORDER BY <order_by_expression_list> [ASC | DESC]). 7949 # This is done "manually", instead of letting _parse_window parse it into an exp.WithinGroup node, so that 7950 # the STRING_AGG call is parsed like in MySQL / SQLite and can thus be transpiled more easily to them. 7951 if not self._match_text_seq("WITHIN", "GROUP"): 7952 self._retreat(index) 7953 return self.validate_expression(exp.GroupConcat.from_arg_list(args), args) 7954 7955 # The corresponding match_r_paren will be called in parse_function (caller) 7956 self._match_l_paren() 7957 7958 return self.expression( 7959 exp.GroupConcat( 7960 this=self._parse_order(this=seq_get(args, 0)), 7961 separator=seq_get(args, 1), 7962 on_overflow=on_overflow, 7963 ) 7964 ) 7965 7966 def _parse_convert(self, strict: bool, safe: bool | None = None) -> exp.Expr | None: 7967 this = self._parse_bitwise() 7968 7969 if self._match(TokenType.USING): 7970 to: exp.Expr | None = exp.DType.CHARACTER_SET.into_expr(kind=self._parse_charset_name()) 7971 elif self._match(TokenType.COMMA): 7972 to = self._parse_types() 7973 else: 7974 to = None 7975 7976 return self.build_cast(strict=strict, this=this, to=to, safe=safe) 7977 7978 def _parse_xml_element(self) -> exp.XMLElement: 7979 if self._match_text_seq("EVALNAME"): 7980 evalname = True 7981 this = self._parse_bitwise() 7982 else: 7983 evalname = None 7984 self._match_text_seq("NAME") 7985 this = self._parse_id_var() 7986 7987 return self.expression( 7988 exp.XMLElement( 7989 this=this, 7990 expressions=self._match(TokenType.COMMA) and self._parse_csv(self._parse_bitwise), 7991 evalname=evalname, 7992 ) 7993 ) 7994 7995 def _parse_xml_table(self) -> exp.XMLTable: 7996 namespaces = None 7997 passing = None 7998 columns = None 7999 8000 if self._match_text_seq("XMLNAMESPACES", "("): 8001 namespaces = self._parse_xml_namespace() 8002 self._match_text_seq(")", ",") 8003 8004 this = self._parse_string() 8005 8006 if self._match_text_seq("PASSING"): 8007 # The BY VALUE keywords are optional and are provided for semantic clarity 8008 self._match_text_seq("BY", "VALUE") 8009 passing = self._parse_csv(self._parse_column) 8010 8011 by_ref = self._match_text_seq("RETURNING", "SEQUENCE", "BY", "REF") 8012 8013 if self._match_text_seq("COLUMNS"): 8014 columns = self._parse_csv(self._parse_field_def) 8015 8016 return self.expression( 8017 exp.XMLTable( 8018 this=this, namespaces=namespaces, passing=passing, columns=columns, by_ref=by_ref 8019 ) 8020 ) 8021 8022 def _parse_xml_namespace(self) -> list[exp.XMLNamespace]: 8023 namespaces = [] 8024 8025 while True: 8026 if self._match(TokenType.DEFAULT): 8027 uri = self._parse_string() 8028 else: 8029 uri = self._parse_alias(self._parse_string()) 8030 namespaces.append(self.expression(exp.XMLNamespace(this=uri))) 8031 if not self._match(TokenType.COMMA): 8032 break 8033 8034 return namespaces 8035 8036 def _parse_decode(self) -> exp.Decode | exp.DecodeCase | None: 8037 args = self._parse_csv(self._parse_disjunction) 8038 8039 if len(args) < 3: 8040 return self.expression(exp.Decode(this=seq_get(args, 0), charset=seq_get(args, 1))) 8041 8042 return self.expression(exp.DecodeCase(expressions=args)) 8043 8044 def _parse_json_key_value(self) -> exp.JSONKeyValue | None: 8045 self._match_text_seq("KEY") 8046 key = self._parse_column() 8047 self._match_set(self.JSON_KEY_VALUE_SEPARATOR_TOKENS) 8048 self._match_text_seq("VALUE") 8049 value = self._parse_bitwise() 8050 8051 if not key and not value: 8052 return None 8053 return self.expression(exp.JSONKeyValue(this=key, expression=value)) 8054 8055 def _parse_format_json(self, this: exp.Expr | None) -> exp.Expr | None: 8056 if not this or not self._match_text_seq("FORMAT", "JSON"): 8057 return this 8058 8059 return self.expression(exp.FormatJson(this=this)) 8060 8061 def _parse_on_condition(self) -> exp.OnCondition | None: 8062 # MySQL uses "X ON EMPTY Y ON ERROR" (e.g. JSON_VALUE) while Oracle uses the opposite (e.g. JSON_EXISTS) 8063 if self.dialect.ON_CONDITION_EMPTY_BEFORE_ERROR: 8064 empty = self._parse_on_handling("EMPTY", *self.ON_CONDITION_TOKENS) 8065 error = self._parse_on_handling("ERROR", *self.ON_CONDITION_TOKENS) 8066 else: 8067 error = self._parse_on_handling("ERROR", *self.ON_CONDITION_TOKENS) 8068 empty = self._parse_on_handling("EMPTY", *self.ON_CONDITION_TOKENS) 8069 8070 null = self._parse_on_handling("NULL", *self.ON_CONDITION_TOKENS) 8071 8072 if not empty and not error and not null: 8073 return None 8074 8075 return self.expression(exp.OnCondition(empty=empty, error=error, null=null)) 8076 8077 def _parse_on_handling(self, on: str, *values: str) -> str | None | exp.Expr | None: 8078 # Parses the "X ON Y" or "DEFAULT <expr> ON Y syntax, e.g. NULL ON NULL (Oracle, T-SQL, MySQL) 8079 for value in values: 8080 if self._match_text_seq(value, "ON", on): 8081 return f"{value} ON {on}" 8082 8083 index = self._index 8084 if self._match(TokenType.DEFAULT): 8085 default_value = self._parse_bitwise() 8086 if self._match_text_seq("ON", on): 8087 return default_value 8088 8089 self._retreat(index) 8090 8091 return None 8092 8093 @t.overload 8094 def _parse_json_object(self, agg: t.Literal[False]) -> exp.JSONObject: ... 8095 8096 @t.overload 8097 def _parse_json_object(self, agg: t.Literal[True]) -> exp.JSONObjectAgg: ... 8098 8099 def _parse_json_object(self, agg=False): 8100 star = self._parse_star() 8101 expressions = ( 8102 [star] 8103 if star 8104 else self._parse_csv(lambda: self._parse_format_json(self._parse_json_key_value())) 8105 ) 8106 null_handling = self._parse_on_handling("NULL", "NULL", "ABSENT") 8107 8108 unique_keys = None 8109 if self._match_text_seq("WITH", "UNIQUE"): 8110 unique_keys = True 8111 elif self._match_text_seq("WITHOUT", "UNIQUE"): 8112 unique_keys = False 8113 8114 self._match_text_seq("KEYS") 8115 8116 return_type = self._match_text_seq("RETURNING") and self._parse_format_json( 8117 self._parse_type() 8118 ) 8119 encoding = self._match_text_seq("ENCODING") and self._parse_var() 8120 8121 return self.expression( 8122 (exp.JSONObjectAgg if agg else exp.JSONObject)( 8123 expressions=expressions, 8124 null_handling=null_handling, 8125 unique_keys=unique_keys, 8126 return_type=return_type, 8127 encoding=encoding, 8128 ) 8129 ) 8130 8131 # Note: this is currently incomplete; it only implements the "JSON_value_column" part 8132 def _parse_json_column_def(self) -> exp.JSONColumnDef: 8133 if not self._match_text_seq("NESTED"): 8134 this = self._parse_id_var() 8135 ordinality = self._match_pair(TokenType.FOR, TokenType.ORDINALITY) 8136 kind = self._parse_types(allow_identifiers=False) 8137 nested = None 8138 else: 8139 this = None 8140 ordinality = None 8141 kind = None 8142 nested = True 8143 8144 format_json = self._match_text_seq("FORMAT", "JSON") 8145 path = self._match_text_seq("PATH") and self._parse_string() 8146 nested_schema = nested and self._parse_json_schema() 8147 8148 return self.expression( 8149 exp.JSONColumnDef( 8150 this=this, 8151 kind=kind, 8152 path=path, 8153 nested_schema=nested_schema, 8154 ordinality=ordinality, 8155 format_json=format_json, 8156 ) 8157 ) 8158 8159 def _parse_json_schema(self) -> exp.JSONSchema: 8160 self._match_text_seq("COLUMNS") 8161 return self.expression( 8162 exp.JSONSchema( 8163 expressions=self._parse_wrapped_csv(self._parse_json_column_def, optional=True) 8164 ) 8165 ) 8166 8167 def _parse_json_table(self) -> exp.JSONTable: 8168 this = self._parse_format_json(self._parse_bitwise()) 8169 path = self._match(TokenType.COMMA) and self._parse_string() 8170 error_handling = self._parse_on_handling("ERROR", "ERROR", "NULL") 8171 empty_handling = self._parse_on_handling("EMPTY", "ERROR", "NULL") 8172 schema = self._parse_json_schema() 8173 8174 return exp.JSONTable( 8175 this=this, 8176 schema=schema, 8177 path=path, 8178 error_handling=error_handling, 8179 empty_handling=empty_handling, 8180 ) 8181 8182 def _parse_match_against(self) -> exp.MatchAgainst: 8183 if self._match_text_seq("TABLE"): 8184 # parse SingleStore MATCH(TABLE ...) syntax 8185 # https://docs.singlestore.com/cloud/reference/sql-reference/full-text-search-functions/match/ 8186 expressions = [] 8187 table = self._parse_table() 8188 if table: 8189 expressions = [table] 8190 else: 8191 expressions = self._parse_csv(self._parse_column) 8192 8193 self._match_text_seq(")", "AGAINST", "(") 8194 8195 this = self._parse_string() 8196 8197 if self._match_text_seq("IN", "NATURAL", "LANGUAGE", "MODE"): 8198 modifier = "IN NATURAL LANGUAGE MODE" 8199 if self._match_text_seq("WITH", "QUERY", "EXPANSION"): 8200 modifier = f"{modifier} WITH QUERY EXPANSION" 8201 elif self._match_text_seq("IN", "BOOLEAN", "MODE"): 8202 modifier = "IN BOOLEAN MODE" 8203 elif self._match_text_seq("WITH", "QUERY", "EXPANSION"): 8204 modifier = "WITH QUERY EXPANSION" 8205 else: 8206 modifier = None 8207 8208 return self.expression( 8209 exp.MatchAgainst(this=this, expressions=expressions, modifier=modifier) 8210 ) 8211 8212 # https://learn.microsoft.com/en-us/sql/t-sql/functions/openjson-transact-sql?view=sql-server-ver16 8213 def _parse_open_json(self) -> exp.OpenJSON: 8214 this = self._parse_bitwise() 8215 path = self._match(TokenType.COMMA) and self._parse_string() 8216 8217 def _parse_open_json_column_def() -> exp.OpenJSONColumnDef: 8218 this = self._parse_field(any_token=True) 8219 kind = self._parse_types() 8220 path = self._parse_string() 8221 as_json = self._match_pair(TokenType.ALIAS, TokenType.JSON) 8222 8223 return self.expression( 8224 exp.OpenJSONColumnDef(this=this, kind=kind, path=path, as_json=as_json) 8225 ) 8226 8227 expressions = None 8228 if self._match_pair(TokenType.R_PAREN, TokenType.WITH): 8229 self._match_l_paren() 8230 expressions = self._parse_csv(_parse_open_json_column_def) 8231 8232 return self.expression(exp.OpenJSON(this=this, path=path, expressions=expressions)) 8233 8234 def _parse_position(self, haystack_first: bool = False) -> exp.StrPosition: 8235 args = self._parse_csv(self._parse_bitwise) 8236 8237 if self._match(TokenType.IN): 8238 return self.expression( 8239 exp.StrPosition(this=self._parse_bitwise(), substr=seq_get(args, 0)) 8240 ) 8241 8242 if haystack_first: 8243 haystack = seq_get(args, 0) 8244 needle = seq_get(args, 1) 8245 else: 8246 haystack = seq_get(args, 1) 8247 needle = seq_get(args, 0) 8248 8249 return self.expression( 8250 exp.StrPosition(this=haystack, substr=needle, position=seq_get(args, 2)) 8251 ) 8252 8253 def _parse_join_hint(self, func_name: str) -> exp.JoinHint: 8254 args = self._parse_csv(self._parse_table) 8255 return exp.JoinHint(this=func_name.upper(), expressions=args) 8256 8257 def _parse_substring(self) -> exp.Substring: 8258 # Postgres supports the form: substring(string [from int] [for int]) 8259 # (despite being undocumented, the reverse order also works) 8260 # https://www.postgresql.org/docs/9.1/functions-string.html @ Table 9-6 8261 8262 args = t.cast(list[t.Optional[exp.Expr]], self._parse_csv(self._parse_bitwise)) 8263 8264 start, length = None, None 8265 8266 while self._curr: 8267 if self._match(TokenType.FROM): 8268 start = self._parse_bitwise() 8269 elif self._match(TokenType.FOR): 8270 if not start: 8271 start = exp.Literal.number(1) 8272 length = self._parse_bitwise() 8273 else: 8274 break 8275 8276 if start: 8277 args.append(start) 8278 if length: 8279 args.append(length) 8280 8281 return self.validate_expression(exp.Substring.from_arg_list(args), args) 8282 8283 def _parse_trim(self) -> exp.Trim: 8284 # https://www.w3resource.com/sql/character-functions/trim.php 8285 # https://docs.oracle.com/javadb/10.8.3.0/ref/rreftrimfunc.html 8286 8287 position = None 8288 collation = None 8289 expression = None 8290 8291 if self._match_texts(self.TRIM_TYPES): 8292 position = self._prev.text.upper() 8293 8294 this = self._parse_bitwise() 8295 if self._match_set((TokenType.FROM, TokenType.COMMA)): 8296 invert_order = self._prev.token_type == TokenType.FROM or self.TRIM_PATTERN_FIRST 8297 expression = self._parse_bitwise() 8298 8299 if invert_order: 8300 this, expression = expression, this 8301 8302 if self._match(TokenType.COLLATE): 8303 collation = self._parse_bitwise() 8304 8305 return self.expression( 8306 exp.Trim(this=this, position=position, expression=expression, collation=collation) 8307 ) 8308 8309 def _parse_window_clause(self) -> list[exp.Expr] | None: 8310 return self._parse_csv(self._parse_named_window) if self._match(TokenType.WINDOW) else None 8311 8312 def _parse_named_window(self) -> exp.Expr | None: 8313 return self._parse_window(self._parse_id_var(), alias=True) 8314 8315 def _parse_respect_or_ignore_nulls(self, this: exp.Expr | None) -> exp.Expr | None: 8316 if self._curr.token_type == TokenType.VAR: 8317 if self._match_text_seq("IGNORE", "NULLS"): 8318 return self.expression(exp.IgnoreNulls(this=this)) 8319 if self._match_text_seq("RESPECT", "NULLS"): 8320 return self.expression(exp.RespectNulls(this=this)) 8321 return this 8322 8323 def _parse_having_max(self, this: exp.Expr | None) -> exp.Expr | None: 8324 if self._match(TokenType.HAVING): 8325 self._match_texts(("MAX", "MIN")) 8326 max = self._prev.text.upper() != "MIN" 8327 return self.expression( 8328 exp.HavingMax(this=this, expression=self._parse_column(), max=max) 8329 ) 8330 8331 return this 8332 8333 def _parse_window(self, this: exp.Expr | None, alias: bool = False) -> exp.Expr | None: 8334 func = this 8335 comments = func.comments if isinstance(func, exp.Expr) else None 8336 8337 # T-SQL allows the OVER (...) syntax after WITHIN GROUP. 8338 # https://learn.microsoft.com/en-us/sql/t-sql/functions/percentile-disc-transact-sql?view=sql-server-ver16 8339 if self._match_text_seq("WITHIN", "GROUP"): 8340 order = self._parse_wrapped(self._parse_order) 8341 this = self.expression(exp.WithinGroup(this=this, expression=order)) 8342 8343 if self._match_pair(TokenType.FILTER, TokenType.L_PAREN): 8344 self._match(TokenType.WHERE) 8345 this = self.expression( 8346 exp.Filter(this=this, expression=self._parse_where(skip_where_token=True)) 8347 ) 8348 self._match_r_paren() 8349 8350 # SQL spec defines an optional [ { IGNORE | RESPECT } NULLS ] OVER 8351 # Some dialects choose to implement and some do not. 8352 # https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html 8353 8354 # There is some code above in _parse_lambda that handles 8355 # SELECT FIRST_VALUE(TABLE.COLUMN IGNORE|RESPECT NULLS) OVER ... 8356 8357 # The below changes handle 8358 # SELECT FIRST_VALUE(TABLE.COLUMN) IGNORE|RESPECT NULLS OVER ... 8359 8360 # Oracle allows both formats 8361 # (https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/img_text/first_value.html) 8362 # and Snowflake chose to do the same for familiarity 8363 # https://docs.snowflake.com/en/sql-reference/functions/first_value.html#usage-notes 8364 if isinstance(this, exp.AggFunc): 8365 ignore_respect = this.find(exp.IgnoreNulls, exp.RespectNulls) 8366 8367 if ignore_respect and ignore_respect is not this: 8368 ignore_respect.replace(ignore_respect.this) 8369 this = self.expression(ignore_respect.__class__(this=this)) 8370 8371 this = self._parse_respect_or_ignore_nulls(this) 8372 8373 # bigquery select from window x AS (partition by ...) 8374 if alias: 8375 over = None 8376 self._match(TokenType.ALIAS) 8377 elif not self._match_set(self.WINDOW_BEFORE_PAREN_TOKENS): 8378 return this 8379 else: 8380 over = self._prev.text.upper() 8381 8382 if comments and isinstance(func, exp.Expr): 8383 func.pop_comments() 8384 8385 if not self._match(TokenType.L_PAREN): 8386 return self.expression( 8387 exp.Window(this=this, alias=self._parse_id_var(False), over=over), comments=comments 8388 ) 8389 8390 window_alias = self._parse_id_var(any_token=False, tokens=self.WINDOW_ALIAS_TOKENS) 8391 8392 first: bool | None = True if self._match(TokenType.FIRST) else None 8393 if self._match_text_seq("LAST"): 8394 first = False 8395 8396 partition, order = self._parse_partition_and_order() 8397 kind = ( 8398 self._match_set((TokenType.ROWS, TokenType.RANGE)) or self._match_text_seq("GROUPS") 8399 ) and self._prev.text 8400 8401 if kind: 8402 self._match(TokenType.BETWEEN) 8403 start = self._parse_window_spec() 8404 8405 end = self._parse_window_spec() if self._match(TokenType.AND) else {} 8406 exclude = ( 8407 self._parse_var_from_options(self.WINDOW_EXCLUDE_OPTIONS) 8408 if self._match_text_seq("EXCLUDE") 8409 else None 8410 ) 8411 8412 spec = self.expression( 8413 exp.WindowSpec( 8414 kind=kind, 8415 start=start["value"], 8416 start_side=start["side"], 8417 end=end.get("value"), 8418 end_side=end.get("side"), 8419 exclude=exclude, 8420 ) 8421 ) 8422 else: 8423 spec = None 8424 8425 self._match_r_paren() 8426 8427 window = self.expression( 8428 exp.Window( 8429 this=this, 8430 partition_by=partition, 8431 order=order, 8432 spec=spec, 8433 alias=window_alias, 8434 over=over, 8435 first=first, 8436 ), 8437 comments=comments, 8438 ) 8439 8440 # This covers Oracle's FIRST/LAST syntax: aggregate KEEP (...) OVER (...) 8441 if self._match_set(self.WINDOW_BEFORE_PAREN_TOKENS, advance=False): 8442 return self._parse_window(window, alias=alias) 8443 8444 return window 8445 8446 def _parse_partition_and_order( 8447 self, 8448 ) -> tuple[list[exp.Expr], exp.Expr | None]: 8449 return self._parse_partition_by(), self._parse_order() 8450 8451 def _parse_window_spec(self) -> dict[str, str | exp.Expr | None]: 8452 self._match(TokenType.BETWEEN) 8453 8454 return { 8455 "value": ( 8456 (self._match_text_seq("UNBOUNDED") and "UNBOUNDED") 8457 or (self._match_text_seq("CURRENT", "ROW") and "CURRENT ROW") 8458 or self._parse_bitwise() 8459 ), 8460 "side": self._prev.text if self._match_texts(self.WINDOW_SIDES) else None, 8461 } 8462 8463 def _parse_alias(self, this: exp.Expr | None, explicit: bool = False) -> exp.Expr | None: 8464 # In some dialects, LIMIT and OFFSET can act as both identifiers and keywords (clauses) 8465 # so this section tries to parse the clause version and if it fails, it treats the token 8466 # as an identifier (alias) 8467 if self._can_parse_limit_or_offset(): 8468 return this 8469 8470 # WINDOW is in ID_VAR_TOKENS, so it can be consumed as an implicit alias. Detect the 8471 # named-window clause shape (`WINDOW <ident> AS (...)`) and avoid swallowing it. 8472 if self._can_parse_named_window(): 8473 return this 8474 8475 any_token = self._match(TokenType.ALIAS) 8476 comments = self._prev_comments 8477 8478 if explicit and not any_token: 8479 return this 8480 8481 if self._match(TokenType.L_PAREN): 8482 aliases = self.expression( 8483 exp.Aliases( 8484 this=this, expressions=self._parse_csv(lambda: self._parse_id_var(any_token)) 8485 ), 8486 comments=comments, 8487 ) 8488 self._match_r_paren(aliases) 8489 return aliases 8490 8491 alias = self._parse_id_var(any_token, tokens=self.ALIAS_TOKENS) or ( 8492 self.STRING_ALIASES and self._parse_string_as_identifier() 8493 ) 8494 8495 if alias: 8496 comments.extend(alias.pop_comments()) 8497 this = self.expression(exp.Alias(this=this, alias=alias), comments=comments) 8498 column = this.this 8499 8500 # Moves the comment next to the alias in `expr /* comment */ AS alias` 8501 if not this.comments and column and column.comments: 8502 this.comments = column.pop_comments() 8503 8504 return this 8505 8506 def _parse_id_var( 8507 self, 8508 any_token: bool = True, 8509 tokens: t.Collection[TokenType] | None = None, 8510 ) -> exp.Expr | None: 8511 expression = self._parse_identifier() 8512 if not expression and ( 8513 (any_token and self._advance_any()) or self._match_set(tokens or self.ID_VAR_TOKENS) 8514 ): 8515 quoted = self._prev.token_type == TokenType.STRING 8516 expression = self._identifier_expression(quoted=quoted) 8517 8518 return expression 8519 8520 def _parse_string(self) -> exp.Expr | None: 8521 if self._match_set(self.STRING_PARSERS): 8522 return self.STRING_PARSERS[self._prev.token_type](self, self._prev) 8523 return self._parse_placeholder() 8524 8525 def _parse_string_as_identifier(self) -> exp.Identifier | None: 8526 if not self._match(TokenType.STRING): 8527 return None 8528 output = exp.to_identifier(self._prev.text, quoted=True) 8529 output.update_positions(self._prev) 8530 return output 8531 8532 def _parse_number(self) -> exp.Expr | None: 8533 if self._match_set(self.NUMERIC_PARSERS): 8534 return self.NUMERIC_PARSERS[self._prev.token_type](self, self._prev) 8535 return self._parse_placeholder() 8536 8537 def _parse_identifier(self) -> exp.Expr | None: 8538 if self._match(TokenType.IDENTIFIER): 8539 return self._identifier_expression(quoted=True) 8540 return self._parse_placeholder() 8541 8542 def _parse_var( 8543 self, 8544 any_token: bool = False, 8545 tokens: t.Collection[TokenType] | None = None, 8546 upper: bool = False, 8547 ) -> exp.Expr | None: 8548 if ( 8549 (any_token and self._advance_any()) 8550 or self._match(TokenType.VAR) 8551 or (self._match_set(tokens) if tokens else False) 8552 ): 8553 return self.expression( 8554 exp.Var(this=self._prev.text.upper() if upper else self._prev.text) 8555 ) 8556 return self._parse_placeholder() 8557 8558 def _advance_any(self, ignore_reserved: bool = False) -> Token | None: 8559 if self._curr and (ignore_reserved or self._curr.token_type not in self.RESERVED_TOKENS): 8560 self._advance() 8561 return self._prev 8562 return None 8563 8564 def _parse_var_or_string(self, upper: bool = False) -> exp.Expr | None: 8565 return self._parse_string() or self._parse_var(any_token=True, upper=upper) 8566 8567 def _parse_primary_or_var(self) -> exp.Expr | None: 8568 return self._parse_primary() or self._parse_var(any_token=True) 8569 8570 def _parse_null(self) -> exp.Expr | None: 8571 if self._match_set((TokenType.NULL, TokenType.UNKNOWN)): 8572 return self.PRIMARY_PARSERS[TokenType.NULL](self, self._prev) 8573 return self._parse_placeholder() 8574 8575 def _parse_boolean(self) -> exp.Expr | None: 8576 if self._match(TokenType.TRUE): 8577 return self.PRIMARY_PARSERS[TokenType.TRUE](self, self._prev) 8578 if self._match(TokenType.FALSE): 8579 return self.PRIMARY_PARSERS[TokenType.FALSE](self, self._prev) 8580 return self._parse_placeholder() 8581 8582 def _parse_star(self) -> exp.Expr | None: 8583 if self._match(TokenType.STAR): 8584 return self.PRIMARY_PARSERS[TokenType.STAR](self, self._prev) 8585 return self._parse_placeholder() 8586 8587 def _parse_parameter(self) -> exp.Parameter: 8588 this = self._parse_identifier() or self._parse_primary_or_var() 8589 return self.expression(exp.Parameter(this=this)) 8590 8591 def _parse_placeholder(self) -> exp.Expr | None: 8592 if self._match_set(self.PLACEHOLDER_PARSERS): 8593 placeholder = self.PLACEHOLDER_PARSERS[self._prev.token_type](self) 8594 if placeholder: 8595 return placeholder 8596 self._advance(-1) 8597 return None 8598 8599 def _parse_star_op(self, *keywords: str) -> list[exp.Expr] | None: 8600 if not self._match_texts(keywords): 8601 return None 8602 if self._match(TokenType.L_PAREN, advance=False): 8603 return self._parse_wrapped_csv(self._parse_expression) 8604 8605 expression = self._parse_alias(self._parse_disjunction(), explicit=True) 8606 return [expression] if expression else None 8607 8608 def _parse_csv( 8609 self, parse_method: t.Callable[[], T | None], sep: TokenType = TokenType.COMMA 8610 ) -> list[T]: 8611 parse_result = parse_method() 8612 items = [parse_result] if parse_result is not None else [] 8613 8614 while self._match(sep): 8615 if isinstance(parse_result, exp.Expr): 8616 self._add_comments(parse_result) 8617 parse_result = parse_method() 8618 if parse_result is not None: 8619 items.append(parse_result) 8620 8621 return items 8622 8623 def _parse_wrapped_id_vars(self, optional: bool = False) -> list[exp.Expr]: 8624 return self._parse_wrapped_csv(self._parse_id_var, optional=optional) 8625 8626 def _parse_wrapped_csv( 8627 self, 8628 parse_method: t.Callable[[], T | None], 8629 sep: TokenType = TokenType.COMMA, 8630 optional: bool = False, 8631 ) -> list[T]: 8632 return self._parse_wrapped( 8633 lambda: self._parse_csv(parse_method, sep=sep), optional=optional 8634 ) 8635 8636 def _parse_wrapped(self, parse_method: t.Callable[[], T], optional: bool = False) -> T: 8637 wrapped = self._match(TokenType.L_PAREN) 8638 if not wrapped and not optional: 8639 self.raise_error("Expecting (") 8640 parse_result = parse_method() 8641 if wrapped: 8642 self._match_r_paren() 8643 return parse_result 8644 8645 def _parse_expressions(self) -> list[exp.Expr]: 8646 return self._parse_csv(self._parse_expression) 8647 8648 def _parse_select_or_expression(self, alias: bool = False) -> exp.Expr | None: 8649 return ( 8650 self._parse_set_operations( 8651 self._parse_alias(self._parse_assignment(), explicit=True) 8652 if alias 8653 else self._parse_assignment() 8654 ) 8655 or self._parse_select() 8656 ) 8657 8658 def _parse_ddl_select(self) -> exp.Expr | None: 8659 return self._parse_query_modifiers( 8660 self._parse_set_operations(self._parse_select(nested=True, parse_subquery_alias=False)) 8661 ) 8662 8663 def _parse_transaction(self) -> exp.Transaction | exp.Command: 8664 this = None 8665 if self._match_texts(self.TRANSACTION_KIND): 8666 this = self._prev.text 8667 8668 self._match_texts(("TRANSACTION", "WORK")) 8669 8670 modes = [] 8671 while True: 8672 mode = [] 8673 while self._match(TokenType.VAR) or self._match(TokenType.NOT): 8674 mode.append(self._prev.text) 8675 8676 if mode: 8677 modes.append(" ".join(mode)) 8678 if not self._match(TokenType.COMMA): 8679 break 8680 8681 return self.expression(exp.Transaction(this=this, modes=modes)) 8682 8683 def _parse_commit_or_rollback(self) -> exp.Commit | exp.Rollback: 8684 chain = None 8685 savepoint = None 8686 is_rollback = self._prev.token_type == TokenType.ROLLBACK 8687 8688 self._match_texts(("TRANSACTION", "WORK")) 8689 8690 if self._match_text_seq("TO"): 8691 self._match_text_seq("SAVEPOINT") 8692 savepoint = self._parse_id_var() 8693 8694 if self._match(TokenType.AND): 8695 chain = not self._match_text_seq("NO") 8696 self._match_text_seq("CHAIN") 8697 8698 if is_rollback: 8699 return self.expression(exp.Rollback(savepoint=savepoint)) 8700 8701 return self.expression(exp.Commit(chain=chain)) 8702 8703 def _parse_refresh(self) -> exp.Refresh | exp.Command: 8704 if self._match(TokenType.TABLE): 8705 kind = "TABLE" 8706 elif self._match_text_seq("MATERIALIZED", "VIEW"): 8707 kind = "MATERIALIZED VIEW" 8708 else: 8709 kind = "" 8710 8711 this = self._parse_string() or self._parse_table() 8712 if not kind and not isinstance(this, exp.Literal): 8713 return self._parse_as_command(self._prev) 8714 8715 return self.expression(exp.Refresh(this=this, kind=kind)) 8716 8717 def _parse_column_def_with_exists(self): 8718 start = self._index 8719 self._match(TokenType.COLUMN) 8720 8721 exists_column = self._parse_exists(not_=True) 8722 expression = self._parse_field_def() 8723 8724 if not isinstance(expression, exp.ColumnDef): 8725 self._retreat(start) 8726 return None 8727 8728 expression.set("exists", exists_column) 8729 8730 return expression 8731 8732 def _parse_add_column(self) -> exp.ColumnDef | None: 8733 if not self._prev.text.upper() == "ADD": 8734 return None 8735 8736 return self._parse_column_def_with_exists() 8737 8738 def _parse_drop_column(self) -> exp.Drop | exp.Command | None: 8739 drop = self._parse_drop() if self._match(TokenType.DROP) else None 8740 if drop and not isinstance(drop, exp.Command): 8741 drop.set("kind", drop.args.get("kind", "COLUMN")) 8742 return drop 8743 8744 def _parse_alter_drop_action(self) -> exp.Expr | None: 8745 return self._parse_drop_column() 8746 8747 # https://docs.aws.amazon.com/athena/latest/ug/alter-table-drop-partition.html 8748 def _parse_drop_partition(self, exists: bool | None = None) -> exp.DropPartition: 8749 return self.expression( 8750 exp.DropPartition(expressions=self._parse_csv(self._parse_partition), exists=exists) 8751 ) 8752 8753 def _parse_alter_table_add(self) -> list[exp.Expr]: 8754 def _parse_add_alteration() -> exp.Expr | None: 8755 self._match_text_seq("ADD") 8756 if self._match_set(self.ADD_CONSTRAINT_TOKENS, advance=False): 8757 return self.expression( 8758 exp.AddConstraint(expressions=self._parse_csv(self._parse_constraint)) 8759 ) 8760 8761 column_def = self._parse_add_column() 8762 if isinstance(column_def, exp.ColumnDef): 8763 return column_def 8764 8765 exists = self._parse_exists(not_=True) 8766 if self._match_pair(TokenType.PARTITION, TokenType.L_PAREN, advance=False): 8767 return self.expression( 8768 exp.AddPartition( 8769 exists=exists, 8770 this=self._parse_field(any_token=True), 8771 location=self._match_text_seq("LOCATION", advance=False) 8772 and self._parse_property(), 8773 ) 8774 ) 8775 8776 return None 8777 8778 if not self._match_set(self.ADD_CONSTRAINT_TOKENS, advance=False) and ( 8779 not self.dialect.ALTER_TABLE_ADD_REQUIRED_FOR_EACH_COLUMN 8780 or self._match_text_seq("COLUMNS") 8781 ): 8782 schema = self._parse_schema() 8783 8784 return ( 8785 ensure_list(schema) 8786 if schema 8787 else self._parse_csv(self._parse_column_def_with_exists) 8788 ) 8789 8790 return self._parse_csv(_parse_add_alteration) 8791 8792 def _parse_alter_table_alter(self) -> exp.Expr | None: 8793 if self._match_texts(self.ALTER_ALTER_PARSERS): 8794 return self.ALTER_ALTER_PARSERS[self._prev.text.upper()](self) 8795 8796 # Many dialects support the ALTER [COLUMN] syntax, so if there is no 8797 # keyword after ALTER we default to parsing this statement 8798 self._match(TokenType.COLUMN) 8799 column = self._parse_field(any_token=True) 8800 8801 if self._match_pair(TokenType.DROP, TokenType.DEFAULT): 8802 return self.expression(exp.AlterColumn(this=column, drop=True)) 8803 if self._match_pair(TokenType.SET, TokenType.DEFAULT): 8804 return self.expression(exp.AlterColumn(this=column, default=self._parse_disjunction())) 8805 if self._match(TokenType.COMMENT): 8806 return self.expression(exp.AlterColumn(this=column, comment=self._parse_string())) 8807 if self._match_text_seq("DROP", "NOT", "NULL"): 8808 return self.expression(exp.AlterColumn(this=column, drop=True, allow_null=True)) 8809 if self._match_text_seq("SET", "NOT", "NULL"): 8810 return self.expression(exp.AlterColumn(this=column, allow_null=False)) 8811 8812 if self._match_text_seq("SET", "VISIBLE"): 8813 return self.expression(exp.AlterColumn(this=column, visible="VISIBLE")) 8814 if self._match_text_seq("SET", "INVISIBLE"): 8815 return self.expression(exp.AlterColumn(this=column, visible="INVISIBLE")) 8816 8817 self._match_text_seq("SET", "DATA") 8818 self._match_text_seq("TYPE") 8819 return self.expression( 8820 exp.AlterColumn( 8821 this=column, 8822 dtype=self._parse_types(), 8823 collate=self._match(TokenType.COLLATE) and self._parse_term(), 8824 using=self._match(TokenType.USING) and self._parse_disjunction(), 8825 ) 8826 ) 8827 8828 def _parse_alter_diststyle(self) -> exp.AlterDistStyle: 8829 if self._match_texts(("ALL", "EVEN", "AUTO")): 8830 return self.expression(exp.AlterDistStyle(this=exp.var(self._prev.text.upper()))) 8831 8832 self._match_text_seq("KEY", "DISTKEY") 8833 return self.expression(exp.AlterDistStyle(this=self._parse_column())) 8834 8835 def _parse_alter_sortkey(self, compound: bool | None = None) -> exp.AlterSortKey: 8836 if compound: 8837 self._match_text_seq("SORTKEY") 8838 8839 if self._match(TokenType.L_PAREN, advance=False): 8840 return self.expression( 8841 exp.AlterSortKey(expressions=self._parse_wrapped_id_vars(), compound=compound) 8842 ) 8843 8844 self._match_texts(("AUTO", "NONE")) 8845 return self.expression( 8846 exp.AlterSortKey(this=exp.var(self._prev.text.upper()), compound=compound) 8847 ) 8848 8849 def _parse_alter_table_drop(self) -> list[exp.Expr]: 8850 index = self._index - 1 8851 8852 partition_exists = self._parse_exists() 8853 if self._match(TokenType.PARTITION, advance=False): 8854 return self._parse_csv(lambda: self._parse_drop_partition(exists=partition_exists)) 8855 8856 self._retreat(index) 8857 return self._parse_csv(self._parse_alter_drop_action) 8858 8859 def _parse_alter_table_rename(self) -> exp.AlterRename | exp.RenameColumn | None: 8860 if self._match(TokenType.COLUMN) or ( 8861 not self.ALTER_RENAME_REQUIRES_COLUMN and not self._match_text_seq("TO", advance=False) 8862 ): 8863 exists = self._parse_exists() 8864 old_column = self._parse_column() 8865 to = self._match_text_seq("TO") 8866 new_column = self._parse_column() 8867 8868 if old_column is None or not to or new_column is None: 8869 return None 8870 8871 return self.expression(exp.RenameColumn(this=old_column, to=new_column, exists=exists)) 8872 8873 self._match_text_seq("TO") 8874 return self.expression(exp.AlterRename(this=self._parse_table(schema=True))) 8875 8876 def _parse_alter_table_set(self) -> exp.AlterSet: 8877 alter_set = self.expression(exp.AlterSet()) 8878 8879 if self._match(TokenType.L_PAREN, advance=False) or self._match_text_seq( 8880 "TABLE", "PROPERTIES" 8881 ): 8882 alter_set.set("expressions", self._parse_wrapped_csv(self._parse_assignment)) 8883 elif self._match_text_seq("FILESTREAM_ON", advance=False): 8884 alter_set.set("expressions", [self._parse_assignment()]) 8885 elif self._match_texts(("LOGGED", "UNLOGGED")): 8886 alter_set.set("option", exp.var(self._prev.text.upper())) 8887 elif self._match_text_seq("WITHOUT") and self._match_texts(("CLUSTER", "OIDS")): 8888 alter_set.set("option", exp.var(f"WITHOUT {self._prev.text.upper()}")) 8889 elif self._match_text_seq("LOCATION"): 8890 alter_set.set("location", self._parse_field()) 8891 elif self._match_text_seq("ACCESS", "METHOD"): 8892 alter_set.set("access_method", self._parse_field()) 8893 elif self._match_text_seq("TABLESPACE"): 8894 alter_set.set("tablespace", self._parse_field()) 8895 elif self._match_text_seq("FILE", "FORMAT") or self._match_text_seq("FILEFORMAT"): 8896 alter_set.set("file_format", [self._parse_field()]) 8897 elif self._match_text_seq("STAGE_FILE_FORMAT"): 8898 alter_set.set("file_format", self._parse_wrapped_options()) 8899 elif self._match_text_seq("STAGE_COPY_OPTIONS"): 8900 alter_set.set("copy_options", self._parse_wrapped_options()) 8901 elif self._match_text_seq("TAG") or self._match_text_seq("TAGS"): 8902 alter_set.set("tag", self._parse_csv(self._parse_assignment)) 8903 else: 8904 if self._match_text_seq("SERDE"): 8905 alter_set.set("serde", self._parse_field()) 8906 8907 properties = self._parse_wrapped(self._parse_properties, optional=True) 8908 alter_set.set("expressions", [properties]) 8909 8910 return alter_set 8911 8912 def _parse_alter_session(self) -> exp.AlterSession: 8913 """Parse ALTER SESSION SET/UNSET statements.""" 8914 if self._match(TokenType.SET): 8915 expressions = self._parse_csv(lambda: self._parse_set_item_assignment()) 8916 return self.expression(exp.AlterSession(expressions=expressions, unset=False)) 8917 8918 self._match_text_seq("UNSET") 8919 expressions = self._parse_csv( 8920 lambda: self.expression(exp.SetItem(this=self._parse_id_var(any_token=True))) 8921 ) 8922 return self.expression(exp.AlterSession(expressions=expressions, unset=True)) 8923 8924 def _parse_alter(self) -> exp.Alter | exp.Command: 8925 start = self._prev 8926 8927 iceberg = self._match_text_seq("ICEBERG") 8928 8929 alter_token = self._match_set(self.ALTERABLES) and self._prev 8930 if not alter_token: 8931 return self._parse_as_command(start) 8932 if iceberg and alter_token.token_type != TokenType.TABLE: 8933 return self._parse_as_command(start) 8934 8935 exists = self._parse_exists() 8936 only = self._match_text_seq("ONLY") 8937 8938 if alter_token.token_type == TokenType.SESSION: 8939 this = None 8940 check = None 8941 cluster = None 8942 else: 8943 this = self._parse_table(schema=True, parse_partition=self.ALTER_TABLE_PARTITIONS) 8944 check = self._match_text_seq("WITH", "CHECK") 8945 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 8946 8947 if self._next: 8948 self._advance() 8949 8950 parser = self.ALTER_PARSERS.get(self._prev.text.upper()) if self._prev else None 8951 if parser: 8952 actions = ensure_list(parser(self)) 8953 not_valid = self._match_text_seq("NOT", "VALID") 8954 options = self._parse_csv(self._parse_property) 8955 cascade = self.dialect.ALTER_TABLE_SUPPORTS_CASCADE and self._match_text_seq("CASCADE") 8956 8957 if not self._curr and actions: 8958 return self.expression( 8959 exp.Alter( 8960 this=this, 8961 kind=alter_token.text.upper(), 8962 exists=exists, 8963 actions=actions, 8964 only=only, 8965 options=options, 8966 cluster=cluster, 8967 not_valid=not_valid, 8968 check=check, 8969 cascade=cascade, 8970 iceberg=iceberg, 8971 ) 8972 ) 8973 8974 return self._parse_as_command(start) 8975 8976 def _parse_analyze(self) -> exp.Analyze | exp.Command: 8977 start = self._prev 8978 # https://duckdb.org/docs/sql/statements/analyze 8979 if not self._curr: 8980 return self.expression(exp.Analyze()) 8981 8982 options = [] 8983 while self._match_texts(self.ANALYZE_STYLES): 8984 if self._prev.text.upper() == "BUFFER_USAGE_LIMIT": 8985 options.append(f"BUFFER_USAGE_LIMIT {self._parse_number()}") 8986 else: 8987 options.append(self._prev.text.upper()) 8988 8989 this: exp.Expr | None = None 8990 inner_expression: exp.Expr | None = None 8991 8992 kind = self._curr.text.upper() if self._curr else None 8993 8994 if self._match(TokenType.TABLE) or self._match(TokenType.INDEX): 8995 this = self._parse_table_parts() 8996 elif self._match_text_seq("TABLES"): 8997 if self._match_set((TokenType.FROM, TokenType.IN)): 8998 kind = f"{kind} {self._prev.text.upper()}" 8999 this = self._parse_table(schema=True, is_db_reference=True) 9000 elif self._match_text_seq("DATABASE"): 9001 this = self._parse_table(schema=True, is_db_reference=True) 9002 elif self._match_text_seq("CLUSTER"): 9003 this = self._parse_table() 9004 # Try matching inner expr keywords before fallback to parse table. 9005 elif self._match_texts(self.ANALYZE_EXPRESSION_PARSERS): 9006 kind = None 9007 inner_expression = self.ANALYZE_EXPRESSION_PARSERS[self._prev.text.upper()](self) 9008 else: 9009 # Empty kind https://prestodb.io/docs/current/sql/analyze.html 9010 kind = None 9011 this = self._parse_table_parts() 9012 9013 partition = self._try_parse(self._parse_partition) 9014 if not partition and self._match_texts(self.PARTITION_KEYWORDS): 9015 return self._parse_as_command(start) 9016 9017 # https://docs.starrocks.io/docs/sql-reference/sql-statements/cbo_stats/ANALYZE_TABLE/ 9018 if self._match_text_seq("WITH", "SYNC", "MODE") or self._match_text_seq( 9019 "WITH", "ASYNC", "MODE" 9020 ): 9021 mode = f"WITH {self._tokens[self._index - 2].text.upper()} MODE" 9022 else: 9023 mode = None 9024 9025 if self._match_texts(self.ANALYZE_EXPRESSION_PARSERS): 9026 inner_expression = self.ANALYZE_EXPRESSION_PARSERS[self._prev.text.upper()](self) 9027 9028 properties = self._parse_properties() 9029 return self.expression( 9030 exp.Analyze( 9031 kind=kind, 9032 this=this, 9033 mode=mode, 9034 partition=partition, 9035 properties=properties, 9036 expression=inner_expression, 9037 options=options, 9038 ) 9039 ) 9040 9041 # https://spark.apache.org/docs/3.5.1/sql-ref-syntax-aux-analyze-table.html 9042 def _parse_analyze_statistics(self) -> exp.AnalyzeStatistics: 9043 this = None 9044 kind = self._prev.text.upper() 9045 option = self._prev.text.upper() if self._match_text_seq("DELTA") else None 9046 expressions = [] 9047 9048 if not self._match_text_seq("STATISTICS"): 9049 self.raise_error("Expecting token STATISTICS") 9050 9051 if self._match_text_seq("NOSCAN"): 9052 this = "NOSCAN" 9053 elif self._match(TokenType.FOR): 9054 if self._match_text_seq("ALL", "COLUMNS"): 9055 this = "FOR ALL COLUMNS" 9056 if self._match_texts("COLUMNS"): 9057 this = "FOR COLUMNS" 9058 expressions = self._parse_csv(self._parse_column_reference) 9059 elif self._match_text_seq("SAMPLE"): 9060 sample = self._parse_number() 9061 expressions = [ 9062 self.expression( 9063 exp.AnalyzeSample( 9064 sample=sample, 9065 kind=self._prev.text.upper() if self._match(TokenType.PERCENT) else None, 9066 ) 9067 ) 9068 ] 9069 9070 return self.expression( 9071 exp.AnalyzeStatistics(kind=kind, option=option, this=this, expressions=expressions) 9072 ) 9073 9074 # https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/ANALYZE.html 9075 def _parse_analyze_validate(self) -> exp.AnalyzeValidate: 9076 kind = None 9077 this = None 9078 expression: exp.Expr | None = None 9079 if self._match_text_seq("REF", "UPDATE"): 9080 kind = "REF" 9081 this = "UPDATE" 9082 if self._match_text_seq("SET", "DANGLING", "TO", "NULL"): 9083 this = "UPDATE SET DANGLING TO NULL" 9084 elif self._match_text_seq("STRUCTURE"): 9085 kind = "STRUCTURE" 9086 if self._match_text_seq("CASCADE", "FAST"): 9087 this = "CASCADE FAST" 9088 elif self._match_text_seq("CASCADE", "COMPLETE") and self._match_texts( 9089 ("ONLINE", "OFFLINE") 9090 ): 9091 this = f"CASCADE COMPLETE {self._prev.text.upper()}" 9092 expression = self._parse_into() 9093 9094 return self.expression(exp.AnalyzeValidate(kind=kind, this=this, expression=expression)) 9095 9096 def _parse_analyze_columns(self) -> exp.AnalyzeColumns | None: 9097 this = self._prev.text.upper() 9098 if self._match_text_seq("COLUMNS"): 9099 return self.expression(exp.AnalyzeColumns(this=f"{this} {self._prev.text.upper()}")) 9100 return None 9101 9102 def _parse_analyze_delete(self) -> exp.AnalyzeDelete | None: 9103 kind = self._prev.text.upper() if self._match_text_seq("SYSTEM") else None 9104 if self._match_text_seq("STATISTICS"): 9105 return self.expression(exp.AnalyzeDelete(kind=kind)) 9106 return None 9107 9108 def _parse_analyze_list(self) -> exp.AnalyzeListChainedRows | None: 9109 if self._match_text_seq("CHAINED", "ROWS"): 9110 return self.expression(exp.AnalyzeListChainedRows(expression=self._parse_into())) 9111 return None 9112 9113 # https://dev.mysql.com/doc/refman/8.4/en/analyze-table.html 9114 def _parse_analyze_histogram(self) -> exp.AnalyzeHistogram: 9115 this = self._prev.text.upper() 9116 expression: exp.Expr | None = None 9117 expressions = [] 9118 update_options = None 9119 9120 if self._match_text_seq("HISTOGRAM", "ON"): 9121 expressions = self._parse_csv(self._parse_column_reference) 9122 with_expressions = [] 9123 while self._match(TokenType.WITH): 9124 # https://docs.starrocks.io/docs/sql-reference/sql-statements/cbo_stats/ANALYZE_TABLE/ 9125 if self._match_texts(("SYNC", "ASYNC")): 9126 if self._match_text_seq("MODE", advance=False): 9127 with_expressions.append(f"{self._prev.text.upper()} MODE") 9128 self._advance() 9129 else: 9130 buckets = self._parse_number() 9131 if self._match_text_seq("BUCKETS"): 9132 with_expressions.append(f"{buckets} BUCKETS") 9133 if with_expressions: 9134 expression = self.expression(exp.AnalyzeWith(expressions=with_expressions)) 9135 9136 if self._match_texts(("MANUAL", "AUTO")) and self._match( 9137 TokenType.UPDATE, advance=False 9138 ): 9139 update_options = self._prev.text.upper() 9140 self._advance() 9141 elif self._match_text_seq("USING", "DATA"): 9142 expression = self.expression(exp.UsingData(this=self._parse_string())) 9143 9144 return self.expression( 9145 exp.AnalyzeHistogram( 9146 this=this, 9147 expressions=expressions, 9148 expression=expression, 9149 update_options=update_options, 9150 ) 9151 ) 9152 9153 def _parse_merge(self) -> exp.Merge: 9154 self._match(TokenType.INTO) 9155 target = self._parse_table() 9156 9157 if target and self._match(TokenType.ALIAS, advance=False): 9158 target.set("alias", self._parse_table_alias()) 9159 9160 self._match(TokenType.USING) 9161 using = self._parse_table() 9162 9163 return self.expression( 9164 exp.Merge( 9165 this=target, 9166 using=using, 9167 on=self._match(TokenType.ON) and self._parse_disjunction(), 9168 using_cond=self._match(TokenType.USING) and self._parse_using_identifiers(), 9169 whens=self._parse_when_matched(), 9170 returning=self._parse_returning(), 9171 ) 9172 ) 9173 9174 def _parse_when_matched(self) -> exp.Whens: 9175 whens = [] 9176 9177 while self._match(TokenType.WHEN): 9178 matched = not self._match(TokenType.NOT) 9179 self._match_text_seq("MATCHED") 9180 source = ( 9181 False 9182 if self._match_text_seq("BY", "TARGET") 9183 else self._match_text_seq("BY", "SOURCE") 9184 ) 9185 condition = self._parse_disjunction() if self._match(TokenType.AND) else None 9186 9187 self._match(TokenType.THEN) 9188 9189 if self._match(TokenType.INSERT): 9190 this = self._parse_star() 9191 if this: 9192 then: exp.Expr | None = self.expression(exp.Insert(this=this)) 9193 else: 9194 then = self.expression( 9195 exp.Insert( 9196 this=exp.var("ROW") 9197 if self._match_text_seq("ROW") 9198 else self._parse_value(values=False), 9199 expression=self._match_text_seq("VALUES") and self._parse_value(), 9200 where=self._parse_where(), 9201 ) 9202 ) 9203 elif self._match(TokenType.UPDATE): 9204 expressions = self._parse_star() 9205 if expressions: 9206 then = self.expression(exp.Update(expressions=expressions)) 9207 else: 9208 then = self.expression( 9209 exp.Update( 9210 expressions=self._match(TokenType.SET) 9211 and self._parse_csv(self._parse_equality), 9212 where=self._parse_where(), 9213 ) 9214 ) 9215 elif self._match(TokenType.DELETE): 9216 then = self.expression(exp.Var(this=self._prev.text)) 9217 else: 9218 then = self._parse_var_from_options(self.CONFLICT_ACTIONS) 9219 9220 whens.append( 9221 self.expression( 9222 exp.When(matched=matched, source=source, condition=condition, then=then) 9223 ) 9224 ) 9225 return self.expression(exp.Whens(expressions=whens)) 9226 9227 def _parse_show(self) -> exp.Expr | None: 9228 parser = self._find_parser(self.SHOW_PARSERS, self.SHOW_TRIE) 9229 if parser: 9230 return parser(self) 9231 return self._parse_as_command(self._prev) 9232 9233 def _parse_set_item_assignment(self, kind: str | None = None) -> exp.Expr | None: 9234 index = self._index 9235 9236 if kind in ("GLOBAL", "SESSION") and self._match_text_seq("TRANSACTION"): 9237 return self._parse_set_transaction(global_=kind == "GLOBAL") 9238 9239 left = self._parse_primary() or self._parse_column() 9240 assignment_delimiter = self._match_texts(self.SET_ASSIGNMENT_DELIMITERS) 9241 9242 if not left or (self.SET_REQUIRES_ASSIGNMENT_DELIMITER and not assignment_delimiter): 9243 self._retreat(index) 9244 return None 9245 9246 right = self._parse_statement() or self._parse_id_var() 9247 if isinstance(right, (exp.Column, exp.Identifier)): 9248 right = exp.var(right.name) 9249 9250 this = self.expression(exp.EQ(this=left, expression=right)) 9251 return self.expression(exp.SetItem(this=this, kind=kind)) 9252 9253 def _parse_set_transaction(self, global_: bool = False) -> exp.Expr: 9254 self._match_text_seq("TRANSACTION") 9255 characteristics = self._parse_csv( 9256 lambda: self._parse_var_from_options(self.TRANSACTION_CHARACTERISTICS) 9257 ) 9258 return self.expression( 9259 exp.SetItem(expressions=characteristics, kind="TRANSACTION", global_=global_) 9260 ) 9261 9262 def _parse_set_item(self) -> exp.Expr | None: 9263 parser = self._find_parser(self.SET_PARSERS, self.SET_TRIE) 9264 return parser(self) if parser else self._parse_set_item_assignment(kind=None) 9265 9266 def _parse_set(self, unset: bool = False, tag: bool = False) -> exp.Set | exp.Command: 9267 index = self._index 9268 set_ = self.expression( 9269 exp.Set(expressions=self._parse_csv(self._parse_set_item), unset=unset, tag=tag) 9270 ) 9271 9272 if self._curr: 9273 self._retreat(index) 9274 return self._parse_as_command(self._prev) 9275 9276 return set_ 9277 9278 def _parse_var_from_options( 9279 self, options: OPTIONS_TYPE, raise_unmatched: bool = True 9280 ) -> exp.Var | None: 9281 start = self._curr 9282 if not start: 9283 return None 9284 9285 option = start.text.upper() 9286 continuations = options.get(option) 9287 9288 index = self._index 9289 self._advance() 9290 for keywords in continuations or []: 9291 if isinstance(keywords, str): 9292 keywords = (keywords,) 9293 9294 if self._match_text_seq(*keywords): 9295 option = f"{option} {' '.join(keywords)}" 9296 break 9297 else: 9298 if continuations or continuations is None: 9299 if raise_unmatched: 9300 self.raise_error(f"Unknown option {option}") 9301 9302 self._retreat(index) 9303 return None 9304 9305 return exp.var(option) 9306 9307 def _parse_as_command(self, start: Token) -> exp.Command: 9308 while self._curr: 9309 self._advance() 9310 text = self._find_sql(start, self._prev) 9311 size = len(start.text) 9312 self._warn_unsupported() 9313 return exp.Command(this=text[:size], expression=text[size:]) 9314 9315 def _parse_dict_property(self, this: str) -> exp.DictProperty: 9316 settings = [] 9317 9318 self._match_l_paren() 9319 kind = self._parse_id_var() 9320 9321 if self._match(TokenType.L_PAREN): 9322 while True: 9323 key = self._parse_id_var() 9324 value = self._parse_function() or self._parse_primary_or_var() 9325 if not key and value is None: 9326 break 9327 settings.append(self.expression(exp.DictSubProperty(this=key, value=value))) 9328 self._match(TokenType.R_PAREN) 9329 9330 self._match_r_paren() 9331 9332 return self.expression( 9333 exp.DictProperty(this=this, kind=kind.this if kind else None, settings=settings) 9334 ) 9335 9336 def _parse_dict_range(self, this: str) -> exp.DictRange: 9337 self._match_l_paren() 9338 has_min = self._match_text_seq("MIN") 9339 if has_min: 9340 min = self._parse_var() or self._parse_primary() 9341 self._match_text_seq("MAX") 9342 max = self._parse_var() or self._parse_primary() 9343 else: 9344 max = self._parse_var() or self._parse_primary() 9345 min = exp.Literal.number(0) 9346 self._match_r_paren() 9347 return self.expression(exp.DictRange(this=this, min=min, max=max)) 9348 9349 def _parse_comprehension(self, this: exp.Expr | None) -> exp.Comprehension | None: 9350 index = self._index 9351 expression = self._parse_column() 9352 position = self._match(TokenType.COMMA) and self._parse_column() 9353 9354 if not self._match(TokenType.IN): 9355 self._retreat(index - 1) 9356 return None 9357 iterator = self._parse_column() 9358 condition = self._parse_disjunction() if self._match_text_seq("IF") else None 9359 return self.expression( 9360 exp.Comprehension( 9361 this=this, 9362 expression=expression, 9363 position=position, 9364 iterator=iterator, 9365 condition=condition, 9366 ) 9367 ) 9368 9369 def _parse_heredoc(self) -> exp.Heredoc | None: 9370 if self._match(TokenType.HEREDOC_STRING): 9371 return self.expression(exp.Heredoc(this=self._prev.text)) 9372 9373 if not self._match_text_seq("$"): 9374 return None 9375 9376 tags = ["$"] 9377 tag_text = None 9378 9379 if self._is_connected(): 9380 self._advance() 9381 tags.append(self._prev.text.upper()) 9382 else: 9383 self.raise_error("No closing $ found") 9384 9385 if tags[-1] != "$": 9386 if self._is_connected() and self._match_text_seq("$"): 9387 tag_text = tags[-1] 9388 tags.append("$") 9389 else: 9390 self.raise_error("No closing $ found") 9391 9392 heredoc_start = self._curr 9393 9394 while self._curr: 9395 if self._match_text_seq(*tags, advance=False): 9396 this = self._find_sql(heredoc_start, self._prev) 9397 self._advance(len(tags)) 9398 return self.expression(exp.Heredoc(this=this, tag=tag_text)) 9399 9400 self._advance() 9401 9402 self.raise_error(f"No closing {''.join(tags)} found") 9403 return None 9404 9405 def _find_parser(self, parsers: dict[str, t.Callable], trie: dict) -> t.Callable | None: 9406 if not self._curr: 9407 return None 9408 9409 index = self._index 9410 this = [] 9411 while True: 9412 # The current token might be multiple words 9413 curr = self._curr.text.upper() 9414 key = curr.split(" ") 9415 this.append(curr) 9416 9417 self._advance() 9418 result, trie = in_trie(trie, key) 9419 if result == TrieResult.FAILED: 9420 break 9421 9422 if result == TrieResult.EXISTS: 9423 subparser = parsers[" ".join(this)] 9424 return subparser 9425 9426 self._retreat(index) 9427 return None 9428 9429 def _match_l_paren(self, expression: exp.Expr | None = None) -> None: 9430 if not self._match(TokenType.L_PAREN, expression=expression): 9431 self.raise_error("Expecting (") 9432 9433 def _match_r_paren(self, expression: exp.Expr | None = None) -> None: 9434 if not self._match(TokenType.R_PAREN, expression=expression): 9435 self.raise_error("Expecting )") 9436 9437 def _replace_lambda( 9438 self, node: exp.Expr | None, expressions: list[exp.Expr] 9439 ) -> exp.Expr | None: 9440 if not node: 9441 return node 9442 9443 lambda_types = {e.name: e.args.get("to") or False for e in expressions} 9444 9445 for column in node.find_all(exp.Column): 9446 typ = lambda_types.get(column.parts[0].name) 9447 if typ is not None: 9448 dot_or_id = column.to_dot() if column.table else column.this 9449 9450 if typ: 9451 dot_or_id = self.expression(exp.Cast(this=dot_or_id, to=typ)) 9452 9453 parent = column.parent 9454 9455 while isinstance(parent, exp.Dot): 9456 if not isinstance(parent.parent, exp.Dot): 9457 parent.replace(dot_or_id) 9458 break 9459 parent = parent.parent 9460 else: 9461 if column is node: 9462 node = dot_or_id 9463 else: 9464 column.replace(dot_or_id) 9465 return node 9466 9467 def _parse_truncate_table(self) -> exp.TruncateTable | None | exp.Expr: 9468 start = self._prev 9469 9470 # Not to be confused with TRUNCATE(number, decimals) function call 9471 if self._match(TokenType.L_PAREN): 9472 self._retreat(self._index - 2) 9473 return self._parse_function() 9474 9475 # Clickhouse supports TRUNCATE DATABASE as well 9476 is_database = self._match(TokenType.DATABASE) 9477 9478 self._match(TokenType.TABLE) 9479 9480 exists = self._parse_exists(not_=False) 9481 9482 expressions = self._parse_csv( 9483 lambda: self._parse_table(schema=True, is_db_reference=is_database) 9484 ) 9485 9486 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 9487 9488 if self._match_text_seq("RESTART", "IDENTITY"): 9489 identity = "RESTART" 9490 elif self._match_text_seq("CONTINUE", "IDENTITY"): 9491 identity = "CONTINUE" 9492 else: 9493 identity = None 9494 9495 if self._match_text_seq("CASCADE") or self._match_text_seq("RESTRICT"): 9496 option = self._prev.text 9497 else: 9498 option = None 9499 9500 partition = self._parse_partition() 9501 9502 # Fallback case 9503 if self._curr: 9504 return self._parse_as_command(start) 9505 9506 return self.expression( 9507 exp.TruncateTable( 9508 expressions=expressions, 9509 is_database=is_database, 9510 exists=exists, 9511 cluster=cluster, 9512 identity=identity, 9513 option=option, 9514 partition=partition, 9515 ) 9516 ) 9517 9518 def _parse_indexed_column(self) -> exp.Expr | None: 9519 return self._parse_ordered(self._parse_opclass) 9520 9521 def _parse_with_operator(self) -> exp.Expr | None: 9522 this = self._parse_indexed_column() 9523 9524 if not self._match(TokenType.WITH): 9525 return this 9526 9527 op = self._parse_var(any_token=True, tokens=self.RESERVED_TOKENS) 9528 9529 return self.expression(exp.WithOperator(this=this, op=op)) 9530 9531 def _parse_wrapped_options(self) -> list[exp.Expr]: 9532 self._match(TokenType.EQ) 9533 self._match(TokenType.L_PAREN) 9534 9535 opts: list[exp.Expr] = [] 9536 option: exp.Expr | list[exp.Expr] | None 9537 while self._curr and not self._match(TokenType.R_PAREN): 9538 if self._match_text_seq("FORMAT_NAME", "="): 9539 # The FORMAT_NAME can be set to an identifier for Snowflake and T-SQL 9540 option = self._parse_format_name() 9541 else: 9542 option = self._parse_property() 9543 9544 if option is None: 9545 self.raise_error("Unable to parse option") 9546 break 9547 9548 opts.extend(ensure_list(option)) 9549 9550 return opts 9551 9552 def _parse_copy_parameters(self) -> list[exp.CopyParameter]: 9553 sep = TokenType.COMMA if self.dialect.COPY_PARAMS_ARE_CSV else None 9554 9555 options = [] 9556 while self._curr and not self._match(TokenType.R_PAREN, advance=False): 9557 option = self._parse_var(any_token=True) 9558 prev = self._prev.text.upper() 9559 9560 # Different dialects might separate options and values by white space, "=" and "AS" 9561 self._match(TokenType.EQ) 9562 self._match(TokenType.ALIAS) 9563 9564 param = self.expression(exp.CopyParameter(this=option)) 9565 9566 if prev in self.COPY_INTO_VARLEN_OPTIONS and self._match( 9567 TokenType.L_PAREN, advance=False 9568 ): 9569 # Snowflake FILE_FORMAT case, Databricks COPY & FORMAT options 9570 param.set("expressions", self._parse_wrapped_options()) 9571 elif prev == "FILE_FORMAT": 9572 # T-SQL's external file format case 9573 param.set("expression", self._parse_field()) 9574 elif ( 9575 prev == "FORMAT" 9576 and self._prev.token_type == TokenType.ALIAS 9577 and self._match_texts(("AVRO", "JSON")) 9578 ): 9579 param.set("this", exp.var(f"FORMAT AS {self._prev.text.upper()}")) 9580 param.set("expression", self._parse_field()) 9581 else: 9582 param.set("expression", self._parse_unquoted_field() or self._parse_bracket()) 9583 9584 options.append(param) 9585 9586 if sep: 9587 self._match(sep) 9588 9589 return options 9590 9591 def _parse_credentials(self) -> exp.Credentials | None: 9592 expr = self.expression(exp.Credentials()) 9593 9594 if self._match_text_seq("STORAGE_INTEGRATION", "="): 9595 expr.set("storage", self._parse_field()) 9596 if self._match_text_seq("CREDENTIALS"): 9597 # Snowflake case: CREDENTIALS = (...), Redshift case: CREDENTIALS <string> 9598 creds = ( 9599 self._parse_wrapped_options() if self._match(TokenType.EQ) else self._parse_field() 9600 ) 9601 expr.set("credentials", creds) 9602 if self._match_text_seq("ENCRYPTION"): 9603 expr.set("encryption", self._parse_wrapped_options()) 9604 if self._match_text_seq("IAM_ROLE"): 9605 expr.set( 9606 "iam_role", 9607 exp.var(self._prev.text) if self._match(TokenType.DEFAULT) else self._parse_field(), 9608 ) 9609 if self._match_text_seq("REGION"): 9610 expr.set("region", self._parse_field()) 9611 9612 return expr 9613 9614 def _parse_file_location(self) -> exp.Expr | None: 9615 return self._parse_field() 9616 9617 def _parse_copy(self) -> exp.Copy | exp.Command: 9618 start = self._prev 9619 9620 self._match(TokenType.INTO) 9621 9622 this = ( 9623 self._parse_select(nested=True, parse_subquery_alias=False) 9624 if self._match(TokenType.L_PAREN, advance=False) 9625 else self._parse_table(schema=True) 9626 ) 9627 9628 kind = self._match(TokenType.FROM) or not self._match_text_seq("TO") 9629 9630 files = self._parse_csv(self._parse_file_location) 9631 if self._match(TokenType.EQ, advance=False): 9632 # Backtrack one token since we've consumed the lhs of a parameter assignment here. 9633 # This can happen for Snowflake dialect. Instead, we'd like to parse the parameter 9634 # list via `_parse_wrapped(..)` below. 9635 self._advance(-1) 9636 files = [] 9637 9638 credentials = self._parse_credentials() 9639 9640 self._match_text_seq("WITH") 9641 9642 params = self._parse_wrapped(self._parse_copy_parameters, optional=True) 9643 9644 # Fallback case 9645 if self._curr: 9646 return self._parse_as_command(start) 9647 9648 return self.expression( 9649 exp.Copy(this=this, kind=kind, credentials=credentials, files=files, params=params) 9650 ) 9651 9652 def _parse_normalize(self) -> exp.Normalize: 9653 return self.expression( 9654 exp.Normalize( 9655 this=self._parse_bitwise(), form=self._match(TokenType.COMMA) and self._parse_var() 9656 ) 9657 ) 9658 9659 def _parse_ceil_floor(self, expr_type: type[TCeilFloor]) -> TCeilFloor: 9660 args = self._parse_csv(lambda: self._parse_lambda()) 9661 9662 this = seq_get(args, 0) 9663 decimals = seq_get(args, 1) 9664 9665 return expr_type( 9666 this=this, 9667 decimals=decimals, 9668 to=self._parse_var() if self._match_text_seq("TO") else None, 9669 ) 9670 9671 def _parse_star_ops(self) -> exp.Expr | None: 9672 star_token = self._prev 9673 9674 if self._match_text_seq("COLUMNS", "(", advance=False): 9675 this = self._parse_function() 9676 if isinstance(this, exp.Columns): 9677 this.set("unpack", True) 9678 return this 9679 9680 ilike = self._parse_string() if self._match(TokenType.ILIKE) else None 9681 9682 return self.expression( 9683 exp.Star( 9684 ilike=ilike, 9685 except_=self._parse_star_op("EXCEPT", "EXCLUDE"), 9686 replace=self._parse_star_op("REPLACE"), 9687 rename=self._parse_star_op("RENAME"), 9688 ) 9689 ).update_positions(star_token) 9690 9691 def _parse_grant_privilege(self) -> exp.GrantPrivilege | None: 9692 privilege_parts = [] 9693 9694 # Keep consuming consecutive keywords until comma (end of this privilege) or ON 9695 # (end of privilege list) or L_PAREN (start of column list) are met 9696 while self._curr and not self._match_set(self.PRIVILEGE_FOLLOW_TOKENS, advance=False): 9697 privilege_parts.append(self._curr.text.upper()) 9698 self._advance() 9699 9700 this = exp.var(" ".join(privilege_parts)) 9701 expressions = ( 9702 self._parse_wrapped_csv(self._parse_column) 9703 if self._match(TokenType.L_PAREN, advance=False) 9704 else None 9705 ) 9706 9707 return self.expression(exp.GrantPrivilege(this=this, expressions=expressions)) 9708 9709 def _parse_grant_principal(self) -> exp.GrantPrincipal | None: 9710 kind = self._match_texts(("ROLE", "GROUP")) and self._prev.text.upper() 9711 principal = self._parse_id_var() 9712 9713 if not principal: 9714 return None 9715 9716 return self.expression(exp.GrantPrincipal(this=principal, kind=kind)) 9717 9718 def _parse_grant_revoke_common( 9719 self, 9720 ) -> tuple[list | None, str | None, exp.Expr | None]: 9721 privileges = self._parse_csv(self._parse_grant_privilege) 9722 9723 self._match(TokenType.ON) 9724 kind = self._prev.text.upper() if self._match_set(self.CREATABLES) else None 9725 9726 # Attempt to parse the securable e.g. MySQL allows names 9727 # such as "foo.*", "*.*" which are not easily parseable yet 9728 securable = self._try_parse(self._parse_table_parts) 9729 9730 return privileges, kind, securable 9731 9732 def _parse_grant(self) -> exp.Grant | exp.Command: 9733 start = self._prev 9734 9735 privileges, kind, securable = self._parse_grant_revoke_common() 9736 9737 if not securable or not self._match_text_seq("TO"): 9738 return self._parse_as_command(start) 9739 9740 principals = self._parse_csv(self._parse_grant_principal) 9741 9742 grant_option = self._match_text_seq("WITH", "GRANT", "OPTION") 9743 9744 if self._curr: 9745 return self._parse_as_command(start) 9746 9747 return self.expression( 9748 exp.Grant( 9749 privileges=privileges, 9750 kind=kind, 9751 securable=securable, 9752 principals=principals, 9753 grant_option=grant_option, 9754 ) 9755 ) 9756 9757 def _parse_revoke(self) -> exp.Revoke | exp.Command: 9758 start = self._prev 9759 9760 grant_option = self._match_text_seq("GRANT", "OPTION", "FOR") 9761 9762 privileges, kind, securable = self._parse_grant_revoke_common() 9763 9764 if not securable or not self._match_text_seq("FROM"): 9765 return self._parse_as_command(start) 9766 9767 principals = self._parse_csv(self._parse_grant_principal) 9768 9769 cascade = None 9770 if self._match_texts(("CASCADE", "RESTRICT")): 9771 cascade = self._prev.text.upper() 9772 9773 if self._curr: 9774 return self._parse_as_command(start) 9775 9776 return self.expression( 9777 exp.Revoke( 9778 privileges=privileges, 9779 kind=kind, 9780 securable=securable, 9781 principals=principals, 9782 grant_option=grant_option, 9783 cascade=cascade, 9784 ) 9785 ) 9786 9787 def _parse_overlay(self) -> exp.Overlay: 9788 def _parse_overlay_arg(text: str) -> exp.Expr | None: 9789 return ( 9790 self._parse_bitwise() 9791 if self._match(TokenType.COMMA) or self._match_text_seq(text) 9792 else None 9793 ) 9794 9795 return self.expression( 9796 exp.Overlay( 9797 this=self._parse_bitwise(), 9798 expression=_parse_overlay_arg("PLACING"), 9799 from_=_parse_overlay_arg("FROM"), 9800 for_=_parse_overlay_arg("FOR"), 9801 ) 9802 ) 9803 9804 def _parse_format_name(self) -> exp.Property: 9805 # Note: Although not specified in the docs, Snowflake does accept a string/identifier 9806 # for FILE_FORMAT = <format_name> 9807 return self.expression( 9808 exp.Property( 9809 this=exp.var("FORMAT_NAME"), value=self._parse_string() or self._parse_table_parts() 9810 ) 9811 ) 9812 9813 def _parse_max_min_by(self, expr_type: type[exp.AggFunc]) -> exp.AggFunc: 9814 args: list[exp.Expr] = [] 9815 9816 if self._match(TokenType.DISTINCT): 9817 args.append(self.expression(exp.Distinct(expressions=[self._parse_lambda()]))) 9818 self._match(TokenType.COMMA) 9819 9820 args.extend(self._parse_function_args()) 9821 9822 return self.expression( 9823 expr_type(this=seq_get(args, 0), expression=seq_get(args, 1), count=seq_get(args, 2)) 9824 ) 9825 9826 def _identifier_expression( 9827 self, token: Token | None = None, quoted: bool | None = None 9828 ) -> exp.Identifier: 9829 token = token or self._prev 9830 return self.expression(exp.Identifier(this=token.text, quoted=quoted), token) 9831 9832 def _build_pipe_cte( 9833 self, 9834 query: exp.Query, 9835 expressions: list[exp.Expr], 9836 alias_cte: exp.TableAlias | None = None, 9837 ) -> exp.Select: 9838 new_cte: str | exp.TableAlias | None 9839 if alias_cte: 9840 new_cte = alias_cte 9841 else: 9842 self._pipe_cte_counter += 1 9843 new_cte = f"__tmp{self._pipe_cte_counter}" 9844 9845 with_ = query.args.get("with_") 9846 ctes = with_.pop() if with_ else None 9847 9848 new_select = exp.select(*expressions, copy=False).from_(new_cte, copy=False) 9849 if ctes: 9850 new_select.set("with_", ctes) 9851 9852 return new_select.with_(new_cte, as_=query, copy=False) 9853 9854 def _parse_pipe_syntax_select(self, query: exp.Select) -> exp.Select: 9855 select = self._parse_select(consume_pipe=False) 9856 if not select: 9857 return query 9858 9859 return self._build_pipe_cte( 9860 query=query.select(*select.expressions, append=False), expressions=[exp.Star()] 9861 ) 9862 9863 def _parse_pipe_syntax_limit(self, query: exp.Select) -> exp.Select: 9864 limit = self._parse_limit() 9865 offset = self._parse_offset() 9866 if limit: 9867 curr_limit = query.args.get("limit", limit) 9868 if curr_limit.expression.to_py() >= limit.expression.to_py(): 9869 query.limit(limit, copy=False) 9870 if offset: 9871 curr_offset = query.args.get("offset") 9872 curr_offset = curr_offset.expression.to_py() if curr_offset else 0 9873 query.offset(exp.Literal.number(curr_offset + offset.expression.to_py()), copy=False) 9874 9875 return query 9876 9877 def _parse_pipe_syntax_aggregate_fields(self) -> exp.Expr | None: 9878 this = self._parse_disjunction() 9879 if self._match_text_seq("GROUP", "AND", advance=False): 9880 return this 9881 9882 this = self._parse_alias(this) 9883 9884 if self._match_set((TokenType.ASC, TokenType.DESC), advance=False): 9885 return self._parse_ordered(lambda: this) 9886 9887 return this 9888 9889 def _parse_pipe_syntax_aggregate_group_order_by( 9890 self, query: exp.Select, group_by_exists: bool = True 9891 ) -> exp.Select: 9892 expr = self._parse_csv(self._parse_pipe_syntax_aggregate_fields) 9893 aggregates_or_groups, orders = [], [] 9894 for element in expr: 9895 if isinstance(element, exp.Ordered): 9896 this = element.this 9897 if isinstance(this, exp.Alias): 9898 element.set("this", this.args["alias"]) 9899 orders.append(element) 9900 else: 9901 this = element 9902 aggregates_or_groups.append(this) 9903 9904 if group_by_exists: 9905 query.select( 9906 *aggregates_or_groups, *query.expressions, append=False, copy=False 9907 ).group_by( 9908 *[projection.args.get("alias", projection) for projection in aggregates_or_groups], 9909 copy=False, 9910 ) 9911 else: 9912 query.select(*aggregates_or_groups, append=False, copy=False) 9913 9914 if orders: 9915 return query.order_by(*orders, append=False, copy=False) 9916 9917 return query 9918 9919 def _parse_pipe_syntax_aggregate(self, query: exp.Select) -> exp.Select: 9920 self._match_text_seq("AGGREGATE") 9921 query = self._parse_pipe_syntax_aggregate_group_order_by(query, group_by_exists=False) 9922 9923 if self._match(TokenType.GROUP_BY) or ( 9924 self._match_text_seq("GROUP", "AND") and self._match(TokenType.ORDER_BY) 9925 ): 9926 query = self._parse_pipe_syntax_aggregate_group_order_by(query) 9927 9928 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 9929 9930 def _parse_pipe_syntax_set_operator(self, query: exp.Query) -> exp.Query | None: 9931 first_setop = self.parse_set_operation(this=query) 9932 if not first_setop: 9933 return None 9934 9935 def _parse_and_unwrap_query() -> exp.Expr | None: 9936 expr = self._parse_paren() 9937 return expr.assert_is(exp.Subquery).unnest() if expr else None 9938 9939 first_setop.this.pop() 9940 9941 setops = [ 9942 first_setop.expression.pop().assert_is(exp.Subquery).unnest(), 9943 *self._parse_csv(_parse_and_unwrap_query), 9944 ] 9945 9946 query = self._build_pipe_cte(query=query, expressions=[exp.Star()]) 9947 with_ = query.args.get("with_") 9948 ctes = with_.pop() if with_ else None 9949 9950 if isinstance(first_setop, exp.Union): 9951 query = query.union(*setops, copy=False, **first_setop.args) 9952 elif isinstance(first_setop, exp.Except): 9953 query = query.except_(*setops, copy=False, **first_setop.args) 9954 else: 9955 query = query.intersect(*setops, copy=False, **first_setop.args) 9956 9957 query.set("with_", ctes) 9958 9959 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 9960 9961 def _parse_pipe_syntax_join(self, query: exp.Query) -> exp.Query | None: 9962 join = self._parse_join() 9963 if not join: 9964 return None 9965 9966 if isinstance(query, exp.Select): 9967 return query.join(join, copy=False) 9968 9969 return query 9970 9971 def _parse_pipe_syntax_pivot(self, query: exp.Select) -> exp.Select: 9972 pivots = self._parse_pivots() 9973 if not pivots: 9974 return query 9975 9976 from_ = query.args.get("from_") 9977 if from_: 9978 from_.this.set("pivots", pivots) 9979 9980 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 9981 9982 def _parse_pipe_syntax_extend(self, query: exp.Select) -> exp.Select: 9983 self._match_text_seq("EXTEND") 9984 query.select(*[exp.Star(), *self._parse_expressions()], append=False, copy=False) 9985 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 9986 9987 def _parse_pipe_syntax_tablesample(self, query: exp.Select) -> exp.Select: 9988 sample = self._parse_table_sample() 9989 9990 with_ = query.args.get("with_") 9991 if with_: 9992 with_.expressions[-1].this.set("sample", sample) 9993 else: 9994 query.set("sample", sample) 9995 9996 return query 9997 9998 def _parse_pipe_syntax_query(self, query: exp.Query) -> exp.Query | None: 9999 if isinstance(query, exp.Subquery): 10000 query = exp.select("*").from_(query, copy=False) 10001 10002 if not query.args.get("from_"): 10003 query = exp.select("*").from_(query.subquery(copy=False), copy=False) 10004 10005 while self._match(TokenType.PIPE_GT): 10006 start_index = self._index 10007 start_text = self._curr.text.upper() 10008 parser = self.PIPE_SYNTAX_TRANSFORM_PARSERS.get(start_text) 10009 if not parser: 10010 # The set operators (UNION, etc) and the JOIN operator have a few common starting 10011 # keywords, making it tricky to disambiguate them without lookahead. The approach 10012 # here is to try and parse a set operation and if that fails, then try to parse a 10013 # join operator. If that fails as well, then the operator is not supported. 10014 parsed_query = self._parse_pipe_syntax_set_operator(query) 10015 parsed_query = parsed_query or self._parse_pipe_syntax_join(query) 10016 if not parsed_query: 10017 self._retreat(start_index) 10018 self.raise_error(f"Unsupported pipe syntax operator: '{start_text}'.") 10019 break 10020 query = parsed_query 10021 else: 10022 query = parser(self, query) 10023 10024 return query 10025 10026 def _parse_declareitem(self) -> exp.DeclareItem | None: 10027 self._match_texts(("VAR", "VARIABLE")) 10028 10029 vars = self._parse_csv(self._parse_id_var) 10030 if not vars: 10031 return None 10032 10033 self._match(TokenType.ALIAS) 10034 kind = self._parse_schema() if self._match(TokenType.TABLE) else self._parse_types() 10035 default = ( 10036 self._match(TokenType.DEFAULT) or self._match(TokenType.EQ) 10037 ) and self._parse_bitwise() 10038 10039 return self.expression(exp.DeclareItem(this=vars, kind=kind, default=default)) 10040 10041 def _parse_declare(self) -> exp.Declare | exp.Command: 10042 start = self._prev 10043 replace = self._match_text_seq("OR", "REPLACE") 10044 expressions = self._try_parse(lambda: self._parse_csv(self._parse_declareitem)) 10045 10046 if not expressions or self._curr: 10047 return self._parse_as_command(start) 10048 10049 return self.expression(exp.Declare(expressions=expressions, replace=replace)) 10050 10051 def build_cast(self, strict: bool, **kwargs) -> exp.Expr: 10052 exp_class = exp.Cast if strict else exp.TryCast 10053 10054 if exp_class == exp.TryCast: 10055 kwargs["requires_string"] = self.dialect.TRY_CAST_REQUIRES_STRING 10056 10057 return self.expression(exp_class(**kwargs)) 10058 10059 def _parse_json_value(self) -> exp.JSONValue: 10060 this = self._parse_bitwise() 10061 self._match(TokenType.COMMA) 10062 path = self._parse_bitwise() 10063 10064 returning = self._match(TokenType.RETURNING) and self._parse_type() 10065 10066 return self.expression( 10067 exp.JSONValue( 10068 this=this, 10069 path=self.dialect.to_json_path(path), 10070 returning=returning, 10071 on_condition=self._parse_on_condition(), 10072 ) 10073 ) 10074 10075 def _parse_group_concat(self) -> exp.Expr | None: 10076 def concat_exprs(node: exp.Expr | None, exprs: list[exp.Expr]) -> exp.Expr: 10077 if isinstance(node, exp.Distinct) and len(node.expressions) > 1: 10078 concat_exprs = [ 10079 self.expression( 10080 exp.Concat( 10081 expressions=node.expressions, 10082 safe=True, 10083 coalesce=self.dialect.CONCAT_COALESCE, 10084 ) 10085 ) 10086 ] 10087 node.set("expressions", concat_exprs) 10088 return node 10089 if len(exprs) == 1: 10090 return exprs[0] 10091 return self.expression( 10092 exp.Concat(expressions=args, safe=True, coalesce=self.dialect.CONCAT_COALESCE) 10093 ) 10094 10095 args = self._parse_csv(self._parse_lambda) 10096 10097 if args: 10098 order = args[-1] if isinstance(args[-1], exp.Order) else None 10099 10100 if order: 10101 # Order By is the last (or only) expression in the list and has consumed the 'expr' before it, 10102 # remove 'expr' from exp.Order and add it back to args 10103 args[-1] = order.this 10104 order.set("this", concat_exprs(order.this, args)) 10105 10106 this = order or concat_exprs(args[0], args) 10107 else: 10108 this = None 10109 10110 separator = self._parse_field() if self._match(TokenType.SEPARATOR) else None 10111 10112 return self.expression(exp.GroupConcat(this=this, separator=separator)) 10113 10114 def _parse_initcap(self) -> exp.Initcap: 10115 expr = exp.Initcap.from_arg_list(self._parse_function_args()) 10116 10117 # attach dialect's default delimiters 10118 if expr.args.get("expression") is None: 10119 expr.set("expression", exp.Literal.string(self.dialect.INITCAP_DEFAULT_DELIMITER_CHARS)) 10120 10121 return expr 10122 10123 def _parse_operator(self, this: exp.Expr | None) -> exp.Expr | None: 10124 while True: 10125 if not self._match(TokenType.L_PAREN): 10126 break 10127 10128 op = "" 10129 while self._curr and not self._match(TokenType.R_PAREN): 10130 op += self._curr.text 10131 self._advance() 10132 10133 comments = self._prev_comments 10134 this = self.expression( 10135 exp.Operator(this=this, operator=op, expression=self._parse_bitwise()), 10136 comments=comments, 10137 ) 10138 10139 if not self._match(TokenType.OPERATOR): 10140 break 10141 10142 return this
Parser consumes a list of tokens produced by the Tokenizer and produces a parsed syntax tree.
Arguments:
- error_level: The desired error level. Default: ErrorLevel.IMMEDIATE
- error_message_context: The amount of context to capture from a query string when displaying the error message (in number of characters). Default: 100
- max_errors: Maximum number of error messages to include in a raised ParseError. This is only relevant if error_level is ErrorLevel.RAISE. Default: 3
- max_nodes: Maximum number of AST nodes to prevent memory exhaustion. Set to -1 (default) to disable the check.
1858 def __init__( 1859 self, 1860 error_level: ErrorLevel | None = None, 1861 error_message_context: int = 100, 1862 max_errors: int = 3, 1863 max_nodes: int = -1, 1864 dialect: DialectType = None, 1865 ): 1866 self.error_level: ErrorLevel = error_level or ErrorLevel.IMMEDIATE 1867 self.error_message_context: int = error_message_context 1868 self.max_errors: int = max_errors 1869 self.max_nodes: int = max_nodes 1870 self.dialect: t.Any = _resolve_dialect(dialect) 1871 self.sql: str = "" 1872 self.errors: list[ParseError] = [] 1873 self._tokens: list[Token] = [] 1874 self._tokens_size: i64 = 0 1875 self._index: i64 = 0 1876 self._curr: Token = SENTINEL_NONE 1877 self._next: Token = SENTINEL_NONE 1878 self._prev: Token = SENTINEL_NONE 1879 self._prev_comments: list[str] = [] 1880 self._pipe_cte_counter: int = 0 1881 self._chunks: list[list[Token]] = [] 1882 self._chunk_index: i64 = 0 1883 self._node_count: int = 0
1885 def reset(self) -> None: 1886 self.sql = "" 1887 self.errors = [] 1888 self._tokens = [] 1889 self._tokens_size = 0 1890 self._index = 0 1891 self._curr = SENTINEL_NONE 1892 self._next = SENTINEL_NONE 1893 self._prev = SENTINEL_NONE 1894 self._prev_comments = [] 1895 self._pipe_cte_counter = 0 1896 self._chunks = [] 1897 self._chunk_index = 0 1898 self._node_count = 0
1988 def raise_error(self, message: str, token: Token = SENTINEL_NONE) -> None: 1989 token = token or self._curr or self._prev or Token.string("") 1990 formatted_sql, start_context, highlight, end_context = highlight_sql( 1991 sql=self.sql, 1992 positions=[(token.start, token.end)], 1993 context_length=self.error_message_context, 1994 ) 1995 formatted_message = f"{message}. Line {token.line}, Col: {token.col}.\n {formatted_sql}" 1996 1997 error = ParseError.new( 1998 formatted_message, 1999 description=message, 2000 line=token.line, 2001 col=token.col, 2002 start_context=start_context, 2003 highlight=highlight, 2004 end_context=end_context, 2005 ) 2006 2007 if self.error_level == ErrorLevel.IMMEDIATE: 2008 raise error 2009 2010 self.errors.append(error)
2012 def validate_expression(self, expression: E, args: list | None = None) -> E: 2013 if self.max_nodes > -1: 2014 self._node_count += 1 2015 if self._node_count > self.max_nodes: 2016 self.raise_error(f"Maximum number of AST nodes ({self.max_nodes}) exceeded") 2017 if self.error_level != ErrorLevel.IGNORE: 2018 for error_message in expression.error_messages(args): 2019 self.raise_error(error_message) 2020 return expression
2039 def parse(self, raw_tokens: list[Token], sql: str) -> list[exp.Expr | None]: 2040 """ 2041 Parses a list of tokens and returns a list of syntax trees, one tree 2042 per parsed SQL statement. 2043 2044 Args: 2045 raw_tokens: The list of tokens. 2046 sql: The original SQL string. 2047 2048 Returns: 2049 The list of the produced syntax trees. 2050 """ 2051 return self._parse( 2052 parse_method=self.__class__._parse_statement, raw_tokens=raw_tokens, sql=sql 2053 )
Parses a list of tokens and returns a list of syntax trees, one tree per parsed SQL statement.
Arguments:
- raw_tokens: The list of tokens.
- sql: The original SQL string.
Returns:
The list of the produced syntax trees.
2055 def parse_into( 2056 self, 2057 expression_types: exp.IntoType, 2058 raw_tokens: list[Token], 2059 sql: str | None = None, 2060 ) -> list[exp.Expr | None]: 2061 """ 2062 Parses a list of tokens into a given Expr type. If a collection of Expr 2063 types is given instead, this method will try to parse the token list into each one 2064 of them, stopping at the first for which the parsing succeeds. 2065 2066 Args: 2067 expression_types: The expression type(s) to try and parse the token list into. 2068 raw_tokens: The list of tokens. 2069 sql: The original SQL string, used to produce helpful debug messages. 2070 2071 Returns: 2072 The target Expr. 2073 """ 2074 errors = [] 2075 for expression_type in ensure_list(expression_types): 2076 parser = self.EXPRESSION_PARSERS.get(t.cast(type[exp.Expr], expression_type)) 2077 if not parser: 2078 raise TypeError(f"No parser registered for {expression_type}") 2079 2080 try: 2081 return self._parse(parser, raw_tokens, sql) 2082 except ParseError as e: 2083 e.errors[0]["into_expression"] = expression_type 2084 errors.append(e) 2085 2086 raise ParseError( 2087 f"Failed to parse '{sql or raw_tokens}' into {expression_types}", 2088 errors=merge_errors(errors), 2089 ) from errors[-1]
Parses a list of tokens into a given Expr type. If a collection of Expr types is given instead, this method will try to parse the token list into each one of them, stopping at the first for which the parsing succeeds.
Arguments:
- expression_types: The expression type(s) to try and parse the token list into.
- raw_tokens: The list of tokens.
- sql: The original SQL string, used to produce helpful debug messages.
Returns:
The target Expr.
2091 def check_errors(self) -> None: 2092 """Logs or raises any found errors, depending on the chosen error level setting.""" 2093 if self.error_level == ErrorLevel.WARN: 2094 for error in self.errors: 2095 logger.error(str(error)) 2096 elif self.error_level == ErrorLevel.RAISE and self.errors: 2097 raise ParseError( 2098 concat_messages(self.errors, self.max_errors), 2099 errors=merge_errors(self.errors), 2100 )
Logs or raises any found errors, depending on the chosen error level setting.
2102 def expression( 2103 self, 2104 instance: E, 2105 token: Token | None = None, 2106 comments: list[str] | None = None, 2107 ) -> E: 2108 if token: 2109 instance.update_positions(token) 2110 instance.add_comments(comments) if comments else self._add_comments(instance) 2111 if not instance.is_primitive: 2112 instance = self.validate_expression(instance) 2113 return instance
5706 def parse_set_operation( 5707 self, this: exp.Expr | None, consume_pipe: bool = False 5708 ) -> exp.Expr | None: 5709 start = self._index 5710 _, side_token, kind_token = self._parse_join_parts() 5711 5712 side = side_token.text if side_token else None 5713 kind = kind_token.text if kind_token else None 5714 5715 if not self._match_set(self.SET_OPERATIONS): 5716 self._retreat(start) 5717 return None 5718 5719 token_type = self._prev.token_type 5720 5721 if token_type == TokenType.UNION: 5722 operation: type[exp.SetOperation] = exp.Union 5723 elif token_type == TokenType.EXCEPT: 5724 operation = exp.Except 5725 else: 5726 operation = exp.Intersect 5727 5728 comments = self._prev.comments 5729 5730 if self._match(TokenType.DISTINCT): 5731 distinct: bool | None = True 5732 elif self._match(TokenType.ALL): 5733 distinct = False 5734 else: 5735 distinct = self.dialect.SET_OP_DISTINCT_BY_DEFAULT[operation] 5736 if distinct is None: 5737 self.raise_error(f"Expected DISTINCT or ALL for {operation.__name__}") 5738 5739 by_name = ( 5740 self._match_text_seq("BY", "NAME") 5741 or self._match_text_seq("STRICT", "CORRESPONDING") 5742 or None 5743 ) 5744 if self._match_text_seq("CORRESPONDING"): 5745 by_name = True 5746 if not side and not kind: 5747 kind = "INNER" 5748 5749 on_column_list = None 5750 if by_name and self._match_texts(("ON", "BY")): 5751 on_column_list = self._parse_wrapped_csv(self._parse_column) 5752 5753 expression = self._parse_select( 5754 nested=True, parse_set_operation=False, consume_pipe=consume_pipe 5755 ) 5756 5757 return self.expression( 5758 operation( 5759 this=this, 5760 distinct=distinct, 5761 by_name=by_name, 5762 expression=expression, 5763 side=side, 5764 kind=kind, 5765 on=on_column_list, 5766 ), 5767 comments=comments, 5768 )