sqlglot.parser
1from __future__ import annotations 2 3import itertools 4import logging 5import re 6import typing as t 7from builtins import type as Type 8from collections import defaultdict 9from collections.abc import Sequence 10 11from sqlglot import exp 12from sqlglot._typing import F 13from sqlglot.errors import ( 14 ErrorLevel, 15 ParseError, 16 TokenError, 17 concat_messages, 18 highlight_sql, 19 merge_errors, 20) 21from sqlglot.expressions import apply_index_offset 22from sqlglot.helper import ensure_list, i64, seq_get 23from sqlglot.time import format_time 24from sqlglot.tokens import Token, Tokenizer, TokenType 25from sqlglot.trie import TrieResult, in_trie, new_trie 26 27if t.TYPE_CHECKING: 28 from re import Pattern 29 30 from sqlglot._typing import BuilderArgs, E 31 from sqlglot.dialects.dialect import Dialect, DialectType 32 from sqlglot.expressions import ExpOrStr 33 34 T = t.TypeVar("T") 35 TCeilFloor = t.TypeVar("TCeilFloor", exp.Ceil, exp.Floor) 36 37logger = logging.getLogger("sqlglot") 38 39OPTIONS_TYPE = dict[str, Sequence[t.Union[Sequence[str], str]]] 40 41# Excludes bare strings, which are also collections of strings, so that a single keyword 42# can't accidentally be matched with substring semantics (e.g. _match_texts("FOO")) 43TEXTS_TYPE = t.Union[tuple[str, ...], list[str], t.AbstractSet[str], t.Mapping[str, t.Any]] 44 45# Used to detect alphabetical characters and +/- in timestamp literals 46TIME_ZONE_RE: Pattern[str] = re.compile(r":.*?[a-zA-Z\+\-]") 47 48 49def build_var_map(args: BuilderArgs) -> exp.StarMap | exp.VarMap: 50 if len(args) == 1 and args[0].is_star: 51 return exp.StarMap(this=args[0]) 52 53 keys: list[ExpOrStr] = [] 54 values: list[ExpOrStr] = [] 55 for i in range(0, len(args), 2): 56 keys.append(args[i]) 57 values.append(args[i + 1]) 58 59 return exp.VarMap(keys=exp.array(*keys, copy=False), values=exp.array(*values, copy=False)) 60 61 62def build_like(args: BuilderArgs) -> exp.Escape | exp.Like: 63 like = exp.Like(this=seq_get(args, 1), expression=seq_get(args, 0)) 64 return exp.Escape(this=like, expression=seq_get(args, 2)) if len(args) > 2 else like 65 66 67def binary_range_parser( 68 expr_type: Type[exp.Expr], reverse_args: bool = False 69) -> t.Callable[[Parser, exp.Expr | None], exp.Expr | None]: 70 def _parse_binary_range(self: Parser, this: exp.Expr | None) -> exp.Expr | None: 71 expression = self._parse_bitwise() 72 if reverse_args: 73 this, expression = expression, this 74 return self._parse_escape(self.expression(expr_type(this=this, expression=expression))) 75 76 return _parse_binary_range 77 78 79def build_logarithm(args: BuilderArgs, dialect: Dialect) -> exp.Func: 80 # Default argument order is base, expression 81 this = seq_get(args, 0) 82 expression = seq_get(args, 1) 83 84 if expression: 85 if not dialect.LOG_BASE_FIRST: 86 this, expression = expression, this 87 return exp.Log(this=this, expression=expression) 88 89 return (exp.Ln if dialect.parser_class.LOG_DEFAULTS_TO_LN else exp.Log)(this=this) 90 91 92def build_hex(args: BuilderArgs, dialect: Dialect) -> exp.Hex | exp.LowerHex: 93 arg = seq_get(args, 0) 94 return exp.LowerHex(this=arg) if dialect.HEX_LOWERCASE else exp.Hex(this=arg) 95 96 97def build_lower(args: BuilderArgs) -> exp.Lower | exp.Hex: 98 # LOWER(HEX(..)) can be simplified to LowerHex to simplify its transpilation 99 arg = seq_get(args, 0) 100 return exp.LowerHex(this=arg.this) if isinstance(arg, exp.Hex) else exp.Lower(this=arg) 101 102 103def build_upper(args: BuilderArgs) -> exp.Upper | exp.Hex: 104 # UPPER(HEX(..)) can be simplified to Hex to simplify its transpilation 105 arg = seq_get(args, 0) 106 return exp.Hex(this=arg.this) if isinstance(arg, exp.Hex) else exp.Upper(this=arg) 107 108 109def build_extract_json_with_path( 110 expr_type: Type[E], 111) -> t.Callable[[BuilderArgs, Dialect], E]: 112 def _builder(args: BuilderArgs, dialect: Dialect) -> E: 113 expression = expr_type( 114 this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1)) 115 ) 116 if len(args) > 2 and expr_type is exp.JSONExtract: 117 expression.set("expressions", args[2:]) 118 if expr_type is exp.JSONExtractScalar: 119 expression.set("scalar_only", dialect.JSON_EXTRACT_SCALAR_SCALAR_ONLY) 120 121 return expression 122 123 return _builder 124 125 126def build_mod(args: BuilderArgs) -> exp.Mod: 127 this = seq_get(args, 0) 128 expression = seq_get(args, 1) 129 130 # Wrap the operands if they are binary nodes, e.g. MOD(a + 1, 7) -> (a + 1) % 7 131 this = exp.Paren(this=this) if isinstance(this, exp.Binary) else this 132 expression = exp.Paren(this=expression) if isinstance(expression, exp.Binary) else expression 133 134 return exp.Mod(this=this, expression=expression) 135 136 137def build_pad(args: BuilderArgs, is_left: bool = True): 138 return exp.Pad( 139 this=seq_get(args, 0), 140 expression=seq_get(args, 1), 141 fill_pattern=seq_get(args, 2), 142 is_left=is_left, 143 ) 144 145 146def build_array_constructor( 147 exp_class: Type[E], args: list[t.Any], bracket_kind: TokenType, dialect: Dialect 148) -> exp.Expr: 149 array_exp = exp_class(expressions=args) 150 151 if exp_class == exp.Array and dialect.HAS_DISTINCT_ARRAY_CONSTRUCTORS: 152 array_exp.set("bracket_notation", bracket_kind == TokenType.L_BRACKET) 153 154 return array_exp 155 156 157def build_convert_timezone( 158 args: BuilderArgs, default_source_tz: str | None = None 159) -> exp.ConvertTimezone | exp.Anonymous: 160 if len(args) == 2: 161 source_tz = exp.Literal.string(default_source_tz) if default_source_tz else None 162 return exp.ConvertTimezone( 163 source_tz=source_tz, target_tz=seq_get(args, 0), timestamp=seq_get(args, 1) 164 ) 165 166 return exp.ConvertTimezone.from_arg_list(args) 167 168 169def build_trim(args: BuilderArgs, is_left: bool = True, reverse_args: bool = False) -> exp.Trim: 170 this, expression = seq_get(args, 0), seq_get(args, 1) 171 172 if expression and reverse_args: 173 this, expression = expression, this 174 175 return exp.Trim(this=this, expression=expression, position="LEADING" if is_left else "TRAILING") 176 177 178def build_coalesce( 179 args: BuilderArgs, is_nvl: bool | None = None, is_null: bool | None = None 180) -> exp.Coalesce: 181 return exp.Coalesce(this=seq_get(args, 0), expressions=args[1:], is_nvl=is_nvl, is_null=is_null) 182 183 184def build_locate_strposition(args: BuilderArgs) -> exp.StrPosition: 185 return exp.StrPosition( 186 this=seq_get(args, 1), 187 substr=seq_get(args, 0), 188 position=seq_get(args, 2), 189 ) 190 191 192def build_array_append(args: BuilderArgs, dialect: Dialect) -> exp.ArrayAppend: 193 """ 194 Builds ArrayAppend with NULL propagation semantics based on the dialect configuration. 195 196 Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. 197 Others (DuckDB, PostgreSQL) create a new single-element array instead. 198 199 Args: 200 args: Function arguments [array, element] 201 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 202 203 Returns: 204 ArrayAppend expression with appropriate null_propagation flag 205 """ 206 return exp.ArrayAppend( 207 this=seq_get(args, 0), 208 expression=seq_get(args, 1), 209 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 210 ) 211 212 213def build_array_prepend(args: BuilderArgs, dialect: Dialect) -> exp.ArrayPrepend: 214 """ 215 Builds ArrayPrepend with NULL propagation semantics based on the dialect configuration. 216 217 Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. 218 Others (DuckDB, PostgreSQL) create a new single-element array instead. 219 220 Args: 221 args: Function arguments [array, element] 222 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 223 224 Returns: 225 ArrayPrepend expression with appropriate null_propagation flag 226 """ 227 return exp.ArrayPrepend( 228 this=seq_get(args, 0), 229 expression=seq_get(args, 1), 230 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 231 ) 232 233 234def build_array_concat(args: BuilderArgs, dialect: Dialect) -> exp.ArrayConcat: 235 """ 236 Builds ArrayConcat with NULL propagation semantics based on the dialect configuration. 237 238 Some dialects (Redshift, Snowflake) return NULL when any input array is NULL. 239 Others (DuckDB, PostgreSQL) skip NULL arrays and continue concatenation. 240 241 Args: 242 args: Function arguments [array1, array2, ...] (variadic) 243 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 244 245 Returns: 246 ArrayConcat expression with appropriate null_propagation flag 247 """ 248 return exp.ArrayConcat( 249 this=seq_get(args, 0), 250 expressions=args[1:], 251 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 252 ) 253 254 255def build_array_remove(args: BuilderArgs, dialect: Dialect) -> exp.ArrayRemove: 256 """ 257 Builds ArrayRemove with NULL propagation semantics based on the dialect configuration. 258 259 Some dialects (Snowflake) return NULL when the removal value is NULL. 260 Others (DuckDB) may return empty array due to NULL comparison semantics. 261 262 Args: 263 args: Function arguments [array, value_to_remove] 264 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 265 266 Returns: 267 ArrayRemove expression with appropriate null_propagation flag 268 """ 269 return exp.ArrayRemove( 270 this=seq_get(args, 0), 271 expression=seq_get(args, 1), 272 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 273 ) 274 275 276def _resolve_dialect(dialect: DialectType) -> Dialect: 277 from sqlglot.dialects.dialect import Dialect 278 279 return Dialect.get_or_raise(dialect) 280 281 282def _unpivot_target(expr: exp.Expr) -> exp.Expr: 283 # UNPIVOT's pre-FOR values and FOR field are new output names, not column references. 284 if isinstance(expr, exp.Column) and not expr.table: 285 return expr.this 286 if isinstance(expr, exp.Tuple): 287 expr.set("expressions", [_unpivot_target(e) for e in expr.expressions]) 288 return expr 289 290 291SENTINEL_NONE: Token = Token(TokenType.SENTINEL, "SENTINEL") 292 293 294class Parser: 295 """ 296 Parser consumes a list of tokens produced by the Tokenizer and produces a parsed syntax tree. 297 298 Args: 299 error_level: The desired error level. 300 Default: ErrorLevel.IMMEDIATE 301 error_message_context: The amount of context to capture from a query string when displaying 302 the error message (in number of characters). 303 Default: 100 304 max_errors: Maximum number of error messages to include in a raised ParseError. 305 This is only relevant if error_level is ErrorLevel.RAISE. 306 Default: 3 307 max_nodes: Maximum number of AST nodes to prevent memory exhaustion. 308 Set to -1 (default) to disable the check. 309 """ 310 311 __slots__ = ( 312 "error_level", 313 "error_message_context", 314 "max_errors", 315 "max_nodes", 316 "dialect", 317 "sql", 318 "errors", 319 "_tokens", 320 "_index", 321 "_curr", 322 "_next", 323 "_prev", 324 "_prev_comments", 325 "_pipe_cte_counter", 326 "_chunks", 327 "_chunk_index", 328 "_tokens_size", 329 "_node_count", 330 ) 331 332 FUNCTIONS: t.ClassVar[dict[str, t.Callable]] = { 333 **{name: func.from_arg_list for name, func in exp.FUNCTION_BY_NAME.items()}, 334 **dict.fromkeys(("COALESCE", "IFNULL", "NVL"), build_coalesce), 335 "ARRAY": lambda args, dialect: exp.Array(expressions=args), 336 "ARRAYAGG": lambda args, dialect: exp.ArrayAgg( 337 this=seq_get(args, 0), nulls_excluded=dialect.ARRAY_AGG_INCLUDES_NULLS is None or None 338 ), 339 "ARRAY_AGG": lambda args, dialect: exp.ArrayAgg( 340 this=seq_get(args, 0), nulls_excluded=dialect.ARRAY_AGG_INCLUDES_NULLS is None or None 341 ), 342 "ARRAY_APPEND": build_array_append, 343 "ARRAY_CAT": build_array_concat, 344 "ARRAY_CONCAT": build_array_concat, 345 "ARRAY_INTERSECT": lambda args: exp.ArrayIntersect(expressions=args), 346 "ARRAY_INTERSECTION": lambda args: exp.ArrayIntersect(expressions=args), 347 "ARRAY_PREPEND": build_array_prepend, 348 "ARRAY_REMOVE": build_array_remove, 349 "COUNT": lambda args: exp.Count(this=seq_get(args, 0), expressions=args[1:], big_int=True), 350 "CONCAT": lambda args, dialect: exp.Concat( 351 expressions=args, 352 safe=not dialect.STRICT_STRING_CONCAT, 353 coalesce=dialect.CONCAT_COALESCE, 354 ), 355 "CONCAT_WS": lambda args, dialect: exp.ConcatWs( 356 expressions=args, 357 safe=not dialect.STRICT_STRING_CONCAT, 358 coalesce=dialect.CONCAT_WS_COALESCE, 359 ), 360 "CONVERT_TIMEZONE": build_convert_timezone, 361 "DATE_TO_DATE_STR": lambda args: exp.Cast( 362 this=seq_get(args, 0), 363 to=exp.DataType(this=exp.DType.TEXT), 364 ), 365 "GENERATE_DATE_ARRAY": lambda args: exp.GenerateDateArray( 366 start=seq_get(args, 0), 367 end=seq_get(args, 1), 368 step=seq_get(args, 2) or exp.Interval(this=exp.Literal.string(1), unit=exp.var("DAY")), 369 ), 370 "GENERATE_UUID": lambda args, dialect: exp.Uuid( 371 is_string=dialect.UUID_IS_STRING_TYPE or None 372 ), 373 "GLOB": lambda args: exp.Glob(this=seq_get(args, 1), expression=seq_get(args, 0)), 374 "GREATEST": lambda args, dialect: exp.Greatest( 375 this=seq_get(args, 0), 376 expressions=args[1:], 377 ignore_nulls=dialect.LEAST_GREATEST_IGNORES_NULLS, 378 ), 379 "LEAST": lambda args, dialect: exp.Least( 380 this=seq_get(args, 0), 381 expressions=args[1:], 382 ignore_nulls=dialect.LEAST_GREATEST_IGNORES_NULLS, 383 ), 384 "HEX": build_hex, 385 "JSON_EXTRACT": build_extract_json_with_path(exp.JSONExtract), 386 "JSON_EXTRACT_SCALAR": build_extract_json_with_path(exp.JSONExtractScalar), 387 "JSON_EXTRACT_PATH_TEXT": build_extract_json_with_path(exp.JSONExtractScalar), 388 "JSON_KEYS": lambda args, dialect: exp.JSONKeys( 389 this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1)) 390 ), 391 "LIKE": build_like, 392 "LOG": build_logarithm, 393 "LOG2": lambda args: exp.Log(this=exp.Literal.number(2), expression=seq_get(args, 0)), 394 "LOG10": lambda args: exp.Log(this=exp.Literal.number(10), expression=seq_get(args, 0)), 395 "LOWER": build_lower, 396 "LPAD": lambda args: build_pad(args), 397 "LEFTPAD": lambda args: build_pad(args), 398 "LTRIM": lambda args: build_trim(args), 399 "MOD": build_mod, 400 "RIGHTPAD": lambda args: build_pad(args, is_left=False), 401 "RPAD": lambda args: build_pad(args, is_left=False), 402 "RTRIM": lambda args: build_trim(args, is_left=False), 403 "SCOPE_RESOLUTION": lambda args: ( 404 exp.ScopeResolution(expression=seq_get(args, 0)) 405 if len(args) != 2 406 else exp.ScopeResolution(this=seq_get(args, 0), expression=seq_get(args, 1)) 407 ), 408 "STRPOS": exp.StrPosition.from_arg_list, 409 "CHARINDEX": lambda args: build_locate_strposition(args), 410 "INSTR": exp.StrPosition.from_arg_list, 411 "LOCATE": lambda args: build_locate_strposition(args), 412 "TIME_TO_TIME_STR": lambda args: exp.Cast( 413 this=seq_get(args, 0), 414 to=exp.DataType(this=exp.DType.TEXT), 415 ), 416 "TO_HEX": build_hex, 417 "TS_OR_DS_TO_DATE_STR": lambda args: exp.Substring( 418 this=exp.Cast( 419 this=seq_get(args, 0), 420 to=exp.DataType(this=exp.DType.TEXT), 421 ), 422 start=exp.Literal.number(1), 423 length=exp.Literal.number(10), 424 ), 425 "UNNEST": lambda args: exp.Unnest(expressions=ensure_list(seq_get(args, 0))), 426 "UPPER": build_upper, 427 "UUID": lambda args, dialect: exp.Uuid(is_string=dialect.UUID_IS_STRING_TYPE or None), 428 "UUID_STRING": lambda args, dialect: exp.Uuid( 429 this=seq_get(args, 0), 430 name=seq_get(args, 1), 431 is_string=dialect.UUID_IS_STRING_TYPE or None, 432 ), 433 "VAR_MAP": build_var_map, 434 } 435 436 NO_PAREN_FUNCTIONS: t.ClassVar[dict] = { 437 TokenType.CURRENT_DATE: exp.CurrentDate, 438 TokenType.CURRENT_DATETIME: exp.CurrentDate, 439 TokenType.CURRENT_TIME: exp.CurrentTime, 440 TokenType.CURRENT_TIMESTAMP: exp.CurrentTimestamp, 441 TokenType.CURRENT_USER: exp.CurrentUser, 442 TokenType.CURRENT_ROLE: exp.CurrentRole, 443 } 444 445 STRUCT_TYPE_TOKENS: t.ClassVar = { 446 TokenType.NESTED, 447 TokenType.OBJECT, 448 TokenType.STRUCT, 449 TokenType.UNION, 450 } 451 452 NESTED_TYPE_TOKENS: t.ClassVar = { 453 TokenType.ARRAY, 454 TokenType.LIST, 455 TokenType.LOWCARDINALITY, 456 TokenType.MAP, 457 TokenType.NULLABLE, 458 TokenType.RANGE, 459 *STRUCT_TYPE_TOKENS, 460 } 461 462 ENUM_TYPE_TOKENS: t.ClassVar = { 463 TokenType.DYNAMIC, 464 TokenType.ENUM, 465 TokenType.ENUM8, 466 TokenType.ENUM16, 467 } 468 469 AGGREGATE_TYPE_TOKENS: t.ClassVar = { 470 TokenType.AGGREGATEFUNCTION, 471 TokenType.SIMPLEAGGREGATEFUNCTION, 472 } 473 474 TYPE_TOKENS: t.ClassVar = { 475 TokenType.BIT, 476 TokenType.BOOLEAN, 477 TokenType.TINYINT, 478 TokenType.UTINYINT, 479 TokenType.SMALLINT, 480 TokenType.USMALLINT, 481 TokenType.INT, 482 TokenType.UINT, 483 TokenType.BIGINT, 484 TokenType.UBIGINT, 485 TokenType.BIGNUM, 486 TokenType.INT128, 487 TokenType.UINT128, 488 TokenType.INT256, 489 TokenType.UINT256, 490 TokenType.MEDIUMINT, 491 TokenType.UMEDIUMINT, 492 TokenType.FIXEDSTRING, 493 TokenType.FLOAT, 494 TokenType.DOUBLE, 495 TokenType.UDOUBLE, 496 TokenType.CHAR, 497 TokenType.NCHAR, 498 TokenType.VARCHAR, 499 TokenType.NVARCHAR, 500 TokenType.BPCHAR, 501 TokenType.TEXT, 502 TokenType.MEDIUMTEXT, 503 TokenType.LONGTEXT, 504 TokenType.BLOB, 505 TokenType.MEDIUMBLOB, 506 TokenType.LONGBLOB, 507 TokenType.BINARY, 508 TokenType.VARBINARY, 509 TokenType.JSON, 510 TokenType.JSONB, 511 TokenType.INTERVAL, 512 TokenType.TINYBLOB, 513 TokenType.TINYTEXT, 514 TokenType.TIME, 515 TokenType.TIMETZ, 516 TokenType.TIME_NS, 517 TokenType.TIMESTAMP, 518 TokenType.TIMESTAMP_S, 519 TokenType.TIMESTAMP_MS, 520 TokenType.TIMESTAMP_NS, 521 TokenType.TIMESTAMPTZ, 522 TokenType.TIMESTAMPLTZ, 523 TokenType.TIMESTAMPNTZ, 524 TokenType.DATETIME, 525 TokenType.DATETIME2, 526 TokenType.DATETIME64, 527 TokenType.SMALLDATETIME, 528 TokenType.DATE, 529 TokenType.DATE32, 530 TokenType.INT4RANGE, 531 TokenType.INT4MULTIRANGE, 532 TokenType.INT8RANGE, 533 TokenType.INT8MULTIRANGE, 534 TokenType.NUMRANGE, 535 TokenType.NUMMULTIRANGE, 536 TokenType.TSRANGE, 537 TokenType.TSMULTIRANGE, 538 TokenType.TSTZRANGE, 539 TokenType.TSTZMULTIRANGE, 540 TokenType.DATERANGE, 541 TokenType.DATEMULTIRANGE, 542 TokenType.DECIMAL, 543 TokenType.DECIMAL32, 544 TokenType.DECIMAL64, 545 TokenType.DECIMAL128, 546 TokenType.DECIMAL256, 547 TokenType.DECFLOAT, 548 TokenType.UDECIMAL, 549 TokenType.BIGDECIMAL, 550 TokenType.UUID, 551 TokenType.GEOGRAPHY, 552 TokenType.GEOGRAPHYPOINT, 553 TokenType.GEOMETRY, 554 TokenType.POINT, 555 TokenType.RING, 556 TokenType.LINESTRING, 557 TokenType.MULTILINESTRING, 558 TokenType.POLYGON, 559 TokenType.MULTIPOLYGON, 560 TokenType.HLLSKETCH, 561 TokenType.HSTORE, 562 TokenType.PSEUDO_TYPE, 563 TokenType.SUPER, 564 TokenType.SERIAL, 565 TokenType.SMALLSERIAL, 566 TokenType.BIGSERIAL, 567 TokenType.XML, 568 TokenType.YEAR, 569 TokenType.USERDEFINED, 570 TokenType.MONEY, 571 TokenType.SMALLMONEY, 572 TokenType.ROWVERSION, 573 TokenType.IMAGE, 574 TokenType.VARIANT, 575 TokenType.VECTOR, 576 TokenType.VOID, 577 TokenType.OBJECT, 578 TokenType.OBJECT_IDENTIFIER, 579 TokenType.INET, 580 TokenType.IPADDRESS, 581 TokenType.IPPREFIX, 582 TokenType.IPV4, 583 TokenType.IPV6, 584 TokenType.UNKNOWN, 585 TokenType.NOTHING, 586 TokenType.NULL, 587 TokenType.NAME, 588 TokenType.TDIGEST, 589 TokenType.DYNAMIC, 590 *ENUM_TYPE_TOKENS, 591 *NESTED_TYPE_TOKENS, 592 *AGGREGATE_TYPE_TOKENS, 593 } 594 595 SIGNED_TO_UNSIGNED_TYPE_TOKEN: t.ClassVar = { 596 TokenType.BIGINT: TokenType.UBIGINT, 597 TokenType.INT: TokenType.UINT, 598 TokenType.MEDIUMINT: TokenType.UMEDIUMINT, 599 TokenType.SMALLINT: TokenType.USMALLINT, 600 TokenType.TINYINT: TokenType.UTINYINT, 601 TokenType.DECIMAL: TokenType.UDECIMAL, 602 TokenType.DOUBLE: TokenType.UDOUBLE, 603 } 604 605 SUBQUERY_PREDICATES: t.ClassVar = { 606 TokenType.ANY: exp.Any, 607 TokenType.ALL: exp.All, 608 TokenType.EXISTS: exp.Exists, 609 TokenType.SOME: exp.Any, 610 } 611 612 SUBQUERY_TOKENS: t.ClassVar = { 613 TokenType.SELECT, 614 TokenType.WITH, 615 TokenType.FROM, 616 } 617 618 RESERVED_TOKENS: t.ClassVar = { 619 *Tokenizer.SINGLE_TOKENS.values(), 620 TokenType.SELECT, 621 } - {TokenType.IDENTIFIER} 622 623 # Tokens whose text is extracted from delimited source text (e.g. quoted identifiers, 624 # string literals), so they must never be treated as keywords when matching by text 625 TEXT_MATCH_EXCLUDED_TOKENS: t.ClassVar[frozenset] = frozenset( 626 { 627 TokenType.BIT_STRING, 628 TokenType.BYTE_STRING, 629 TokenType.HEREDOC_STRING, 630 TokenType.HEX_STRING, 631 TokenType.IDENTIFIER, 632 TokenType.NATIONAL_STRING, 633 TokenType.RAW_STRING, 634 TokenType.STRING, 635 TokenType.UNICODE_STRING, 636 } 637 ) 638 639 DB_CREATABLES: t.ClassVar = { 640 TokenType.DATABASE, 641 TokenType.DICTIONARY, 642 TokenType.FILE_FORMAT, 643 TokenType.MODEL, 644 TokenType.NAMESPACE, 645 TokenType.SCHEMA, 646 TokenType.SEMANTIC_VIEW, 647 TokenType.SEQUENCE, 648 TokenType.SINK, 649 TokenType.SOURCE, 650 TokenType.STAGE, 651 TokenType.STORAGE_INTEGRATION, 652 TokenType.STREAMLIT, 653 TokenType.TABLE, 654 TokenType.TAG, 655 TokenType.VIEW, 656 TokenType.WAREHOUSE, 657 } 658 659 CREATABLES: t.ClassVar = { 660 TokenType.COLUMN, 661 TokenType.CONSTRAINT, 662 TokenType.FOREIGN_KEY, 663 TokenType.FUNCTION, 664 TokenType.INDEX, 665 TokenType.PROCEDURE, 666 TokenType.TRIGGER, 667 TokenType.TYPE, 668 *DB_CREATABLES, 669 } 670 671 TRIGGER_EVENTS: t.ClassVar = { 672 TokenType.INSERT, 673 TokenType.UPDATE, 674 TokenType.DELETE, 675 TokenType.TRUNCATE, 676 } 677 678 ALTERABLES: t.ClassVar = { 679 TokenType.INDEX, 680 TokenType.TABLE, 681 TokenType.VIEW, 682 TokenType.SESSION, 683 } 684 685 # Tokens that can represent identifiers 686 ID_VAR_TOKENS: t.ClassVar[set] = { 687 TokenType.ALL, 688 TokenType.ANALYZE, 689 TokenType.ATTACH, 690 TokenType.VAR, 691 TokenType.ANTI, 692 TokenType.APPLY, 693 TokenType.ASC, 694 TokenType.ASOF, 695 TokenType.AUTO_INCREMENT, 696 TokenType.BEGIN, 697 TokenType.BPCHAR, 698 TokenType.CACHE, 699 TokenType.CASE, 700 TokenType.COLLATE, 701 TokenType.COMMAND, 702 TokenType.COMMENT, 703 TokenType.COMMIT, 704 TokenType.CONSTRAINT, 705 TokenType.COPY, 706 TokenType.CUBE, 707 TokenType.CURRENT_SCHEMA, 708 TokenType.DEFAULT, 709 TokenType.DELETE, 710 TokenType.DESC, 711 TokenType.DESCRIBE, 712 TokenType.DETACH, 713 TokenType.DICTIONARY, 714 TokenType.DIV, 715 TokenType.END, 716 TokenType.EXECUTE, 717 TokenType.EXPORT, 718 TokenType.ESCAPE, 719 TokenType.FALSE, 720 TokenType.FIRST, 721 TokenType.FILE, 722 TokenType.FILTER, 723 TokenType.FINAL, 724 TokenType.FORMAT, 725 TokenType.FULL, 726 TokenType.GET, 727 TokenType.IDENTIFIER, 728 TokenType.INOUT, 729 TokenType.IS, 730 TokenType.ISNULL, 731 TokenType.INTERVAL, 732 TokenType.KEEP, 733 TokenType.KILL, 734 TokenType.LEFT, 735 TokenType.LIMIT, 736 TokenType.LOAD, 737 TokenType.LOCK, 738 TokenType.MATCH, 739 TokenType.MERGE, 740 TokenType.NATURAL, 741 TokenType.NEXT, 742 TokenType.OFFSET, 743 TokenType.OPERATOR, 744 TokenType.ORDINALITY, 745 TokenType.OUT, 746 TokenType.OVER, 747 TokenType.OVERLAPS, 748 TokenType.OVERWRITE, 749 TokenType.PARTITION, 750 TokenType.PERCENT, 751 TokenType.PIVOT, 752 TokenType.PROJECTION, 753 TokenType.PRAGMA, 754 TokenType.PUT, 755 TokenType.RANGE, 756 TokenType.RECURSIVE, 757 TokenType.REFERENCES, 758 TokenType.REFRESH, 759 TokenType.RENAME, 760 TokenType.REPLACE, 761 TokenType.RIGHT, 762 TokenType.ROLLUP, 763 TokenType.ROW, 764 TokenType.ROWS, 765 TokenType.SEMI, 766 TokenType.SET, 767 TokenType.SETTINGS, 768 TokenType.SHOW, 769 TokenType.STREAM, 770 TokenType.STREAMLIT, 771 TokenType.TEMPORARY, 772 TokenType.TOP, 773 TokenType.TRUE, 774 TokenType.TRUNCATE, 775 TokenType.UNIQUE, 776 TokenType.UNNEST, 777 TokenType.UNPIVOT, 778 TokenType.UPDATE, 779 TokenType.USE, 780 TokenType.VOLATILE, 781 TokenType.WINDOW, 782 TokenType.CURRENT_CATALOG, 783 TokenType.LOCALTIME, 784 TokenType.LOCALTIMESTAMP, 785 TokenType.SESSION_USER, 786 TokenType.STRAIGHT_JOIN, 787 *ALTERABLES, 788 *CREATABLES, 789 *SUBQUERY_PREDICATES, 790 *TYPE_TOKENS, 791 *NO_PAREN_FUNCTIONS, 792 } - {TokenType.UNION} 793 794 TABLE_ALIAS_TOKENS: t.ClassVar[set] = ID_VAR_TOKENS - { 795 TokenType.ANTI, 796 TokenType.ASOF, 797 TokenType.FULL, 798 TokenType.LEFT, 799 TokenType.LOCK, 800 TokenType.NATURAL, 801 TokenType.RIGHT, 802 TokenType.SEMI, 803 TokenType.WINDOW, 804 } 805 806 ALIAS_TOKENS: t.ClassVar = ID_VAR_TOKENS 807 808 COLON_PLACEHOLDER_TOKENS: t.ClassVar = ID_VAR_TOKENS 809 810 ARRAY_CONSTRUCTORS: t.ClassVar = { 811 "ARRAY": exp.Array, 812 "LIST": exp.List, 813 } 814 815 COMMENT_TABLE_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - {TokenType.IS} 816 817 UPDATE_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - {TokenType.SET} 818 819 TRIM_TYPES: t.ClassVar = {"LEADING", "TRAILING", "BOTH"} 820 821 # Tokens that indicate a simple column reference 822 IDENTIFIER_TOKENS: t.ClassVar[frozenset] = frozenset({TokenType.VAR, TokenType.IDENTIFIER}) 823 824 BRACKETS: t.ClassVar[frozenset] = frozenset({TokenType.L_BRACKET, TokenType.L_BRACE}) 825 826 # Postfix tokens that prevent the bare column fast path 827 COLUMN_POSTFIX_TOKENS: t.ClassVar[frozenset] = frozenset( 828 { 829 TokenType.L_PAREN, 830 TokenType.L_BRACKET, 831 TokenType.L_BRACE, 832 TokenType.COLON, 833 TokenType.JOIN_MARKER, 834 } 835 ) 836 837 TABLE_POSTFIX_TOKENS: t.ClassVar[frozenset] = frozenset( 838 { 839 TokenType.L_PAREN, 840 TokenType.L_BRACKET, 841 TokenType.L_BRACE, 842 TokenType.PIVOT, 843 TokenType.UNPIVOT, 844 TokenType.TABLE_SAMPLE, 845 } 846 ) 847 848 FUNC_TOKENS: t.ClassVar = { 849 TokenType.COLLATE, 850 TokenType.COMMAND, 851 TokenType.CURRENT_DATE, 852 TokenType.CURRENT_DATETIME, 853 TokenType.CURRENT_SCHEMA, 854 TokenType.CURRENT_TIMESTAMP, 855 TokenType.CURRENT_TIME, 856 TokenType.CURRENT_USER, 857 TokenType.CURRENT_CATALOG, 858 TokenType.FILTER, 859 TokenType.FIRST, 860 TokenType.FORMAT, 861 TokenType.GET, 862 TokenType.GLOB, 863 TokenType.IDENTIFIER, 864 TokenType.INDEX, 865 TokenType.ISNULL, 866 TokenType.ILIKE, 867 TokenType.INSERT, 868 TokenType.LIKE, 869 TokenType.LOCALTIME, 870 TokenType.LOCALTIMESTAMP, 871 TokenType.MERGE, 872 TokenType.NEXT, 873 TokenType.OFFSET, 874 TokenType.PRIMARY_KEY, 875 TokenType.RANGE, 876 TokenType.REPLACE, 877 TokenType.RLIKE, 878 TokenType.ROW, 879 TokenType.SESSION_USER, 880 TokenType.UNNEST, 881 TokenType.VAR, 882 TokenType.LEFT, 883 TokenType.RIGHT, 884 TokenType.SEQUENCE, 885 TokenType.DATE, 886 TokenType.DATETIME, 887 TokenType.TABLE, 888 TokenType.TIMESTAMP, 889 TokenType.TIMESTAMPTZ, 890 TokenType.TRUNCATE, 891 TokenType.UTC_DATE, 892 TokenType.UTC_TIME, 893 TokenType.UTC_TIMESTAMP, 894 TokenType.WINDOW, 895 TokenType.XOR, 896 *TYPE_TOKENS, 897 *SUBQUERY_PREDICATES, 898 } 899 900 CONJUNCTION: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 901 TokenType.AND: exp.And, 902 } 903 904 ASSIGNMENT: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 905 TokenType.COLON_EQ: exp.PropertyEQ, 906 } 907 908 DISJUNCTION: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 909 TokenType.OR: exp.Or, 910 } 911 912 EQUALITY: t.ClassVar = { 913 TokenType.EQ: exp.EQ, 914 TokenType.NEQ: exp.NEQ, 915 TokenType.NULLSAFE_EQ: exp.NullSafeEQ, 916 } 917 918 COMPARISON: t.ClassVar = { 919 TokenType.GT: exp.GT, 920 TokenType.GTE: exp.GTE, 921 TokenType.LT: exp.LT, 922 TokenType.LTE: exp.LTE, 923 } 924 925 BITWISE: t.ClassVar = { 926 TokenType.AMP: exp.BitwiseAnd, 927 TokenType.CARET: exp.BitwiseXor, 928 TokenType.PIPE: exp.BitwiseOr, 929 } 930 931 TERM: t.ClassVar = { 932 TokenType.DASH: exp.Sub, 933 TokenType.PLUS: exp.Add, 934 TokenType.MOD: exp.Mod, 935 TokenType.COLLATE: exp.Collate, 936 } 937 938 FACTOR: t.ClassVar = { 939 TokenType.DIV: exp.IntDiv, 940 TokenType.LR_ARROW: exp.Distance, 941 TokenType.LLRR_ARROW: exp.DistanceNd, 942 TokenType.SLASH: exp.Div, 943 TokenType.STAR: exp.Mul, 944 } 945 946 EXPONENT: t.ClassVar[dict[TokenType, type[exp.Expr]]] = {} 947 948 TIMES: t.ClassVar = { 949 TokenType.TIME, 950 TokenType.TIMETZ, 951 } 952 953 TIMESTAMPS: t.ClassVar = { 954 TokenType.TIMESTAMP, 955 TokenType.TIMESTAMPNTZ, 956 TokenType.TIMESTAMPTZ, 957 TokenType.TIMESTAMPLTZ, 958 *TIMES, 959 } 960 961 SET_OPERATIONS: t.ClassVar = { 962 TokenType.UNION, 963 TokenType.INTERSECT, 964 TokenType.EXCEPT, 965 } 966 967 JOIN_METHODS: t.ClassVar = { 968 TokenType.ASOF, 969 TokenType.NATURAL, 970 TokenType.POSITIONAL, 971 } 972 973 JOIN_SIDES: t.ClassVar = { 974 TokenType.LEFT, 975 TokenType.RIGHT, 976 TokenType.FULL, 977 } 978 979 JOIN_KINDS: t.ClassVar = { 980 TokenType.ANTI, 981 TokenType.CROSS, 982 TokenType.INNER, 983 TokenType.OUTER, 984 TokenType.SEMI, 985 TokenType.STRAIGHT_JOIN, 986 } 987 988 JOIN_HINTS: t.ClassVar[set[str]] = set() 989 990 # Tokens that unambiguously end a table reference on the fast path 991 TABLE_TERMINATORS: t.ClassVar[frozenset] = frozenset( 992 { 993 TokenType.COMMA, 994 TokenType.GROUP_BY, 995 TokenType.HAVING, 996 TokenType.JOIN, 997 TokenType.LIMIT, 998 TokenType.ON, 999 TokenType.ORDER_BY, 1000 TokenType.R_PAREN, 1001 TokenType.SEMICOLON, 1002 TokenType.SENTINEL, 1003 TokenType.WHERE, 1004 *SET_OPERATIONS, 1005 *JOIN_KINDS, 1006 *JOIN_METHODS, 1007 *JOIN_SIDES, 1008 } 1009 ) 1010 1011 LAMBDAS: t.ClassVar = { 1012 TokenType.ARROW: lambda self, expressions: self.expression( 1013 exp.Lambda( 1014 this=self._replace_lambda( 1015 self._parse_disjunction(), 1016 expressions, 1017 ), 1018 expressions=expressions, 1019 ) 1020 ), 1021 TokenType.FARROW: lambda self, expressions: self.expression( 1022 exp.Kwarg( 1023 this=exp.var(expressions[0].name), 1024 expression=self._parse_disjunction() or self._parse_select(), 1025 ) 1026 ), 1027 } 1028 1029 # Whether lambda args include type annotations, e.g. TRANSFORM(arr, x INT -> x + 1) in Snowflake 1030 TYPED_LAMBDA_ARGS: t.ClassVar[bool] = False 1031 1032 LAMBDA_ARG_TERMINATORS: t.ClassVar[frozenset] = frozenset({TokenType.COMMA, TokenType.R_PAREN}) 1033 1034 COLUMN_OPERATORS: t.ClassVar = { 1035 TokenType.DOT: None, 1036 TokenType.DOTCOLON: lambda self, this, to: self.expression(exp.JSONCast(this=this, to=to)), 1037 TokenType.DCOLON: lambda self, this, to: self.build_cast( 1038 strict=self.STRICT_CAST, this=this, to=to 1039 ), 1040 TokenType.ARROW: lambda self, this, path: self.expression( 1041 exp.JSONExtract( 1042 this=this, 1043 expression=self.dialect.to_json_path(path), 1044 only_json_types=self.JSON_ARROWS_REQUIRE_JSON_TYPE, 1045 ) 1046 ), 1047 TokenType.DARROW: lambda self, this, path: self.expression( 1048 exp.JSONExtractScalar( 1049 this=this, 1050 expression=self.dialect.to_json_path(path), 1051 only_json_types=self.JSON_ARROWS_REQUIRE_JSON_TYPE, 1052 scalar_only=self.dialect.JSON_EXTRACT_SCALAR_SCALAR_ONLY, 1053 ) 1054 ), 1055 TokenType.HASH_ARROW: lambda self, this, path: self.expression( 1056 exp.JSONBExtract(this=this, expression=path) 1057 ), 1058 TokenType.DHASH_ARROW: lambda self, this, path: self.expression( 1059 exp.JSONBExtractScalar(this=this, expression=path) 1060 ), 1061 TokenType.PLACEHOLDER: lambda self, this, key: self.expression( 1062 exp.JSONBContains(this=this, expression=key) 1063 ), 1064 } 1065 1066 CAST_COLUMN_OPERATORS: t.ClassVar = { 1067 TokenType.DOTCOLON, 1068 TokenType.DCOLON, 1069 } 1070 1071 EXPRESSION_PARSERS: t.ClassVar = { 1072 exp.Cluster: lambda self: self._parse_sort(exp.Cluster, TokenType.CLUSTER_BY), 1073 exp.Column: lambda self: self._parse_column(), 1074 exp.ColumnDef: lambda self: self._parse_column_def(self._parse_column()), 1075 exp.Condition: lambda self: self._parse_disjunction(), 1076 exp.DataType: lambda self: self._parse_types(allow_identifiers=False, schema=True), 1077 exp.Expr: lambda self: self._parse_expression(), 1078 exp.From: lambda self: self._parse_from(joins=True), 1079 exp.GrantPrincipal: lambda self: self._parse_grant_principal(), 1080 exp.GrantPrivilege: lambda self: self._parse_grant_privilege(), 1081 exp.Group: lambda self: self._parse_group(), 1082 exp.Having: lambda self: self._parse_having(), 1083 exp.Hint: lambda self: self._parse_hint_body(), 1084 exp.Identifier: lambda self: self._parse_id_var(), 1085 exp.Join: lambda self: self._parse_join(), 1086 exp.Lambda: lambda self: self._parse_lambda(), 1087 exp.Lateral: lambda self: self._parse_lateral(), 1088 exp.Limit: lambda self: self._parse_limit(), 1089 exp.Offset: lambda self: self._parse_offset(), 1090 exp.Order: lambda self: self._parse_order(), 1091 exp.Ordered: lambda self: self._parse_ordered(), 1092 exp.Properties: lambda self: self._parse_properties(), 1093 exp.PartitionedByProperty: lambda self: self._parse_partitioned_by(), 1094 exp.Qualify: lambda self: self._parse_qualify(), 1095 exp.Returning: lambda self: self._parse_returning(), 1096 exp.Select: lambda self: self._parse_select(), 1097 exp.Sort: lambda self: self._parse_sort(exp.Sort, TokenType.SORT_BY), 1098 exp.Table: lambda self: self._parse_table_parts(), 1099 exp.TableAlias: lambda self: self._parse_table_alias(), 1100 exp.Tuple: lambda self: self._parse_value(values=False), 1101 exp.Whens: lambda self: self._parse_when_matched(), 1102 exp.Where: lambda self: self._parse_where(), 1103 exp.Window: lambda self: self._parse_named_window(), 1104 exp.With: lambda self: self._parse_with(), 1105 } 1106 1107 STATEMENT_PARSERS: t.ClassVar = { 1108 TokenType.ALTER: lambda self: self._parse_alter(), 1109 TokenType.ANALYZE: lambda self: self._parse_analyze(), 1110 TokenType.BEGIN: lambda self: self._parse_transaction(), 1111 TokenType.CACHE: lambda self: self._parse_cache(), 1112 TokenType.COMMENT: lambda self: self._parse_comment(), 1113 TokenType.COMMIT: lambda self: self._parse_commit_or_rollback(), 1114 TokenType.COPY: lambda self: self._parse_copy(), 1115 TokenType.CREATE: lambda self: self._parse_create(), 1116 TokenType.DELETE: lambda self: self._parse_delete(), 1117 TokenType.DESC: lambda self: self._parse_describe(), 1118 TokenType.DESCRIBE: lambda self: self._parse_describe(), 1119 TokenType.DROP: lambda self: self._parse_drop(), 1120 TokenType.GRANT: lambda self: self._parse_grant(), 1121 TokenType.REVOKE: lambda self: self._parse_revoke(), 1122 TokenType.INSERT: lambda self: self._parse_insert(), 1123 TokenType.KILL: lambda self: self._parse_kill(), 1124 TokenType.LOAD: lambda self: self._parse_load(), 1125 TokenType.MERGE: lambda self: self._parse_merge(), 1126 TokenType.PIVOT: lambda self: self._parse_simplified_pivot(), 1127 TokenType.PRAGMA: lambda self: self.expression(exp.Pragma(this=self._parse_expression())), 1128 TokenType.REFRESH: lambda self: self._parse_refresh(), 1129 TokenType.ROLLBACK: lambda self: self._parse_commit_or_rollback(), 1130 TokenType.SET: lambda self: self._parse_set(), 1131 TokenType.TRUNCATE: lambda self: self._parse_truncate_table(), 1132 TokenType.UNCACHE: lambda self: self._parse_uncache(), 1133 TokenType.UNPIVOT: lambda self: self._parse_simplified_pivot(is_unpivot=True), 1134 TokenType.UPDATE: lambda self: self._parse_update(), 1135 TokenType.USE: lambda self: self._parse_use(), 1136 TokenType.SEMICOLON: lambda self: exp.Semicolon(), 1137 } 1138 1139 UNARY_PARSERS: t.ClassVar = { 1140 TokenType.PLUS: lambda self: self._parse_unary(), # Unary + is handled as a no-op 1141 TokenType.NOT: lambda self: self.expression(exp.Not(this=self._parse_equality())), 1142 TokenType.TILDE: lambda self: self.expression(exp.BitwiseNot(this=self._parse_unary())), 1143 TokenType.DASH: lambda self: self.expression(exp.Neg(this=self._parse_unary())), 1144 TokenType.PIPE_SLASH: lambda self: self.expression(exp.Sqrt(this=self._parse_unary())), 1145 TokenType.DPIPE_SLASH: lambda self: self.expression(exp.Cbrt(this=self._parse_unary())), 1146 } 1147 1148 STRING_PARSERS: t.ClassVar = { 1149 TokenType.HEREDOC_STRING: lambda self, token: self.expression( 1150 exp.RawString(this=token.text), token 1151 ), 1152 TokenType.NATIONAL_STRING: lambda self, token: self.expression( 1153 exp.National(this=token.text), token 1154 ), 1155 TokenType.RAW_STRING: lambda self, token: self.expression( 1156 exp.RawString(this=token.text), token 1157 ), 1158 TokenType.STRING: lambda self, token: self.expression( 1159 exp.Literal(this=token.text, is_string=True), token 1160 ), 1161 TokenType.UNICODE_STRING: lambda self, token: self.expression( 1162 exp.UnicodeString( 1163 this=token.text, escape=self._match_text_seq("UESCAPE") and self._parse_string() 1164 ), 1165 token, 1166 ), 1167 } 1168 1169 NUMERIC_PARSERS: t.ClassVar = { 1170 TokenType.BIT_STRING: lambda self, token: self.expression( 1171 exp.BitString(this=token.text), token 1172 ), 1173 TokenType.BYTE_STRING: lambda self, token: self.expression( 1174 exp.ByteString( 1175 this=token.text, is_bytes=self.dialect.BYTE_STRING_IS_BYTES_TYPE or None 1176 ), 1177 token, 1178 ), 1179 TokenType.HEX_STRING: lambda self, token: self.expression( 1180 exp.HexString( 1181 this=token.text, is_integer=self.dialect.HEX_STRING_IS_INTEGER_TYPE or None 1182 ), 1183 token, 1184 ), 1185 TokenType.NUMBER: lambda self, token: self.expression( 1186 exp.Literal(this=token.text, is_string=False), token 1187 ), 1188 } 1189 1190 PRIMARY_PARSERS: t.ClassVar = { 1191 **STRING_PARSERS, 1192 **NUMERIC_PARSERS, 1193 TokenType.INTRODUCER: lambda self, token: self._parse_introducer(token), 1194 TokenType.NULL: lambda self, _: self.expression(exp.Null()), 1195 TokenType.TRUE: lambda self, _: self.expression(exp.Boolean(this=True)), 1196 TokenType.FALSE: lambda self, _: self.expression(exp.Boolean(this=False)), 1197 TokenType.SESSION_PARAMETER: lambda self, _: self._parse_session_parameter(), 1198 TokenType.STAR: lambda self, _: self._parse_star_ops(), 1199 } 1200 1201 PLACEHOLDER_PARSERS: t.ClassVar = { 1202 TokenType.PLACEHOLDER: lambda self: self.expression(exp.Placeholder()), 1203 TokenType.PARAMETER: lambda self: self._parse_parameter(), 1204 TokenType.COLON: lambda self: ( 1205 self.expression(exp.Placeholder(this=self._prev.text)) 1206 if self._match_set(self.COLON_PLACEHOLDER_TOKENS) 1207 else None 1208 ), 1209 } 1210 1211 RANGE_PARSERS: t.ClassVar = { 1212 TokenType.AT_GT: binary_range_parser(exp.ArrayContainsAll), 1213 TokenType.BETWEEN: lambda self, this: self._parse_between(this), 1214 TokenType.GLOB: binary_range_parser(exp.Glob), 1215 TokenType.ILIKE: binary_range_parser(exp.ILike), 1216 TokenType.IN: lambda self, this: self._parse_in(this), 1217 TokenType.IRLIKE: binary_range_parser(exp.RegexpILike), 1218 TokenType.IS: lambda self, this: self._parse_is(this), 1219 TokenType.LIKE: binary_range_parser(exp.Like), 1220 TokenType.LT_AT: binary_range_parser(exp.ArrayContainedBy), 1221 TokenType.OVERLAPS: binary_range_parser(exp.Overlaps), 1222 TokenType.RLIKE: binary_range_parser(exp.RegexpLike), 1223 TokenType.SIMILAR_TO: binary_range_parser(exp.SimilarTo), 1224 TokenType.FOR: lambda self, this: self._parse_comprehension(this), 1225 TokenType.QMARK_AMP: binary_range_parser(exp.JSONBContainsAllTopKeys), 1226 TokenType.QMARK_PIPE: binary_range_parser(exp.JSONBContainsAnyTopKeys), 1227 TokenType.HASH_DASH: binary_range_parser(exp.JSONBDeleteAtPath), 1228 TokenType.AT_QMARK: binary_range_parser(exp.JSONBPathExists), 1229 TokenType.ADJACENT: binary_range_parser(exp.Adjacent), 1230 TokenType.OPERATOR: lambda self, this: self._parse_operator(this), 1231 TokenType.AMP_LT: binary_range_parser(exp.ExtendsLeft), 1232 TokenType.AMP_GT: binary_range_parser(exp.ExtendsRight), 1233 } 1234 1235 PIPE_SYNTAX_TRANSFORM_PARSERS: t.ClassVar = { 1236 "AGGREGATE": lambda self, query: self._parse_pipe_syntax_aggregate(query), 1237 "AS": lambda self, query: self._build_pipe_cte( 1238 query, [exp.Star()], self._parse_table_alias() 1239 ), 1240 "DISTINCT": lambda self, query: self._advance() or query.distinct(copy=False), 1241 "EXTEND": lambda self, query: self._parse_pipe_syntax_extend(query), 1242 "LIMIT": lambda self, query: self._parse_pipe_syntax_limit(query), 1243 "ORDER BY": lambda self, query: query.order_by( 1244 self._parse_order(), append=False, copy=False 1245 ), 1246 "PIVOT": lambda self, query: self._parse_pipe_syntax_pivot(query), 1247 "SELECT": lambda self, query: self._parse_pipe_syntax_select(query), 1248 "TABLESAMPLE": lambda self, query: self._parse_pipe_syntax_tablesample(query), 1249 "UNPIVOT": lambda self, query: self._parse_pipe_syntax_pivot(query), 1250 "WHERE": lambda self, query: query.where(self._parse_where(), copy=False), 1251 } 1252 1253 PROPERTY_PARSERS: t.ClassVar[dict[str, t.Callable]] = { 1254 "ALLOWED_VALUES": lambda self: self.expression( 1255 exp.AllowedValuesProperty(expressions=self._parse_csv(self._parse_primary)) 1256 ), 1257 "ALGORITHM": lambda self: self._parse_property_assignment(exp.AlgorithmProperty), 1258 "AUTO": lambda self: self._parse_auto_property(), 1259 "AUTO_INCREMENT": lambda self: self._parse_property_assignment(exp.AutoIncrementProperty), 1260 "BACKUP": lambda self: self.expression( 1261 exp.BackupProperty(this=self._parse_var(any_token=True)) 1262 ), 1263 "BLOCKCOMPRESSION": lambda self: self._parse_blockcompression(), 1264 "CALLED": lambda self: self._parse_called_on_null_input_property(), 1265 "CHARSET": lambda self, **kwargs: self._parse_character_set(**kwargs), 1266 "CHARACTER SET": lambda self, **kwargs: self._parse_character_set(**kwargs), 1267 "CHECKSUM": lambda self: self._parse_checksum(), 1268 "CLUSTER BY": lambda self: self._parse_cluster_property(), 1269 "CLUSTERED": lambda self: self._parse_clustered_by(), 1270 "COLLATE": lambda self, **kwargs: self._parse_property_assignment( 1271 exp.CollateProperty, **kwargs 1272 ), 1273 "COMMENT": lambda self: self._parse_property_assignment(exp.SchemaCommentProperty), 1274 "CONTAINS": lambda self: self._parse_contains_property(), 1275 "COPY": lambda self: self._parse_copy_property(), 1276 "DATABLOCKSIZE": lambda self, **kwargs: self._parse_datablocksize(**kwargs), 1277 "DATA_DELETION": lambda self: self._parse_data_deletion_property(), 1278 "DEFINER": lambda self: self._parse_definer(), 1279 "DETERMINISTIC": lambda self: self.expression( 1280 exp.StabilityProperty(this=exp.Literal.string("IMMUTABLE")) 1281 ), 1282 "DISTRIBUTED": lambda self: self._parse_distributed_property(), 1283 "DUPLICATE": lambda self: self._parse_composite_key_property(exp.DuplicateKeyProperty), 1284 "DYNAMIC": lambda self: self.expression(exp.DynamicProperty()), 1285 "DISTKEY": lambda self: self._parse_distkey(), 1286 "DISTSTYLE": lambda self: self._parse_property_assignment(exp.DistStyleProperty), 1287 "EMPTY": lambda self: self.expression(exp.EmptyProperty()), 1288 "ENGINE": lambda self: self._parse_property_assignment(exp.EngineProperty), 1289 "ENVIRONMENT": lambda self: self.expression( 1290 exp.EnviromentProperty(expressions=self._parse_wrapped_csv(self._parse_assignment)) 1291 ), 1292 "HANDLER": lambda self: self._parse_property_assignment(exp.HandlerProperty), 1293 "EXECUTE": lambda self: self._parse_property_assignment(exp.ExecuteAsProperty), 1294 "EXTERNAL": lambda self: self.expression(exp.ExternalProperty()), 1295 "FALLBACK": lambda self, **kwargs: self._parse_fallback(**kwargs), 1296 "FORMAT": lambda self: self._parse_property_assignment(exp.FileFormatProperty), 1297 "FREESPACE": lambda self: self._parse_freespace(), 1298 "GLOBAL": lambda self: self.expression(exp.GlobalProperty()), 1299 "HEAP": lambda self: self.expression(exp.HeapProperty()), 1300 "ICEBERG": lambda self: self.expression(exp.IcebergProperty()), 1301 "IMMUTABLE": lambda self: self.expression( 1302 exp.StabilityProperty(this=exp.Literal.string("IMMUTABLE")) 1303 ), 1304 "INHERITS": lambda self: self.expression( 1305 exp.InheritsProperty(expressions=self._parse_wrapped_csv(self._parse_table)) 1306 ), 1307 "INPUT": lambda self: self.expression(exp.InputModelProperty(this=self._parse_schema())), 1308 "JOURNAL": lambda self, **kwargs: self._parse_journal(**kwargs), 1309 "LANGUAGE": lambda self: self._parse_property_assignment(exp.LanguageProperty), 1310 "LAYOUT": lambda self: self._parse_dict_property(this="LAYOUT"), 1311 "LIFETIME": lambda self: self._parse_dict_range(this="LIFETIME"), 1312 "LIKE": lambda self: self._parse_create_like(), 1313 "LOCATION": lambda self: self._parse_property_assignment(exp.LocationProperty), 1314 "LOCK": lambda self: self._parse_locking(), 1315 "LOCKING": lambda self: self._parse_locking(), 1316 "LOG": lambda self, **kwargs: self._parse_log(**kwargs), 1317 "MATERIALIZED": lambda self: self.expression(exp.MaterializedProperty()), 1318 "MERGEBLOCKRATIO": lambda self, **kwargs: self._parse_mergeblockratio(**kwargs), 1319 "MODIFIES": lambda self: self._parse_modifies_property(), 1320 "MULTISET": lambda self: self.expression(exp.SetProperty(multi=True)), 1321 "NO": lambda self: self._parse_no_property(), 1322 "ON": lambda self: self._parse_on_property(), 1323 "ORDER BY": lambda self: self._parse_order(skip_order_token=True), 1324 "OUTPUT": lambda self: self.expression(exp.OutputModelProperty(this=self._parse_schema())), 1325 "PARTITION": lambda self: self._parse_partitioned_of(), 1326 "PARTITION BY": lambda self: self._parse_partitioned_by(), 1327 "PARTITIONED BY": lambda self: self._parse_partitioned_by(), 1328 "PARTITIONED_BY": lambda self: self._parse_partitioned_by(), 1329 "PRIMARY KEY": lambda self: self._parse_primary_key(in_props=True), 1330 "RANGE": lambda self: self._parse_dict_range(this="RANGE"), 1331 "READS": lambda self: self._parse_reads_property(), 1332 "REMOTE": lambda self: self._parse_remote_with_connection(), 1333 "RETURNS": lambda self: self._parse_returns(), 1334 "STRICT": lambda self: self.expression(exp.StrictProperty()), 1335 "STREAMING": lambda self: self.expression(exp.StreamingTableProperty()), 1336 "ROW": lambda self: self._parse_row(), 1337 "ROW_FORMAT": lambda self: self._parse_property_assignment(exp.RowFormatProperty), 1338 "SAMPLE": lambda self: self.expression( 1339 exp.SampleProperty(this=self._match_text_seq("BY") and self._parse_bitwise()) 1340 ), 1341 "SECURE": lambda self: self.expression(exp.SecureProperty()), 1342 "SECURITY": lambda self: self._parse_sql_security(), 1343 "SQL SECURITY": lambda self: self._parse_sql_security(), 1344 "SET": lambda self: self.expression(exp.SetProperty(multi=False)), 1345 "SETTINGS": lambda self: self._parse_settings_property(), 1346 "SHARING": lambda self: self._parse_property_assignment(exp.SharingProperty), 1347 "SORTKEY": lambda self: self._parse_sortkey(), 1348 "SOURCE": lambda self: self._parse_dict_property(this="SOURCE"), 1349 "STABLE": lambda self: self.expression( 1350 exp.StabilityProperty(this=exp.Literal.string("STABLE")) 1351 ), 1352 "STORED": lambda self: self._parse_stored(), 1353 "SYSTEM_VERSIONING": lambda self: self._parse_system_versioning_property(), 1354 "TBLPROPERTIES": lambda self: self._parse_wrapped_properties(), 1355 "TEMP": lambda self: self.expression(exp.TemporaryProperty()), 1356 "TEMPORARY": lambda self: self.expression(exp.TemporaryProperty()), 1357 "TO": lambda self: self._parse_to_table(), 1358 "TRANSIENT": lambda self: self.expression(exp.TransientProperty()), 1359 "TRANSFORM": lambda self: self.expression( 1360 exp.TransformModelProperty(expressions=self._parse_wrapped_csv(self._parse_expression)) 1361 ), 1362 "TTL": lambda self: self._parse_ttl(), 1363 "USING": lambda self: self._parse_property_assignment(exp.FileFormatProperty), 1364 "UNLOGGED": lambda self: self.expression(exp.UnloggedProperty()), 1365 "VOLATILE": lambda self: self._parse_volatile_property(), 1366 "WITH": lambda self: self._parse_with_property(), 1367 } 1368 1369 CONSTRAINT_PARSERS: t.ClassVar = { 1370 "AUTOINCREMENT": lambda self: self._parse_auto_increment(), 1371 "AUTO_INCREMENT": lambda self: self._parse_auto_increment(), 1372 "CASESPECIFIC": lambda self: self.expression(exp.CaseSpecificColumnConstraint(not_=False)), 1373 "CHARACTER SET": lambda self: self.expression( 1374 exp.CharacterSetColumnConstraint(this=self._parse_var_or_string()) 1375 ), 1376 "CHECK": lambda self: self._parse_check_constraint(), 1377 "COLLATE": lambda self: self.expression( 1378 exp.CollateColumnConstraint(this=self._parse_identifier() or self._parse_column()) 1379 ), 1380 "COMMENT": lambda self: self.expression( 1381 exp.CommentColumnConstraint(this=self._parse_string()) 1382 ), 1383 "COMPRESS": lambda self: self._parse_compress(), 1384 "CLUSTERED": lambda self: self.expression( 1385 exp.ClusteredColumnConstraint(this=self._parse_wrapped_csv(self._parse_ordered)) 1386 ), 1387 "NONCLUSTERED": lambda self: self.expression( 1388 exp.NonClusteredColumnConstraint(this=self._parse_wrapped_csv(self._parse_ordered)) 1389 ), 1390 "DEFAULT": lambda self: self.expression( 1391 exp.DefaultColumnConstraint(this=self._parse_bitwise()) 1392 ), 1393 "ENCODE": lambda self: self.expression(exp.EncodeColumnConstraint(this=self._parse_var())), 1394 "EPHEMERAL": lambda self: self.expression( 1395 exp.EphemeralColumnConstraint(this=self._parse_bitwise()) 1396 ), 1397 "EXCLUDE": lambda self: self.expression( 1398 exp.ExcludeColumnConstraint(this=self._parse_index_params()) 1399 ), 1400 "FOREIGN KEY": lambda self: self._parse_foreign_key(), 1401 "FORMAT": lambda self: self.expression( 1402 exp.DateFormatColumnConstraint(this=self._parse_var_or_string()) 1403 ), 1404 "GENERATED": lambda self: self._parse_generated_as_identity(), 1405 "IDENTITY": lambda self: self._parse_auto_increment(), 1406 "INLINE": lambda self: self._parse_inline(), 1407 "LIKE": lambda self: self._parse_create_like(), 1408 "NOT": lambda self: self._parse_not_constraint(), 1409 "NULL": lambda self: self.expression(exp.NotNullColumnConstraint(allow_null=True)), 1410 "ON": lambda self: ( 1411 ( 1412 self._match(TokenType.UPDATE) 1413 and self.expression(exp.OnUpdateColumnConstraint(this=self._parse_function())) 1414 ) 1415 or self.expression(exp.OnProperty(this=self._parse_id_var())) 1416 ), 1417 "PATH": lambda self: self.expression(exp.PathColumnConstraint(this=self._parse_string())), 1418 "PERIOD": lambda self: self._parse_period_for_system_time(), 1419 "PRIMARY KEY": lambda self: self._parse_primary_key(), 1420 "REFERENCES": lambda self: self._parse_references(match=False), 1421 "TITLE": lambda self: self.expression( 1422 exp.TitleColumnConstraint(this=self._parse_var_or_string()) 1423 ), 1424 "TTL": lambda self: self.expression(exp.MergeTreeTTL(expressions=[self._parse_bitwise()])), 1425 "UNIQUE": lambda self: self._parse_unique(), 1426 "UPPERCASE": lambda self: self.expression(exp.UppercaseColumnConstraint()), 1427 "WITH": lambda self: self.expression( 1428 exp.Properties(expressions=self._parse_wrapped_properties()) 1429 ), 1430 "BUCKET": lambda self: self._parse_partitioned_by_bucket_or_truncate(), 1431 "TRUNCATE": lambda self: self._parse_partitioned_by_bucket_or_truncate(), 1432 } 1433 1434 def _parse_partitioned_by_bucket_or_truncate(self) -> exp.Expr | None: 1435 if not self._match(TokenType.L_PAREN, advance=False): 1436 # Partitioning by bucket or truncate follows the syntax: 1437 # PARTITION BY (BUCKET(..) | TRUNCATE(..)) 1438 # If we don't have parenthesis after each keyword, we should instead parse this as an identifier 1439 self._retreat(self._index - 1) 1440 return None 1441 1442 klass = ( 1443 exp.PartitionedByBucket 1444 if self._prev.text.upper() == "BUCKET" 1445 else exp.PartitionByTruncate 1446 ) 1447 1448 args = self._parse_wrapped_csv(lambda: self._parse_primary() or self._parse_column()) 1449 this, expression = seq_get(args, 0), seq_get(args, 1) 1450 1451 if isinstance(this, exp.Literal): 1452 # Check for Iceberg partition transforms (bucket / truncate) and ensure their arguments are in the right order 1453 # - For Hive, it's `bucket(<num buckets>, <col name>)` or `truncate(<num_chars>, <col_name>)` 1454 # - For Trino, it's reversed - `bucket(<col name>, <num buckets>)` or `truncate(<col_name>, <num_chars>)` 1455 # Both variants are canonicalized in the latter i.e `bucket(<col name>, <num buckets>)` 1456 # 1457 # Hive ref: https://docs.aws.amazon.com/athena/latest/ug/querying-iceberg-creating-tables.html#querying-iceberg-partitioning 1458 # Trino ref: https://docs.aws.amazon.com/athena/latest/ug/create-table-as.html#ctas-table-properties 1459 this, expression = expression, this 1460 1461 return self.expression(klass(this=this, expression=expression)) 1462 1463 ALTER_PARSERS: t.ClassVar = { 1464 "ADD": lambda self: self._parse_alter_table_add(), 1465 "AS": lambda self: self._parse_select(), 1466 "ALTER": lambda self: self._parse_alter_table_alter(), 1467 "CLUSTER BY": lambda self: self._parse_cluster_property(), 1468 "DELETE": lambda self: self.expression(exp.Delete(where=self._parse_where())), 1469 "DROP": lambda self: self._parse_alter_table_drop(), 1470 "RENAME": lambda self: self._parse_alter_table_rename(), 1471 "SET": lambda self: self._parse_alter_table_set(), 1472 "SWAP": lambda self: self.expression( 1473 exp.SwapTable(this=self._match(TokenType.WITH) and self._parse_table(schema=True)) 1474 ), 1475 } 1476 1477 ALTER_ALTER_PARSERS: t.ClassVar = { 1478 "DISTKEY": lambda self: self._parse_alter_diststyle(), 1479 "DISTSTYLE": lambda self: self._parse_alter_diststyle(), 1480 "SORTKEY": lambda self: self._parse_alter_sortkey(), 1481 "COMPOUND": lambda self: self._parse_alter_sortkey(compound=True), 1482 } 1483 1484 SCHEMA_UNNAMED_CONSTRAINTS: t.ClassVar = { 1485 "CHECK", 1486 "EXCLUDE", 1487 "FOREIGN KEY", 1488 "LIKE", 1489 "PERIOD", 1490 "PRIMARY KEY", 1491 "UNIQUE", 1492 "BUCKET", 1493 "TRUNCATE", 1494 } 1495 1496 NO_PAREN_FUNCTION_PARSERS: t.ClassVar = { 1497 "ANY": lambda self: self.expression(exp.Any(this=self._parse_bitwise())), 1498 "CASE": lambda self: self._parse_case(), 1499 "CONNECT_BY_ROOT": lambda self: self.expression( 1500 exp.ConnectByRoot(this=self._parse_column()) 1501 ), 1502 "IF": lambda self: self._parse_if(), 1503 } 1504 1505 INVALID_FUNC_NAME_TOKENS: t.ClassVar = { 1506 TokenType.IDENTIFIER, 1507 TokenType.STRING, 1508 } 1509 1510 FUNCTIONS_WITH_ALIASED_ARGS: t.ClassVar = {"STRUCT"} 1511 1512 KEY_VALUE_DEFINITIONS: t.ClassVar = (exp.Alias, exp.EQ, exp.PropertyEQ, exp.Slice) 1513 1514 FUNCTION_PARSERS: t.ClassVar[dict[str, t.Callable]] = { 1515 **{ 1516 name: lambda self: self._parse_distinct_arg_function(exp.ArgMax) 1517 for name in exp.ArgMax.sql_names() 1518 }, 1519 **{ 1520 name: lambda self: self._parse_distinct_arg_function(exp.ArgMin) 1521 for name in exp.ArgMin.sql_names() 1522 }, 1523 "CAST": lambda self: self._parse_cast(self.STRICT_CAST), 1524 "CEIL": lambda self: self._parse_ceil_floor(exp.Ceil), 1525 "CONVERT": lambda self: self._parse_convert(self.STRICT_CAST), 1526 "CHAR": lambda self: self._parse_char(), 1527 "CHR": lambda self: self._parse_char(), 1528 "DECODE": lambda self: self._parse_decode(), 1529 "EXTRACT": lambda self: self._parse_extract(), 1530 "FLOOR": lambda self: self._parse_ceil_floor(exp.Floor), 1531 "GAP_FILL": lambda self: self._parse_gap_fill(), 1532 "INITCAP": lambda self: self._parse_initcap(), 1533 "JSON_OBJECT": lambda self: self._parse_json_object(), 1534 "JSON_OBJECTAGG": lambda self: self._parse_json_object(agg=True), 1535 "JSON_TABLE": lambda self: self._parse_json_table(), 1536 "MATCH": lambda self: self._parse_match_against(), 1537 "NORMALIZE": lambda self: self._parse_normalize(), 1538 "OPENJSON": lambda self: self._parse_open_json(), 1539 "OVERLAY": lambda self: self._parse_overlay(), 1540 "POSITION": lambda self: self._parse_position(), 1541 "SAFE_CAST": lambda self: self._parse_cast(False, safe=True), 1542 "STRING_AGG": lambda self: self._parse_string_agg(), 1543 "SUBSTRING": lambda self: self._parse_substring(), 1544 "TRIM": lambda self: self._parse_trim(), 1545 "TRY_CAST": lambda self: self._parse_cast(False, safe=True), 1546 "TRY_CONVERT": lambda self: self._parse_convert(False, safe=True), 1547 "XMLELEMENT": lambda self: self._parse_xml_element(), 1548 "XMLTABLE": lambda self: self._parse_xml_table(), 1549 } 1550 1551 QUERY_MODIFIER_PARSERS: t.ClassVar = { 1552 TokenType.MATCH_RECOGNIZE: lambda self: ("match", self._parse_match_recognize()), 1553 TokenType.PREWHERE: lambda self: ("prewhere", self._parse_prewhere()), 1554 TokenType.WHERE: lambda self: ("where", self._parse_where()), 1555 TokenType.GROUP_BY: lambda self: ("group", self._parse_group()), 1556 TokenType.HAVING: lambda self: ("having", self._parse_having()), 1557 TokenType.QUALIFY: lambda self: ("qualify", self._parse_qualify()), 1558 TokenType.WINDOW: lambda self: ("windows", self._parse_window_clause()), 1559 TokenType.ORDER_BY: lambda self: ("order", self._parse_order()), 1560 TokenType.LIMIT: lambda self: ("limit", self._parse_limit()), 1561 TokenType.FETCH: lambda self: ("limit", self._parse_limit()), 1562 TokenType.OFFSET: lambda self: ("offset", self._parse_offset()), 1563 TokenType.FOR: lambda self: ("locks", self._parse_locks()), 1564 TokenType.LOCK: lambda self: ("locks", self._parse_locks()), 1565 TokenType.TABLE_SAMPLE: lambda self: ("sample", self._parse_table_sample(as_modifier=True)), 1566 TokenType.USING: lambda self: ("sample", self._parse_table_sample(as_modifier=True)), 1567 TokenType.CLUSTER_BY: lambda self: ( 1568 "cluster", 1569 self._parse_cluster(), 1570 ), 1571 TokenType.DISTRIBUTE_BY: lambda self: ( 1572 "distribute", 1573 self._parse_sort(exp.Distribute, TokenType.DISTRIBUTE_BY), 1574 ), 1575 TokenType.SORT_BY: lambda self: ("sort", self._parse_sort(exp.Sort, TokenType.SORT_BY)), 1576 TokenType.CONNECT_BY: lambda self: ("connect", self._parse_connect(skip_start_token=True)), 1577 TokenType.START_WITH: lambda self: ("connect", self._parse_connect()), 1578 } 1579 QUERY_MODIFIER_TOKENS: t.ClassVar = set(QUERY_MODIFIER_PARSERS) 1580 1581 SET_PARSERS: t.ClassVar = { 1582 "GLOBAL": lambda self: self._parse_set_item_assignment("GLOBAL"), 1583 "LOCAL": lambda self: self._parse_set_item_assignment("LOCAL"), 1584 "SESSION": lambda self: self._parse_set_item_assignment("SESSION"), 1585 "TRANSACTION": lambda self: self._parse_set_transaction(), 1586 } 1587 1588 SHOW_PARSERS: t.ClassVar[dict[str, t.Callable]] = {} 1589 1590 TYPE_LITERAL_PARSERS: t.ClassVar = { 1591 exp.DType.JSON: lambda self, this, _: self.expression(exp.ParseJSON(this=this)), 1592 } 1593 1594 TYPE_CONVERTERS: t.ClassVar[dict[exp.DType, t.Callable[[exp.DataType], exp.DataType]]] = {} 1595 1596 DDL_SELECT_TOKENS: t.ClassVar = {TokenType.SELECT, TokenType.WITH, TokenType.L_PAREN} 1597 1598 PRE_VOLATILE_TOKENS: t.ClassVar = {TokenType.CREATE, TokenType.REPLACE, TokenType.UNIQUE} 1599 1600 TRANSACTION_KIND: t.ClassVar = {"DEFERRED", "IMMEDIATE", "EXCLUSIVE"} 1601 TRANSACTION_CHARACTERISTICS: t.ClassVar[OPTIONS_TYPE] = { 1602 "ISOLATION": ( 1603 ("LEVEL", "REPEATABLE", "READ"), 1604 ("LEVEL", "READ", "COMMITTED"), 1605 ("LEVEL", "READ", "UNCOMITTED"), 1606 ("LEVEL", "SERIALIZABLE"), 1607 ), 1608 "READ": ("WRITE", "ONLY"), 1609 } 1610 1611 CONFLICT_ACTIONS: t.ClassVar[OPTIONS_TYPE] = { 1612 **dict.fromkeys(("ABORT", "FAIL", "IGNORE", "REPLACE", "ROLLBACK", "UPDATE"), tuple()), 1613 "DO": ("NOTHING", "UPDATE"), 1614 } 1615 1616 TRIGGER_TIMING: t.ClassVar[OPTIONS_TYPE] = { 1617 "INSTEAD": (("OF",),), 1618 "BEFORE": tuple(), 1619 "AFTER": tuple(), 1620 } 1621 1622 TRIGGER_DEFERRABLE: t.ClassVar[OPTIONS_TYPE] = { 1623 "NOT": (("DEFERRABLE",),), 1624 "DEFERRABLE": tuple(), 1625 } 1626 1627 CREATE_SEQUENCE: t.ClassVar[OPTIONS_TYPE] = { 1628 "SCALE": ("EXTEND", "NOEXTEND"), 1629 "SHARD": ("EXTEND", "NOEXTEND"), 1630 "NO": ("CYCLE", "CACHE", "MAXVALUE", "MINVALUE"), 1631 **dict.fromkeys( 1632 ( 1633 "SESSION", 1634 "GLOBAL", 1635 "KEEP", 1636 "NOKEEP", 1637 "ORDER", 1638 "NOORDER", 1639 "NOCACHE", 1640 "CYCLE", 1641 "NOCYCLE", 1642 "NOMINVALUE", 1643 "NOMAXVALUE", 1644 "NOSCALE", 1645 "NOSHARD", 1646 ), 1647 tuple(), 1648 ), 1649 } 1650 1651 ISOLATED_LOADING_OPTIONS: t.ClassVar[OPTIONS_TYPE] = {"FOR": ("ALL", "INSERT", "NONE")} 1652 1653 USABLES: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys( 1654 ("ROLE", "WAREHOUSE", "DATABASE", "SCHEMA", "CATALOG"), tuple() 1655 ) 1656 1657 CAST_ACTIONS: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys(("RENAME", "ADD"), ("FIELDS",)) 1658 1659 SCHEMA_BINDING_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1660 "TYPE": ("EVOLUTION",), 1661 **dict.fromkeys(("BINDING", "COMPENSATION", "EVOLUTION"), tuple()), 1662 } 1663 1664 PROCEDURE_OPTIONS: t.ClassVar[OPTIONS_TYPE] = {} 1665 1666 EXECUTE_AS_OPTIONS: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys( 1667 ("CALLER", "SELF", "OWNER"), tuple() 1668 ) 1669 1670 KEY_CONSTRAINT_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1671 "NOT": ("ENFORCED",), 1672 "MATCH": ( 1673 "FULL", 1674 "PARTIAL", 1675 "SIMPLE", 1676 ), 1677 "INITIALLY": ("DEFERRED", "IMMEDIATE"), 1678 "USING": ( 1679 "BTREE", 1680 "HASH", 1681 ), 1682 **dict.fromkeys(("DEFERRABLE", "NORELY", "RELY"), tuple()), 1683 } 1684 1685 WINDOW_EXCLUDE_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1686 "NO": ("OTHERS",), 1687 "CURRENT": ("ROW",), 1688 **dict.fromkeys(("GROUP", "TIES"), tuple()), 1689 } 1690 1691 INSERT_ALTERNATIVES: t.ClassVar = {"ABORT", "FAIL", "IGNORE", "REPLACE", "ROLLBACK"} 1692 1693 CLONE_KEYWORDS: t.ClassVar = {"CLONE", "COPY"} 1694 HISTORICAL_DATA_PREFIX: t.ClassVar = {"AT", "BEFORE", "END"} 1695 HISTORICAL_DATA_KIND: t.ClassVar = {"OFFSET", "STATEMENT", "STREAM", "TIMESTAMP", "VERSION"} 1696 1697 OPCLASS_FOLLOW_KEYWORDS: t.ClassVar = {"ASC", "DESC", "NULLS", "WITH"} 1698 1699 OPTYPE_FOLLOW_TOKENS: t.ClassVar = {TokenType.COMMA, TokenType.R_PAREN} 1700 1701 TABLE_INDEX_HINT_TOKENS: t.ClassVar = {TokenType.FORCE, TokenType.IGNORE, TokenType.USE} 1702 1703 VIEW_ATTRIBUTES: t.ClassVar = {"ENCRYPTION", "SCHEMABINDING", "VIEW_METADATA"} 1704 1705 WINDOW_ALIAS_TOKENS: t.ClassVar = ID_VAR_TOKENS - {TokenType.RANGE, TokenType.ROWS} 1706 WINDOW_BEFORE_PAREN_TOKENS: t.ClassVar = {TokenType.OVER} 1707 WINDOW_SIDES: t.ClassVar = {"FOLLOWING", "PRECEDING"} 1708 1709 JSON_KEY_VALUE_SEPARATOR_TOKENS: t.ClassVar = {TokenType.COLON, TokenType.COMMA, TokenType.IS} 1710 1711 FETCH_TOKENS: t.ClassVar = ID_VAR_TOKENS - {TokenType.ROW, TokenType.ROWS, TokenType.PERCENT} 1712 1713 ADD_CONSTRAINT_TOKENS: t.ClassVar = { 1714 TokenType.CONSTRAINT, 1715 TokenType.FOREIGN_KEY, 1716 TokenType.INDEX, 1717 TokenType.KEY, 1718 TokenType.PRIMARY_KEY, 1719 TokenType.UNIQUE, 1720 } 1721 1722 DISTINCT_TOKENS: t.ClassVar = {TokenType.DISTINCT} 1723 1724 UNNEST_OFFSET_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - SET_OPERATIONS 1725 1726 SELECT_START_TOKENS: t.ClassVar = {TokenType.L_PAREN, TokenType.WITH, TokenType.SELECT} 1727 1728 COPY_INTO_VARLEN_OPTIONS: t.ClassVar = { 1729 "FILE_FORMAT", 1730 "COPY_OPTIONS", 1731 "FORMAT_OPTIONS", 1732 "CREDENTIAL", 1733 } 1734 1735 IS_JSON_PREDICATE_KIND: t.ClassVar = {"VALUE", "SCALAR", "ARRAY", "OBJECT"} 1736 1737 ODBC_DATETIME_LITERALS: t.ClassVar[dict[str, type[exp.Expr]]] = {} 1738 1739 ON_CONDITION_TOKENS: t.ClassVar = {"ERROR", "NULL", "TRUE", "FALSE", "EMPTY"} 1740 1741 PRIVILEGE_FOLLOW_TOKENS: t.ClassVar = {TokenType.ON, TokenType.COMMA, TokenType.L_PAREN} 1742 1743 # The style options for the DESCRIBE statement 1744 DESCRIBE_STYLES: t.ClassVar = {"ANALYZE", "EXTENDED", "FORMATTED", "HISTORY"} 1745 1746 SET_ASSIGNMENT_DELIMITERS: t.ClassVar = {"=", ":=", "TO"} 1747 1748 # The style options for the ANALYZE statement 1749 ANALYZE_STYLES: t.ClassVar = { 1750 "BUFFER_USAGE_LIMIT", 1751 "FULL", 1752 "LOCAL", 1753 "NO_WRITE_TO_BINLOG", 1754 "SAMPLE", 1755 "SKIP_LOCKED", 1756 "VERBOSE", 1757 } 1758 1759 ANALYZE_EXPRESSION_PARSERS: t.ClassVar = { 1760 "ALL": lambda self: self._parse_analyze_columns(), 1761 "COMPUTE": lambda self: self._parse_analyze_statistics(), 1762 "DELETE": lambda self: self._parse_analyze_delete(), 1763 "DROP": lambda self: self._parse_analyze_histogram(), 1764 "ESTIMATE": lambda self: self._parse_analyze_statistics(), 1765 "LIST": lambda self: self._parse_analyze_list(), 1766 "PREDICATE": lambda self: self._parse_analyze_columns(), 1767 "UPDATE": lambda self: self._parse_analyze_histogram(), 1768 "VALIDATE": lambda self: self._parse_analyze_validate(), 1769 } 1770 1771 PARTITION_KEYWORDS: t.ClassVar = {"PARTITION", "SUBPARTITION"} 1772 1773 AMBIGUOUS_ALIAS_TOKENS: t.ClassVar = (TokenType.LIMIT, TokenType.OFFSET) 1774 1775 OPERATION_MODIFIERS: t.ClassVar[set[str]] = set() 1776 1777 RECURSIVE_CTE_SEARCH_KIND: t.ClassVar = {"BREADTH", "DEPTH", "CYCLE"} 1778 1779 SECURITY_PROPERTY_KEYWORDS: t.ClassVar = {"DEFINER", "INVOKER", "NONE"} 1780 1781 MODIFIABLES: t.ClassVar = (exp.Query, exp.Table, exp.TableFromRows, exp.Values) 1782 1783 STRICT_CAST: t.ClassVar = True 1784 1785 PREFIXED_PIVOT_COLUMNS: t.ClassVar = False 1786 IDENTIFY_PIVOT_STRINGS: t.ClassVar = False 1787 # Controls when an aggregation's name is included in a pivoted column's name: 1788 # "agg_name_if_aliased" - only for aggregations that carry an explicit alias 1789 # "agg_name_if_aliased_or_multiple" - if aliased, or whenever there are multiple aggregations 1790 # "agg_name_if_multiple" - only when there are multiple aggregations (a lone agg is value-only) 1791 PIVOT_COLUMN_NAMING: t.ClassVar[str] = "agg_name_if_aliased" 1792 1793 LOG_DEFAULTS_TO_LN: t.ClassVar = False 1794 1795 # Whether the table sample clause expects CSV syntax 1796 TABLESAMPLE_CSV: t.ClassVar = False 1797 1798 # The default method used for table sampling 1799 DEFAULT_SAMPLING_METHOD: t.ClassVar[str | None] = None 1800 1801 # Whether the SET command needs a delimiter (e.g. "=") for assignments 1802 SET_REQUIRES_ASSIGNMENT_DELIMITER: t.ClassVar = True 1803 1804 # Whether the TRIM function expects the characters to trim as its first argument 1805 TRIM_PATTERN_FIRST: t.ClassVar = False 1806 1807 # Whether string aliases are supported `SELECT COUNT(*) 'count'` 1808 STRING_ALIASES: t.ClassVar = False 1809 1810 # Whether query modifiers such as LIMIT are attached to the UNION node (vs its right operand) 1811 MODIFIERS_ATTACHED_TO_SET_OP: t.ClassVar = True 1812 SET_OP_MODIFIERS: t.ClassVar = {"order", "limit", "offset"} 1813 1814 # Whether to parse IF statements that aren't followed by a left parenthesis as commands 1815 NO_PAREN_IF_COMMANDS: t.ClassVar = True 1816 1817 # Whether the -> and ->> operators expect documents of type JSON (e.g. Postgres) 1818 JSON_ARROWS_REQUIRE_JSON_TYPE: t.ClassVar = False 1819 1820 # Whether the `:` operator is used to extract a value from a VARIANT column 1821 COLON_IS_VARIANT_EXTRACT: t.ClassVar = False 1822 1823 # Whether a chain of colon extractions (x:y:z) is a single extraction with a merged 1824 # path (x:y.z, e.g. Snowflake) or each colon extracts from the previous result (e.g. Databricks) 1825 COLON_CHAIN_IS_SINGLE_EXTRACT: t.ClassVar = True 1826 1827 # Whether or not a VALUES keyword needs to be followed by '(' to form a VALUES clause. 1828 # If this is True and '(' is not found, the keyword will be treated as an identifier 1829 VALUES_FOLLOWED_BY_PAREN: t.ClassVar = True 1830 1831 # Whether implicit unnesting is supported, e.g. SELECT 1 FROM y.z AS z, z.a (Redshift) 1832 SUPPORTS_IMPLICIT_UNNEST: t.ClassVar = False 1833 1834 # Whether or not interval spans are supported, INTERVAL 1 YEAR TO MONTHS 1835 INTERVAL_SPANS: t.ClassVar = True 1836 1837 # Whether a PARTITION clause can follow a table reference 1838 SUPPORTS_PARTITION_SELECTION: t.ClassVar = False 1839 1840 # Whether the `name AS expr` schema/column constraint requires parentheses around `expr` 1841 WRAPPED_TRANSFORM_COLUMN_CONSTRAINT: t.ClassVar = True 1842 1843 # Whether the 'AS' keyword is optional in the CTE definition syntax 1844 OPTIONAL_ALIAS_TOKEN_CTE: t.ClassVar = True 1845 1846 # Whether renaming a column with an ALTER statement requires the presence of the COLUMN keyword 1847 ALTER_RENAME_REQUIRES_COLUMN: t.ClassVar = True 1848 1849 # Whether Alter statements are allowed to contain Partition specifications 1850 ALTER_TABLE_PARTITIONS: t.ClassVar = False 1851 1852 # Whether all join types have the same precedence, i.e., they "naturally" produce a left-deep tree. 1853 # In standard SQL, joins that use the JOIN keyword take higher precedence than comma-joins. That is 1854 # to say, JOIN operators happen before comma operators. This is not the case in some dialects, such 1855 # as BigQuery, where all joins have the same precedence. 1856 JOINS_HAVE_EQUAL_PRECEDENCE: t.ClassVar = False 1857 1858 # Whether TIMESTAMP <literal> can produce a zone-aware timestamp 1859 ZONE_AWARE_TIMESTAMP_CONSTRUCTOR: t.ClassVar = False 1860 1861 # Whether map literals support arbitrary expressions as keys. 1862 # When True, allows complex keys like arrays or literals: {[1, 2]: 3}, {1: 2} (e.g. DuckDB). 1863 # When False, keys are typically restricted to identifiers. 1864 MAP_KEYS_ARE_ARBITRARY_EXPRESSIONS: t.ClassVar = False 1865 1866 # Whether JSON_EXTRACT requires a JSON expression as the first argument, e.g this 1867 # is true for Snowflake but not for BigQuery which can also process strings 1868 JSON_EXTRACT_REQUIRES_JSON_EXPRESSION: t.ClassVar = False 1869 1870 # Dialects like Databricks support JOINS without join criteria 1871 # Adding an ON TRUE, makes transpilation semantically correct for other dialects 1872 ADD_JOIN_ON_TRUE: t.ClassVar = False 1873 1874 # Whether INTERVAL spans with literal format '\d+ hh:[mm:[ss[.ff]]]' 1875 # can omit the span unit `DAY TO MINUTE` or `DAY TO SECOND` 1876 SUPPORTS_OMITTED_INTERVAL_SPAN_UNIT: t.ClassVar = False 1877 1878 # Whether adjacent string literals like 'foo' 'bar' require a whitespace or comment between them 1879 # to be considered valid syntactically. Such expressions evaluate to the strings' concatenation. 1880 ADJACENT_STRINGS_CANNOT_BE_CONNECTED: t.ClassVar = False 1881 1882 SHOW_TRIE: t.ClassVar[dict] = new_trie(key.split(" ") for key in SHOW_PARSERS) 1883 SET_TRIE: t.ClassVar[dict] = new_trie(key.split(" ") for key in SET_PARSERS) 1884 1885 def __init__( 1886 self, 1887 error_level: ErrorLevel | None = None, 1888 error_message_context: int = 100, 1889 max_errors: int = 3, 1890 max_nodes: int = -1, 1891 dialect: DialectType = None, 1892 ): 1893 self.error_level: ErrorLevel = error_level or ErrorLevel.IMMEDIATE 1894 self.error_message_context: int = error_message_context 1895 self.max_errors: int = max_errors 1896 self.max_nodes: int = max_nodes 1897 self.dialect: t.Any = _resolve_dialect(dialect) 1898 self.sql: str = "" 1899 self.errors: list[ParseError] = [] 1900 self._tokens: list[Token] = [] 1901 self._tokens_size: i64 = 0 1902 self._index: i64 = 0 1903 self._curr: Token = SENTINEL_NONE 1904 self._next: Token = SENTINEL_NONE 1905 self._prev: Token = SENTINEL_NONE 1906 self._prev_comments: list[str] = [] 1907 self._pipe_cte_counter: int = 0 1908 self._chunks: list[list[Token]] = [] 1909 self._chunk_index: i64 = 0 1910 self._node_count: int = 0 1911 1912 def reset(self) -> None: 1913 self.sql = "" 1914 self.errors = [] 1915 self._tokens = [] 1916 self._tokens_size = 0 1917 self._index = 0 1918 self._curr = SENTINEL_NONE 1919 self._next = SENTINEL_NONE 1920 self._prev = SENTINEL_NONE 1921 self._prev_comments = [] 1922 self._pipe_cte_counter = 0 1923 self._chunks = [] 1924 self._chunk_index = 0 1925 self._node_count = 0 1926 1927 def _advance(self, times: i64 = 1) -> None: 1928 index = self._index + times 1929 self._index = index 1930 tokens = self._tokens 1931 size = self._tokens_size 1932 self._curr = tokens[index] if index < size else SENTINEL_NONE 1933 self._next = tokens[index + 1] if index + 1 < size else SENTINEL_NONE 1934 1935 if index > 0: 1936 prev = tokens[index - 1] 1937 self._prev = prev 1938 self._prev_comments = prev.comments 1939 else: 1940 self._prev = SENTINEL_NONE 1941 self._prev_comments = [] 1942 1943 def _advance_chunk(self) -> None: 1944 self._index = -1 1945 self._tokens = self._chunks[self._chunk_index] 1946 self._tokens_size = i64(len(self._tokens)) 1947 self._chunk_index += 1 1948 self._advance() 1949 1950 def _retreat(self, index: i64) -> None: 1951 if index != self._index: 1952 self._advance(index - self._index) 1953 1954 def _add_comments(self, expression: exp.Expr | None) -> None: 1955 if expression and self._prev_comments: 1956 expression.add_comments(self._prev_comments) 1957 self._prev_comments = [] 1958 1959 def _match( 1960 self, token_type: TokenType, advance: bool = True, expression: exp.Expr | None = None 1961 ) -> bool: 1962 if self._curr.token_type == token_type: 1963 if advance: 1964 self._advance() 1965 self._add_comments(expression) 1966 return True 1967 return False 1968 1969 def _match_set(self, types: t.Collection[TokenType], advance: bool = True) -> bool: 1970 if self._curr.token_type in types: 1971 if advance: 1972 self._advance() 1973 return True 1974 return False 1975 1976 def _match_pair( 1977 self, token_type_a: TokenType, token_type_b: TokenType, advance: bool = True 1978 ) -> bool: 1979 if self._curr.token_type == token_type_a and self._next.token_type == token_type_b: 1980 if advance: 1981 self._advance(2) 1982 return True 1983 return False 1984 1985 def _match_texts(self, texts: TEXTS_TYPE, advance: bool = True) -> bool: 1986 if ( 1987 self._curr.token_type not in self.TEXT_MATCH_EXCLUDED_TOKENS 1988 and self._curr.text.upper() in texts 1989 ): 1990 if advance: 1991 self._advance() 1992 return True 1993 return False 1994 1995 def _match_text_seq(self, *texts: str, advance: bool = True) -> bool: 1996 index = self._index 1997 excluded_tokens = self.TEXT_MATCH_EXCLUDED_TOKENS 1998 for text in texts: 1999 if self._curr.token_type not in excluded_tokens and self._curr.text.upper() == text: 2000 self._advance() 2001 else: 2002 self._retreat(index) 2003 return False 2004 2005 if not advance: 2006 self._retreat(index) 2007 2008 return True 2009 2010 def _is_connected(self) -> bool: 2011 prev = self._prev 2012 curr = self._curr 2013 return bool(prev and curr and prev.end + 1 == curr.start) 2014 2015 def _find_sql(self, start: Token, end: Token) -> str: 2016 return self.sql[start.start : end.end + 1] 2017 2018 def raise_error(self, message: str, token: Token = SENTINEL_NONE) -> None: 2019 token = token or self._curr or self._prev or Token.string("") 2020 formatted_sql, start_context, highlight, end_context = highlight_sql( 2021 sql=self.sql, 2022 positions=[(token.start, token.end)], 2023 context_length=self.error_message_context, 2024 ) 2025 formatted_message = f"{message}. Line {token.line}, Col: {token.col}.\n {formatted_sql}" 2026 2027 error = ParseError.new( 2028 formatted_message, 2029 description=message, 2030 line=token.line, 2031 col=token.col, 2032 start_context=start_context, 2033 highlight=highlight, 2034 end_context=end_context, 2035 ) 2036 2037 if self.error_level == ErrorLevel.IMMEDIATE: 2038 raise error 2039 2040 self.errors.append(error) 2041 2042 def validate_expression(self, expression: E, args: list | None = None) -> E: 2043 if self.max_nodes > -1: 2044 self._node_count += 1 2045 if self._node_count > self.max_nodes: 2046 self.raise_error(f"Maximum number of AST nodes ({self.max_nodes}) exceeded") 2047 if self.error_level != ErrorLevel.IGNORE: 2048 for error_message in expression.error_messages(args): 2049 self.raise_error(error_message) 2050 return expression 2051 2052 def _try_parse(self, parse_method: t.Callable[[], T], retreat: bool = False) -> T | None: 2053 index = self._index 2054 error_level = self.error_level 2055 this: T | None = None 2056 2057 self.error_level = ErrorLevel.IMMEDIATE 2058 try: 2059 this = parse_method() 2060 except ParseError: 2061 this = None 2062 finally: 2063 if not this or retreat: 2064 self._retreat(index) 2065 self.error_level = error_level 2066 2067 return this 2068 2069 def parse(self, raw_tokens: list[Token], sql: str) -> list[exp.Expr | None]: 2070 """ 2071 Parses a list of tokens and returns a list of syntax trees, one tree 2072 per parsed SQL statement. 2073 2074 Args: 2075 raw_tokens: The list of tokens. 2076 sql: The original SQL string. 2077 2078 Returns: 2079 The list of the produced syntax trees. 2080 """ 2081 return self._parse( 2082 parse_method=self.__class__._parse_statement, raw_tokens=raw_tokens, sql=sql 2083 ) 2084 2085 def parse_into( 2086 self, 2087 expression_types: exp.IntoType, 2088 raw_tokens: list[Token], 2089 sql: str | None = None, 2090 ) -> list[exp.Expr | None]: 2091 """ 2092 Parses a list of tokens into a given Expr type. If a collection of Expr 2093 types is given instead, this method will try to parse the token list into each one 2094 of them, stopping at the first for which the parsing succeeds. 2095 2096 Args: 2097 expression_types: The expression type(s) to try and parse the token list into. 2098 raw_tokens: The list of tokens. 2099 sql: The original SQL string, used to produce helpful debug messages. 2100 2101 Returns: 2102 The target Expr. 2103 """ 2104 errors = [] 2105 for expression_type in ensure_list(expression_types): 2106 parser = self.EXPRESSION_PARSERS.get(t.cast(type[exp.Expr], expression_type)) 2107 if not parser: 2108 raise TypeError(f"No parser registered for {expression_type}") 2109 2110 try: 2111 return self._parse(parser, raw_tokens, sql) 2112 except ParseError as e: 2113 e.errors[0]["into_expression"] = expression_type 2114 errors.append(e) 2115 2116 raise ParseError( 2117 f"Failed to parse '{sql or raw_tokens}' into {expression_types}", 2118 errors=merge_errors(errors), 2119 ) from errors[-1] 2120 2121 def check_errors(self) -> None: 2122 """Logs or raises any found errors, depending on the chosen error level setting.""" 2123 if self.error_level == ErrorLevel.WARN: 2124 for error in self.errors: 2125 logger.error(str(error)) 2126 elif self.error_level == ErrorLevel.RAISE and self.errors: 2127 raise ParseError( 2128 concat_messages(self.errors, self.max_errors), 2129 errors=merge_errors(self.errors), 2130 ) 2131 2132 def expression( 2133 self, 2134 instance: E, 2135 token: Token | None = None, 2136 comments: list[str] | None = None, 2137 ) -> E: 2138 if token: 2139 instance.update_positions(token) 2140 instance.add_comments(comments) if comments else self._add_comments(instance) 2141 if not instance.is_primitive: 2142 instance = self.validate_expression(instance) 2143 return instance 2144 2145 def _parse_batch_statements( 2146 self, 2147 parse_method: t.Callable[[Parser], exp.Expr | None], 2148 sep_first_statement: bool = True, 2149 ) -> list[exp.Expr | None]: 2150 expressions = [] 2151 2152 # Chunkification binds if/while statements with the first statement of the body 2153 if sep_first_statement: 2154 self._match(TokenType.BEGIN) 2155 expressions.append(parse_method(self)) 2156 2157 chunks_length = len(self._chunks) 2158 while self._chunk_index < chunks_length: 2159 self._advance_chunk() 2160 2161 if self._match(TokenType.ELSE, advance=False): 2162 return expressions 2163 2164 if expressions and not self._next and self._match(TokenType.END): 2165 expressions.append(exp.EndStatement()) 2166 continue 2167 2168 expressions.append(parse_method(self)) 2169 2170 if self._index < self._tokens_size: 2171 self.raise_error("Invalid expression / Unexpected token") 2172 2173 self.check_errors() 2174 2175 return expressions 2176 2177 def _parse( 2178 self, 2179 parse_method: t.Callable[[Parser], exp.Expr | None], 2180 raw_tokens: list[Token], 2181 sql: str | None = None, 2182 ) -> list[exp.Expr | None]: 2183 self.reset() 2184 self.sql = sql or "" 2185 2186 total = len(raw_tokens) 2187 chunks: list[list[Token]] = [[]] 2188 2189 for i, token in enumerate(raw_tokens): 2190 if token.token_type == TokenType.SEMICOLON: 2191 if token.comments: 2192 chunks.append([token]) 2193 2194 if i < total - 1: 2195 chunks.append([]) 2196 else: 2197 chunks[-1].append(token) 2198 2199 self._chunks = chunks 2200 2201 return self._parse_batch_statements(parse_method=parse_method, sep_first_statement=False) 2202 2203 def _warn_unsupported(self) -> None: 2204 if self._tokens_size <= 1: 2205 return 2206 2207 # We use _find_sql because self.sql may comprise multiple chunks, and we're only 2208 # interested in emitting a warning for the one being currently processed. 2209 sql = self._find_sql(self._tokens[0], self._tokens[-1])[: self.error_message_context] 2210 2211 logger.warning( 2212 f"'{sql}' contains unsupported syntax. Falling back to parsing as a 'Command'." 2213 ) 2214 2215 def _parse_command(self) -> exp.Command: 2216 self._warn_unsupported() 2217 comments = self._prev_comments 2218 return self.expression( 2219 exp.Command(this=self._prev.text.upper(), expression=self._parse_string()), 2220 comments=comments, 2221 ) 2222 2223 def _parse_comment(self, allow_exists: bool = True) -> exp.Expr: 2224 start = self._prev 2225 exists = self._parse_exists() if allow_exists else None 2226 2227 self._match(TokenType.ON) 2228 2229 materialized = self._match_text_seq("MATERIALIZED") 2230 kind = self._match_set(self.CREATABLES) and self._prev 2231 if not kind: 2232 return self._parse_as_command(start) 2233 2234 if kind.token_type in (TokenType.FUNCTION, TokenType.PROCEDURE): 2235 this = self._parse_user_defined_function(kind=kind.token_type) 2236 elif kind.token_type == TokenType.TABLE: 2237 this = self._parse_table(alias_tokens=self.COMMENT_TABLE_ALIAS_TOKENS) 2238 elif kind.token_type == TokenType.COLUMN: 2239 this = self._parse_column() 2240 else: 2241 this = self._parse_table_parts(schema=True) 2242 2243 self._match(TokenType.IS) 2244 2245 return self.expression( 2246 exp.Comment( 2247 this=this, 2248 kind=kind.text, 2249 expression=self._parse_string(), 2250 exists=exists, 2251 materialized=materialized, 2252 ) 2253 ) 2254 2255 def _parse_to_table( 2256 self, 2257 ) -> exp.ToTableProperty: 2258 table = self._parse_table_parts(schema=True) 2259 return self.expression(exp.ToTableProperty(this=table)) 2260 2261 # https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#mergetree-table-ttl 2262 def _parse_ttl(self) -> exp.Expr: 2263 def _parse_ttl_action() -> exp.Expr | None: 2264 this = self._parse_bitwise() 2265 2266 if self._match_text_seq("DELETE"): 2267 return self.expression(exp.MergeTreeTTLAction(this=this, delete=True)) 2268 if self._match_text_seq("RECOMPRESS"): 2269 return self.expression( 2270 exp.MergeTreeTTLAction(this=this, recompress=self._parse_bitwise()) 2271 ) 2272 if self._match_text_seq("TO", "DISK"): 2273 return self.expression( 2274 exp.MergeTreeTTLAction(this=this, to_disk=self._parse_string()) 2275 ) 2276 if self._match_text_seq("TO", "VOLUME"): 2277 return self.expression( 2278 exp.MergeTreeTTLAction(this=this, to_volume=self._parse_string()) 2279 ) 2280 2281 return this 2282 2283 expressions = self._parse_csv(_parse_ttl_action) 2284 where = self._parse_where() 2285 group = self._parse_group() 2286 2287 aggregates = None 2288 if group and self._match(TokenType.SET): 2289 aggregates = self._parse_csv(self._parse_set_item) 2290 2291 return self.expression( 2292 exp.MergeTreeTTL( 2293 expressions=expressions, where=where, group=group, aggregates=aggregates 2294 ) 2295 ) 2296 2297 def _parse_condition(self) -> exp.Expr | None: 2298 return self._parse_wrapped(parse_method=self._parse_expression, optional=True) 2299 2300 def _parse_block(self) -> exp.Block: 2301 return self.expression( 2302 exp.Block( 2303 expressions=self._parse_batch_statements( 2304 parse_method=lambda self: self._parse_statement() 2305 ) 2306 ) 2307 ) 2308 2309 def _parse_whileblock(self) -> exp.WhileBlock: 2310 return self.expression( 2311 exp.WhileBlock(this=self._parse_condition(), body=self._parse_block()) 2312 ) 2313 2314 def _parse_statement(self) -> exp.Expr | None: 2315 if not self._curr: 2316 return None 2317 2318 if self._match_set(self.STATEMENT_PARSERS): 2319 comments = self._prev_comments 2320 stmt = self.STATEMENT_PARSERS[self._prev.token_type](self) 2321 stmt.add_comments(comments, prepend=True) 2322 return stmt 2323 2324 if self._match_set(self.dialect.tokenizer_class.COMMANDS): 2325 return self._parse_command() 2326 2327 if self._match_text_seq("WHILE"): 2328 return self._parse_whileblock() 2329 2330 expression = self._parse_expression() 2331 expression = self._parse_set_operations(expression) if expression else self._parse_select() 2332 2333 if isinstance(expression, exp.Subquery) and self._match(TokenType.PIPE_GT, advance=False): 2334 expression = self._parse_pipe_syntax_query(expression) 2335 2336 return self._parse_query_modifiers(expression) 2337 2338 def _parse_drop(self, exists: bool = False) -> exp.Drop | exp.Command: 2339 start = self._prev 2340 temporary = self._match(TokenType.TEMPORARY) 2341 materialized = self._match_text_seq("MATERIALIZED") 2342 iceberg = self._match_text_seq("ICEBERG") 2343 2344 kind = self._match_set(self.CREATABLES) and self._prev.text.upper() 2345 if not kind or (iceberg and kind and kind != "TABLE"): 2346 return self._parse_as_command(start) 2347 2348 concurrently = self._match_text_seq("CONCURRENTLY") 2349 if_exists = exists or self._parse_exists() 2350 2351 if kind == "COLUMN": 2352 this = self._parse_column() 2353 else: 2354 this = self._parse_table_parts(schema=True, is_db_reference=kind == "SCHEMA") 2355 2356 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 2357 2358 if self._match(TokenType.L_PAREN, advance=False): 2359 expressions = self._parse_wrapped_csv(self._parse_types) 2360 else: 2361 expressions = None 2362 2363 cascade_or_restrict = self._match_texts(("CASCADE", "RESTRICT")) and self._prev.text.upper() 2364 2365 return self.expression( 2366 exp.Drop( 2367 exists=if_exists, 2368 this=this, 2369 expressions=expressions, 2370 kind=self.dialect.CREATABLE_KIND_MAPPING.get(kind) or kind, 2371 temporary=temporary, 2372 materialized=materialized, 2373 cascade=cascade_or_restrict == "CASCADE", 2374 restrict=cascade_or_restrict == "RESTRICT", 2375 constraints=self._match_text_seq("CONSTRAINTS"), 2376 purge=self._match_text_seq("PURGE"), 2377 cluster=cluster, 2378 concurrently=concurrently, 2379 sync=self._match_text_seq("SYNC"), 2380 iceberg=iceberg, 2381 ) 2382 ) 2383 2384 def _parse_exists(self, not_: bool = False) -> bool | None: 2385 return ( 2386 self._match_text_seq("IF") 2387 and (not not_ or self._match(TokenType.NOT)) 2388 and self._match(TokenType.EXISTS) 2389 ) 2390 2391 def _parse_create(self) -> exp.Create | exp.Command: 2392 # Note: this can't be None because we've matched a statement parser 2393 start = self._prev 2394 2395 replace = ( 2396 start.token_type == TokenType.REPLACE 2397 or self._match_pair(TokenType.OR, TokenType.REPLACE) 2398 or self._match_pair(TokenType.OR, TokenType.ALTER) 2399 ) 2400 refresh = self._match_pair(TokenType.OR, TokenType.REFRESH) 2401 2402 unique = self._match(TokenType.UNIQUE) 2403 2404 if self._match_text_seq("CLUSTERED", "COLUMNSTORE"): 2405 clustered = True 2406 elif self._match_text_seq("NONCLUSTERED", "COLUMNSTORE") or self._match_text_seq( 2407 "COLUMNSTORE" 2408 ): 2409 clustered = False 2410 else: 2411 clustered = None 2412 2413 if self._match_pair(TokenType.TABLE, TokenType.FUNCTION, advance=False): 2414 self._advance() 2415 2416 properties = None 2417 create_token = self._match_set(self.CREATABLES) and self._prev 2418 2419 if not create_token: 2420 # exp.Properties.Location.POST_CREATE 2421 properties = self._parse_properties() 2422 create_token = self._match_set(self.CREATABLES) and self._prev 2423 2424 if not properties or not create_token: 2425 return self._parse_as_command(start) 2426 2427 create_token_type = t.cast(Token, create_token).token_type 2428 2429 concurrently = self._match_text_seq("CONCURRENTLY") 2430 exists = self._parse_exists(not_=True) 2431 this = None 2432 expression: exp.Expr | None = None 2433 indexes = None 2434 no_schema_binding = None 2435 begin = None 2436 clone = None 2437 2438 def extend_props(temp_props: exp.Properties | None) -> None: 2439 nonlocal properties 2440 if properties and temp_props: 2441 properties.expressions.extend(temp_props.expressions) 2442 elif temp_props: 2443 properties = temp_props 2444 2445 if create_token_type in (TokenType.FUNCTION, TokenType.PROCEDURE): 2446 this = self._parse_user_defined_function(kind=create_token_type) 2447 2448 # exp.Properties.Location.POST_SCHEMA ("schema" here is the UDF's type signature) 2449 extend_props(self._parse_properties()) 2450 2451 expression = self._parse_heredoc() if self._match(TokenType.ALIAS) else None 2452 2453 if ( 2454 not expression 2455 and create_token_type == TokenType.FUNCTION 2456 and isinstance(this, exp.UserDefinedFunction) 2457 and this.args.get("wrapped") 2458 ): 2459 pre_table_index = self._index 2460 is_table = self._match(TokenType.TABLE) 2461 2462 expression = self._parse_expression() 2463 overload_mode = bool( 2464 expression 2465 and self._curr.token_type == TokenType.COMMA 2466 and self._next.token_type == TokenType.L_PAREN 2467 ) 2468 if not overload_mode: 2469 self._retreat(pre_table_index) 2470 is_table = False 2471 expression = None 2472 else: 2473 is_table = False 2474 overload_mode = False 2475 2476 extend_props(self._parse_function_properties()) 2477 2478 if not expression: 2479 if self._match(TokenType.COMMAND): 2480 expression = self._parse_as_command(self._prev) 2481 else: 2482 begin = self._match(TokenType.BEGIN) 2483 return_ = self._match_text_seq("RETURN") 2484 2485 if self._match(TokenType.STRING, advance=False): 2486 # Takes care of BigQuery's JavaScript UDF definitions that end in an OPTIONS property 2487 # # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_function_statement 2488 expression = self._parse_string() 2489 extend_props(self._parse_properties()) 2490 else: 2491 expression = ( 2492 self._parse_user_defined_function_expression() 2493 if create_token_type == TokenType.FUNCTION 2494 else self._parse_block() 2495 ) 2496 2497 if return_: 2498 expression = self.expression(exp.Return(this=expression)) 2499 2500 if overload_mode and expression: 2501 expression = self._parse_macro_overloads( 2502 t.cast(exp.UserDefinedFunction, this), expression, is_table 2503 ) 2504 elif create_token_type == TokenType.INDEX: 2505 # Postgres allows anonymous indexes, eg. CREATE INDEX IF NOT EXISTS ON t(c) 2506 if not self._match(TokenType.ON): 2507 index = self._parse_id_var() 2508 anonymous = False 2509 else: 2510 index = None 2511 anonymous = True 2512 2513 this = self._parse_index(index=index, anonymous=anonymous) 2514 elif ( 2515 create_token_type == TokenType.CONSTRAINT and self._match(TokenType.TRIGGER) 2516 ) or create_token_type == TokenType.TRIGGER: 2517 if is_constraint := (create_token_type == TokenType.CONSTRAINT): 2518 create_token = self._prev 2519 2520 trigger_name = self._parse_id_var() 2521 if not trigger_name: 2522 return self._parse_as_command(start) 2523 2524 timing_var = self._parse_var_from_options(self.TRIGGER_TIMING, raise_unmatched=False) 2525 timing = timing_var.this if timing_var else None 2526 if not timing: 2527 return self._parse_as_command(start) 2528 2529 events = self._parse_trigger_events() 2530 if not self._match(TokenType.ON): 2531 self.raise_error("Expected ON in trigger definition") 2532 2533 table = self._parse_table_parts() 2534 referenced_table = self._parse_table_parts() if self._match(TokenType.FROM) else None 2535 deferrable, initially = self._parse_trigger_deferrable() 2536 referencing = self._parse_trigger_referencing() 2537 for_each = self._parse_trigger_for_each() 2538 when = self._match_text_seq("WHEN") and self._parse_wrapped( 2539 self._parse_disjunction, optional=True 2540 ) 2541 execute = self._parse_trigger_execute() 2542 2543 if execute is None: 2544 return self._parse_as_command(start) 2545 2546 trigger_props = self.expression( 2547 exp.TriggerProperties( 2548 table=table, 2549 timing=timing, 2550 events=events, 2551 execute=execute, 2552 constraint=is_constraint, 2553 referenced_table=referenced_table, 2554 deferrable=deferrable, 2555 initially=initially, 2556 referencing=referencing, 2557 for_each=for_each, 2558 when=when, 2559 ) 2560 ) 2561 2562 this = trigger_name 2563 extend_props(exp.Properties(expressions=[trigger_props] if trigger_props else [])) 2564 elif create_token_type == TokenType.TYPE: 2565 this = self._parse_table_parts(schema=True) 2566 if not this or not self._match(TokenType.ALIAS): 2567 return self._parse_as_command(start) 2568 2569 if self._match(TokenType.ENUM): 2570 expression = exp.DataType( 2571 this=exp.DType.ENUM, 2572 expressions=self._parse_wrapped_csv(self._parse_string), 2573 ) 2574 elif self._match(TokenType.L_PAREN, advance=False): 2575 expression = self._parse_schema() 2576 else: 2577 return self._parse_as_command(start) 2578 elif create_token_type in self.DB_CREATABLES: 2579 table_parts = self._parse_table_parts( 2580 schema=True, is_db_reference=create_token_type == TokenType.SCHEMA 2581 ) 2582 2583 # exp.Properties.Location.POST_NAME 2584 self._match(TokenType.COMMA) 2585 extend_props(self._parse_properties(before=True)) 2586 2587 this = self._parse_schema(this=table_parts) 2588 2589 # exp.Properties.Location.POST_SCHEMA and POST_WITH 2590 extend_props(self._parse_properties()) 2591 2592 has_alias = self._match(TokenType.ALIAS) 2593 if not self._match_set(self.DDL_SELECT_TOKENS, advance=False): 2594 # exp.Properties.Location.POST_ALIAS 2595 extend_props(self._parse_properties()) 2596 2597 if create_token_type == TokenType.SEQUENCE: 2598 expression = self._parse_types() 2599 props = self._parse_properties() 2600 if props: 2601 sequence_props = exp.SequenceProperties() 2602 options = [] 2603 for prop in props: 2604 if isinstance(prop, exp.SequenceProperties): 2605 for arg, value in prop.args.items(): 2606 if arg == "options": 2607 options.extend(value) 2608 else: 2609 sequence_props.set(arg, value) 2610 prop.pop() 2611 2612 if options: 2613 sequence_props.set("options", options) 2614 2615 props.append("expressions", sequence_props) 2616 extend_props(props) 2617 else: 2618 expression = self._parse_ddl_select() 2619 2620 # Some dialects also support using a table as an alias instead of a SELECT. 2621 # Here we fallback to this as an alternative. 2622 if not expression and has_alias: 2623 expression = self._try_parse(self._parse_table_parts) 2624 2625 if create_token_type == TokenType.TABLE: 2626 # exp.Properties.Location.POST_EXPRESSION 2627 extend_props(self._parse_properties()) 2628 2629 indexes = [] 2630 while True: 2631 index = self._parse_index() 2632 2633 # exp.Properties.Location.POST_INDEX 2634 extend_props(self._parse_properties()) 2635 if not index: 2636 break 2637 else: 2638 self._match(TokenType.COMMA) 2639 indexes.append(index) 2640 elif create_token_type == TokenType.VIEW: 2641 if self._match_text_seq("WITH", "NO", "SCHEMA", "BINDING"): 2642 no_schema_binding = True 2643 elif create_token_type in (TokenType.SINK, TokenType.SOURCE): 2644 extend_props(self._parse_properties()) 2645 2646 shallow = self._match_text_seq("SHALLOW") 2647 2648 if self._match_texts(self.CLONE_KEYWORDS): 2649 copy = self._prev.text.lower() == "copy" 2650 clone = self.expression( 2651 exp.Clone(this=self._parse_table(schema=True), shallow=shallow, copy=copy) 2652 ) 2653 2654 if self._curr and not self._match_set((TokenType.R_PAREN, TokenType.COMMA), advance=False): 2655 return self._parse_as_command(start) 2656 2657 create_kind_text = create_token.text.upper() 2658 return self.expression( 2659 exp.Create( 2660 this=this, 2661 kind=self.dialect.CREATABLE_KIND_MAPPING.get(create_kind_text) or create_kind_text, 2662 replace=replace, 2663 refresh=refresh, 2664 unique=unique, 2665 expression=expression, 2666 exists=exists, 2667 properties=properties, 2668 indexes=indexes, 2669 no_schema_binding=no_schema_binding, 2670 begin=begin, 2671 clone=clone, 2672 concurrently=concurrently, 2673 clustered=clustered, 2674 ) 2675 ) 2676 2677 def _parse_sequence_properties(self) -> exp.SequenceProperties | None: 2678 seq = exp.SequenceProperties() 2679 2680 options = [] 2681 index = self._index 2682 2683 while self._curr: 2684 self._match(TokenType.COMMA) 2685 if self._match_text_seq("INCREMENT"): 2686 self._match_text_seq("BY") 2687 self._match_text_seq("=") 2688 seq.set("increment", self._parse_term()) 2689 elif self._match_text_seq("MINVALUE"): 2690 seq.set("minvalue", self._parse_term()) 2691 elif self._match_text_seq("MAXVALUE"): 2692 seq.set("maxvalue", self._parse_term()) 2693 elif self._match(TokenType.START_WITH) or self._match_text_seq("START"): 2694 self._match_text_seq("=") 2695 seq.set("start", self._parse_term()) 2696 elif self._match_text_seq("CACHE"): 2697 # T-SQL allows empty CACHE which is initialized dynamically 2698 seq.set("cache", self._parse_number() or True) 2699 elif self._match_text_seq("OWNED", "BY"): 2700 # "OWNED BY NONE" is the default 2701 seq.set("owned", None if self._match_text_seq("NONE") else self._parse_column()) 2702 else: 2703 opt = self._parse_var_from_options(self.CREATE_SEQUENCE, raise_unmatched=False) 2704 if opt: 2705 options.append(opt) 2706 else: 2707 break 2708 2709 seq.set("options", options if options else None) 2710 return None if self._index == index else seq 2711 2712 def _parse_trigger_events(self) -> list[exp.TriggerEvent]: 2713 events = [] 2714 2715 while True: 2716 event_type = self._match_set(self.TRIGGER_EVENTS) and self._prev.text.upper() 2717 2718 if not event_type: 2719 self.raise_error("Expected trigger event (INSERT, UPDATE, DELETE, TRUNCATE)") 2720 2721 columns = ( 2722 self._parse_csv(self._parse_column) 2723 if event_type == "UPDATE" and self._match_text_seq("OF") 2724 else None 2725 ) 2726 2727 events.append(self.expression(exp.TriggerEvent(this=event_type, columns=columns))) 2728 2729 if not self._match(TokenType.OR): 2730 break 2731 2732 return events 2733 2734 def _parse_trigger_deferrable( 2735 self, 2736 ) -> tuple[str | None, str | None]: 2737 deferrable_var = self._parse_var_from_options( 2738 self.TRIGGER_DEFERRABLE, raise_unmatched=False 2739 ) 2740 deferrable = deferrable_var.this if deferrable_var else None 2741 2742 initially = None 2743 if deferrable and self._match_text_seq("INITIALLY"): 2744 initially = ( 2745 self._prev.text.upper() if self._match_texts(("IMMEDIATE", "DEFERRED")) else None 2746 ) 2747 2748 return deferrable, initially 2749 2750 def _parse_trigger_referencing_clause(self, keyword: str) -> exp.Expr | None: 2751 if not self._match_text_seq(keyword): 2752 return None 2753 if not self._match_text_seq("TABLE"): 2754 self.raise_error(f"Expected TABLE after {keyword} in REFERENCING clause") 2755 self._match_text_seq("AS") 2756 return self._parse_id_var() 2757 2758 def _parse_trigger_referencing(self) -> exp.TriggerReferencing | None: 2759 if not self._match_text_seq("REFERENCING"): 2760 return None 2761 2762 old_alias = None 2763 new_alias = None 2764 2765 while True: 2766 if alias := self._parse_trigger_referencing_clause("OLD"): 2767 if old_alias is not None: 2768 self.raise_error("Duplicate OLD clause in REFERENCING") 2769 old_alias = alias 2770 elif alias := self._parse_trigger_referencing_clause("NEW"): 2771 if new_alias is not None: 2772 self.raise_error("Duplicate NEW clause in REFERENCING") 2773 new_alias = alias 2774 else: 2775 break 2776 2777 if old_alias is None and new_alias is None: 2778 self.raise_error("REFERENCING clause requires at least OLD TABLE or NEW TABLE") 2779 2780 return self.expression(exp.TriggerReferencing(old=old_alias, new=new_alias)) 2781 2782 def _parse_trigger_for_each(self) -> str | None: 2783 if not self._match_text_seq("FOR", "EACH"): 2784 return None 2785 2786 return self._prev.text.upper() if self._match_texts(("ROW", "STATEMENT")) else None 2787 2788 def _parse_trigger_execute(self) -> exp.TriggerExecute | None: 2789 if not self._match(TokenType.EXECUTE): 2790 return None 2791 2792 if not self._match_set((TokenType.FUNCTION, TokenType.PROCEDURE)): 2793 self.raise_error("Expected FUNCTION or PROCEDURE after EXECUTE") 2794 2795 func_call = self._parse_column() 2796 return self.expression(exp.TriggerExecute(this=func_call)) 2797 2798 def _parse_property_before(self) -> exp.Expr | list[exp.Expr] | None: 2799 # only used for teradata currently 2800 self._match(TokenType.COMMA) 2801 2802 kwargs = { 2803 "no": self._match_text_seq("NO"), 2804 "dual": self._match_text_seq("DUAL"), 2805 "before": self._match_text_seq("BEFORE"), 2806 "default": self._match_text_seq("DEFAULT"), 2807 "local": (self._match_text_seq("LOCAL") and "LOCAL") 2808 or (self._match_text_seq("NOT", "LOCAL") and "NOT LOCAL"), 2809 "after": self._match_text_seq("AFTER"), 2810 "minimum": self._match_texts(("MIN", "MINIMUM")), 2811 "maximum": self._match_texts(("MAX", "MAXIMUM")), 2812 } 2813 2814 if self._match_texts(self.PROPERTY_PARSERS): 2815 parser = self.PROPERTY_PARSERS[self._prev.text.upper()] 2816 try: 2817 return parser(self, **{k: v for k, v in kwargs.items() if v}) 2818 except TypeError: 2819 self.raise_error(f"Cannot parse property '{self._prev.text}'") 2820 2821 return None 2822 2823 def _parse_wrapped_properties(self) -> list[exp.Expr | list[exp.Expr]]: 2824 return self._parse_wrapped_csv(self._parse_property) 2825 2826 def _parse_property(self) -> exp.Expr | list[exp.Expr] | None: 2827 if self._match_texts(self.PROPERTY_PARSERS): 2828 return self.PROPERTY_PARSERS[self._prev.text.upper()](self) 2829 2830 if self._match(TokenType.DEFAULT) and self._match_texts(self.PROPERTY_PARSERS): 2831 return self.PROPERTY_PARSERS[self._prev.text.upper()](self, default=True) 2832 2833 if self._match_text_seq("COMPOUND", "SORTKEY"): 2834 return self._parse_sortkey(compound=True) 2835 2836 if self._match_text_seq("PARAMETER", "STYLE", "PANDAS"): 2837 return self.expression(exp.ParameterStyleProperty(this="PANDAS")) 2838 2839 index = self._index 2840 2841 seq_props = self._parse_sequence_properties() 2842 if seq_props: 2843 return seq_props 2844 2845 self._retreat(index) 2846 return self._parse_key_value_property() 2847 2848 def _parse_key_value_property( 2849 self, parse_value: t.Callable[[], exp.Expr | None] | None = None 2850 ) -> exp.Property | None: 2851 index = self._index 2852 key = self._parse_column() 2853 2854 if not self._match(TokenType.EQ): 2855 self._retreat(index) 2856 return None 2857 2858 # Transform the key to exp.Dot if it's dotted identifiers wrapped in exp.Column or to exp.Var otherwise 2859 if isinstance(key, exp.Column): 2860 key = key.to_dot() if len(key.parts) > 1 else exp.var(key.name) 2861 2862 value = ( 2863 parse_value() 2864 if parse_value 2865 else self._parse_bitwise() or self._parse_var(any_token=True) 2866 ) 2867 2868 # Transform the value to exp.Var if it was parsed as exp.Column(exp.Identifier()) 2869 if isinstance(value, exp.Column): 2870 value = exp.var(value.name) 2871 2872 return self.expression(exp.Property(this=key, value=value)) 2873 2874 def _parse_stored(self) -> exp.FileFormatProperty | exp.StorageHandlerProperty: 2875 if self._match_text_seq("BY"): 2876 return self.expression(exp.StorageHandlerProperty(this=self._parse_var_or_string())) 2877 2878 self._match(TokenType.ALIAS) 2879 input_format = self._parse_string() if self._match_text_seq("INPUTFORMAT") else None 2880 output_format = self._parse_string() if self._match_text_seq("OUTPUTFORMAT") else None 2881 2882 return self.expression( 2883 exp.FileFormatProperty( 2884 this=( 2885 self.expression( 2886 exp.InputOutputFormat( 2887 input_format=input_format, output_format=output_format 2888 ) 2889 ) 2890 if input_format or output_format 2891 else self._parse_var_or_string() or self._parse_number() or self._parse_id_var() 2892 ), 2893 hive_format=True, 2894 ) 2895 ) 2896 2897 def _parse_unquoted_field(self) -> exp.Expr | None: 2898 field = self._parse_field() 2899 if isinstance(field, exp.Identifier) and not field.quoted: 2900 field = exp.var(field) 2901 2902 return field 2903 2904 def _parse_property_assignment(self, exp_class: type[E], **kwargs: t.Any) -> E: 2905 self._match(TokenType.EQ) 2906 self._match(TokenType.ALIAS) 2907 2908 return self.expression(exp_class(this=self._parse_unquoted_field(), **kwargs)) 2909 2910 def _parse_properties(self, before: bool | None = None) -> exp.Properties | None: 2911 properties = [] 2912 while True: 2913 if before: 2914 prop = self._parse_property_before() 2915 else: 2916 prop = self._parse_property() 2917 if not prop: 2918 break 2919 for p in ensure_list(prop): 2920 properties.append(p) 2921 2922 if properties: 2923 return self.expression(exp.Properties(expressions=properties)) 2924 2925 return None 2926 2927 def _parse_fallback(self, no: bool = False) -> exp.FallbackProperty: 2928 return self.expression( 2929 exp.FallbackProperty(no=no, protection=self._match_text_seq("PROTECTION")) 2930 ) 2931 2932 def _parse_sql_security(self) -> exp.SqlSecurityProperty: 2933 return self.expression( 2934 exp.SqlSecurityProperty( 2935 this=self._match_texts(self.SECURITY_PROPERTY_KEYWORDS) and self._prev.text.upper() 2936 ) 2937 ) 2938 2939 def _parse_settings_property(self) -> exp.SettingsProperty: 2940 return self.expression( 2941 exp.SettingsProperty(expressions=self._parse_csv(self._parse_assignment)) 2942 ) 2943 2944 def _parse_called_on_null_input_property(self) -> exp.CalledOnNullInputProperty | None: 2945 if not self._match_text_seq("ON", "NULL", "INPUT"): 2946 self._retreat(self._index - 1) 2947 return None 2948 2949 return self.expression(exp.CalledOnNullInputProperty()) 2950 2951 def _parse_volatile_property(self) -> exp.VolatileProperty | exp.StabilityProperty: 2952 if self._index >= 2: 2953 pre_volatile_token = self._tokens[self._index - 2] 2954 else: 2955 pre_volatile_token = None 2956 2957 if pre_volatile_token and pre_volatile_token.token_type in self.PRE_VOLATILE_TOKENS: 2958 return exp.VolatileProperty() 2959 2960 return self.expression(exp.StabilityProperty(this=exp.Literal.string("VOLATILE"))) 2961 2962 def _parse_retention_period(self) -> exp.Var: 2963 # Parse TSQL's HISTORY_RETENTION_PERIOD: {INFINITE | <number> DAY | DAYS | MONTH ...} 2964 number = self._parse_number() 2965 number_str = f"{number} " if number else "" 2966 unit = self._parse_var(any_token=True) 2967 return exp.var(f"{number_str}{unit}") 2968 2969 def _parse_system_versioning_property( 2970 self, with_: bool = False 2971 ) -> exp.WithSystemVersioningProperty: 2972 self._match(TokenType.EQ) 2973 prop = self.expression(exp.WithSystemVersioningProperty(on=True, with_=with_)) 2974 2975 if self._match_text_seq("OFF"): 2976 prop.set("on", False) 2977 return prop 2978 2979 self._match(TokenType.ON) 2980 if self._match(TokenType.L_PAREN): 2981 while self._curr and not self._match(TokenType.R_PAREN): 2982 if self._match_text_seq("HISTORY_TABLE", "="): 2983 prop.set("this", self._parse_table_parts()) 2984 elif self._match_text_seq("DATA_CONSISTENCY_CHECK", "="): 2985 prop.set("data_consistency", self._advance_any() and self._prev.text.upper()) 2986 elif self._match_text_seq("HISTORY_RETENTION_PERIOD", "="): 2987 prop.set("retention_period", self._parse_retention_period()) 2988 2989 self._match(TokenType.COMMA) 2990 2991 return prop 2992 2993 def _parse_data_deletion_property(self) -> exp.DataDeletionProperty: 2994 self._match(TokenType.EQ) 2995 on = self._match_text_seq("ON") or not self._match_text_seq("OFF") 2996 prop = self.expression(exp.DataDeletionProperty(on=on)) 2997 2998 if self._match(TokenType.L_PAREN): 2999 while self._curr and not self._match(TokenType.R_PAREN): 3000 if self._match_text_seq("FILTER_COLUMN", "="): 3001 prop.set("filter_column", self._parse_column()) 3002 elif self._match_text_seq("RETENTION_PERIOD", "="): 3003 prop.set("retention_period", self._parse_retention_period()) 3004 3005 self._match(TokenType.COMMA) 3006 3007 return prop 3008 3009 def _parse_distributed_property(self) -> exp.DistributedByProperty: 3010 kind = "HASH" 3011 expressions: list[exp.Expr] | None = None 3012 if self._match_text_seq("BY", "HASH"): 3013 expressions = self._parse_wrapped_csv(self._parse_id_var) 3014 elif self._match_text_seq("BY", "RANDOM"): 3015 kind = "RANDOM" 3016 3017 # If the BUCKETS keyword is not present, the number of buckets is AUTO 3018 buckets: exp.Expr | None = None 3019 if self._match_text_seq("BUCKETS") and not self._match_text_seq("AUTO"): 3020 buckets = self._parse_number() 3021 3022 return self.expression( 3023 exp.DistributedByProperty( 3024 expressions=expressions, kind=kind, buckets=buckets, order=self._parse_order() 3025 ) 3026 ) 3027 3028 def _parse_composite_key_property(self, expr_type: type[E]) -> E: 3029 self._match_text_seq("KEY") 3030 expressions = self._parse_wrapped_id_vars() 3031 return self.expression(expr_type(expressions=expressions)) 3032 3033 def _parse_with_property(self) -> exp.Expr | None | list[exp.Expr]: 3034 if self._match_text_seq("(", "SYSTEM_VERSIONING"): 3035 prop = self._parse_system_versioning_property(with_=True) 3036 self._match_r_paren() 3037 return prop 3038 3039 if self._match(TokenType.L_PAREN, advance=False): 3040 result: list[exp.Expr] = [] 3041 for i in self._parse_wrapped_properties(): 3042 result.extend(i) if isinstance(i, list) else result.append(i) 3043 return result 3044 3045 if self._match_text_seq("JOURNAL"): 3046 return self._parse_withjournaltable() 3047 3048 if self._match_texts(self.VIEW_ATTRIBUTES): 3049 return self.expression(exp.ViewAttributeProperty(this=self._prev.text.upper())) 3050 3051 if self._match_text_seq("DATA"): 3052 return self._parse_withdata(no=False) 3053 elif self._match_text_seq("NO", "DATA"): 3054 return self._parse_withdata(no=True) 3055 3056 if self._match(TokenType.SERDE_PROPERTIES, advance=False): 3057 return self._parse_serde_properties(with_=True) 3058 3059 if self._match(TokenType.SCHEMA): 3060 return self.expression( 3061 exp.WithSchemaBindingProperty( 3062 this=self._parse_var_from_options(self.SCHEMA_BINDING_OPTIONS) 3063 ) 3064 ) 3065 3066 if self._match_texts(self.PROCEDURE_OPTIONS, advance=False): 3067 return self.expression( 3068 exp.WithProcedureOptions(expressions=self._parse_csv(self._parse_procedure_option)) 3069 ) 3070 3071 if not self._next: 3072 return None 3073 3074 return self._parse_withisolatedloading() 3075 3076 def _parse_procedure_option(self) -> exp.Expr | None: 3077 if self._match_text_seq("EXECUTE", "AS"): 3078 return self.expression( 3079 exp.ExecuteAsProperty( 3080 this=self._parse_var_from_options( 3081 self.EXECUTE_AS_OPTIONS, raise_unmatched=False 3082 ) 3083 or self._parse_string() 3084 ) 3085 ) 3086 3087 return self._parse_var_from_options(self.PROCEDURE_OPTIONS) 3088 3089 # https://dev.mysql.com/doc/refman/8.0/en/create-view.html 3090 def _parse_definer(self) -> exp.DefinerProperty | None: 3091 self._match(TokenType.EQ) 3092 3093 user = self._parse_id_var() 3094 self._match(TokenType.PARAMETER) 3095 host = self._parse_id_var() or (self._match(TokenType.MOD) and self._prev.text) 3096 3097 if not user or not host: 3098 return None 3099 3100 return exp.DefinerProperty(this=f"{user}@{host}") 3101 3102 def _parse_withjournaltable(self) -> exp.WithJournalTableProperty: 3103 self._match(TokenType.TABLE) 3104 self._match(TokenType.EQ) 3105 return self.expression(exp.WithJournalTableProperty(this=self._parse_table_parts())) 3106 3107 def _parse_log(self, no: bool = False) -> exp.LogProperty: 3108 return self.expression(exp.LogProperty(no=no)) 3109 3110 def _parse_journal(self, **kwargs) -> exp.JournalProperty: 3111 return self.expression(exp.JournalProperty(**kwargs)) 3112 3113 def _parse_checksum(self) -> exp.ChecksumProperty: 3114 self._match(TokenType.EQ) 3115 3116 on = None 3117 if self._match(TokenType.ON): 3118 on = True 3119 elif self._match_text_seq("OFF"): 3120 on = False 3121 3122 return self.expression(exp.ChecksumProperty(on=on, default=self._match(TokenType.DEFAULT))) 3123 3124 def _parse_cluster(self) -> exp.Cluster: 3125 self._match(TokenType.CLUSTER_BY) 3126 return self.expression( 3127 exp.Cluster( 3128 expressions=self._parse_csv(self._parse_column), 3129 ) 3130 ) 3131 3132 def _parse_cluster_property(self) -> exp.ClusterProperty: 3133 return self.expression( 3134 exp.ClusterProperty( 3135 expressions=self._parse_wrapped_csv(self._parse_column), 3136 ) 3137 ) 3138 3139 def _parse_clustered_by(self) -> exp.ClusteredByProperty: 3140 self._match_text_seq("BY") 3141 3142 self._match_l_paren() 3143 expressions = self._parse_csv(self._parse_column) 3144 self._match_r_paren() 3145 3146 if self._match_text_seq("SORTED", "BY"): 3147 self._match_l_paren() 3148 sorted_by = self._parse_csv(self._parse_ordered) 3149 self._match_r_paren() 3150 else: 3151 sorted_by = None 3152 3153 self._match(TokenType.INTO) 3154 buckets = self._parse_number() 3155 self._match_text_seq("BUCKETS") 3156 3157 return self.expression( 3158 exp.ClusteredByProperty(expressions=expressions, sorted_by=sorted_by, buckets=buckets) 3159 ) 3160 3161 def _parse_copy_property(self) -> exp.CopyGrantsProperty | None: 3162 if not self._match_text_seq("GRANTS"): 3163 self._retreat(self._index - 1) 3164 return None 3165 3166 return self.expression(exp.CopyGrantsProperty()) 3167 3168 def _parse_freespace(self) -> exp.FreespaceProperty: 3169 self._match(TokenType.EQ) 3170 return self.expression( 3171 exp.FreespaceProperty(this=self._parse_number(), percent=self._match(TokenType.PERCENT)) 3172 ) 3173 3174 def _parse_mergeblockratio( 3175 self, no: bool = False, default: bool = False 3176 ) -> exp.MergeBlockRatioProperty: 3177 if self._match(TokenType.EQ): 3178 return self.expression( 3179 exp.MergeBlockRatioProperty( 3180 this=self._parse_number(), percent=self._match(TokenType.PERCENT) 3181 ) 3182 ) 3183 3184 return self.expression(exp.MergeBlockRatioProperty(no=no, default=default)) 3185 3186 def _parse_datablocksize( 3187 self, 3188 default: bool | None = None, 3189 minimum: bool | None = None, 3190 maximum: bool | None = None, 3191 ) -> exp.DataBlocksizeProperty: 3192 self._match(TokenType.EQ) 3193 size = self._parse_number() 3194 3195 units = None 3196 if self._match_texts(("BYTES", "KBYTES", "KILOBYTES")): 3197 units = self._prev.text 3198 3199 return self.expression( 3200 exp.DataBlocksizeProperty( 3201 size=size, units=units, default=default, minimum=minimum, maximum=maximum 3202 ) 3203 ) 3204 3205 def _parse_blockcompression(self) -> exp.BlockCompressionProperty: 3206 self._match(TokenType.EQ) 3207 always = self._match_text_seq("ALWAYS") 3208 manual = self._match_text_seq("MANUAL") 3209 never = self._match_text_seq("NEVER") 3210 default = self._match_text_seq("DEFAULT") 3211 3212 autotemp = None 3213 if self._match_text_seq("AUTOTEMP"): 3214 autotemp = self._parse_schema() 3215 3216 return self.expression( 3217 exp.BlockCompressionProperty( 3218 always=always, manual=manual, never=never, default=default, autotemp=autotemp 3219 ) 3220 ) 3221 3222 def _parse_withisolatedloading(self) -> exp.IsolatedLoadingProperty | None: 3223 index = self._index 3224 no = self._match_text_seq("NO") 3225 concurrent = self._match_text_seq("CONCURRENT") 3226 3227 if not self._match_text_seq("ISOLATED", "LOADING"): 3228 self._retreat(index) 3229 return None 3230 3231 target = self._parse_var_from_options(self.ISOLATED_LOADING_OPTIONS, raise_unmatched=False) 3232 return self.expression( 3233 exp.IsolatedLoadingProperty(no=no, concurrent=concurrent, target=target) 3234 ) 3235 3236 def _parse_locking(self) -> exp.LockingProperty: 3237 if self._match(TokenType.TABLE): 3238 kind = "TABLE" 3239 elif self._match(TokenType.VIEW): 3240 kind = "VIEW" 3241 elif self._match(TokenType.ROW): 3242 kind = "ROW" 3243 elif self._match_text_seq("DATABASE"): 3244 kind = "DATABASE" 3245 else: 3246 kind = None 3247 3248 if kind in ("DATABASE", "TABLE", "VIEW"): 3249 this = self._parse_table_parts() 3250 else: 3251 this = None 3252 3253 if self._match(TokenType.FOR): 3254 for_or_in = "FOR" 3255 elif self._match(TokenType.IN): 3256 for_or_in = "IN" 3257 else: 3258 for_or_in = None 3259 3260 if self._match_text_seq("ACCESS"): 3261 lock_type = "ACCESS" 3262 elif self._match_texts(("EXCL", "EXCLUSIVE")): 3263 lock_type = "EXCLUSIVE" 3264 elif self._match_text_seq("SHARE"): 3265 lock_type = "SHARE" 3266 elif self._match_text_seq("READ"): 3267 lock_type = "READ" 3268 elif self._match_text_seq("WRITE"): 3269 lock_type = "WRITE" 3270 elif self._match_text_seq("CHECKSUM"): 3271 lock_type = "CHECKSUM" 3272 else: 3273 lock_type = None 3274 3275 override = self._match_text_seq("OVERRIDE") 3276 3277 return self.expression( 3278 exp.LockingProperty( 3279 this=this, kind=kind, for_or_in=for_or_in, lock_type=lock_type, override=override 3280 ) 3281 ) 3282 3283 def _parse_partition_by(self) -> list[exp.Expr]: 3284 if self._match(TokenType.PARTITION_BY): 3285 return self._parse_csv(self._parse_disjunction) 3286 return [] 3287 3288 def _parse_partition_bound_spec(self) -> exp.PartitionBoundSpec: 3289 def _parse_partition_bound_expr() -> exp.Expr | None: 3290 if self._match_text_seq("MINVALUE"): 3291 return exp.var("MINVALUE") 3292 if self._match_text_seq("MAXVALUE"): 3293 return exp.var("MAXVALUE") 3294 return self._parse_bitwise() 3295 3296 this: exp.Expr | list[exp.Expr] | None = None 3297 expression = None 3298 from_expressions = None 3299 to_expressions = None 3300 3301 if self._match(TokenType.IN): 3302 this = self._parse_wrapped_csv(self._parse_bitwise) 3303 elif self._match(TokenType.FROM): 3304 from_expressions = self._parse_wrapped_csv(_parse_partition_bound_expr) 3305 self._match_text_seq("TO") 3306 to_expressions = self._parse_wrapped_csv(_parse_partition_bound_expr) 3307 elif self._match_text_seq("WITH", "(", "MODULUS"): 3308 this = self._parse_number() 3309 self._match_text_seq(",", "REMAINDER") 3310 expression = self._parse_number() 3311 self._match_r_paren() 3312 else: 3313 self.raise_error("Failed to parse partition bound spec.") 3314 3315 return self.expression( 3316 exp.PartitionBoundSpec( 3317 this=this, 3318 expression=expression, 3319 from_expressions=from_expressions, 3320 to_expressions=to_expressions, 3321 ) 3322 ) 3323 3324 # https://www.postgresql.org/docs/current/sql-createtable.html 3325 def _parse_partitioned_of(self) -> exp.PartitionedOfProperty | None: 3326 if not self._match_text_seq("OF"): 3327 self._retreat(self._index - 1) 3328 return None 3329 3330 this = self._parse_table(schema=True) 3331 3332 if self._match(TokenType.DEFAULT): 3333 expression: exp.Var | exp.PartitionBoundSpec = exp.var("DEFAULT") 3334 elif self._match_text_seq("FOR", "VALUES"): 3335 expression = self._parse_partition_bound_spec() 3336 else: 3337 self.raise_error("Expecting either DEFAULT or FOR VALUES clause.") 3338 3339 return self.expression(exp.PartitionedOfProperty(this=this, expression=expression)) 3340 3341 def _parse_partitioned_by(self) -> exp.PartitionedByProperty: 3342 self._match(TokenType.EQ) 3343 return self.expression( 3344 exp.PartitionedByProperty( 3345 this=self._parse_schema() or self._parse_bracket(self._parse_field()) 3346 ) 3347 ) 3348 3349 def _parse_withdata(self, no: bool = False) -> exp.WithDataProperty: 3350 if self._match_text_seq("AND", "STATISTICS"): 3351 statistics = True 3352 elif self._match_text_seq("AND", "NO", "STATISTICS"): 3353 statistics = False 3354 else: 3355 statistics = None 3356 3357 return self.expression(exp.WithDataProperty(no=no, statistics=statistics)) 3358 3359 def _parse_contains_property(self) -> exp.SqlReadWriteProperty | None: 3360 if self._match_text_seq("SQL"): 3361 return self.expression(exp.SqlReadWriteProperty(this="CONTAINS SQL")) 3362 return None 3363 3364 def _parse_modifies_property(self) -> exp.SqlReadWriteProperty | None: 3365 if self._match_text_seq("SQL", "DATA"): 3366 return self.expression(exp.SqlReadWriteProperty(this="MODIFIES SQL DATA")) 3367 return None 3368 3369 def _parse_no_property(self) -> exp.Expr | None: 3370 if self._match_text_seq("PRIMARY", "INDEX"): 3371 return exp.NoPrimaryIndexProperty() 3372 if self._match_text_seq("SQL"): 3373 return self.expression(exp.SqlReadWriteProperty(this="NO SQL")) 3374 return None 3375 3376 def _parse_on_property(self) -> exp.Expr | None: 3377 if self._match_text_seq("COMMIT", "PRESERVE", "ROWS"): 3378 return exp.OnCommitProperty() 3379 if self._match_text_seq("COMMIT", "DELETE", "ROWS"): 3380 return exp.OnCommitProperty(delete=True) 3381 return self.expression(exp.OnProperty(this=self._parse_schema(self._parse_id_var()))) 3382 3383 def _parse_reads_property(self) -> exp.SqlReadWriteProperty | None: 3384 if self._match_text_seq("SQL", "DATA"): 3385 return self.expression(exp.SqlReadWriteProperty(this="READS SQL DATA")) 3386 return None 3387 3388 def _parse_distkey(self) -> exp.DistKeyProperty: 3389 return self.expression(exp.DistKeyProperty(this=self._parse_wrapped(self._parse_id_var))) 3390 3391 def _parse_create_like(self) -> exp.LikeProperty | None: 3392 table = self._parse_table(schema=True) 3393 3394 options = [] 3395 while self._match_texts(("INCLUDING", "EXCLUDING")): 3396 this = self._prev.text.upper() 3397 3398 id_var = self._parse_id_var() 3399 if not id_var: 3400 return None 3401 3402 options.append( 3403 self.expression(exp.Property(this=this, value=exp.var(id_var.this.upper()))) 3404 ) 3405 3406 return self.expression(exp.LikeProperty(this=table, expressions=options)) 3407 3408 def _parse_sortkey(self, compound: bool = False) -> exp.SortKeyProperty: 3409 return self.expression( 3410 exp.SortKeyProperty(this=self._parse_wrapped_id_vars(), compound=compound) 3411 ) 3412 3413 def _parse_character_set(self, default: bool = False) -> exp.CharacterSetProperty: 3414 self._match(TokenType.EQ) 3415 return self.expression( 3416 exp.CharacterSetProperty(this=self._parse_var_or_string(), default=default) 3417 ) 3418 3419 def _parse_remote_with_connection(self) -> exp.RemoteWithConnectionModelProperty: 3420 self._match_text_seq("WITH", "CONNECTION") 3421 return self.expression( 3422 exp.RemoteWithConnectionModelProperty(this=self._parse_table_parts()) 3423 ) 3424 3425 def _parse_returns(self) -> exp.ReturnsProperty: 3426 value: exp.Expr | None 3427 null = None 3428 is_table = self._match(TokenType.TABLE) 3429 3430 if is_table: 3431 if self._match(TokenType.LT): 3432 value = self.expression( 3433 exp.Schema(this="TABLE", expressions=self._parse_csv(self._parse_struct_types)) 3434 ) 3435 if not self._match(TokenType.GT): 3436 self.raise_error("Expecting >") 3437 else: 3438 value = self._parse_schema(exp.var("TABLE")) 3439 elif self._match_text_seq("NULL", "ON", "NULL", "INPUT"): 3440 null = True 3441 value = None 3442 else: 3443 value = self._parse_types() 3444 3445 return self.expression(exp.ReturnsProperty(this=value, is_table=is_table, null=null)) 3446 3447 def _parse_describe(self) -> exp.Describe: 3448 kind = self._prev.text if self._match_set(self.CREATABLES) else None 3449 style: str | None = ( 3450 self._prev.text.upper() if self._match_texts(self.DESCRIBE_STYLES) else None 3451 ) 3452 if self._match(TokenType.DOT): 3453 style = None 3454 self._retreat(self._index - 2) 3455 3456 format = self._parse_property() if self._match(TokenType.FORMAT, advance=False) else None 3457 3458 if self._match_set(self.STATEMENT_PARSERS, advance=False): 3459 this = self._parse_statement() 3460 else: 3461 this = self._parse_table(schema=True) 3462 3463 properties = self._parse_properties() 3464 expressions = properties.expressions if properties else None 3465 partition = self._parse_partition() 3466 return self.expression( 3467 exp.Describe( 3468 this=this, 3469 style=style, 3470 kind=kind, 3471 expressions=expressions, 3472 partition=partition, 3473 format=format, 3474 as_json=self._match_text_seq("AS", "JSON"), 3475 ) 3476 ) 3477 3478 def _parse_multitable_inserts(self, comments: list[str] | None) -> exp.MultitableInserts: 3479 kind = self._prev.text.upper() 3480 expressions = [] 3481 3482 def parse_conditional_insert() -> exp.ConditionalInsert | None: 3483 if self._match(TokenType.WHEN): 3484 expression = self._parse_disjunction() 3485 self._match(TokenType.THEN) 3486 else: 3487 expression = None 3488 3489 else_ = self._match(TokenType.ELSE) 3490 3491 if not self._match(TokenType.INTO): 3492 return None 3493 3494 return self.expression( 3495 exp.ConditionalInsert( 3496 this=self.expression( 3497 exp.Insert( 3498 this=self._parse_table(schema=True), 3499 expression=self._parse_derived_table_values(), 3500 ) 3501 ), 3502 expression=expression, 3503 else_=else_, 3504 ) 3505 ) 3506 3507 expression = parse_conditional_insert() 3508 while expression is not None: 3509 expressions.append(expression) 3510 expression = parse_conditional_insert() 3511 3512 return self.expression( 3513 exp.MultitableInserts(kind=kind, expressions=expressions, source=self._parse_table()), 3514 comments=comments, 3515 ) 3516 3517 def _parse_insert(self) -> exp.Insert | exp.MultitableInserts: 3518 comments: list[str] = [] 3519 hint = self._parse_hint() 3520 overwrite = self._match(TokenType.OVERWRITE) 3521 ignore = self._match(TokenType.IGNORE) 3522 local = self._match_text_seq("LOCAL") 3523 alternative = None 3524 is_function = None 3525 3526 if self._match_text_seq("DIRECTORY"): 3527 this: exp.Expr | None = self.expression( 3528 exp.Directory( 3529 this=self._parse_var_or_string(), 3530 local=local, 3531 row_format=self._parse_row_format(match_row=True), 3532 ) 3533 ) 3534 else: 3535 if self._match_set((TokenType.FIRST, TokenType.ALL)): 3536 comments += ensure_list(self._prev_comments) 3537 return self._parse_multitable_inserts(comments) 3538 3539 if self._match(TokenType.OR): 3540 alternative = self._match_texts(self.INSERT_ALTERNATIVES) and self._prev.text 3541 3542 self._match(TokenType.INTO) 3543 comments += ensure_list(self._prev_comments) 3544 self._match(TokenType.TABLE) 3545 is_function = self._match(TokenType.FUNCTION) 3546 3547 this = self._parse_function() if is_function else self._parse_insert_table() 3548 3549 # MySQL's INSERT ... SET is normalized into the INSERT ... (cols) VALUES (vals) variant 3550 set_values = None 3551 if self._match(TokenType.SET): 3552 columns = [] 3553 values = [] 3554 3555 def _parse_set_assignment() -> exp.Expr | None: 3556 target = self._parse_column() 3557 if isinstance(target, exp.Column) and self._match(TokenType.EQ): 3558 if self.dialect.SUPPORTS_VALUES_DEFAULT and self._match(TokenType.DEFAULT): 3559 value: exp.Expr | None = exp.var(self._prev.text.upper()) 3560 else: 3561 value = self._parse_disjunction() 3562 3563 if value: 3564 columns.append(target.this) 3565 values.append(value) 3566 return value 3567 3568 self.raise_error("Expected column assignment in INSERT ... SET") 3569 return None 3570 3571 self._parse_csv(_parse_set_assignment) 3572 3573 this = self.expression(exp.Schema(this=this, expressions=columns)) 3574 set_values = self.expression( 3575 exp.Values( 3576 expressions=[exp.Tuple(expressions=values)], 3577 alias=self._parse_table_alias(), 3578 ) 3579 ) 3580 3581 returning = self._parse_returning() # TSQL allows RETURNING before source 3582 3583 return self.expression( 3584 exp.Insert( 3585 hint=hint, 3586 is_function=is_function, 3587 this=this, 3588 stored=self._match_text_seq("STORED") and self._parse_stored(), 3589 by_name=self._match_text_seq("BY", "NAME"), 3590 exists=self._parse_exists(), 3591 where=self._match_pair(TokenType.REPLACE, TokenType.WHERE) 3592 and self._parse_disjunction(), 3593 partition=self._match(TokenType.PARTITION_BY) and self._parse_partitioned_by(), 3594 settings=self._match_text_seq("SETTINGS") and self._parse_settings_property(), 3595 default=self._match_text_seq("DEFAULT", "VALUES"), 3596 expression=set_values 3597 or self._parse_derived_table_values() 3598 or self._parse_ddl_select(), 3599 conflict=self._parse_on_conflict(), 3600 returning=returning or self._parse_returning(), 3601 overwrite=overwrite, 3602 alternative=alternative, 3603 ignore=ignore, 3604 source=self._match(TokenType.TABLE) and self._parse_table(), 3605 ), 3606 comments=comments, 3607 ) 3608 3609 def _parse_insert_table(self) -> exp.Expr | None: 3610 this = self._parse_table(schema=True, parse_partition=True) 3611 if isinstance(this, exp.Table) and self._match(TokenType.ALIAS, advance=False): 3612 this.set("alias", self._parse_table_alias()) 3613 return this 3614 3615 def _parse_kill(self) -> exp.Kill: 3616 kind = exp.var(self._prev.text) if self._match_texts(("CONNECTION", "QUERY")) else None 3617 3618 return self.expression(exp.Kill(this=self._parse_primary(), kind=kind)) 3619 3620 def _parse_on_conflict(self) -> exp.OnConflict | None: 3621 conflict = self._match_text_seq("ON", "CONFLICT") 3622 duplicate = self._match_text_seq("ON", "DUPLICATE", "KEY") 3623 3624 if not conflict and not duplicate: 3625 return None 3626 3627 conflict_keys = None 3628 constraint = None 3629 3630 if conflict: 3631 if self._match_text_seq("ON", "CONSTRAINT"): 3632 constraint = self._parse_id_var() 3633 elif self._match(TokenType.L_PAREN): 3634 conflict_keys = self._parse_csv(self._parse_indexed_column) 3635 self._match_r_paren() 3636 3637 index_predicate = self._parse_where() 3638 3639 action = self._parse_var_from_options(self.CONFLICT_ACTIONS) 3640 if self._prev.token_type == TokenType.UPDATE: 3641 self._match(TokenType.SET) 3642 expressions = self._parse_csv(self._parse_equality) 3643 else: 3644 expressions = None 3645 3646 return self.expression( 3647 exp.OnConflict( 3648 duplicate=duplicate, 3649 expressions=expressions, 3650 action=action, 3651 conflict_keys=conflict_keys, 3652 index_predicate=index_predicate, 3653 constraint=constraint, 3654 where=self._parse_where(), 3655 ) 3656 ) 3657 3658 def _parse_returning(self) -> exp.Returning | None: 3659 if not self._match(TokenType.RETURNING): 3660 return None 3661 return self.expression( 3662 exp.Returning( 3663 expressions=self._parse_csv(self._parse_expression), 3664 into=self._match(TokenType.INTO) and self._parse_table_part(), 3665 ) 3666 ) 3667 3668 def _parse_row(self) -> exp.RowFormatSerdeProperty | exp.RowFormatDelimitedProperty | None: 3669 if not self._match(TokenType.FORMAT): 3670 return None 3671 return self._parse_row_format() 3672 3673 def _parse_serde_properties(self, with_: bool = False) -> exp.SerdeProperties | None: 3674 index = self._index 3675 with_ = with_ or self._match_text_seq("WITH") 3676 3677 if not self._match(TokenType.SERDE_PROPERTIES): 3678 self._retreat(index) 3679 return None 3680 return self.expression( 3681 exp.SerdeProperties(expressions=self._parse_wrapped_properties(), with_=with_) 3682 ) 3683 3684 def _parse_row_format( 3685 self, match_row: bool = False 3686 ) -> exp.RowFormatSerdeProperty | exp.RowFormatDelimitedProperty | None: 3687 if match_row and not self._match_pair(TokenType.ROW, TokenType.FORMAT): 3688 return None 3689 3690 if self._match_text_seq("SERDE"): 3691 this = self._parse_string() 3692 3693 serde_properties = self._parse_serde_properties() 3694 3695 return self.expression( 3696 exp.RowFormatSerdeProperty(this=this, serde_properties=serde_properties) 3697 ) 3698 3699 self._match_text_seq("DELIMITED") 3700 3701 kwargs = {} 3702 3703 if self._match_text_seq("FIELDS", "TERMINATED", "BY"): 3704 kwargs["fields"] = self._parse_string() 3705 if self._match_text_seq("ESCAPED", "BY"): 3706 kwargs["escaped"] = self._parse_string() 3707 if self._match_text_seq("COLLECTION", "ITEMS", "TERMINATED", "BY"): 3708 kwargs["collection_items"] = self._parse_string() 3709 if self._match_text_seq("MAP", "KEYS", "TERMINATED", "BY"): 3710 kwargs["map_keys"] = self._parse_string() 3711 if self._match_text_seq("LINES", "TERMINATED", "BY"): 3712 kwargs["lines"] = self._parse_string() 3713 if self._match_text_seq("NULL", "DEFINED", "AS"): 3714 kwargs["null"] = self._parse_string() 3715 3716 return self.expression(exp.RowFormatDelimitedProperty(**kwargs)) # type: ignore 3717 3718 def _parse_load(self) -> exp.LoadData | exp.Command: 3719 if self._match_text_seq("DATA"): 3720 local = self._match_text_seq("LOCAL") 3721 self._match_text_seq("INPATH") 3722 inpath = self._parse_string() 3723 overwrite = self._match(TokenType.OVERWRITE) 3724 temp: bool | None = None 3725 if self._match(TokenType.INTO): 3726 temp = self._match(TokenType.TEMPORARY) 3727 self._match(TokenType.TABLE) 3728 3729 return self.expression( 3730 exp.LoadData( 3731 this=self._parse_table(schema=True), 3732 local=local, 3733 overwrite=overwrite, 3734 temp=temp, 3735 inpath=inpath, 3736 files=self._match_text_seq("FROM", "FILES") 3737 and exp.Properties(expressions=self._parse_wrapped_properties()), 3738 partition=self._parse_partition(), 3739 input_format=self._match_text_seq("INPUTFORMAT") and self._parse_string(), 3740 serde=self._match_text_seq("SERDE") and self._parse_string(), 3741 ) 3742 ) 3743 return self._parse_as_command(self._prev) 3744 3745 def _parse_delete(self) -> exp.Delete: 3746 hint = self._parse_hint() 3747 3748 # This handles MySQL's "Multiple-Table Syntax" 3749 # https://dev.mysql.com/doc/refman/8.0/en/delete.html 3750 tables = None 3751 if not self._match(TokenType.FROM, advance=False): 3752 tables = self._parse_csv(self._parse_table) or None 3753 3754 returning = self._parse_returning() 3755 3756 return self.expression( 3757 exp.Delete( 3758 hint=hint, 3759 tables=tables, 3760 this=self._match(TokenType.FROM) and self._parse_table(joins=True), 3761 using=self._match(TokenType.USING) 3762 and self._parse_csv(lambda: self._parse_table(joins=True)), 3763 cluster=self._match(TokenType.ON) and self._parse_on_property(), 3764 where=self._parse_where(), 3765 returning=returning or self._parse_returning(), 3766 order=self._parse_order(), 3767 limit=self._parse_limit(), 3768 ) 3769 ) 3770 3771 def _parse_update(self) -> exp.Update: 3772 hint = self._parse_hint() 3773 kwargs: dict[str, object] = { 3774 "hint": hint, 3775 "this": self._parse_table(joins=True, alias_tokens=self.UPDATE_ALIAS_TOKENS), 3776 } 3777 while self._curr: 3778 if self._match(TokenType.SET): 3779 kwargs["expressions"] = self._parse_csv(self._parse_equality) 3780 elif self._match(TokenType.RETURNING, advance=False): 3781 kwargs["returning"] = self._parse_returning() 3782 elif self._match(TokenType.FROM, advance=False): 3783 from_ = self._parse_from(joins=True) 3784 table = from_.this if from_ else None 3785 if isinstance(table, exp.Subquery) and self._match(TokenType.JOIN, advance=False): 3786 table.set("joins", list(self._parse_joins()) or None) 3787 3788 kwargs["from_"] = from_ 3789 elif self._match(TokenType.WHERE, advance=False): 3790 kwargs["where"] = self._parse_where() 3791 elif self._match(TokenType.ORDER_BY, advance=False): 3792 kwargs["order"] = self._parse_order() 3793 elif self._match(TokenType.LIMIT, advance=False): 3794 kwargs["limit"] = self._parse_limit() 3795 else: 3796 break 3797 3798 return self.expression(exp.Update(**kwargs)) 3799 3800 def _parse_use(self) -> exp.Use: 3801 return self.expression( 3802 exp.Use( 3803 kind=self._parse_var_from_options(self.USABLES, raise_unmatched=False), 3804 this=self._parse_table(schema=False), 3805 ) 3806 ) 3807 3808 def _parse_uncache(self) -> exp.Uncache: 3809 if not self._match(TokenType.TABLE): 3810 self.raise_error("Expecting TABLE after UNCACHE") 3811 3812 return self.expression( 3813 exp.Uncache(exists=self._parse_exists(), this=self._parse_table(schema=True)) 3814 ) 3815 3816 def _parse_cache(self) -> exp.Cache: 3817 lazy = self._match_text_seq("LAZY") 3818 self._match(TokenType.TABLE) 3819 table = self._parse_table(schema=True) 3820 3821 options = [] 3822 if self._match_text_seq("OPTIONS"): 3823 self._match_l_paren() 3824 k = self._parse_string() 3825 self._match(TokenType.EQ) 3826 v = self._parse_string() 3827 options = [k, v] 3828 self._match_r_paren() 3829 3830 self._match(TokenType.ALIAS) 3831 return self.expression( 3832 exp.Cache( 3833 this=table, lazy=lazy, options=options, expression=self._parse_select(nested=True) 3834 ) 3835 ) 3836 3837 def _parse_partition(self) -> exp.Partition | None: 3838 if not self._match_texts(self.PARTITION_KEYWORDS): 3839 return None 3840 3841 return self.expression( 3842 exp.Partition( 3843 subpartition=self._prev.text.upper() == "SUBPARTITION", 3844 expressions=self._parse_wrapped_csv(self._parse_disjunction), 3845 ) 3846 ) 3847 3848 def _parse_value(self, values: bool = True) -> exp.Tuple | None: 3849 def _parse_value_expression() -> exp.Expr | None: 3850 if self.dialect.SUPPORTS_VALUES_DEFAULT and self._match(TokenType.DEFAULT): 3851 return exp.var(self._prev.text.upper()) 3852 return self._parse_expression() 3853 3854 if self._match(TokenType.L_PAREN): 3855 expressions = self._parse_csv(_parse_value_expression) 3856 self._match_r_paren() 3857 return self.expression(exp.Tuple(expressions=expressions)) 3858 3859 # In some dialects we can have VALUES 1, 2 which results in 1 column & 2 rows. 3860 expression = self._parse_expression() 3861 if expression: 3862 return self.expression(exp.Tuple(expressions=[expression])) 3863 return None 3864 3865 def _parse_projections( 3866 self, 3867 ) -> tuple[list[exp.Expr], list[exp.Expr] | None]: 3868 return self._parse_expressions(), None 3869 3870 def _parse_wrapped_select(self, table: bool = False) -> exp.Expr | None: 3871 if self._match_set((TokenType.PIVOT, TokenType.UNPIVOT)): 3872 this: exp.Expr | None = self._parse_simplified_pivot( 3873 is_unpivot=self._prev.token_type == TokenType.UNPIVOT 3874 ) 3875 elif self._match(TokenType.FROM): 3876 from_ = self._parse_from(joins=True, skip_from_token=True, consume_pipe=True) 3877 # Support parentheses for duckdb FROM-first syntax 3878 select = self._parse_select(from_=from_) 3879 if select: 3880 if not select.args.get("from_"): 3881 select.set("from_", from_) 3882 this = select 3883 else: 3884 this = exp.select("*").from_(t.cast(exp.From, from_)) 3885 this = self._parse_query_modifiers(self._parse_set_operations(this)) 3886 else: 3887 this = ( 3888 self._parse_table(consume_pipe=True) 3889 if table 3890 else self._parse_select(nested=True, parse_set_operation=False) 3891 ) 3892 3893 # Transform exp.Values into a exp.Table to pass through parse_query_modifiers 3894 # in case a modifier (e.g. join) is following 3895 if table and isinstance(this, exp.Values) and this.alias: 3896 alias = this.args["alias"].pop() 3897 this = exp.Table(this=this, alias=alias) 3898 3899 this = self._parse_query_modifiers(self._parse_set_operations(this)) 3900 3901 return this 3902 3903 def _parse_select( 3904 self, 3905 nested: bool = False, 3906 table: bool = False, 3907 parse_subquery_alias: bool = True, 3908 parse_set_operation: bool = True, 3909 consume_pipe: bool = True, 3910 from_: exp.From | None = None, 3911 ) -> exp.Expr | None: 3912 query = self._parse_select_query( 3913 nested=nested, 3914 table=table, 3915 parse_subquery_alias=parse_subquery_alias, 3916 parse_set_operation=parse_set_operation, 3917 ) 3918 3919 if consume_pipe and self._match(TokenType.PIPE_GT, advance=False): 3920 if not query and from_: 3921 query = exp.select("*").from_(from_) 3922 if isinstance(query, exp.Query): 3923 query = self._parse_pipe_syntax_query(query) 3924 query = query.subquery(copy=False) if query and table else query 3925 3926 return query 3927 3928 def _parse_select_query( 3929 self, 3930 nested: bool = False, 3931 table: bool = False, 3932 parse_subquery_alias: bool = True, 3933 parse_set_operation: bool = True, 3934 ) -> exp.Expr | None: 3935 cte = self._parse_with() 3936 3937 if cte: 3938 this = self._parse_statement() 3939 3940 if not this: 3941 self.raise_error("Failed to parse any statement following CTE") 3942 return cte 3943 3944 while isinstance(this, exp.Subquery) and this.is_wrapper: 3945 this = this.this 3946 3947 assert this is not None 3948 if "with_" in this.arg_types: 3949 if inner_cte := this.args.get("with_"): 3950 cte.set("expressions", cte.expressions + inner_cte.expressions) 3951 if inner_cte.args.get("recursive"): 3952 cte.set("recursive", True) 3953 this.set("with_", cte) 3954 else: 3955 self.raise_error(f"{this.key} does not support CTE") 3956 this = cte 3957 3958 return this 3959 3960 # duckdb supports leading with FROM x 3961 from_ = ( 3962 self._parse_from(joins=True, consume_pipe=True) 3963 if self._match(TokenType.FROM, advance=False) 3964 else None 3965 ) 3966 3967 if self._match(TokenType.SELECT): 3968 comments = self._prev_comments 3969 3970 hint = self._parse_hint() 3971 3972 if self._next and not self._next.token_type == TokenType.DOT: 3973 all_ = self._match(TokenType.ALL) 3974 matched_distinct = self._match_set(self.DISTINCT_TOKENS) 3975 else: 3976 all_, matched_distinct = None, False 3977 3978 kind = ( 3979 self._prev.text.upper() 3980 if self._match(TokenType.ALIAS) and self._match_texts(("STRUCT", "VALUE")) 3981 else None 3982 ) 3983 3984 distinct: exp.Expr | None = ( 3985 self.expression( 3986 exp.Distinct( 3987 on=self._parse_value(values=False) if self._match(TokenType.ON) else None 3988 ) 3989 ) 3990 if matched_distinct 3991 else None 3992 ) 3993 3994 operation_modifiers = [] 3995 while self._curr and self._match_texts(self.OPERATION_MODIFIERS): 3996 operation_modifiers.append(exp.var(self._prev.text.upper())) 3997 3998 limit = self._parse_limit(top=True) 3999 4000 # Some dialects (e.g. Redshift, T-SQL) allow SELECT TOP N DISTINCT ... 4001 if limit and not matched_distinct and not all_: 4002 matched_distinct = self._match_set(self.DISTINCT_TOKENS) 4003 if matched_distinct: 4004 distinct = self.expression( 4005 exp.Distinct( 4006 on=self._parse_value(values=False) 4007 if self._match(TokenType.ON) 4008 else None 4009 ) 4010 ) 4011 else: 4012 all_ = self._match(TokenType.ALL) 4013 4014 if all_ and distinct: 4015 self.raise_error("Cannot specify both ALL and DISTINCT after SELECT") 4016 4017 projections, exclude = self._parse_projections() 4018 4019 this = self.expression( 4020 exp.Select( 4021 kind=kind, 4022 hint=hint, 4023 distinct=distinct, 4024 expressions=projections, 4025 limit=limit, 4026 exclude=exclude, 4027 operation_modifiers=operation_modifiers or None, 4028 ) 4029 ) 4030 this.comments = comments 4031 4032 into = self._parse_into() 4033 if into: 4034 this.set("into", into) 4035 4036 if not from_: 4037 from_ = self._parse_from() 4038 4039 if from_: 4040 this.set("from_", from_) 4041 4042 this = self._parse_query_modifiers(this) 4043 elif (table or nested) and self._match(TokenType.L_PAREN): 4044 comments = self._prev_comments 4045 this = self._parse_wrapped_select(table=table) 4046 4047 if this: 4048 this.add_comments(comments, prepend=True) 4049 4050 # We return early here so that the UNION isn't attached to the subquery by the 4051 # following call to _parse_set_operations, but instead becomes the parent node 4052 self._match_r_paren() 4053 return self._parse_subquery(this, parse_alias=parse_subquery_alias) 4054 elif self._match(TokenType.VALUES, advance=False): 4055 this = self._parse_derived_table_values() 4056 elif from_: 4057 this = exp.select("*").from_(from_.this, copy=False) 4058 this = self._parse_query_modifiers(this) 4059 elif self._match(TokenType.SUMMARIZE): 4060 table = self._match(TokenType.TABLE) 4061 this = self._parse_select() or self._parse_string() or self._parse_table() 4062 return self.expression(exp.Summarize(this=this, table=table)) 4063 elif self._match(TokenType.DESCRIBE): 4064 this = self._parse_describe() 4065 else: 4066 this = None 4067 4068 return self._parse_set_operations(this) if parse_set_operation else this 4069 4070 def _parse_recursive_with_search(self) -> exp.RecursiveWithSearch | None: 4071 self._match_text_seq("SEARCH") 4072 4073 kind = self._match_texts(self.RECURSIVE_CTE_SEARCH_KIND) and self._prev.text.upper() 4074 4075 if not kind: 4076 return None 4077 4078 self._match_text_seq("FIRST", "BY") 4079 4080 return self.expression( 4081 exp.RecursiveWithSearch( 4082 kind=kind, 4083 this=self._parse_id_var(), 4084 expression=self._match_text_seq("SET") and self._parse_id_var(), 4085 using=self._match_text_seq("USING") and self._parse_id_var(), 4086 ) 4087 ) 4088 4089 def _parse_with(self, skip_with_token: bool = False) -> exp.With | None: 4090 if not skip_with_token and not self._match(TokenType.WITH): 4091 return None 4092 4093 comments = self._prev_comments 4094 recursive = self._match(TokenType.RECURSIVE) 4095 4096 last_comments = None 4097 expressions = [] 4098 while True: 4099 cte = self._parse_cte() 4100 if isinstance(cte, exp.CTE): 4101 expressions.append(cte) 4102 if last_comments: 4103 cte.add_comments(last_comments) 4104 4105 if not self._match(TokenType.COMMA) and not self._match(TokenType.WITH): 4106 break 4107 else: 4108 self._match(TokenType.WITH) 4109 4110 last_comments = self._prev_comments 4111 4112 return self.expression( 4113 exp.With( 4114 expressions=expressions, 4115 recursive=recursive or None, 4116 search=self._parse_recursive_with_search(), 4117 ), 4118 comments=comments, 4119 ) 4120 4121 def _parse_cte(self) -> exp.CTE | None: 4122 index = self._index 4123 4124 alias = self._parse_table_alias(self.ID_VAR_TOKENS) 4125 if not alias or not alias.this: 4126 self.raise_error("Expected CTE to have alias") 4127 4128 key_expressions = ( 4129 self._parse_wrapped_id_vars() if self._match_text_seq("USING", "KEY") else None 4130 ) 4131 4132 if not self._match(TokenType.ALIAS) and not self.OPTIONAL_ALIAS_TOKEN_CTE: 4133 self._retreat(index) 4134 return None 4135 4136 comments = self._prev_comments 4137 4138 if self._match_text_seq("NOT", "MATERIALIZED"): 4139 materialized = False 4140 elif self._match_text_seq("MATERIALIZED"): 4141 materialized = True 4142 else: 4143 materialized = None 4144 4145 cte = self.expression( 4146 exp.CTE( 4147 this=self._parse_wrapped(self._parse_statement), 4148 alias=alias, 4149 materialized=materialized, 4150 key_expressions=key_expressions, 4151 ), 4152 comments=comments, 4153 ) 4154 4155 values = cte.this 4156 if isinstance(values, exp.Values): 4157 cte.set("this", self._values_to_select(values)) 4158 4159 return cte 4160 4161 def _values_to_select(self, values: exp.Values) -> exp.Select: 4162 if values.alias: 4163 return exp.select("*").from_(values) 4164 return exp.select("*").from_(exp.alias_(values, "_values", table=True)) 4165 4166 def _parse_table_alias( 4167 self, alias_tokens: t.Collection[TokenType] | None = None 4168 ) -> exp.TableAlias | None: 4169 # In some dialects, LIMIT and OFFSET can act as both identifiers and keywords (clauses) 4170 # so this section tries to parse the clause version and if it fails, it treats the token 4171 # as an identifier (alias) 4172 if self._can_parse_limit_or_offset(): 4173 return None 4174 4175 any_token = self._match(TokenType.ALIAS) 4176 alias = ( 4177 self._parse_id_var(any_token=any_token, tokens=alias_tokens or self.TABLE_ALIAS_TOKENS) 4178 or self._parse_string_as_identifier() 4179 ) 4180 4181 index = self._index 4182 if self._match(TokenType.L_PAREN): 4183 columns = self._parse_csv(self._parse_function_parameter) 4184 self._match_r_paren() if columns else self._retreat(index) 4185 else: 4186 columns = None 4187 4188 if not alias and not columns: 4189 return None 4190 4191 table_alias = self.expression(exp.TableAlias(this=alias, columns=columns)) 4192 4193 # We bubble up comments from the Identifier to the TableAlias 4194 if isinstance(alias, exp.Identifier): 4195 table_alias.add_comments(alias.pop_comments()) 4196 4197 return table_alias 4198 4199 def _parse_subquery( 4200 self, this: exp.Expr | None, parse_alias: bool = True 4201 ) -> exp.Subquery | None: 4202 if not this: 4203 return None 4204 4205 return self.expression( 4206 exp.Subquery( 4207 this=this, 4208 pivots=self._parse_pivots(), 4209 alias=self._parse_table_alias() if parse_alias else None, 4210 sample=self._parse_table_sample(), 4211 ) 4212 ) 4213 4214 def _implicit_unnests_to_explicit(self, this: E) -> E: 4215 from sqlglot.optimizer.normalize_identifiers import normalize_identifiers as _norm 4216 4217 refs = {_norm(this.args["from_"].this.copy(), dialect=self.dialect).alias_or_name} 4218 for i, join in enumerate(this.args.get("joins") or []): 4219 table = join.this 4220 normalized_table = table.copy() 4221 normalized_table.meta["maybe_column"] = True 4222 normalized_table = _norm(normalized_table, dialect=self.dialect) 4223 4224 if isinstance(table, exp.Table) and not join.args.get("on"): 4225 if len(normalized_table.parts) > 1 and normalized_table.parts[0].name in refs: 4226 table_as_column = table.to_column() 4227 unnest = exp.Unnest(expressions=[table_as_column]) 4228 4229 # Table.to_column creates a parent Alias node that we want to convert to 4230 # a TableAlias and attach to the Unnest, so it matches the parser's output 4231 if isinstance(table.args.get("alias"), exp.TableAlias): 4232 table_as_column.replace(table_as_column.this) 4233 exp.alias_(unnest, None, table=[table.args["alias"].this], copy=False) 4234 4235 table.replace(unnest) 4236 4237 refs.add(normalized_table.alias_or_name) 4238 4239 return this 4240 4241 @t.overload 4242 def _parse_query_modifiers(self, this: E) -> E: ... 4243 4244 @t.overload 4245 def _parse_query_modifiers(self, this: None) -> None: ... 4246 4247 def _parse_query_modifiers(self, this): 4248 if isinstance(this, self.MODIFIABLES): 4249 for join in self._parse_joins(): 4250 this.append("joins", join) 4251 for lateral in iter(self._parse_lateral, None): 4252 this.append("laterals", lateral) 4253 4254 while True: 4255 if self._match_set(self.QUERY_MODIFIER_PARSERS, advance=False): 4256 modifier_token = self._curr 4257 parser = self.QUERY_MODIFIER_PARSERS[modifier_token.token_type] 4258 key, expression = parser(self) 4259 4260 if expression: 4261 if this.args.get(key): 4262 self.raise_error( 4263 f"Found multiple '{modifier_token.text.upper()}' clauses", 4264 token=modifier_token, 4265 ) 4266 4267 this.set(key, expression) 4268 if key == "limit": 4269 offset = expression.args.get("offset") 4270 expression.set("offset", None) 4271 4272 if offset: 4273 offset = exp.Offset(expression=offset) 4274 this.set("offset", offset) 4275 4276 limit_by_expressions = expression.expressions 4277 expression.set("expressions", None) 4278 offset.set("expressions", limit_by_expressions) 4279 continue 4280 break 4281 4282 if self.SUPPORTS_IMPLICIT_UNNEST and this and this.args.get("from_"): 4283 this = self._implicit_unnests_to_explicit(this) 4284 4285 return this 4286 4287 def _parse_hint_fallback_to_string(self) -> exp.Hint | None: 4288 start = self._curr 4289 while self._curr: 4290 self._advance() 4291 4292 end = self._tokens[self._index - 1] 4293 return exp.Hint(expressions=[self._find_sql(start, end)]) 4294 4295 def _parse_hint_function_call(self) -> exp.Expr | None: 4296 return self._parse_function_call() 4297 4298 def _parse_hint_body(self) -> exp.Hint | None: 4299 start_index = self._index 4300 should_fallback_to_string = False 4301 4302 hints = [] 4303 try: 4304 for hint in iter( 4305 lambda: self._parse_csv( 4306 lambda: self._parse_hint_function_call() or self._parse_var(upper=True), 4307 ), 4308 [], 4309 ): 4310 hints.extend(hint) 4311 except ParseError: 4312 should_fallback_to_string = True 4313 4314 if should_fallback_to_string or self._curr: 4315 self._retreat(start_index) 4316 return self._parse_hint_fallback_to_string() 4317 4318 return self.expression(exp.Hint(expressions=hints)) 4319 4320 def _parse_hint(self) -> exp.Hint | None: 4321 if self._match(TokenType.HINT) and self._prev_comments: 4322 return exp.maybe_parse(self._prev_comments[0], into=exp.Hint, dialect=self.dialect) 4323 4324 return None 4325 4326 def _parse_into(self) -> exp.Into | None: 4327 if not self._match(TokenType.INTO): 4328 return None 4329 4330 temp = self._match(TokenType.TEMPORARY) 4331 unlogged = self._match_text_seq("UNLOGGED") 4332 self._match(TokenType.TABLE) 4333 4334 return self.expression( 4335 exp.Into(this=self._parse_table(schema=True), temporary=temp, unlogged=unlogged) 4336 ) 4337 4338 def _parse_from( 4339 self, 4340 joins: bool = False, 4341 skip_from_token: bool = False, 4342 consume_pipe: bool = False, 4343 ) -> exp.From | None: 4344 if not skip_from_token and not self._match(TokenType.FROM): 4345 return None 4346 4347 comments = self._prev_comments 4348 return self.expression( 4349 exp.From(this=self._parse_table(joins=joins, consume_pipe=consume_pipe)), 4350 comments=comments, 4351 ) 4352 4353 def _parse_match_recognize_measure(self) -> exp.MatchRecognizeMeasure: 4354 return self.expression( 4355 exp.MatchRecognizeMeasure( 4356 window_frame=self._match_texts(("FINAL", "RUNNING")) and self._prev.text.upper(), 4357 this=self._parse_expression(), 4358 ) 4359 ) 4360 4361 def _parse_match_recognize(self) -> exp.MatchRecognize | None: 4362 if not self._match(TokenType.MATCH_RECOGNIZE): 4363 return None 4364 4365 self._match_l_paren() 4366 4367 partition = self._parse_partition_by() 4368 order = self._parse_order() 4369 4370 measures = ( 4371 self._parse_csv(self._parse_match_recognize_measure) 4372 if self._match_text_seq("MEASURES") 4373 else None 4374 ) 4375 4376 if self._match_text_seq("ONE", "ROW", "PER", "MATCH"): 4377 rows = exp.var("ONE ROW PER MATCH") 4378 elif self._match_text_seq("ALL", "ROWS", "PER", "MATCH"): 4379 text = "ALL ROWS PER MATCH" 4380 if self._match_text_seq("SHOW", "EMPTY", "MATCHES"): 4381 text += " SHOW EMPTY MATCHES" 4382 elif self._match_text_seq("OMIT", "EMPTY", "MATCHES"): 4383 text += " OMIT EMPTY MATCHES" 4384 elif self._match_text_seq("WITH", "UNMATCHED", "ROWS"): 4385 text += " WITH UNMATCHED ROWS" 4386 rows = exp.var(text) 4387 else: 4388 rows = None 4389 4390 if self._match_text_seq("AFTER", "MATCH", "SKIP"): 4391 text = "AFTER MATCH SKIP" 4392 if self._match_text_seq("PAST", "LAST", "ROW"): 4393 text += " PAST LAST ROW" 4394 elif self._match_text_seq("TO", "NEXT", "ROW"): 4395 text += " TO NEXT ROW" 4396 elif self._match_text_seq("TO", "FIRST"): 4397 text += f" TO FIRST {self._advance_any().text}" # type: ignore 4398 elif self._match_text_seq("TO", "LAST"): 4399 text += f" TO LAST {self._advance_any().text}" # type: ignore 4400 after = exp.var(text) 4401 else: 4402 after = None 4403 4404 if self._match_text_seq("PATTERN"): 4405 self._match_l_paren() 4406 4407 if not self._curr: 4408 self.raise_error("Expecting )", self._curr) 4409 4410 paren = 1 4411 start = self._curr 4412 4413 while self._curr and paren > 0: 4414 if self._curr.token_type == TokenType.L_PAREN: 4415 paren += 1 4416 if self._curr.token_type == TokenType.R_PAREN: 4417 paren -= 1 4418 4419 end = self._prev 4420 self._advance() 4421 4422 if paren > 0: 4423 self.raise_error("Expecting )", self._curr) 4424 4425 pattern = exp.var(self._find_sql(start, end)) 4426 else: 4427 pattern = None 4428 4429 define = ( 4430 self._parse_csv(self._parse_name_as_expression) 4431 if self._match_text_seq("DEFINE") 4432 else None 4433 ) 4434 4435 self._match_r_paren() 4436 4437 return self.expression( 4438 exp.MatchRecognize( 4439 partition_by=partition, 4440 order=order, 4441 measures=measures, 4442 rows=rows, 4443 after=after, 4444 pattern=pattern, 4445 define=define, 4446 alias=self._parse_table_alias(), 4447 ) 4448 ) 4449 4450 def _parse_lateral(self) -> exp.Lateral | None: 4451 cross_apply: bool | None = None 4452 if self._match_pair(TokenType.CROSS, TokenType.APPLY): 4453 cross_apply = True 4454 elif self._match_pair(TokenType.OUTER, TokenType.APPLY): 4455 cross_apply = False 4456 4457 if cross_apply is not None: 4458 this = self._parse_select(table=True) 4459 view = None 4460 outer = None 4461 elif self._match(TokenType.LATERAL): 4462 this = self._parse_select(table=True) 4463 view = self._match(TokenType.VIEW) 4464 outer = self._match(TokenType.OUTER) 4465 else: 4466 return None 4467 4468 if not this: 4469 this = ( 4470 self._parse_unnest() 4471 or self._parse_function() 4472 or self._parse_id_var(any_token=False) 4473 ) 4474 4475 while self._match(TokenType.DOT): 4476 this = exp.Dot( 4477 this=this, 4478 expression=self._parse_function() or self._parse_id_var(any_token=False), 4479 ) 4480 4481 ordinality: bool | None = None 4482 4483 if view: 4484 table = self._parse_id_var(any_token=False) 4485 columns = self._parse_csv(self._parse_id_var) if self._match(TokenType.ALIAS) else [] 4486 table_alias: exp.TableAlias | None = self.expression( 4487 exp.TableAlias(this=table, columns=columns) 4488 ) 4489 elif isinstance(this, (exp.Subquery, exp.Unnest)) and this.alias: 4490 # We move the alias from the lateral's child node to the lateral itself 4491 table_alias = this.args["alias"].pop() 4492 else: 4493 ordinality = self._match_pair(TokenType.WITH, TokenType.ORDINALITY) 4494 table_alias = self._parse_table_alias() 4495 4496 return self.expression( 4497 exp.Lateral( 4498 this=this, 4499 view=view, 4500 outer=outer, 4501 alias=table_alias, 4502 cross_apply=cross_apply, 4503 ordinality=ordinality, 4504 ) 4505 ) 4506 4507 def _parse_stream(self) -> exp.Stream | None: 4508 index = self._index 4509 if self._match(TokenType.STREAM): 4510 if this := self._try_parse(self._parse_table): 4511 return self.expression(exp.Stream(this=this)) 4512 self._retreat(index) 4513 return None 4514 4515 def _parse_join_parts( 4516 self, 4517 ) -> tuple[Token | None, Token | None, Token | None]: 4518 return ( 4519 self._prev if self._match_set(self.JOIN_METHODS) else None, 4520 self._prev if self._match_set(self.JOIN_SIDES) else None, 4521 self._prev if self._match_set(self.JOIN_KINDS) else None, 4522 ) 4523 4524 def _parse_using_identifiers(self) -> list[exp.Expr]: 4525 def _parse_column_as_identifier() -> exp.Expr | None: 4526 this = self._parse_column() 4527 if isinstance(this, exp.Column): 4528 return this.this 4529 return this 4530 4531 return self._parse_wrapped_csv(_parse_column_as_identifier, optional=True) 4532 4533 def _parse_join( 4534 self, 4535 skip_join_token: bool = False, 4536 parse_bracket: bool = False, 4537 alias_tokens: t.Collection[TokenType] | None = None, 4538 ) -> exp.Join | None: 4539 if self._match(TokenType.COMMA): 4540 table = self._try_parse(lambda: self._parse_table(alias_tokens=alias_tokens)) 4541 cross_join = self.expression(exp.Join(this=table)) if table else None 4542 4543 if cross_join and self.JOINS_HAVE_EQUAL_PRECEDENCE: 4544 cross_join.set("kind", "CROSS") 4545 4546 return cross_join 4547 4548 index = self._index 4549 method, side, kind = self._parse_join_parts() 4550 directed = self._match_text_seq("DIRECTED") 4551 hint = self._prev.text if self._match_texts(self.JOIN_HINTS) else None 4552 join = self._match(TokenType.JOIN) or (kind and kind.token_type == TokenType.STRAIGHT_JOIN) 4553 join_comments = self._prev_comments 4554 4555 if not skip_join_token and not join: 4556 self._retreat(index) 4557 kind = None 4558 method = None 4559 side = None 4560 4561 outer_apply = self._match_pair(TokenType.OUTER, TokenType.APPLY, False) 4562 cross_apply = self._match_pair(TokenType.CROSS, TokenType.APPLY, False) 4563 4564 if not skip_join_token and not join and not outer_apply and not cross_apply: 4565 return None 4566 4567 kwargs: dict[str, t.Any] = { 4568 "this": self._parse_table(parse_bracket=parse_bracket, alias_tokens=alias_tokens) 4569 } 4570 if kind and kind.token_type == TokenType.ARRAY and self._match(TokenType.COMMA): 4571 kwargs["expressions"] = self._parse_csv( 4572 lambda: self._parse_table(parse_bracket=parse_bracket, alias_tokens=alias_tokens) 4573 ) 4574 4575 if method: 4576 kwargs["method"] = method.text.upper() 4577 if side: 4578 kwargs["side"] = side.text.upper() 4579 if kind: 4580 kwargs["kind"] = kind.text.upper() 4581 if hint: 4582 kwargs["hint"] = hint 4583 4584 if self._match(TokenType.MATCH_CONDITION): 4585 kwargs["match_condition"] = self._parse_wrapped(self._parse_comparison) 4586 4587 if self._match(TokenType.ON): 4588 kwargs["on"] = self._parse_disjunction() 4589 elif self._match(TokenType.USING): 4590 kwargs["using"] = self._parse_using_identifiers() 4591 elif ( 4592 not method 4593 and not (outer_apply or cross_apply) 4594 and not isinstance(kwargs["this"], exp.Unnest) 4595 and not (kind and kind.token_type in (TokenType.CROSS, TokenType.ARRAY)) 4596 ): 4597 index = self._index 4598 joins: list | None = list(self._parse_joins(alias_tokens=alias_tokens)) 4599 4600 if joins and self._match(TokenType.ON): 4601 kwargs["on"] = self._parse_disjunction() 4602 elif joins and self._match(TokenType.USING): 4603 kwargs["using"] = self._parse_using_identifiers() 4604 else: 4605 joins = None 4606 self._retreat(index) 4607 4608 kwargs["this"].set("joins", joins if joins else None) 4609 4610 kwargs["pivots"] = self._parse_pivots() 4611 4612 comments = [c for token in (method, side, kind) if token for c in token.comments] 4613 comments = (join_comments or []) + comments 4614 4615 if ( 4616 self.ADD_JOIN_ON_TRUE 4617 and not kwargs.get("on") 4618 and not kwargs.get("using") 4619 and not kwargs.get("method") 4620 and kwargs.get("kind") in (None, "INNER", "OUTER") 4621 ): 4622 kwargs["on"] = exp.true() 4623 4624 if directed: 4625 kwargs["directed"] = directed 4626 4627 return self.expression(exp.Join(**kwargs), comments=comments) 4628 4629 def _parse_opclass(self) -> exp.Expr | None: 4630 this = self._parse_disjunction() 4631 4632 if self._match_texts(self.OPCLASS_FOLLOW_KEYWORDS, advance=False): 4633 return this 4634 4635 if not self._match_set(self.OPTYPE_FOLLOW_TOKENS, advance=False): 4636 return self.expression(exp.Opclass(this=this, expression=self._parse_table_parts())) 4637 4638 return this 4639 4640 def _parse_index_params(self) -> exp.IndexParameters: 4641 using = self._parse_var(any_token=True) if self._match(TokenType.USING) else None 4642 4643 if self._match(TokenType.L_PAREN, advance=False): 4644 columns = self._parse_wrapped_csv(self._parse_with_operator) 4645 else: 4646 columns = None 4647 4648 include = self._parse_wrapped_id_vars() if self._match_text_seq("INCLUDE") else None 4649 partition_by = self._parse_partition_by() 4650 with_storage = self._match(TokenType.WITH) and self._parse_wrapped_properties() 4651 tablespace = ( 4652 self._parse_var(any_token=True) 4653 if self._match_text_seq("USING", "INDEX", "TABLESPACE") 4654 else None 4655 ) 4656 where = self._parse_where() 4657 4658 on = self._parse_field() if self._match(TokenType.ON) else None 4659 4660 return self.expression( 4661 exp.IndexParameters( 4662 using=using, 4663 columns=columns, 4664 include=include, 4665 partition_by=partition_by, 4666 where=where, 4667 with_storage=with_storage, 4668 tablespace=tablespace, 4669 on=on, 4670 ) 4671 ) 4672 4673 def _parse_index( 4674 self, index: exp.Expr | None = None, anonymous: bool = False 4675 ) -> exp.Index | None: 4676 if index or anonymous: 4677 unique = None 4678 primary = None 4679 amp = None 4680 4681 self._match(TokenType.ON) 4682 self._match(TokenType.TABLE) # hive 4683 table = self._parse_table_parts(schema=True) 4684 else: 4685 unique = self._match(TokenType.UNIQUE) 4686 primary = self._match_text_seq("PRIMARY") 4687 amp = self._match_text_seq("AMP") 4688 4689 if not self._match(TokenType.INDEX): 4690 return None 4691 4692 index = self._parse_id_var() 4693 table = None 4694 4695 params = self._parse_index_params() 4696 4697 return self.expression( 4698 exp.Index( 4699 this=index, table=table, unique=unique, primary=primary, amp=amp, params=params 4700 ) 4701 ) 4702 4703 def _parse_table_hints(self) -> list[exp.Expr] | None: 4704 hints: list[exp.Expr] = [] 4705 if self._match_pair(TokenType.WITH, TokenType.L_PAREN): 4706 # https://learn.microsoft.com/en-us/sql/t-sql/queries/hints-transact-sql-table?view=sql-server-ver16 4707 hints.append( 4708 self.expression( 4709 exp.WithTableHint( 4710 expressions=self._parse_csv( 4711 lambda: self._parse_function() or self._parse_var(any_token=True) 4712 ) 4713 ) 4714 ) 4715 ) 4716 self._match_r_paren() 4717 else: 4718 # https://dev.mysql.com/doc/refman/8.0/en/index-hints.html 4719 while self._match_set(self.TABLE_INDEX_HINT_TOKENS): 4720 hint = exp.IndexTableHint(this=self._prev.text.upper()) 4721 4722 self._match_set((TokenType.INDEX, TokenType.KEY)) 4723 if self._match(TokenType.FOR): 4724 hint.set("target", self._advance_any() and self._prev.text.upper()) 4725 4726 hint.set("expressions", self._parse_wrapped_id_vars()) 4727 hints.append(hint) 4728 4729 return hints or None 4730 4731 def _parse_table_part(self, schema: bool = False) -> exp.Expr | None: 4732 return ( 4733 (not schema and self._parse_function(optional_parens=False)) 4734 or self._parse_id_var(any_token=False) 4735 or self._parse_string_as_identifier() 4736 or self._parse_placeholder() 4737 ) 4738 4739 def _parse_table_parts_fast(self) -> exp.Table | None: 4740 index = self._index 4741 parts: list[exp.Identifier] | None = None 4742 all_comments: list[str] | None = None 4743 4744 while self._match_set(self.IDENTIFIER_TOKENS): 4745 token = self._prev 4746 comments = self._prev_comments 4747 4748 has_dot = self._match(TokenType.DOT) 4749 curr_tt = self._curr.token_type 4750 4751 if not has_dot: 4752 if curr_tt in self.TABLE_POSTFIX_TOKENS: 4753 self._retreat(index) 4754 return None 4755 elif curr_tt not in self.IDENTIFIER_TOKENS: 4756 self._retreat(index) 4757 return None 4758 4759 if parts is None: 4760 parts = [] 4761 4762 if comments: 4763 if all_comments is None: 4764 all_comments = [] 4765 all_comments.extend(comments) 4766 self._prev_comments = [] 4767 4768 parts.append( 4769 self.expression( 4770 exp.Identifier( 4771 this=token.text, quoted=token.token_type == TokenType.IDENTIFIER 4772 ), 4773 token, 4774 ) 4775 ) 4776 4777 if not has_dot: 4778 break 4779 4780 if parts is None: 4781 return None 4782 4783 n = len(parts) 4784 4785 if n == 1: 4786 table: exp.Table = exp.Table(this=parts[0]) 4787 elif n == 2: 4788 table = exp.Table(this=parts[1], db=parts[0]) 4789 elif n >= 3: 4790 this: exp.Identifier | exp.Dot = parts[2] 4791 for i in range(3, n): 4792 this = exp.Dot(this=this, expression=parts[i]) 4793 4794 table = exp.Table(this=this, db=parts[1], catalog=parts[0]) 4795 4796 if table is None: 4797 self._retreat(index) 4798 elif all_comments: 4799 table.add_comments(all_comments) 4800 return table 4801 4802 def _parse_table_parts( 4803 self, 4804 schema: bool = False, 4805 is_db_reference: bool = False, 4806 wildcard: bool = False, 4807 fast: bool = False, 4808 ) -> exp.Table | exp.Dot | None: 4809 if fast: 4810 return self._parse_table_parts_fast() 4811 4812 catalog: exp.Expr | str | None = None 4813 db: exp.Expr | str | None = None 4814 table: exp.Expr | str | None = self._parse_table_part(schema=schema) 4815 4816 while self._match(TokenType.DOT): 4817 if catalog: 4818 # This allows nesting the table in arbitrarily many dot expressions if needed 4819 table = self.expression( 4820 exp.Dot(this=table, expression=self._parse_table_part(schema=schema)) 4821 ) 4822 else: 4823 catalog = db 4824 db = table 4825 # "" used for tsql FROM a..b case 4826 table = self._parse_table_part(schema=schema) or "" 4827 4828 if ( 4829 wildcard 4830 and self._is_connected() 4831 and (isinstance(table, exp.Identifier) or not table) 4832 and self._match(TokenType.STAR) 4833 ): 4834 if isinstance(table, exp.Identifier): 4835 table.args["this"] += "*" 4836 else: 4837 table = exp.Identifier(this="*") 4838 4839 if is_db_reference: 4840 catalog = db 4841 db = table 4842 table = None 4843 4844 if not table and not is_db_reference: 4845 self.raise_error(f"Expected table name but got {self._curr}") 4846 if not db and is_db_reference: 4847 self.raise_error(f"Expected database name but got {self._curr}") 4848 4849 table = self.expression(exp.Table(this=table, db=db, catalog=catalog)) 4850 4851 # Bubble up comments from identifier parts to the Table 4852 comments = [] 4853 for part in table.parts: 4854 if part_comments := part.pop_comments(): 4855 comments.extend(part_comments) 4856 if comments: 4857 table.add_comments(comments) 4858 4859 changes = self._parse_changes() 4860 if changes: 4861 table.set("changes", changes) 4862 4863 at_before = self._parse_historical_data() 4864 if at_before: 4865 table.set("when", at_before) 4866 4867 pivots = self._parse_pivots() 4868 if pivots: 4869 table.set("pivots", pivots) 4870 4871 return table 4872 4873 def _parse_table( 4874 self, 4875 schema: bool = False, 4876 joins: bool = False, 4877 alias_tokens: t.Collection[TokenType] | None = None, 4878 parse_bracket: bool = False, 4879 is_db_reference: bool = False, 4880 parse_partition: bool = False, 4881 consume_pipe: bool = False, 4882 ) -> exp.Expr | None: 4883 if not schema and not is_db_reference and not consume_pipe and not joins: 4884 index = self._index 4885 table = self._parse_table_parts(fast=True) 4886 4887 if table is not None: 4888 curr_tt = self._curr.token_type 4889 next_tt = self._next.token_type 4890 4891 fast_terminators = self.TABLE_TERMINATORS 4892 4893 # only return the table if we're sure there are no other operators 4894 # MATCH_CONDITION is a special case because it accepts any alias before it like LIMIT 4895 if curr_tt in fast_terminators and next_tt != TokenType.MATCH_CONDITION: 4896 return table 4897 4898 postfix_tokens = self.TABLE_POSTFIX_TOKENS 4899 4900 if curr_tt not in postfix_tokens and next_tt not in postfix_tokens: 4901 if alias := self._parse_table_alias( 4902 alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS 4903 ): 4904 table.set("alias", alias) 4905 4906 if self._curr.token_type in fast_terminators: 4907 return table 4908 4909 self._retreat(index) 4910 4911 if stream := self._parse_stream(): 4912 return stream 4913 4914 if lateral := self._parse_lateral(): 4915 return lateral 4916 4917 if unnest := self._parse_unnest(): 4918 return unnest 4919 4920 if values := self._parse_derived_table_values(): 4921 return values 4922 4923 if subquery := self._parse_select(table=True, consume_pipe=consume_pipe): 4924 if not subquery.args.get("pivots"): 4925 subquery.set("pivots", self._parse_pivots()) 4926 if joins: 4927 for join in self._parse_joins(): 4928 subquery.append("joins", join) 4929 return subquery 4930 4931 bracket = parse_bracket and self._parse_bracket(None) 4932 bracket = self.expression(exp.Table(this=bracket)) if bracket else None 4933 4934 rows_from_tables = ( 4935 self._parse_wrapped_csv(self._parse_table) 4936 if self._match_text_seq("ROWS", "FROM") 4937 else None 4938 ) 4939 rows_from = ( 4940 self.expression(exp.Table(rows_from=rows_from_tables)) if rows_from_tables else None 4941 ) 4942 4943 only = self._match(TokenType.ONLY) 4944 4945 this = t.cast( 4946 exp.Expr, 4947 bracket 4948 or rows_from 4949 or self._parse_bracket( 4950 self._parse_table_parts(schema=schema, is_db_reference=is_db_reference) 4951 ), 4952 ) 4953 4954 if only: 4955 this.set("only", only) 4956 4957 # Postgres supports a wildcard (table) suffix operator, which is a no-op in this context 4958 self._match(TokenType.STAR) 4959 4960 parse_partition = parse_partition or self.SUPPORTS_PARTITION_SELECTION 4961 if parse_partition and self._match(TokenType.PARTITION, advance=False): 4962 this.set("partition", self._parse_partition()) 4963 4964 if schema: 4965 return self._parse_schema(this=this) 4966 4967 if self.dialect.ALIAS_POST_VERSION: 4968 this.set("version", self._parse_version()) 4969 4970 if self.dialect.ALIAS_POST_TABLESAMPLE: 4971 this.set("sample", self._parse_table_sample()) 4972 4973 alias = self._parse_table_alias(alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS) 4974 if alias: 4975 this.set("alias", alias) 4976 4977 # DuckDB requires the time-travel clause to come after the alias, e.g. 4978 # SELECT * FROM t AS a AT (VERSION => 1) 4979 if isinstance(this, exp.Table) and not this.args.get("when"): 4980 this.set("when", self._parse_historical_data()) 4981 4982 if self._match(TokenType.INDEXED_BY): 4983 this.set("indexed", self._parse_table_parts()) 4984 elif self._match_text_seq("NOT", "INDEXED"): 4985 this.set("indexed", False) 4986 4987 if isinstance(this, exp.Table) and self._match_text_seq("AT"): 4988 return self.expression( 4989 exp.AtIndex(this=this.to_column(copy=False), expression=self._parse_id_var()) 4990 ) 4991 4992 this.set("hints", self._parse_table_hints()) 4993 4994 if not this.args.get("pivots"): 4995 this.set("pivots", self._parse_pivots()) 4996 4997 if not self.dialect.ALIAS_POST_TABLESAMPLE: 4998 this.set("sample", self._parse_table_sample()) 4999 5000 if not self.dialect.ALIAS_POST_VERSION: 5001 this.set("version", self._parse_version()) 5002 5003 if joins: 5004 for join in self._parse_joins(alias_tokens=alias_tokens): 5005 this.append("joins", join) 5006 5007 if self._match_pair(TokenType.WITH, TokenType.ORDINALITY): 5008 this.set("ordinality", True) 5009 this.set("alias", self._parse_table_alias()) 5010 5011 return this 5012 5013 def _parse_version(self) -> exp.Version | None: 5014 if self._match(TokenType.TIMESTAMP_SNAPSHOT): 5015 this = "TIMESTAMP" 5016 elif self._match(TokenType.VERSION_SNAPSHOT): 5017 this = "VERSION" 5018 else: 5019 return None 5020 5021 if self._match_set((TokenType.FROM, TokenType.BETWEEN)): 5022 kind = self._prev.text.upper() 5023 start = self._parse_bitwise() 5024 self._match_texts(("TO", "AND")) 5025 end = self._parse_bitwise() 5026 expression: exp.Expr | None = self.expression(exp.Tuple(expressions=[start, end])) 5027 elif self._match_text_seq("CONTAINED", "IN"): 5028 kind = "CONTAINED IN" 5029 expression = self.expression( 5030 exp.Tuple(expressions=self._parse_wrapped_csv(self._parse_bitwise)) 5031 ) 5032 elif self._match(TokenType.ALL): 5033 kind = "ALL" 5034 expression = None 5035 else: 5036 self._match_text_seq("AS", "OF") 5037 kind = "AS OF" 5038 expression = self._parse_type() 5039 5040 return self.expression(exp.Version(this=this, expression=expression, kind=kind)) 5041 5042 def _parse_historical_data(self) -> exp.HistoricalData | None: 5043 # https://docs.snowflake.com/en/sql-reference/constructs/at-before 5044 index = self._index 5045 historical_data = None 5046 if self._match_texts(self.HISTORICAL_DATA_PREFIX): 5047 this = self._prev.text.upper() 5048 kind = ( 5049 self._match(TokenType.L_PAREN) 5050 and self._match_texts(self.HISTORICAL_DATA_KIND) 5051 and self._prev.text.upper() 5052 ) 5053 expression = self._match(TokenType.FARROW) and self._parse_bitwise() 5054 5055 if expression: 5056 self._match_r_paren() 5057 historical_data = self.expression( 5058 exp.HistoricalData(this=this, kind=kind, expression=expression) 5059 ) 5060 else: 5061 self._retreat(index) 5062 5063 return historical_data 5064 5065 def _parse_changes(self) -> exp.Changes | None: 5066 if not self._match_text_seq("CHANGES", "(", "INFORMATION", "=>"): 5067 return None 5068 5069 information = self._parse_var(any_token=True) 5070 self._match_r_paren() 5071 5072 return self.expression( 5073 exp.Changes( 5074 information=information, 5075 at_before=self._parse_historical_data(), 5076 end=self._parse_historical_data(), 5077 ) 5078 ) 5079 5080 def _parse_unnest(self, with_alias: bool = True) -> exp.Unnest | None: 5081 if not self._match_pair(TokenType.UNNEST, TokenType.L_PAREN, advance=False): 5082 return None 5083 5084 self._advance() 5085 5086 expressions = self._parse_wrapped_csv(self._parse_equality) 5087 offset: bool | exp.Expr = self._match_pair(TokenType.WITH, TokenType.ORDINALITY) 5088 5089 alias = self._parse_table_alias() if with_alias else None 5090 5091 if alias: 5092 if self.dialect.UNNEST_COLUMN_ONLY: 5093 if alias.args.get("columns"): 5094 self.raise_error("Unexpected extra column alias in unnest.") 5095 5096 alias.set("columns", [alias.this]) 5097 alias.set("this", None) 5098 5099 columns = alias.args.get("columns") or [] 5100 if offset and len(expressions) < len(columns): 5101 offset = columns.pop() 5102 5103 if not offset and self._match_pair(TokenType.WITH, TokenType.OFFSET): 5104 self._match(TokenType.ALIAS) 5105 offset = self._parse_id_var( 5106 any_token=False, tokens=self.UNNEST_OFFSET_ALIAS_TOKENS 5107 ) or exp.to_identifier("offset") 5108 5109 return self.expression(exp.Unnest(expressions=expressions, alias=alias, offset=offset)) 5110 5111 def _parse_derived_table_values(self) -> exp.Values | None: 5112 is_derived = self._match_pair(TokenType.L_PAREN, TokenType.VALUES) 5113 if not is_derived and not ( 5114 # ClickHouse's `FORMAT Values` is equivalent to `VALUES` 5115 self._match_text_seq("VALUES") or self._match_text_seq("FORMAT", "VALUES") 5116 ): 5117 return None 5118 5119 expressions = self._parse_csv(self._parse_value) 5120 alias = self._parse_table_alias() 5121 5122 if is_derived: 5123 self._match_r_paren() 5124 5125 return self.expression( 5126 exp.Values(expressions=expressions, alias=alias or self._parse_table_alias()) 5127 ) 5128 5129 def _parse_table_sample(self, as_modifier: bool = False) -> exp.TableSample | None: 5130 if not self._match(TokenType.TABLE_SAMPLE) and not ( 5131 as_modifier and self._match_text_seq("USING", "SAMPLE") 5132 ): 5133 return None 5134 5135 bucket_numerator = None 5136 bucket_denominator = None 5137 bucket_field = None 5138 percent = None 5139 size = None 5140 seed = None 5141 5142 method = self._parse_var(tokens=(TokenType.ROW,), upper=True) 5143 matched_l_paren = self._match(TokenType.L_PAREN) 5144 5145 if self.TABLESAMPLE_CSV: 5146 num = None 5147 expressions = self._parse_csv(self._parse_primary) 5148 else: 5149 expressions = None 5150 num = ( 5151 self._parse_factor() 5152 if self._match(TokenType.NUMBER, advance=False) 5153 else self._parse_primary() or self._parse_placeholder() 5154 ) 5155 5156 if self._match_text_seq("BUCKET"): 5157 bucket_numerator = self._parse_number() 5158 self._match_text_seq("OUT", "OF") 5159 bucket_denominator = bucket_denominator = self._parse_number() 5160 self._match(TokenType.ON) 5161 bucket_field = self._parse_field() 5162 elif self._match_set((TokenType.PERCENT, TokenType.MOD)): 5163 percent = num 5164 elif self._match(TokenType.ROWS) or not self.dialect.TABLESAMPLE_SIZE_IS_PERCENT: 5165 size = num 5166 else: 5167 percent = num 5168 5169 if matched_l_paren: 5170 self._match_r_paren() 5171 5172 if self._match(TokenType.L_PAREN): 5173 method = self._parse_var(upper=True) 5174 seed = self._match(TokenType.COMMA) and self._parse_number() 5175 self._match_r_paren() 5176 elif self._match_texts(("SEED", "REPEATABLE")): 5177 seed = self._parse_wrapped(self._parse_number) 5178 5179 if not method and self.DEFAULT_SAMPLING_METHOD: 5180 method = exp.var(self.DEFAULT_SAMPLING_METHOD) 5181 5182 return self.expression( 5183 exp.TableSample( 5184 expressions=expressions, 5185 method=method, 5186 bucket_numerator=bucket_numerator, 5187 bucket_denominator=bucket_denominator, 5188 bucket_field=bucket_field, 5189 percent=percent, 5190 size=size, 5191 seed=seed, 5192 ) 5193 ) 5194 5195 def _parse_pivots(self) -> list[exp.Pivot] | None: 5196 if self._curr.token_type not in (TokenType.PIVOT, TokenType.UNPIVOT): 5197 return None 5198 return list(iter(self._parse_pivot, None)) or None 5199 5200 def _parse_joins( 5201 self, alias_tokens: t.Collection[TokenType] | None = None 5202 ) -> t.Iterator[exp.Join]: 5203 return iter(lambda: self._parse_join(alias_tokens=alias_tokens), None) 5204 5205 def _parse_unpivot_columns(self) -> exp.UnpivotColumns | None: 5206 if not self._match(TokenType.INTO): 5207 return None 5208 5209 return self.expression( 5210 exp.UnpivotColumns( 5211 this=self._match_text_seq("NAME") and self._parse_column(), 5212 expressions=self._match_text_seq("VALUE") and self._parse_csv(self._parse_column), 5213 ) 5214 ) 5215 5216 # https://duckdb.org/docs/sql/statements/pivot 5217 def _parse_simplified_pivot(self, is_unpivot: bool | None = None) -> exp.Pivot: 5218 def _parse_on() -> exp.Expr | None: 5219 this = self._parse_bitwise() 5220 5221 if self._match(TokenType.IN): 5222 # PIVOT ... ON col IN (row_val1, row_val2) 5223 return self._parse_in(this) 5224 if self._match(TokenType.ALIAS, advance=False): 5225 # UNPIVOT ... ON (col1, col2, col3) AS row_val 5226 return self._parse_alias(this) 5227 5228 return this 5229 5230 this = self._parse_table() 5231 expressions = self._match(TokenType.ON) and self._parse_csv(_parse_on) 5232 into = self._parse_unpivot_columns() 5233 using = self._match(TokenType.USING) and self._parse_csv( 5234 lambda: self._parse_alias(self._parse_column()) 5235 ) 5236 group = self._parse_group() 5237 5238 return self.expression( 5239 exp.Pivot( 5240 this=this, 5241 expressions=expressions, 5242 using=using, 5243 group=group, 5244 unpivot=is_unpivot, 5245 into=into, 5246 ) 5247 ) 5248 5249 def _parse_pivot_in(self) -> exp.In: 5250 def _parse_aliased_expression() -> exp.Expr | None: 5251 this = self._parse_select_or_expression() 5252 5253 self._match(TokenType.ALIAS) 5254 alias = self._parse_bitwise() 5255 if alias: 5256 if isinstance(alias, exp.Column) and not alias.db: 5257 alias = alias.this 5258 return self.expression(exp.PivotAlias(this=this, alias=alias)) 5259 5260 return this 5261 5262 value = self._parse_column() 5263 5264 if not self._match(TokenType.IN): 5265 self.raise_error("Expecting IN") 5266 5267 if self._match(TokenType.L_PAREN): 5268 if self._match(TokenType.ANY): 5269 exprs: list[exp.Expr] = ensure_list(exp.PivotAny(this=self._parse_order())) 5270 else: 5271 exprs = self._parse_csv(_parse_aliased_expression) 5272 self._match_r_paren() 5273 return self.expression(exp.In(this=value, expressions=exprs)) 5274 5275 return self.expression(exp.In(this=value, field=self._parse_id_var())) 5276 5277 def _parse_pivot_aggregation(self) -> exp.Expr | None: 5278 func = self._parse_function() 5279 if not func: 5280 if self._prev.token_type == TokenType.COMMA: 5281 return None 5282 self.raise_error("Expecting an aggregation function in PIVOT") 5283 5284 return self._parse_alias(func) 5285 5286 def _parse_pivot(self) -> exp.Pivot | None: 5287 index = self._index 5288 include_nulls = None 5289 5290 if self._match(TokenType.PIVOT): 5291 unpivot = False 5292 elif self._match(TokenType.UNPIVOT): 5293 unpivot = True 5294 5295 # https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-qry-select-unpivot.html#syntax 5296 if self._match_text_seq("INCLUDE", "NULLS"): 5297 include_nulls = True 5298 elif self._match_text_seq("EXCLUDE", "NULLS"): 5299 include_nulls = False 5300 else: 5301 return None 5302 5303 expressions = [] 5304 5305 if not self._match(TokenType.L_PAREN): 5306 self._retreat(index) 5307 return None 5308 5309 if unpivot: 5310 expressions = self._parse_csv(self._parse_column) 5311 else: 5312 expressions = self._parse_csv(self._parse_pivot_aggregation) 5313 5314 if not expressions: 5315 self.raise_error("Failed to parse PIVOT's aggregation list") 5316 5317 if not self._match(TokenType.FOR): 5318 self.raise_error("Expecting FOR") 5319 5320 fields = [] 5321 while True: 5322 field = self._try_parse(self._parse_pivot_in) 5323 if not field: 5324 break 5325 fields.append(field) 5326 5327 default_on_null = self._match_text_seq("DEFAULT", "ON", "NULL") and self._parse_wrapped( 5328 self._parse_bitwise 5329 ) 5330 5331 group = self._parse_group() 5332 5333 self._match_r_paren() 5334 5335 pivot = self.expression( 5336 exp.Pivot( 5337 expressions=expressions, 5338 fields=fields, 5339 unpivot=unpivot, 5340 include_nulls=include_nulls, 5341 default_on_null=default_on_null, 5342 group=group, 5343 ) 5344 ) 5345 5346 if unpivot: 5347 pivot.set("expressions", [_unpivot_target(e) for e in pivot.expressions]) 5348 for pivot_field in pivot.fields: 5349 if isinstance(pivot_field, exp.In): 5350 pivot_field.set("this", _unpivot_target(pivot_field.this)) 5351 5352 if not self._match_set((TokenType.PIVOT, TokenType.UNPIVOT), advance=False): 5353 pivot.set("alias", self._parse_table_alias()) 5354 5355 if not unpivot: 5356 names = self._pivot_column_names(t.cast(list[exp.Expr], expressions)) 5357 5358 columns: list[exp.Expr] = [] 5359 all_fields = [] 5360 for pivot_field in pivot.fields: 5361 pivot_field_expressions = pivot_field.expressions 5362 5363 # The `PivotAny` expression corresponds to `ANY ORDER BY <column>`; we can't infer in this case. 5364 if isinstance(seq_get(pivot_field_expressions, 0), exp.PivotAny): 5365 continue 5366 5367 all_fields.append( 5368 [ 5369 fld.sql() if self.IDENTIFY_PIVOT_STRINGS else fld.alias_or_name 5370 for fld in pivot_field_expressions 5371 ] 5372 ) 5373 5374 if all_fields: 5375 if names: 5376 all_fields.append(names) 5377 5378 # Generate all possible combinations of the pivot columns 5379 # e.g PIVOT(sum(...) as total FOR year IN (2000, 2010) FOR country IN ('NL', 'US')) 5380 # generates the product between [[2000, 2010], ['NL', 'US'], ['total']] 5381 for fld_parts_tuple in itertools.product(*all_fields): 5382 fld_parts = list(fld_parts_tuple) 5383 5384 if names and self.PREFIXED_PIVOT_COLUMNS: 5385 # Move the "name" to the front of the list 5386 fld_parts.insert(0, fld_parts.pop(-1)) 5387 5388 columns.append(exp.to_identifier("_".join(fld_parts))) 5389 5390 pivot.set("columns", columns) 5391 pivot.set("identify_pivot_strings", self.IDENTIFY_PIVOT_STRINGS) 5392 pivot.set("prefixed_pivot_columns", self.PREFIXED_PIVOT_COLUMNS) 5393 pivot.set("pivot_column_naming", self.PIVOT_COLUMN_NAMING) 5394 5395 return pivot 5396 5397 def _pivot_column_names(self, aggregations: list[exp.Expr]) -> list[str]: 5398 return [agg.alias for agg in aggregations if agg.alias] 5399 5400 def _parse_prewhere(self, skip_where_token: bool = False) -> exp.PreWhere | None: 5401 if not skip_where_token and not self._match(TokenType.PREWHERE): 5402 return None 5403 5404 comments = self._prev_comments 5405 return self.expression( 5406 exp.PreWhere(this=self._parse_disjunction()), 5407 comments=comments, 5408 ) 5409 5410 def _parse_where(self, skip_where_token: bool = False) -> exp.Where | None: 5411 if not skip_where_token and not self._match(TokenType.WHERE): 5412 return None 5413 5414 comments = self._prev_comments 5415 return self.expression( 5416 exp.Where(this=self._parse_disjunction()), 5417 comments=comments, 5418 ) 5419 5420 def _parse_group(self, skip_group_by_token: bool = False) -> exp.Group | None: 5421 if not skip_group_by_token and not self._match(TokenType.GROUP_BY): 5422 return None 5423 comments = self._prev_comments 5424 5425 elements: dict[str, t.Any] = defaultdict(list) 5426 5427 if self._match(TokenType.ALL): 5428 elements["all"] = True 5429 elif self._match(TokenType.DISTINCT): 5430 elements["all"] = False 5431 5432 if self._match_set(self.QUERY_MODIFIER_TOKENS, advance=False): 5433 return self.expression(exp.Group(**elements), comments=comments) # type: ignore 5434 5435 while True: 5436 index = self._index 5437 5438 elements["expressions"].extend( 5439 self._parse_csv( 5440 lambda: ( 5441 None 5442 if self._match_set((TokenType.CUBE, TokenType.ROLLUP), advance=False) 5443 else self._parse_disjunction() 5444 ) 5445 ) 5446 ) 5447 5448 before_with_index = self._index 5449 with_prefix = self._match(TokenType.WITH) 5450 5451 if cube_or_rollup := self._parse_cube_or_rollup(with_prefix=with_prefix): 5452 key = "rollup" if isinstance(cube_or_rollup, exp.Rollup) else "cube" 5453 elements[key].append(cube_or_rollup) 5454 elif grouping_sets := self._parse_grouping_sets(): 5455 elements["grouping_sets"].append(grouping_sets) 5456 elif self._match_text_seq("TOTALS"): 5457 elements["totals"] = True # type: ignore 5458 5459 if before_with_index <= self._index <= before_with_index + 1: 5460 self._retreat(before_with_index) 5461 break 5462 5463 if index == self._index: 5464 break 5465 5466 return self.expression(exp.Group(**elements), comments=comments) # type: ignore 5467 5468 def _parse_cube_or_rollup(self, with_prefix: bool = False) -> exp.Cube | exp.Rollup | None: 5469 if self._match(TokenType.CUBE): 5470 kind: type[exp.Cube | exp.Rollup] = exp.Cube 5471 elif self._match(TokenType.ROLLUP): 5472 kind = exp.Rollup 5473 else: 5474 return None 5475 5476 return self.expression( 5477 kind(expressions=[] if with_prefix else self._parse_wrapped_csv(self._parse_bitwise)) 5478 ) 5479 5480 def _parse_grouping_sets(self) -> exp.GroupingSets | None: 5481 if self._match(TokenType.GROUPING_SETS): 5482 return self.expression( 5483 exp.GroupingSets(expressions=self._parse_wrapped_csv(self._parse_grouping_set)) 5484 ) 5485 return None 5486 5487 def _parse_grouping_set(self) -> exp.Expr | None: 5488 return self._parse_grouping_sets() or self._parse_cube_or_rollup() or self._parse_bitwise() 5489 5490 def _parse_having(self, skip_having_token: bool = False) -> exp.Having | None: 5491 if not skip_having_token and not self._match(TokenType.HAVING): 5492 return None 5493 comments = self._prev_comments 5494 return self.expression( 5495 exp.Having(this=self._parse_disjunction()), 5496 comments=comments, 5497 ) 5498 5499 def _parse_qualify(self) -> exp.Qualify | None: 5500 if not self._match(TokenType.QUALIFY): 5501 return None 5502 return self.expression(exp.Qualify(this=self._parse_disjunction())) 5503 5504 def _parse_connect_with_prior(self) -> exp.Expr | None: 5505 self.NO_PAREN_FUNCTION_PARSERS["PRIOR"] = lambda self: self.expression( 5506 exp.Prior(this=self._parse_bitwise()) 5507 ) 5508 connect = self._parse_disjunction() 5509 self.NO_PAREN_FUNCTION_PARSERS.pop("PRIOR") 5510 return connect 5511 5512 def _parse_connect(self, skip_start_token: bool = False) -> exp.Connect | None: 5513 if skip_start_token: 5514 start = None 5515 elif self._match(TokenType.START_WITH): 5516 start = self._parse_disjunction() 5517 else: 5518 return None 5519 5520 self._match(TokenType.CONNECT_BY) 5521 nocycle = self._match_text_seq("NOCYCLE") 5522 connect = self._parse_connect_with_prior() 5523 5524 if not start and self._match(TokenType.START_WITH): 5525 start = self._parse_disjunction() 5526 5527 return self.expression(exp.Connect(start=start, connect=connect, nocycle=nocycle)) 5528 5529 def _parse_name_as_expression(self) -> exp.Expr | None: 5530 this = self._parse_id_var(any_token=True) 5531 if self._match(TokenType.ALIAS): 5532 this = self.expression(exp.Alias(alias=this, this=self._parse_disjunction())) 5533 return this 5534 5535 def _parse_interpolate(self) -> list[exp.Expr] | None: 5536 if self._match_text_seq("INTERPOLATE"): 5537 return self._parse_wrapped_csv(self._parse_name_as_expression) 5538 return None 5539 5540 def _parse_order( 5541 self, this: exp.Expr | None = None, skip_order_token: bool = False 5542 ) -> exp.Expr | None: 5543 siblings = None 5544 if not skip_order_token and not self._match(TokenType.ORDER_BY): 5545 if not self._match(TokenType.ORDER_SIBLINGS_BY): 5546 return this 5547 5548 siblings = True 5549 5550 comments = self._prev_comments 5551 return self.expression( 5552 exp.Order( 5553 this=this, 5554 expressions=self._parse_csv(self._parse_ordered), 5555 siblings=siblings, 5556 ), 5557 comments=comments, 5558 ) 5559 5560 def _parse_sort(self, exp_class: type[E], token: TokenType) -> E | None: 5561 if not self._match(token): 5562 return None 5563 return self.expression(exp_class(expressions=self._parse_csv(self._parse_ordered))) 5564 5565 def _parse_ordered( 5566 self, parse_method: t.Callable[[], exp.Expr | None] | None = None 5567 ) -> exp.Ordered | None: 5568 this = parse_method() if parse_method else self._parse_disjunction() 5569 if not this: 5570 return None 5571 5572 if this.name.upper() == "ALL" and self.dialect.SUPPORTS_ORDER_BY_ALL: 5573 this = exp.var("ALL") 5574 5575 asc = self._match(TokenType.ASC) 5576 desc: bool | None = True if self._match(TokenType.DESC) else (False if asc else None) 5577 5578 is_nulls_first = self._match_text_seq("NULLS", "FIRST") 5579 is_nulls_last = self._match_text_seq("NULLS", "LAST") 5580 5581 nulls_first = is_nulls_first or False 5582 explicitly_null_ordered = is_nulls_first or is_nulls_last 5583 5584 if ( 5585 not explicitly_null_ordered 5586 and ( 5587 (not desc and self.dialect.NULL_ORDERING == "nulls_are_small") 5588 or (desc and self.dialect.NULL_ORDERING != "nulls_are_small") 5589 ) 5590 and self.dialect.NULL_ORDERING != "nulls_are_last" 5591 ): 5592 nulls_first = True 5593 5594 if self._match_text_seq("WITH", "FILL"): 5595 with_fill = self.expression( 5596 exp.WithFill( 5597 from_=self._match(TokenType.FROM) and self._parse_bitwise(), 5598 to=self._match_text_seq("TO") and self._parse_bitwise(), 5599 step=self._match_text_seq("STEP") and self._parse_bitwise(), 5600 interpolate=self._parse_interpolate(), 5601 ) 5602 ) 5603 else: 5604 with_fill = None 5605 5606 return self.expression( 5607 exp.Ordered(this=this, desc=desc, nulls_first=nulls_first, with_fill=with_fill) 5608 ) 5609 5610 def _parse_limit_options(self) -> exp.LimitOptions | None: 5611 percent = self._match_set((TokenType.PERCENT, TokenType.MOD)) 5612 rows = self._match_set((TokenType.ROW, TokenType.ROWS)) 5613 self._match_text_seq("ONLY") 5614 with_ties = self._match_text_seq("WITH", "TIES") 5615 5616 if not (percent or rows or with_ties): 5617 return None 5618 5619 return self.expression(exp.LimitOptions(percent=percent, rows=rows, with_ties=with_ties)) 5620 5621 def _parse_limit( 5622 self, 5623 this: exp.Expr | None = None, 5624 top: bool = False, 5625 skip_limit_token: bool = False, 5626 ) -> exp.Expr | None: 5627 if skip_limit_token or self._match(TokenType.TOP if top else TokenType.LIMIT): 5628 comments = self._prev_comments 5629 if top: 5630 limit_paren = self._match(TokenType.L_PAREN) 5631 expression = ( 5632 self._parse_term() or self._parse_select() 5633 if limit_paren 5634 else self._parse_number() 5635 ) 5636 5637 if limit_paren: 5638 self._match_r_paren() 5639 5640 else: 5641 if self.dialect.SUPPORTS_LIMIT_ALL and self._match(TokenType.ALL): 5642 return this 5643 5644 # Parsing LIMIT x% (i.e x PERCENT) as a term leads to an error, since 5645 # we try to build an exp.Mod expr. For that matter, we backtrack and instead 5646 # consume the factor plus parse the percentage separately 5647 index = self._index 5648 expression = self._try_parse(self._parse_term) 5649 if isinstance(expression, exp.Mod): 5650 self._retreat(index) 5651 expression = self._parse_factor() 5652 elif not expression: 5653 expression = self._parse_factor() 5654 limit_options = self._parse_limit_options() 5655 5656 if self._match(TokenType.COMMA): 5657 offset = expression 5658 expression = self._parse_term() 5659 else: 5660 offset = None 5661 5662 limit_exp = self.expression( 5663 exp.Limit( 5664 this=this, 5665 expression=expression, 5666 offset=offset, 5667 limit_options=limit_options, 5668 expressions=self._parse_limit_by(), 5669 ), 5670 comments=comments, 5671 ) 5672 5673 return limit_exp 5674 5675 if self._match(TokenType.FETCH): 5676 direction = ( 5677 self._prev.text.upper() 5678 if self._match_set((TokenType.FIRST, TokenType.NEXT)) 5679 else "FIRST" 5680 ) 5681 5682 count = self._parse_field(tokens=self.FETCH_TOKENS) 5683 5684 return self.expression( 5685 exp.Fetch( 5686 direction=direction, count=count, limit_options=self._parse_limit_options() 5687 ) 5688 ) 5689 5690 return this 5691 5692 def _parse_offset(self, this: exp.Expr | None = None) -> exp.Expr | None: 5693 if not self._match(TokenType.OFFSET): 5694 return this 5695 5696 count = self._parse_term() 5697 self._match_set((TokenType.ROW, TokenType.ROWS)) 5698 5699 return self.expression( 5700 exp.Offset(this=this, expression=count, expressions=self._parse_limit_by()) 5701 ) 5702 5703 def _can_parse_limit_or_offset(self) -> bool: 5704 if not self._match_set(self.AMBIGUOUS_ALIAS_TOKENS, advance=False): 5705 return False 5706 5707 index = self._index 5708 result = bool( 5709 self._try_parse(self._parse_limit, retreat=True) 5710 or self._try_parse(self._parse_offset, retreat=True) 5711 ) 5712 self._retreat(index) 5713 5714 # MATCH_CONDITION (...) is a special construct that should not be consumed by limit/offset 5715 if self._next.token_type == TokenType.MATCH_CONDITION: 5716 result = False 5717 5718 return result 5719 5720 def _can_parse_named_window(self) -> bool: 5721 # `WINDOW` is in ID_VAR_TOKENS so it could be mistakenly consumed as an implicit alias. 5722 # Refuse only when the following tokens look like a named-window clause: `WINDOW <id> AS (`. 5723 if not self._match(TokenType.WINDOW, advance=False): 5724 return False 5725 5726 name = self._tokens[self._index + 1] if self._index + 1 < len(self._tokens) else None 5727 if name is None or name.token_type not in self.ID_VAR_TOKENS: 5728 return False 5729 5730 alias_tok = self._tokens[self._index + 2] if self._index + 2 < len(self._tokens) else None 5731 if alias_tok is None or alias_tok.token_type != TokenType.ALIAS: 5732 return False 5733 5734 body = self._tokens[self._index + 3] if self._index + 3 < len(self._tokens) else None 5735 return body is not None and body.token_type == TokenType.L_PAREN 5736 5737 def _parse_limit_by(self) -> list[exp.Expr] | None: 5738 return self._parse_csv(self._parse_bitwise) if self._match_text_seq("BY") else None 5739 5740 def _parse_locks(self) -> list[exp.Lock]: 5741 locks = [] 5742 while True: 5743 update, key = None, None 5744 if self._match_text_seq("FOR", "UPDATE"): 5745 update = True 5746 elif self._match_text_seq("FOR", "SHARE") or self._match_text_seq( 5747 "LOCK", "IN", "SHARE", "MODE" 5748 ): 5749 update = False 5750 elif self._match_text_seq("FOR", "KEY", "SHARE"): 5751 update, key = False, True 5752 elif self._match_text_seq("FOR", "NO", "KEY", "UPDATE"): 5753 update, key = True, True 5754 else: 5755 break 5756 5757 expressions = None 5758 if self._match_text_seq("OF"): 5759 expressions = self._parse_csv(lambda: self._parse_table(schema=True)) 5760 5761 wait: bool | exp.Expr | None = None 5762 if self._match_text_seq("NOWAIT"): 5763 wait = True 5764 elif self._match_text_seq("WAIT"): 5765 wait = self._parse_primary() 5766 elif self._match_text_seq("SKIP", "LOCKED"): 5767 wait = False 5768 5769 locks.append( 5770 self.expression( 5771 exp.Lock(update=update, expressions=expressions, wait=wait, key=key) 5772 ) 5773 ) 5774 5775 return locks 5776 5777 def parse_set_operation( 5778 self, this: exp.Expr | None, consume_pipe: bool = False 5779 ) -> exp.Expr | None: 5780 start = self._index 5781 _, side_token, kind_token = self._parse_join_parts() 5782 5783 side = side_token.text if side_token else None 5784 kind = kind_token.text if kind_token else None 5785 5786 if not self._match_set(self.SET_OPERATIONS): 5787 self._retreat(start) 5788 return None 5789 5790 token_type = self._prev.token_type 5791 5792 if token_type == TokenType.UNION: 5793 operation: type[exp.SetOperation] = exp.Union 5794 elif token_type == TokenType.EXCEPT: 5795 operation = exp.Except 5796 else: 5797 operation = exp.Intersect 5798 5799 comments = self._prev.comments 5800 5801 if self._match(TokenType.DISTINCT): 5802 distinct: bool | None = True 5803 elif self._match(TokenType.ALL): 5804 distinct = False 5805 else: 5806 distinct = self.dialect.SET_OP_DISTINCT_BY_DEFAULT[operation] 5807 if distinct is None: 5808 self.raise_error(f"Expected DISTINCT or ALL for {operation.__name__}") 5809 5810 by_name = ( 5811 self._match_text_seq("BY", "NAME") 5812 or self._match_text_seq("STRICT", "CORRESPONDING") 5813 or None 5814 ) 5815 if self._match_text_seq("CORRESPONDING"): 5816 by_name = True 5817 if not side and not kind: 5818 kind = "INNER" 5819 5820 on_column_list = None 5821 if by_name and self._match_texts(("ON", "BY")): 5822 on_column_list = self._parse_wrapped_csv(self._parse_column) 5823 5824 expression = self._parse_select( 5825 nested=True, parse_set_operation=False, consume_pipe=consume_pipe 5826 ) 5827 5828 # Wrap VALUES operands in selects, both for consistency with the CTE canonicalization 5829 # in _parse_cte and so that alias pushdown can reach into set operation branches 5830 if isinstance(this, exp.Values): 5831 this = self._values_to_select(this) 5832 if isinstance(expression, exp.Values): 5833 expression = self._values_to_select(expression) 5834 5835 return self.expression( 5836 operation( 5837 this=this, 5838 distinct=distinct, 5839 by_name=by_name, 5840 expression=expression, 5841 side=side, 5842 kind=kind, 5843 on=on_column_list, 5844 ), 5845 comments=comments, 5846 ) 5847 5848 def _parse_set_operations(self, this: exp.Expr | None) -> exp.Expr | None: 5849 while this: 5850 setop = self.parse_set_operation(this) 5851 if not setop: 5852 break 5853 this = setop 5854 5855 if isinstance(this, exp.SetOperation) and self.MODIFIERS_ATTACHED_TO_SET_OP: 5856 expression = this.expression 5857 5858 if expression: 5859 for arg in self.SET_OP_MODIFIERS: 5860 expr = expression.args.get(arg) 5861 if expr: 5862 this.set(arg, expr.pop()) 5863 5864 return this 5865 5866 def _parse_expression(self) -> exp.Expr | None: 5867 return self._parse_alias(self._parse_assignment()) 5868 5869 def _parse_assignment(self) -> exp.Expr | None: 5870 this = self._parse_disjunction() 5871 if not this and self._next.token_type in self.ASSIGNMENT: 5872 # This allows us to parse <non-identifier token> := <expr> 5873 this = exp.column( 5874 t.cast(str, self._advance_any(ignore_reserved=True) and self._prev.text) 5875 ) 5876 5877 while self._match_set(self.ASSIGNMENT): 5878 if isinstance(this, exp.Column) and len(this.parts) == 1: 5879 this = this.this 5880 5881 comments = self._prev_comments 5882 this = self.expression( 5883 self.ASSIGNMENT[self._prev.token_type]( 5884 this=this, expression=self._parse_assignment() 5885 ), 5886 comments=comments, 5887 ) 5888 5889 return this 5890 5891 def _parse_disjunction(self) -> exp.Expr | None: 5892 this = self._parse_conjunction() 5893 while self._match_set(self.DISJUNCTION): 5894 comments = self._prev_comments 5895 this = self.expression( 5896 self.DISJUNCTION[self._prev.token_type]( 5897 this=this, expression=self._parse_conjunction() 5898 ), 5899 comments=comments, 5900 ) 5901 return this 5902 5903 def _parse_conjunction(self) -> exp.Expr | None: 5904 this = self._parse_equality() 5905 while self._match_set(self.CONJUNCTION): 5906 comments = self._prev_comments 5907 this = self.expression( 5908 self.CONJUNCTION[self._prev.token_type]( 5909 this=this, expression=self._parse_equality() 5910 ), 5911 comments=comments, 5912 ) 5913 return this 5914 5915 def _parse_equality(self) -> exp.Expr | None: 5916 this = self._parse_comparison() 5917 while self._match_set(self.EQUALITY): 5918 comments = self._prev_comments 5919 this = self.expression( 5920 self.EQUALITY[self._prev.token_type]( 5921 this=this, expression=self._parse_comparison() 5922 ), 5923 comments=comments, 5924 ) 5925 return this 5926 5927 def _parse_comparison(self) -> exp.Expr | None: 5928 this = self._parse_range() 5929 while self._match_set(self.COMPARISON): 5930 comments = self._prev_comments 5931 this = self.expression( 5932 self.COMPARISON[self._prev.token_type](this=this, expression=self._parse_range()), 5933 comments=comments, 5934 ) 5935 return this 5936 5937 def _parse_range(self, this: exp.Expr | None = None) -> exp.Expr | None: 5938 this = this or self._parse_bitwise() 5939 5940 while True: 5941 negate = self._match(TokenType.NOT) 5942 if self._match_set(self.RANGE_PARSERS): 5943 expression = self.RANGE_PARSERS[self._prev.token_type](self, this) 5944 if not expression: 5945 return this 5946 5947 this = expression 5948 elif self._match(TokenType.ISNULL) or (negate and self._match(TokenType.NULL)): 5949 this = self.expression(exp.Is(this=this, expression=exp.Null())) 5950 elif self._match(TokenType.NOTNULL): 5951 # Postgres supports ISNULL and NOTNULL for conditions. 5952 # https://blog.andreiavram.ro/postgresql-null-composite-type/ 5953 if self.dialect.NORMALIZE_NOT_NULL: 5954 this = self.expression(exp.Is(this=this, expression=exp.Null())) 5955 this = self.expression(exp.Not(this=this)) 5956 else: 5957 this = self.expression(exp.Is(this=this, expression=exp.Null(), negate=True)) 5958 else: 5959 if negate: 5960 self._retreat(self._index - 1) 5961 break 5962 5963 if negate: 5964 this = self._negate_range(this) 5965 if self._curr and ( 5966 self._curr.token_type == TokenType.NOT 5967 or self._curr.token_type in self.RANGE_PARSERS 5968 ): 5969 this = self.expression(exp.Paren(this=this)) 5970 5971 return this 5972 5973 def _negate_range(self, this: exp.Expr | None = None) -> exp.Expr | None: 5974 if not this: 5975 return this 5976 5977 expression = this.this if isinstance(this, exp.Escape) else this 5978 if isinstance(expression, (exp.Like, exp.ILike)): 5979 expression.set("negate", True) 5980 return this 5981 5982 return self.expression(exp.Not(this=this)) 5983 5984 def _parse_is(self, this: exp.Expr | None) -> exp.Expr | None: 5985 index = self._index - 1 5986 negate = self._match(TokenType.NOT) 5987 5988 if self._match_text_seq("DISTINCT", "FROM"): 5989 klass = exp.NullSafeEQ if negate else exp.NullSafeNEQ 5990 return self.expression(klass(this=this, expression=self._parse_bitwise())) 5991 5992 if self._match(TokenType.JSON): 5993 kind = self._match_texts(self.IS_JSON_PREDICATE_KIND) and self._prev.text.upper() 5994 5995 if self._match_text_seq("WITH"): 5996 _with = True 5997 elif self._match_text_seq("WITHOUT"): 5998 _with = False 5999 else: 6000 _with = None 6001 6002 unique = self._match(TokenType.UNIQUE) 6003 self._match_text_seq("KEYS") 6004 expression: exp.Expr | None = self.expression( 6005 exp.JSON(this=kind, with_=_with, unique=unique) 6006 ) 6007 else: 6008 expression = self._parse_null() or self._parse_bitwise() 6009 if not expression: 6010 self._retreat(index) 6011 return None 6012 6013 if negate and isinstance(expression, exp.Null) and not self.dialect.NORMALIZE_NOT_NULL: 6014 this = self.expression(exp.Is(this=this, expression=expression, negate=True)) 6015 else: 6016 this = self.expression(exp.Is(this=this, expression=expression)) 6017 this = self.expression(exp.Not(this=this)) if negate else this 6018 6019 return self._parse_column_ops(this) 6020 6021 def _parse_in(self, this: exp.Expr | None, alias: bool = False) -> exp.In: 6022 unnest = self._parse_unnest(with_alias=False) 6023 if unnest: 6024 this = self.expression(exp.In(this=this, unnest=unnest)) 6025 elif self._match_set((TokenType.L_PAREN, TokenType.L_BRACKET)): 6026 matched_l_paren = self._prev.token_type == TokenType.L_PAREN 6027 expressions = self._parse_csv(lambda: self._parse_select_or_expression(alias=alias)) 6028 6029 if len(expressions) == 1 and isinstance(query := expressions[0], exp.Query): 6030 this = self.expression( 6031 exp.In(this=this, query=self._parse_query_modifiers(query).subquery(copy=False)) 6032 ) 6033 else: 6034 this = self.expression(exp.In(this=this, expressions=expressions)) 6035 6036 if matched_l_paren: 6037 self._match_r_paren(this) 6038 elif not self._match(TokenType.R_BRACKET, expression=this): 6039 self.raise_error("Expecting ]") 6040 else: 6041 this = self.expression(exp.In(this=this, field=self._parse_column())) 6042 6043 return this 6044 6045 def _parse_between(self, this: exp.Expr | None) -> exp.Between: 6046 symmetric = None 6047 if self._match_text_seq("SYMMETRIC"): 6048 symmetric = True 6049 elif self._match_text_seq("ASYMMETRIC"): 6050 symmetric = False 6051 6052 low = self._parse_bitwise() 6053 self._match(TokenType.AND) 6054 high = self._parse_bitwise() 6055 6056 return self.expression(exp.Between(this=this, low=low, high=high, symmetric=symmetric)) 6057 6058 def _parse_escape(self, this: exp.Expr | None) -> exp.Expr | None: 6059 if not self._match(TokenType.ESCAPE): 6060 return this 6061 return self.expression( 6062 exp.Escape(this=this, expression=self._parse_string() or self._parse_null()) 6063 ) 6064 6065 def _parse_interval_span(self, this: exp.Expr) -> exp.Interval: 6066 # handle day-time format interval span with omitted units: 6067 # INTERVAL '<number days> hh[:][mm[:ss[.ff]]]' <maybe `unit TO unit`> 6068 interval_span_units_omitted = None 6069 if ( 6070 this 6071 and this.is_string 6072 and self.SUPPORTS_OMITTED_INTERVAL_SPAN_UNIT 6073 and exp.INTERVAL_DAY_TIME_RE.match(this.name) 6074 ): 6075 index = self._index 6076 6077 # Var "TO" Var 6078 first_unit = self._parse_var(any_token=True, upper=True) 6079 second_unit = None 6080 if first_unit and self._match_text_seq("TO"): 6081 second_unit = self._parse_var(any_token=True, upper=True) 6082 6083 interval_span_units_omitted = not (first_unit and second_unit) 6084 6085 self._retreat(index) 6086 6087 if interval_span_units_omitted: 6088 unit = None 6089 else: 6090 unit = self._parse_function() 6091 if not unit and ( 6092 self._curr.token_type == TokenType.VAR 6093 or self._curr.text.upper() in self.dialect.VALID_INTERVAL_UNITS 6094 ): 6095 unit = self._parse_var(any_token=True, upper=True) 6096 6097 # Most dialects support, e.g., the form INTERVAL '5' day, thus we try to parse 6098 # each INTERVAL expression into this canonical form so it's easy to transpile 6099 if this and this.is_number: 6100 this = exp.Literal.string(this.to_py()) 6101 elif this and this.is_string: 6102 parts = exp.INTERVAL_STRING_RE.findall(this.name) 6103 if parts and unit: 6104 # Unconsume the eagerly-parsed unit, since the real unit was part of the string 6105 unit = None 6106 self._retreat(self._index - 1) 6107 6108 if len(parts) == 1: 6109 this = exp.Literal.string(parts[0][0]) 6110 unit = self.expression(exp.Var(this=parts[0][1].upper())) 6111 6112 if self.INTERVAL_SPANS and self._match_text_seq("TO"): 6113 unit = self.expression( 6114 exp.IntervalSpan( 6115 this=unit, 6116 expression=self._parse_function() 6117 or self._parse_var(any_token=True, upper=True), 6118 ) 6119 ) 6120 6121 return self.expression(exp.Interval(this=this, unit=unit)) 6122 6123 def _parse_interval(self, require_interval: bool = True) -> exp.Add | exp.Interval | None: 6124 index = self._index 6125 6126 if not self._match(TokenType.INTERVAL) and require_interval: 6127 return None 6128 6129 if self._match(TokenType.STRING, advance=False): 6130 this = self._parse_primary() 6131 else: 6132 this = self._parse_term() 6133 6134 if not this or ( 6135 isinstance(this, exp.Column) 6136 and not this.table 6137 and not this.this.quoted 6138 and self._curr 6139 and self._curr.text.upper() not in self.dialect.VALID_INTERVAL_UNITS 6140 ): 6141 self._retreat(index) 6142 return None 6143 6144 interval = self._parse_interval_span(this) 6145 6146 index = self._index 6147 self._match(TokenType.PLUS) 6148 6149 # Convert INTERVAL 'val_1' unit_1 [+] ... [+] 'val_n' unit_n into a sum of intervals 6150 if self._match_set((TokenType.STRING, TokenType.NUMBER), advance=False): 6151 return self.expression(exp.Add(this=interval, expression=self._parse_interval(False))) 6152 6153 self._retreat(index) 6154 return interval 6155 6156 def _parse_bitwise(self) -> exp.Expr | None: 6157 this = self._parse_term() 6158 6159 while True: 6160 if self._match_set(self.BITWISE): 6161 this = self.expression( 6162 self.BITWISE[self._prev.token_type](this=this, expression=self._parse_term()) 6163 ) 6164 elif self.dialect.DPIPE_IS_STRING_CONCAT and self._match(TokenType.DPIPE): 6165 this = self.expression( 6166 exp.DPipe( 6167 this=this, 6168 expression=self._parse_term(), 6169 safe=not self.dialect.STRICT_STRING_CONCAT, 6170 ) 6171 ) 6172 elif self._match(TokenType.DQMARK): 6173 this = self.expression( 6174 exp.Coalesce(this=this, expressions=ensure_list(self._parse_term())) 6175 ) 6176 elif self._match_pair(TokenType.LT, TokenType.LT): 6177 this = self.expression( 6178 exp.BitwiseLeftShift(this=this, expression=self._parse_term()) 6179 ) 6180 elif self._match_pair(TokenType.GT, TokenType.GT): 6181 this = self.expression( 6182 exp.BitwiseRightShift(this=this, expression=self._parse_term()) 6183 ) 6184 else: 6185 break 6186 6187 return this 6188 6189 def _parse_term(self) -> exp.Expr | None: 6190 this = self._parse_factor() 6191 6192 while self._match_set(self.TERM): 6193 klass = self.TERM[self._prev.token_type] 6194 comments = self._prev_comments 6195 expression = self._parse_factor() 6196 6197 this = self.expression(klass(this=this, expression=expression), comments=comments) 6198 6199 if isinstance(this, exp.Collate): 6200 expr = this.expression 6201 6202 # Preserve collations such as pg_catalog."default" (Postgres) as columns, otherwise 6203 # fallback to Identifier / Var 6204 if isinstance(expr, exp.Column) and len(expr.parts) == 1: 6205 ident = expr.this 6206 if isinstance(ident, exp.Identifier): 6207 this.set("expression", ident if ident.quoted else exp.var(ident.name)) 6208 6209 return this 6210 6211 def _parse_factor(self) -> exp.Expr | None: 6212 parse_method = self._parse_exponent if self.EXPONENT else self._parse_unary 6213 this = self._parse_at_time_zone(parse_method()) 6214 6215 while self._match_set(self.FACTOR): 6216 klass = self.FACTOR[self._prev.token_type] 6217 comments = self._prev_comments 6218 expression = parse_method() 6219 6220 if not expression and klass is exp.IntDiv and self._prev.text.isalpha(): 6221 self._retreat(self._index - 1) 6222 return this 6223 6224 this = self.expression(klass(this=this, expression=expression), comments=comments) 6225 6226 if isinstance(this, exp.Div): 6227 this.set("typed", self.dialect.TYPED_DIVISION) 6228 this.set("safe", self.dialect.SAFE_DIVISION) 6229 6230 return this 6231 6232 def _parse_exponent(self) -> exp.Expr | None: 6233 this = self._parse_unary() 6234 while self._match_set(self.EXPONENT): 6235 comments = self._prev_comments 6236 this = self.expression( 6237 self.EXPONENT[self._prev.token_type](this=this, expression=self._parse_unary()), 6238 comments=comments, 6239 ) 6240 return this 6241 6242 def _parse_unary(self) -> exp.Expr | None: 6243 if self._match_set(self.UNARY_PARSERS): 6244 return self.UNARY_PARSERS[self._prev.token_type](self) 6245 return self._parse_type() 6246 6247 def _parse_type( 6248 self, parse_interval: bool = True, fallback_to_identifier: bool = False 6249 ) -> exp.Expr | None: 6250 if not fallback_to_identifier and (atom := self._parse_atom()) is not None: 6251 return atom 6252 6253 if interval := parse_interval and self._parse_interval(): 6254 return self._parse_column_ops(interval) 6255 6256 index = self._index 6257 data_type = self._parse_types(check_func=True, allow_identifiers=False) 6258 6259 # parse_types() returns a Cast if we parsed BQ's inline constructor <type>(<values>) e.g. 6260 # STRUCT<a INT, b STRING>(1, 'foo'), which is canonicalized to CAST(<values> AS <type>) 6261 if isinstance(data_type, exp.Cast): 6262 # This constructor can contain ops directly after it, for instance struct unnesting: 6263 # STRUCT<a INT, b STRING>(1, 'foo').* --> CAST(STRUCT(1, 'foo') AS STRUCT<a iNT, b STRING).* 6264 return self._parse_column_ops(data_type) 6265 6266 if data_type: 6267 index2 = self._index 6268 this = self._parse_primary() 6269 6270 if isinstance(this, exp.Literal): 6271 literal = this.name 6272 this = self._parse_column_ops(this) 6273 6274 parser = self.TYPE_LITERAL_PARSERS.get(data_type.this) 6275 if parser: 6276 return parser(self, this, data_type) 6277 6278 if ( 6279 self.ZONE_AWARE_TIMESTAMP_CONSTRUCTOR 6280 and data_type.is_type(exp.DType.TIMESTAMP) 6281 and TIME_ZONE_RE.search(literal) 6282 ): 6283 data_type = exp.DType.TIMESTAMPTZ.into_expr() 6284 6285 return self.expression(exp.Cast(this=this, to=data_type)) 6286 6287 # The expressions arg gets set by the parser when we have something like DECIMAL(38, 0) 6288 # in the input SQL. In that case, we'll produce these tokens: DECIMAL ( 38 , 0 ) 6289 # 6290 # If the index difference here is greater than 1, that means the parser itself must have 6291 # consumed additional tokens such as the DECIMAL scale and precision in the above example. 6292 # 6293 # If it's not greater than 1, then it must be 1, because we've consumed at least the type 6294 # keyword, meaning that the expressions arg of the DataType must have gotten set by a 6295 # callable in the TYPE_CONVERTERS mapping. For example, Snowflake converts DECIMAL to 6296 # DECIMAL(38, 0)) in order to facilitate the data type's transpilation. 6297 # 6298 # In these cases, we don't really want to return the converted type, but instead retreat 6299 # and try to parse a Column or Identifier in the section below. 6300 if data_type.expressions and index2 - index > 1: 6301 self._retreat(index2) 6302 return self._parse_column_ops(data_type) 6303 6304 self._retreat(index) 6305 6306 if fallback_to_identifier: 6307 return self._parse_id_var() 6308 6309 return self._parse_column() 6310 6311 def _parse_type_size(self) -> exp.DataTypeParam | None: 6312 this = self._parse_type() 6313 if not this: 6314 return None 6315 6316 if isinstance(this, exp.Column) and not this.table: 6317 this = exp.var(this.name.upper()) 6318 6319 return self.expression( 6320 exp.DataTypeParam(this=this, expression=self._parse_var(any_token=True)) 6321 ) 6322 6323 def _parse_user_defined_type(self, identifier: exp.Identifier) -> exp.Expr | None: 6324 type_name = identifier.name 6325 6326 while self._match(TokenType.DOT): 6327 type_name = f"{type_name}.{self._advance_any() and self._prev.text}" 6328 6329 return exp.DataType.from_str(type_name, dialect=self.dialect, udt=True) 6330 6331 def _parse_types( 6332 self, 6333 check_func: bool = False, 6334 schema: bool = False, 6335 allow_identifiers: bool = True, 6336 with_collation: bool = False, 6337 ) -> exp.Expr | None: 6338 index = self._index 6339 this: exp.Expr | None = None 6340 6341 if self._match_set(self.TYPE_TOKENS): 6342 type_token = self._prev.token_type 6343 else: 6344 type_token = None 6345 identifier = allow_identifiers and self._parse_id_var( 6346 any_token=False, tokens=(TokenType.VAR,) 6347 ) 6348 if isinstance(identifier, exp.Identifier): 6349 try: 6350 tokens = self.dialect.tokenize(identifier.name) 6351 except TokenError: 6352 tokens = None 6353 6354 if tokens and (type_token := tokens[0].token_type) in self.TYPE_TOKENS: 6355 if len(tokens) > 1: 6356 return exp.DataType.from_str(identifier.name, dialect=self.dialect) 6357 elif self.dialect.SUPPORTS_USER_DEFINED_TYPES: 6358 this = self._parse_user_defined_type(identifier) 6359 else: 6360 self._retreat(self._index - 1) 6361 return None 6362 else: 6363 return None 6364 6365 if type_token == TokenType.PSEUDO_TYPE: 6366 return self.expression(exp.PseudoType(this=self._prev.text.upper())) 6367 6368 if type_token == TokenType.OBJECT_IDENTIFIER: 6369 return self.expression(exp.ObjectIdentifier(this=self._prev.text.upper())) 6370 6371 # https://materialize.com/docs/sql/types/map/ 6372 if type_token == TokenType.MAP and self._match(TokenType.L_BRACKET): 6373 key_type = self._parse_types( 6374 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6375 ) 6376 if not self._match(TokenType.FARROW): 6377 self._retreat(index) 6378 return None 6379 6380 value_type = self._parse_types( 6381 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6382 ) 6383 if not self._match(TokenType.R_BRACKET): 6384 self._retreat(index) 6385 return None 6386 6387 return exp.DataType( 6388 this=exp.DType.MAP, 6389 expressions=[key_type, value_type], 6390 nested=True, 6391 ) 6392 6393 nested = type_token in self.NESTED_TYPE_TOKENS 6394 is_struct = type_token in self.STRUCT_TYPE_TOKENS 6395 is_aggregate = type_token in self.AGGREGATE_TYPE_TOKENS 6396 expressions = None 6397 maybe_func = False 6398 6399 if self._match(TokenType.L_PAREN): 6400 if is_struct: 6401 expressions = self._parse_csv(lambda: self._parse_struct_types(type_required=True)) 6402 elif nested: 6403 expressions = self._parse_csv( 6404 lambda: self._parse_types( 6405 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6406 ) 6407 ) 6408 if type_token == TokenType.NULLABLE and len(expressions) == 1: 6409 this = expressions[0] 6410 this.set("nullable", True) 6411 self._match_r_paren() 6412 return this 6413 elif type_token in self.ENUM_TYPE_TOKENS: 6414 expressions = self._parse_csv(self._parse_equality) 6415 elif type_token == TokenType.JSON: 6416 # ClickHouse JSON type supports arguments: JSON(col Type, SKIP col, param=value) 6417 # https://clickhouse.com/docs/sql-reference/data-types/newjson 6418 expressions = self._parse_csv(self._parse_json_type_arg) 6419 elif is_aggregate: 6420 func_or_ident = self._parse_function(anonymous=True) or self._parse_id_var( 6421 any_token=False, tokens=(TokenType.VAR, TokenType.ANY) 6422 ) 6423 if not func_or_ident: 6424 return None 6425 expressions = [func_or_ident] 6426 if self._match(TokenType.COMMA): 6427 expressions.extend( 6428 self._parse_csv( 6429 lambda: self._parse_types( 6430 check_func=check_func, 6431 schema=schema, 6432 allow_identifiers=allow_identifiers, 6433 ) 6434 ) 6435 ) 6436 else: 6437 expressions = self._parse_csv(self._parse_type_size) 6438 6439 # https://docs.snowflake.com/en/sql-reference/data-types-vector 6440 if type_token == TokenType.VECTOR and len(expressions) == 2: 6441 expressions = self._parse_vector_expressions(expressions) 6442 6443 if not self._match(TokenType.R_PAREN): 6444 self._retreat(index) 6445 return None 6446 6447 maybe_func = True 6448 6449 values: list[exp.Expr] | None = None 6450 6451 if nested and self._match(TokenType.LT): 6452 if is_struct: 6453 expressions = self._parse_csv(lambda: self._parse_struct_types(type_required=True)) 6454 else: 6455 expressions = self._parse_csv( 6456 lambda: self._parse_types( 6457 check_func=check_func, 6458 schema=schema, 6459 allow_identifiers=allow_identifiers, 6460 with_collation=True, 6461 ) 6462 ) 6463 6464 if not self._match(TokenType.GT): 6465 self.raise_error("Expecting >") 6466 6467 if self._match_set((TokenType.L_BRACKET, TokenType.L_PAREN)): 6468 values = self._parse_csv(self._parse_disjunction) 6469 if not values and is_struct: 6470 values = None 6471 self._retreat(self._index - 1) 6472 else: 6473 self._match_set((TokenType.R_BRACKET, TokenType.R_PAREN)) 6474 6475 if type_token in self.TIMESTAMPS: 6476 if self._match_text_seq("WITH", "TIME", "ZONE"): 6477 maybe_func = False 6478 tz_type = exp.DType.TIMETZ if type_token in self.TIMES else exp.DType.TIMESTAMPTZ 6479 this = exp.DataType(this=tz_type, expressions=expressions) 6480 elif self._match_text_seq("WITH", "LOCAL", "TIME", "ZONE"): 6481 maybe_func = False 6482 this = exp.DataType(this=exp.DType.TIMESTAMPLTZ, expressions=expressions) 6483 elif self._match_text_seq("WITHOUT", "TIME", "ZONE"): 6484 maybe_func = False 6485 elif type_token == TokenType.INTERVAL: 6486 if self._curr.text.upper() in self.dialect.VALID_INTERVAL_UNITS: 6487 unit = self._parse_var(upper=True) 6488 if self._match_text_seq("TO"): 6489 unit = exp.IntervalSpan(this=unit, expression=self._parse_var(upper=True)) 6490 6491 this = self.expression(exp.DataType(this=self.expression(exp.Interval(unit=unit)))) 6492 else: 6493 this = self.expression(exp.DataType(this=exp.DType.INTERVAL)) 6494 elif type_token == TokenType.VOID: 6495 this = exp.DataType(this=exp.DType.NULL) 6496 6497 if maybe_func and check_func: 6498 index2 = self._index 6499 peek = self._parse_string() 6500 6501 if not peek: 6502 self._retreat(index) 6503 return None 6504 6505 self._retreat(index2) 6506 6507 if not this: 6508 assert type_token is not None 6509 if self._match_text_seq("UNSIGNED"): 6510 unsigned_type_token = self.SIGNED_TO_UNSIGNED_TYPE_TOKEN.get(type_token) 6511 if not unsigned_type_token: 6512 self.raise_error(f"Cannot convert {type_token.name} to unsigned.") 6513 6514 type_token = unsigned_type_token or type_token 6515 6516 # NULLABLE without parentheses can be a column (Presto/Trino) 6517 if type_token == TokenType.NULLABLE and not expressions: 6518 self._retreat(index) 6519 return None 6520 6521 this = exp.DataType( 6522 this=exp.DType[type_token.name], 6523 expressions=expressions, 6524 nested=nested, 6525 ) 6526 6527 # Empty arrays/structs are allowed 6528 if values is not None: 6529 cls = exp.Struct if is_struct else exp.Array 6530 this = exp.cast(cls(expressions=values), this, copy=False) 6531 6532 elif expressions: 6533 this.set("expressions", expressions) 6534 6535 # https://materialize.com/docs/sql/types/list/#type-name 6536 while self._match(TokenType.LIST): 6537 this = exp.DataType(this=exp.DType.LIST, expressions=[this], nested=True) 6538 6539 index = self._index 6540 6541 # Postgres supports the INT ARRAY[3] syntax as a synonym for INT[3] 6542 matched_array = self._match(TokenType.ARRAY) 6543 6544 while self._curr: 6545 datatype_token = self._prev.token_type 6546 matched_l_bracket = self._match(TokenType.L_BRACKET) 6547 6548 if (not matched_l_bracket and not matched_array) or ( 6549 datatype_token == TokenType.ARRAY and self._match(TokenType.R_BRACKET) 6550 ): 6551 # Postgres allows casting empty arrays such as ARRAY[]::INT[], 6552 # not to be confused with the fixed size array parsing 6553 break 6554 6555 matched_array = False 6556 values = self._parse_csv(self._parse_disjunction) or None 6557 if ( 6558 values 6559 and not schema 6560 and ( 6561 not self.dialect.SUPPORTS_FIXED_SIZE_ARRAYS 6562 or datatype_token == TokenType.ARRAY 6563 or not self._match(TokenType.R_BRACKET, advance=False) 6564 ) 6565 ): 6566 # Retreating here means that we should not parse the following values as part of the data type, e.g. in DuckDB 6567 # ARRAY[1] should retreat and instead be parsed into exp.Array in contrast to INT[x][y] which denotes a fixed-size array data type 6568 self._retreat(index) 6569 break 6570 6571 this = exp.DataType( 6572 this=exp.DType.ARRAY, expressions=[this], values=values, nested=True 6573 ) 6574 self._match(TokenType.R_BRACKET) 6575 6576 if self.TYPE_CONVERTERS and isinstance(this.this, exp.DType): 6577 converter = self.TYPE_CONVERTERS.get(this.this) 6578 if converter: 6579 this = converter(t.cast(exp.DataType, this)) 6580 6581 if with_collation and isinstance(this, exp.DataType) and self._match(TokenType.COLLATE): 6582 this.set("collate", self._parse_identifier() or self._parse_column()) 6583 6584 return this 6585 6586 def _parse_json_type_arg(self) -> exp.Expr | None: 6587 """Parse a single argument to ClickHouse's JSON type.""" 6588 6589 # SKIP col or SKIP REGEXP 'pattern' 6590 if self._match_text_seq("SKIP"): 6591 regexp = self._match(TokenType.RLIKE) 6592 arg = self._parse_column() 6593 if isinstance(arg, exp.Column): 6594 arg = arg.to_dot() 6595 return self.expression(exp.SkipJSONColumn(regexp=regexp, expression=arg)) 6596 6597 param_or_col = self._parse_column() 6598 if not isinstance(param_or_col, exp.Column): 6599 return None 6600 6601 # Parameter: name=value (e.g., max_dynamic_paths=2) 6602 if len(param_or_col.parts) == 1 and self._match(TokenType.EQ): 6603 param = param_or_col.name 6604 value = self._parse_primary() 6605 return self.expression(exp.EQ(this=exp.var(param), expression=value)) 6606 6607 # Column type hint: col_name Type 6608 col = param_or_col.to_dot() 6609 kind = self._parse_types(check_func=False, allow_identifiers=False) 6610 return self.expression(exp.ColumnDef(this=col, kind=kind)) 6611 6612 def _parse_vector_expressions(self, expressions: list[exp.Expr]) -> list[exp.Expr]: 6613 return [exp.DataType.from_str(expressions[0].name, dialect=self.dialect), *expressions[1:]] 6614 6615 def _parse_struct_types(self, type_required: bool = False) -> exp.Expr | None: 6616 index = self._index 6617 6618 if ( 6619 self._curr 6620 and self._next 6621 and self._curr.token_type in self.TYPE_TOKENS 6622 and self._next.token_type in self.TYPE_TOKENS 6623 ): 6624 # Takes care of special cases like `STRUCT<list ARRAY<...>>` where the identifier is also a 6625 # type token. Without this, the list will be parsed as a type and we'll eventually crash 6626 this = self._parse_id_var() 6627 else: 6628 this = ( 6629 self._parse_type(parse_interval=False, fallback_to_identifier=True) 6630 or self._parse_id_var() 6631 ) 6632 6633 self._match(TokenType.COLON) 6634 6635 if ( 6636 type_required 6637 and not isinstance(this, exp.DataType) 6638 and not self._match_set(self.TYPE_TOKENS, advance=False) 6639 ): 6640 self._retreat(index) 6641 return self._parse_types() 6642 6643 return self._parse_column_def(this) 6644 6645 def _parse_at_time_zone(self, this: exp.Expr | None) -> exp.Expr | None: 6646 if not self._match_text_seq("AT", "TIME", "ZONE"): 6647 return this 6648 return self._parse_at_time_zone( 6649 self.expression(exp.AtTimeZone(this=this, zone=self._parse_unary())) 6650 ) 6651 6652 def _parse_atom(self) -> exp.Expr | None: 6653 if ( 6654 self._curr.token_type in self.IDENTIFIER_TOKENS 6655 and (column := self._parse_column()) is not None 6656 ): 6657 return column 6658 6659 token = self._curr 6660 token_type = token.token_type 6661 6662 if not (primary_parser := self.PRIMARY_PARSERS.get(token_type)): 6663 return None 6664 6665 next_type = self._next.token_type 6666 6667 if ( 6668 next_type in self.COLUMN_OPERATORS 6669 or next_type in self.COLUMN_POSTFIX_TOKENS 6670 or (token_type == TokenType.STRING and next_type == TokenType.STRING) 6671 ): 6672 return None 6673 6674 self._advance() 6675 return primary_parser(self, token) 6676 6677 def _parse_column(self) -> exp.Expr | None: 6678 column: exp.Expr | None = self._parse_column_parts_fast() 6679 if column is None: 6680 this = self._parse_column_reference() 6681 if not this: 6682 this = self._parse_bracket(this) 6683 column = self._parse_column_ops(this) if this else this 6684 6685 if column: 6686 if self.dialect.SUPPORTS_COLUMN_JOIN_MARKS: 6687 column.set("join_mark", self._match(TokenType.JOIN_MARKER)) 6688 if self.COLON_IS_VARIANT_EXTRACT: 6689 column = self._parse_colon_as_variant_extract(column) 6690 6691 return column 6692 6693 def _parse_column_parts_fast(self) -> exp.Column | exp.Dot | None: 6694 """Fast path for simple column and dot references (a, a.b, ...). 6695 6696 Greedily consumes VAR/IDENTIFIER tokens separated by DOTs, then checks 6697 that nothing complex follows. If it does, retreats and returns None so 6698 the slow path can handle it. For >4 parts, wraps in exp.Dot nodes. 6699 """ 6700 index = self._index 6701 parts: list[exp.Identifier] | None = None 6702 all_comments: list[str] | None = None 6703 6704 while self._match_set(self.IDENTIFIER_TOKENS): 6705 token = self._prev 6706 comments = self._prev_comments 6707 6708 if parts is None and token.text.upper() in self.NO_PAREN_FUNCTION_PARSERS: 6709 self._retreat(index) 6710 return None 6711 6712 has_dot = self._match(TokenType.DOT) 6713 curr_tt = self._curr.token_type 6714 6715 if not has_dot: 6716 if curr_tt in self.COLUMN_OPERATORS or curr_tt in self.COLUMN_POSTFIX_TOKENS: 6717 self._retreat(index) 6718 return None 6719 elif curr_tt not in self.IDENTIFIER_TOKENS: 6720 self._retreat(index) 6721 return None 6722 6723 if parts is None: 6724 parts = [] 6725 6726 if comments: 6727 if all_comments is None: 6728 all_comments = [] 6729 all_comments.extend(comments) 6730 self._prev_comments = [] 6731 6732 parts.append( 6733 self.expression( 6734 exp.Identifier( 6735 this=token.text, quoted=token.token_type == TokenType.IDENTIFIER 6736 ), 6737 token, 6738 ) 6739 ) 6740 6741 if not has_dot: 6742 break 6743 6744 if parts is None: 6745 return None 6746 6747 n = len(parts) 6748 6749 if n == 1: 6750 column: exp.Column | exp.Dot = exp.Column(this=parts[0]) 6751 elif n == 2: 6752 column = exp.Column(this=parts[1], table=parts[0]) 6753 elif n == 3: 6754 column = exp.Column(this=parts[2], table=parts[1], db=parts[0]) 6755 else: 6756 column = exp.Column(this=parts[3], table=parts[2], db=parts[1], catalog=parts[0]) 6757 6758 for i in range(4, n): 6759 column = exp.Dot(this=column, expression=parts[i]) 6760 6761 if all_comments: 6762 column.add_comments(all_comments) 6763 6764 return column 6765 6766 def _parse_column_reference(self) -> exp.Expr | None: 6767 this = self._parse_field() 6768 if ( 6769 not this 6770 and self._match(TokenType.VALUES, advance=False) 6771 and self.VALUES_FOLLOWED_BY_PAREN 6772 and (not self._next or self._next.token_type != TokenType.L_PAREN) 6773 ): 6774 this = self._parse_id_var() 6775 6776 if isinstance(this, exp.Identifier): 6777 # We bubble up comments from the Identifier to the Column 6778 this = self.expression(exp.Column(this=this), comments=this.pop_comments()) 6779 6780 return this 6781 6782 def _build_json_extract( 6783 self, 6784 this: exp.Expr | None, 6785 path_parts: list[exp.JSONPathPart], 6786 ) -> tuple[exp.Expr | None, list[exp.JSONPathPart]]: 6787 if len(path_parts) > 1: 6788 this = self.expression( 6789 exp.JSONExtract( 6790 this=this, 6791 expression=exp.JSONPath(expressions=path_parts), 6792 variant_extract=True, 6793 requires_json=self.JSON_EXTRACT_REQUIRES_JSON_EXPRESSION, 6794 ) 6795 ) 6796 path_parts = [exp.JSONPathRoot()] 6797 6798 return this, path_parts 6799 6800 def _parse_colon_as_variant_extract(self, this: exp.Expr | None) -> exp.Expr | None: 6801 path_parts: list[exp.JSONPathPart] = [exp.JSONPathRoot()] 6802 6803 while self._match(TokenType.COLON): 6804 if not self.COLON_CHAIN_IS_SINGLE_EXTRACT: 6805 this, path_parts = self._build_json_extract(this, path_parts) 6806 6807 key = self._parse_id_var(any_token=True, tokens=(TokenType.SELECT,)) 6808 6809 if key: 6810 quoted = isinstance(key, exp.Identifier) and key.quoted 6811 path_parts.append(exp.JSONPathKey(this=key.name, quoted=quoted)) 6812 6813 while True: 6814 if self._match(TokenType.DOT): 6815 next_key = self._parse_id_var(any_token=True, tokens=(TokenType.SELECT,)) 6816 6817 if next_key: 6818 quoted = isinstance(next_key, exp.Identifier) and next_key.quoted 6819 path_parts.append(exp.JSONPathKey(this=next_key.name, quoted=quoted)) 6820 elif self._match(TokenType.L_BRACKET): 6821 bracket_expr = self._parse_bracket_key_value() 6822 6823 if not self._match(TokenType.R_BRACKET): 6824 self.raise_error("Expected ]") 6825 6826 if bracket_expr: 6827 if bracket_expr.is_string: 6828 path_parts.append(exp.JSONPathKey(this=bracket_expr.name, quoted=True)) 6829 elif bracket_expr.is_star: 6830 path_parts.append(exp.JSONPathSubscript(this=exp.JSONPathWildcard())) 6831 elif bracket_expr.is_number: 6832 path_parts.append(exp.JSONPathSubscript(this=bracket_expr.to_py())) 6833 else: 6834 this, path_parts = self._build_json_extract(this, path_parts) 6835 6836 this = self.expression( 6837 exp.Bracket( 6838 this=this, expressions=[bracket_expr], json_access=True 6839 ), 6840 ) 6841 6842 elif self._match(TokenType.DCOLON): 6843 this, path_parts = self._build_json_extract(this, path_parts) 6844 6845 cast_type = self._parse_types() 6846 if cast_type: 6847 this = self.expression(exp.Cast(this=this, to=cast_type)) 6848 else: 6849 self.raise_error("Expected type after '::'") 6850 else: 6851 break 6852 6853 this, _ = self._build_json_extract(this, path_parts) 6854 6855 return this 6856 6857 def _parse_dcolon(self) -> exp.Expr | None: 6858 return self._parse_types() 6859 6860 def _parse_column_ops(self, this: exp.Expr | None) -> exp.Expr | None: 6861 while self._curr.token_type in self.BRACKETS: 6862 this = self._parse_bracket(this) 6863 6864 column_operators = self.COLUMN_OPERATORS 6865 cast_column_operators = self.CAST_COLUMN_OPERATORS 6866 while self._curr: 6867 op_token = self._curr.token_type 6868 6869 if op_token not in column_operators: 6870 break 6871 op = column_operators[op_token] 6872 self._advance() 6873 6874 if op_token in cast_column_operators: 6875 field = self._parse_dcolon() 6876 if not field: 6877 self.raise_error("Expected type") 6878 elif op and self._curr: 6879 field = self._parse_column_reference() or self._parse_bitwise() 6880 if isinstance(field, exp.Column) and self._match(TokenType.DOT, advance=False): 6881 field = self._parse_column_ops(field) 6882 else: 6883 dot = self._is_connected() and self._prev.token_type == TokenType.DOT 6884 field = self._parse_field(any_token=True, anonymous_func=True) 6885 6886 # In t.true, t.null we should produce an Identifier node 6887 if dot and isinstance(field, (exp.Null, exp.Boolean)): 6888 field = self.expression( 6889 exp.Identifier(this=self._prev.text), 6890 comments=field.comments, 6891 ) 6892 6893 # Function calls can be qualified, e.g., x.y.FOO() 6894 # This converts the final AST to a series of Dots leading to the function call 6895 # https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-reference#function_call_rules 6896 if isinstance(field, (exp.Func, exp.Window)) and this: 6897 this = this.transform( 6898 lambda n: n.to_dot(include_dots=False) if isinstance(n, exp.Column) else n 6899 ) 6900 6901 if op: 6902 this = op(self, this, field) 6903 elif isinstance(this, exp.Column) and not this.args.get("catalog"): 6904 this = self.expression( 6905 exp.Column( 6906 this=field, 6907 table=this.this, 6908 db=this.args.get("table"), 6909 catalog=this.args.get("db"), 6910 ), 6911 comments=this.comments, 6912 ) 6913 elif isinstance(field, exp.Window): 6914 # Move the exp.Dot's to the window's function 6915 window_func = self.expression(exp.Dot(this=this, expression=field.this)) 6916 field.set("this", window_func) 6917 this = field 6918 else: 6919 this = self.expression(exp.Dot(this=this, expression=field)) 6920 6921 if field and field.comments: 6922 t.cast(exp.Expr, this).add_comments(field.pop_comments()) 6923 6924 this = self._parse_bracket(this) 6925 6926 return this 6927 6928 def _parse_paren(self) -> exp.Expr | None: 6929 if not self._match(TokenType.L_PAREN): 6930 return None 6931 6932 comments = self._prev_comments 6933 query = self._parse_select() 6934 6935 if query: 6936 expressions = [query] 6937 else: 6938 expressions = self._parse_expressions() 6939 6940 this = seq_get(expressions, 0) 6941 6942 if not this and self._match(TokenType.R_PAREN, advance=False): 6943 this = self.expression(exp.Tuple()) 6944 elif len(expressions) > 1 or self._prev.token_type == TokenType.COMMA: 6945 this = self.expression(exp.Tuple(expressions=expressions)) 6946 elif isinstance(this, exp.UNWRAPPED_QUERIES): 6947 this = self._parse_subquery(this=this, parse_alias=False) 6948 elif isinstance(this, (exp.Subquery, exp.Values)): 6949 this = self._parse_subquery( 6950 this=self._parse_query_modifiers(self._parse_set_operations(this)), 6951 parse_alias=False, 6952 ) 6953 else: 6954 this = self.expression(exp.Paren(this=this)) 6955 6956 if this: 6957 this.add_comments(comments) 6958 6959 self._match_r_paren(expression=this) 6960 6961 if isinstance(this, exp.Paren) and isinstance(this.this, exp.AggFunc): 6962 return self._parse_window(this) 6963 6964 return this 6965 6966 def _parse_primary(self) -> exp.Expr | None: 6967 if self._match_set(self.PRIMARY_PARSERS): 6968 token_type = self._prev.token_type 6969 primary = self.PRIMARY_PARSERS[token_type](self, self._prev) 6970 6971 if token_type == TokenType.STRING: 6972 expressions = [primary] 6973 while self._match(TokenType.STRING, advance=False): 6974 if self._is_connected() and self.ADJACENT_STRINGS_CANNOT_BE_CONNECTED: 6975 self.raise_error( 6976 "Adjacent string literals need to be separated by whitespace or comments" 6977 ) 6978 6979 self._advance() 6980 expressions.append(exp.Literal.string(self._prev.text)) 6981 6982 if len(expressions) > 1: 6983 return self.expression( 6984 exp.Concat(expressions=expressions, coalesce=self.dialect.CONCAT_COALESCE) 6985 ) 6986 6987 return primary 6988 6989 if self._match_pair(TokenType.DOT, TokenType.NUMBER): 6990 return exp.Literal.number(f"0.{self._prev.text}") 6991 6992 return self._parse_paren() 6993 6994 def _parse_field( 6995 self, 6996 any_token: bool = False, 6997 tokens: t.Collection[TokenType] | None = None, 6998 anonymous_func: bool = False, 6999 ) -> exp.Expr | None: 7000 if anonymous_func: 7001 field = ( 7002 self._parse_function(anonymous=anonymous_func, any_token=any_token) 7003 or self._parse_primary() 7004 ) 7005 else: 7006 field = self._parse_primary() or self._parse_function( 7007 anonymous=anonymous_func, any_token=any_token 7008 ) 7009 return field or self._parse_id_var(any_token=any_token, tokens=tokens) 7010 7011 def _parse_function( 7012 self, 7013 functions: dict[str, t.Callable] | None = None, 7014 anonymous: bool = False, 7015 optional_parens: bool = True, 7016 any_token: bool = False, 7017 ) -> exp.Expr | None: 7018 # This allows us to also parse {fn <function>} syntax (Snowflake, MySQL support this) 7019 # See: https://community.snowflake.com/s/article/SQL-Escape-Sequences 7020 fn_syntax = False 7021 if ( 7022 self._match(TokenType.L_BRACE, advance=False) 7023 and self._next 7024 and self._next.text.upper() == "FN" 7025 ): 7026 self._advance(2) 7027 fn_syntax = True 7028 7029 func = self._parse_function_call( 7030 functions=functions, 7031 anonymous=anonymous, 7032 optional_parens=optional_parens, 7033 any_token=any_token, 7034 ) 7035 7036 if fn_syntax: 7037 self._match(TokenType.R_BRACE) 7038 7039 return func 7040 7041 def _parse_function_args(self, alias: bool = False) -> list[exp.Expr]: 7042 return self._parse_csv(lambda: self._parse_lambda(alias=alias)) 7043 7044 def _parse_function_call( 7045 self, 7046 functions: dict[str, t.Callable] | None = None, 7047 anonymous: bool = False, 7048 optional_parens: bool = True, 7049 any_token: bool = False, 7050 ) -> exp.Expr | None: 7051 if not self._curr: 7052 return None 7053 7054 comments = self._curr.comments 7055 prev = self._prev 7056 token = self._curr 7057 token_type = self._curr.token_type 7058 this: str | exp.Expr = self._curr.text 7059 upper = self._curr.text.upper() 7060 7061 after_dot = prev.token_type == TokenType.DOT 7062 parser = self.NO_PAREN_FUNCTION_PARSERS.get(upper) 7063 if ( 7064 optional_parens 7065 and parser 7066 and token_type not in self.INVALID_FUNC_NAME_TOKENS 7067 and not after_dot 7068 ): 7069 self._advance() 7070 return self._parse_window(parser(self)) 7071 7072 if self._next.token_type != TokenType.L_PAREN: 7073 if optional_parens and token_type in self.NO_PAREN_FUNCTIONS and not after_dot: 7074 self._advance() 7075 return self.expression(self.NO_PAREN_FUNCTIONS[token_type]()) 7076 7077 return None 7078 7079 if any_token: 7080 if token_type in self.RESERVED_TOKENS: 7081 return None 7082 elif token_type not in self.FUNC_TOKENS: 7083 return None 7084 7085 self._advance(2) 7086 7087 parser = self.FUNCTION_PARSERS.get(upper) 7088 if parser and not anonymous: 7089 result = parser(self) 7090 else: 7091 subquery_predicate = self.SUBQUERY_PREDICATES.get(token_type) 7092 7093 if subquery_predicate: 7094 expr = None 7095 if self._curr.token_type in self.SUBQUERY_TOKENS: 7096 expr = self._parse_select() 7097 self._match_r_paren() 7098 elif prev and prev.token_type in (TokenType.LIKE, TokenType.ILIKE): 7099 # Backtrack one token since we've consumed the L_PAREN here. Instead, we'd like 7100 # to parse "LIKE [ANY | ALL] (...)" as a whole into an exp.Tuple or exp.Paren 7101 self._advance(-1) 7102 expr = self._parse_bitwise() 7103 7104 if expr: 7105 return self.expression(subquery_predicate(this=expr), comments=comments) 7106 7107 if functions is None: 7108 functions = self.FUNCTIONS 7109 7110 function = functions.get(upper) 7111 known_function = function and not anonymous 7112 7113 alias = not known_function or upper in self.FUNCTIONS_WITH_ALIASED_ARGS 7114 args = self._parse_function_args(alias) 7115 7116 post_func_comments = self._curr.comments if self._curr else None 7117 if known_function and post_func_comments: 7118 # If the user-inputted comment "/* sqlglot.anonymous */" is following the function 7119 # call we'll construct it as exp.Anonymous, even if it's "known" 7120 if any( 7121 comment.lstrip().startswith(exp.SQLGLOT_ANONYMOUS) 7122 for comment in post_func_comments 7123 ): 7124 known_function = False 7125 7126 if alias and known_function: 7127 args = self._kv_to_prop_eq(args) 7128 7129 if known_function: 7130 func_builder = t.cast(t.Callable, function) 7131 7132 # mypyc compiled functions don't have __code__, so we use 7133 # try/except to check if func_builder accepts 'dialect'. 7134 try: 7135 func = func_builder(args) 7136 except TypeError: 7137 func = func_builder(args, dialect=self.dialect) 7138 7139 func = self.validate_expression(func, args) 7140 if self.dialect.PRESERVE_ORIGINAL_NAMES: 7141 func.meta["name"] = this 7142 7143 result = func 7144 else: 7145 if token_type == TokenType.IDENTIFIER: 7146 this = exp.Identifier(this=this, quoted=True).update_positions(token) 7147 7148 result = self.expression(exp.Anonymous(this=this, expressions=args)) 7149 7150 result = result.update_positions(token) 7151 7152 if isinstance(result, exp.Expr): 7153 result.add_comments(comments) 7154 7155 if parser: 7156 self._match(TokenType.R_PAREN, expression=result) 7157 else: 7158 self._match_r_paren(result) 7159 return self._parse_window(result) 7160 7161 def _to_prop_eq(self, expression: exp.Expr, index: int) -> exp.Expr: 7162 return expression 7163 7164 def _kv_to_prop_eq( 7165 self, expressions: list[exp.Expr], parse_map: bool = False 7166 ) -> list[exp.Expr]: 7167 transformed = [] 7168 7169 for index, e in enumerate(expressions): 7170 if isinstance(e, self.KEY_VALUE_DEFINITIONS): 7171 if isinstance(e, exp.Alias): 7172 e = self.expression(exp.PropertyEQ(this=e.args.get("alias"), expression=e.this)) 7173 7174 if not isinstance(e, exp.PropertyEQ): 7175 e = self.expression( 7176 exp.PropertyEQ( 7177 this=e.this if parse_map else exp.to_identifier(e.this.name), 7178 expression=e.expression, 7179 ) 7180 ) 7181 7182 if isinstance(e.this, exp.Column): 7183 e.this.replace(e.this.this) 7184 else: 7185 e = self._to_prop_eq(e, index) 7186 7187 transformed.append(e) 7188 7189 return transformed 7190 7191 def _parse_function_properties(self) -> exp.Properties | None: 7192 # Skip the generic `key = value` fallback in _parse_property since this 7193 # runs post-AS where a function body like `name = expr` can be misread 7194 # as a property. 7195 properties = [] 7196 while True: 7197 if self._match_texts(self.PROPERTY_PARSERS): 7198 prop = self.PROPERTY_PARSERS[self._prev.text.upper()](self) 7199 elif self._match(TokenType.DEFAULT) and self._match_texts(self.PROPERTY_PARSERS): 7200 prop = self.PROPERTY_PARSERS[self._prev.text.upper()](self, default=True) 7201 else: 7202 break 7203 for p in ensure_list(prop): 7204 properties.append(p) 7205 7206 return self.expression(exp.Properties(expressions=properties)) if properties else None 7207 7208 def _parse_user_defined_function_expression(self) -> exp.Expr | None: 7209 return self._parse_statement() 7210 7211 def _parse_function_parameter(self) -> exp.Expr | None: 7212 return self._parse_column_def(this=self._parse_id_var(), computed_column=False) 7213 7214 def _parse_user_defined_function(self, kind: TokenType | None = None) -> exp.Expr | None: 7215 this = self._parse_table_parts(schema=True) 7216 7217 if not self._match(TokenType.L_PAREN): 7218 return this 7219 7220 expressions = self._parse_csv(self._parse_function_parameter) 7221 self._match_r_paren() 7222 return self.expression( 7223 exp.UserDefinedFunction(this=this, expressions=expressions, wrapped=True) 7224 ) 7225 7226 def _parse_macro_overloads( 7227 self, 7228 this: exp.UserDefinedFunction, 7229 first_body: exp.Expr, 7230 first_is_table: bool = False, 7231 ) -> exp.MacroOverloads: 7232 overloads = [ 7233 self.expression( 7234 exp.MacroOverload( 7235 this=first_body, 7236 expressions=this.expressions or None, 7237 is_table=first_is_table, 7238 ) 7239 ) 7240 ] 7241 this.set("expressions", None) 7242 this.set("wrapped", False) 7243 7244 while self._match(TokenType.COMMA): 7245 if not self._match(TokenType.L_PAREN): 7246 break 7247 7248 params = self._parse_csv(self._parse_function_parameter) 7249 self._match_r_paren() 7250 7251 if not self._match(TokenType.ALIAS): 7252 break 7253 7254 is_table = self._match(TokenType.TABLE) 7255 body = self._parse_expression() 7256 macro = exp.MacroOverload(this=body, expressions=params, is_table=is_table) 7257 overloads.append(self.expression(macro)) 7258 7259 return self.expression(exp.MacroOverloads(expressions=overloads)) 7260 7261 def _parse_introducer(self, token: Token) -> exp.Introducer | exp.Identifier: 7262 literal = self._parse_primary() 7263 if literal: 7264 return self.expression(exp.Introducer(this=token.text, expression=literal), token) 7265 7266 return self._identifier_expression(token) 7267 7268 def _parse_session_parameter(self) -> exp.SessionParameter: 7269 kind = None 7270 this = self._parse_id_var() or self._parse_primary() 7271 7272 if this and self._match(TokenType.DOT): 7273 kind = this.name 7274 this = self._parse_var() or self._parse_primary() 7275 7276 return self.expression(exp.SessionParameter(this=this, kind=kind)) 7277 7278 def _parse_lambda_arg(self) -> exp.Expr | None: 7279 return self._parse_id_var() 7280 7281 def _parse_lambda(self, alias: bool = False) -> exp.Expr | None: 7282 next_token_type = self._next.token_type 7283 7284 # Fast path: simple atom (column, literal, null, bool) followed by , or ) 7285 if ( 7286 next_token_type in self.LAMBDA_ARG_TERMINATORS 7287 and (atom := self._parse_atom()) is not None 7288 ): 7289 return atom 7290 7291 index = self._index 7292 7293 if self._match(TokenType.L_PAREN): 7294 expressions = t.cast( 7295 list[t.Optional[exp.Expr]], self._parse_csv(self._parse_lambda_arg) 7296 ) 7297 7298 if not self._match(TokenType.R_PAREN): 7299 self._retreat(index) 7300 elif self._match_set(self.LAMBDAS): 7301 return self.LAMBDAS[self._prev.token_type](self, expressions) 7302 else: 7303 self._retreat(index) 7304 elif self.TYPED_LAMBDA_ARGS or next_token_type in self.LAMBDAS: 7305 expressions = [self._parse_lambda_arg()] 7306 7307 if self._match_set(self.LAMBDAS): 7308 return self.LAMBDAS[self._prev.token_type](self, expressions) 7309 7310 self._retreat(index) 7311 7312 this: exp.Expr | None 7313 7314 if self._match(TokenType.DISTINCT): 7315 this = self.expression( 7316 exp.Distinct(expressions=self._parse_csv(self._parse_disjunction)) 7317 ) 7318 else: 7319 self._match(TokenType.ALL) # ALL is the default/no-op aggregate modifier (SQL-92) 7320 this = self._parse_select_or_expression(alias=alias) 7321 7322 return self._parse_limit( 7323 self._parse_respect_or_ignore_nulls( 7324 self._parse_order(self._parse_having_max(self._parse_respect_or_ignore_nulls(this))) 7325 ) 7326 ) 7327 7328 def _parse_schema(self, this: exp.Expr | None = None) -> exp.Expr | None: 7329 index = self._index 7330 if not self._match(TokenType.L_PAREN): 7331 return this 7332 7333 # Disambiguate between schema and subquery/CTE, e.g. in INSERT INTO table (<expr>), 7334 # expr can be of both types 7335 if self._match_set(self.SELECT_START_TOKENS): 7336 self._retreat(index) 7337 return this 7338 args = self._parse_csv(lambda: self._parse_constraint() or self._parse_field_def()) 7339 self._match_r_paren() 7340 return self.expression(exp.Schema(this=this, expressions=args)) 7341 7342 def _parse_field_def(self) -> exp.Expr | None: 7343 return self._parse_column_def(self._parse_field(any_token=True)) 7344 7345 def _parse_column_def( 7346 self, this: exp.Expr | None, computed_column: bool = True 7347 ) -> exp.Expr | None: 7348 # column defs are not really columns, they're identifiers 7349 if isinstance(this, exp.Column): 7350 this = this.this 7351 7352 if not computed_column: 7353 self._match(TokenType.ALIAS) 7354 7355 kind = self._parse_types(schema=True) 7356 7357 if self._match_text_seq("FOR", "ORDINALITY"): 7358 return self.expression(exp.ColumnDef(this=this, ordinality=True)) 7359 7360 constraints: list[exp.Expr] = [] 7361 7362 if (not kind and self._match(TokenType.ALIAS)) or self._match_texts( 7363 ("ALIAS", "MATERIALIZED") 7364 ): 7365 persisted = self._prev.text.upper() == "MATERIALIZED" 7366 constraint_kind = exp.ComputedColumnConstraint( 7367 this=self._parse_disjunction(), 7368 persisted=persisted or self._match_text_seq("PERSISTED"), 7369 data_type=exp.Var(this="AUTO") 7370 if self._match_text_seq("AUTO") 7371 else self._parse_types(), 7372 not_null=self._match_pair(TokenType.NOT, TokenType.NULL), 7373 ) 7374 constraints.append(self.expression(exp.ColumnConstraint(kind=constraint_kind))) 7375 elif not kind and self._match_set({TokenType.IN, TokenType.OUT}, advance=False): 7376 in_out_constraint = self.expression( 7377 exp.InOutColumnConstraint( 7378 input_=self._match(TokenType.IN), output=self._match(TokenType.OUT) 7379 ) 7380 ) 7381 constraints.append(in_out_constraint) 7382 kind = self._parse_types() 7383 elif ( 7384 kind 7385 and self._match(TokenType.ALIAS, advance=False) 7386 and ( 7387 not self.WRAPPED_TRANSFORM_COLUMN_CONSTRAINT 7388 or self._next.token_type == TokenType.L_PAREN 7389 ) 7390 ): 7391 self._advance() 7392 constraints.append( 7393 self.expression( 7394 exp.ColumnConstraint( 7395 kind=exp.ComputedColumnConstraint( 7396 this=self._parse_disjunction(), 7397 persisted=self._match_texts(("STORED", "VIRTUAL")) 7398 and self._prev.text.upper() == "STORED", 7399 ) 7400 ) 7401 ) 7402 ) 7403 7404 while True: 7405 constraint = self._parse_column_constraint() 7406 if not constraint: 7407 break 7408 constraints.append(constraint) 7409 7410 if not kind and not constraints: 7411 return this 7412 7413 position = None 7414 if self._match_texts(("FIRST", "AFTER")): 7415 pos = self._prev.text 7416 position = self.expression(exp.ColumnPosition(this=self._parse_column(), position=pos)) 7417 7418 return self.expression( 7419 exp.ColumnDef(this=this, kind=kind, constraints=constraints, position=position) 7420 ) 7421 7422 def _parse_auto_increment( 7423 self, 7424 ) -> exp.GeneratedAsIdentityColumnConstraint | exp.AutoIncrementColumnConstraint: 7425 start = None 7426 increment = None 7427 order = None 7428 7429 if self._match(TokenType.L_PAREN, advance=False): 7430 args = self._parse_wrapped_csv(self._parse_bitwise) 7431 start = seq_get(args, 0) 7432 increment = seq_get(args, 1) 7433 7434 # The remaining parts form an unordered bag and any of them can be omitted, in which 7435 # case the engine falls back to its own default, so they're parsed independently. 7436 while True: 7437 if self._match_text_seq("START"): 7438 start = self._parse_bitwise() 7439 elif self._match_text_seq("INCREMENT"): 7440 increment = self._parse_bitwise() 7441 elif self._match_text_seq("ORDER"): 7442 order = True 7443 elif self._match_text_seq("NOORDER"): 7444 order = False 7445 else: 7446 break 7447 7448 if start or increment or order is not None: 7449 return exp.GeneratedAsIdentityColumnConstraint( 7450 start=start, increment=increment, this=False, order=order 7451 ) 7452 7453 return exp.AutoIncrementColumnConstraint() 7454 7455 def _parse_check_constraint(self) -> exp.CheckColumnConstraint | None: 7456 if not self._match(TokenType.L_PAREN, advance=False): 7457 return None 7458 7459 return self.expression( 7460 exp.CheckColumnConstraint( 7461 this=self._parse_wrapped(self._parse_assignment), 7462 enforced=self._match_text_seq("ENFORCED"), 7463 ) 7464 ) 7465 7466 def _parse_auto_property(self) -> exp.AutoRefreshProperty | None: 7467 if not self._match_text_seq("REFRESH"): 7468 self._retreat(self._index - 1) 7469 return None 7470 return self.expression(exp.AutoRefreshProperty(this=self._parse_var(upper=True))) 7471 7472 def _parse_compress(self) -> exp.CompressColumnConstraint: 7473 if self._match(TokenType.L_PAREN, advance=False): 7474 return self.expression( 7475 exp.CompressColumnConstraint(this=self._parse_wrapped_csv(self._parse_bitwise)) 7476 ) 7477 7478 return self.expression(exp.CompressColumnConstraint(this=self._parse_bitwise())) 7479 7480 def _parse_generated_as_identity( 7481 self, 7482 ) -> ( 7483 exp.GeneratedAsIdentityColumnConstraint 7484 | exp.ComputedColumnConstraint 7485 | exp.GeneratedAsRowColumnConstraint 7486 ): 7487 if self._match_text_seq("BY", "DEFAULT"): 7488 on_null = self._match_pair(TokenType.ON, TokenType.NULL) 7489 this = self.expression( 7490 exp.GeneratedAsIdentityColumnConstraint(this=False, on_null=on_null) 7491 ) 7492 else: 7493 self._match_text_seq("ALWAYS") 7494 this = self.expression(exp.GeneratedAsIdentityColumnConstraint(this=True)) 7495 7496 self._match(TokenType.ALIAS) 7497 7498 if self._match_text_seq("ROW"): 7499 start = self._match_text_seq("START") 7500 if not start: 7501 self._match(TokenType.END) 7502 hidden = self._match_text_seq("HIDDEN") 7503 return self.expression(exp.GeneratedAsRowColumnConstraint(start=start, hidden=hidden)) 7504 7505 identity = self._match_text_seq("IDENTITY") 7506 7507 if self._match(TokenType.L_PAREN): 7508 if self._match(TokenType.START_WITH): 7509 this.set("start", self._parse_bitwise()) 7510 if self._match_text_seq("INCREMENT", "BY"): 7511 this.set("increment", self._parse_bitwise()) 7512 if self._match_text_seq("MINVALUE"): 7513 this.set("minvalue", self._parse_bitwise()) 7514 if self._match_text_seq("MAXVALUE"): 7515 this.set("maxvalue", self._parse_bitwise()) 7516 7517 if self._match_text_seq("CYCLE"): 7518 this.set("cycle", True) 7519 elif self._match_text_seq("NO", "CYCLE"): 7520 this.set("cycle", False) 7521 7522 if not identity: 7523 this.set("expression", self._parse_range()) 7524 elif not this.args.get("start") and self._match(TokenType.NUMBER, advance=False): 7525 args = self._parse_csv(self._parse_bitwise) 7526 this.set("start", seq_get(args, 0)) 7527 this.set("increment", seq_get(args, 1)) 7528 7529 self._match_r_paren() 7530 7531 return this 7532 7533 def _parse_inline(self) -> exp.InlineLengthColumnConstraint: 7534 self._match_text_seq("LENGTH") 7535 return self.expression(exp.InlineLengthColumnConstraint(this=self._parse_bitwise())) 7536 7537 def _parse_not_constraint(self) -> exp.Expr | None: 7538 if self._match_text_seq("NULL"): 7539 return self.expression(exp.NotNullColumnConstraint()) 7540 if self._match_text_seq("CASESPECIFIC"): 7541 return self.expression(exp.CaseSpecificColumnConstraint(not_=True)) 7542 if self._match_text_seq("FOR", "REPLICATION"): 7543 return self.expression(exp.NotForReplicationColumnConstraint()) 7544 7545 # Unconsume the `NOT` token 7546 self._retreat(self._index - 1) 7547 return None 7548 7549 def _parse_column_constraint(self) -> exp.Expr | None: 7550 this = self._parse_id_var() if self._match(TokenType.CONSTRAINT) else None 7551 7552 procedure_option_follows = ( 7553 self._match(TokenType.WITH, advance=False) 7554 and self._next 7555 and self._next.text.upper() in self.PROCEDURE_OPTIONS 7556 ) 7557 7558 if not procedure_option_follows and self._match_texts(self.CONSTRAINT_PARSERS): 7559 constraint = self.CONSTRAINT_PARSERS[self._prev.text.upper()](self) 7560 if not constraint: 7561 self._retreat(self._index - 1) 7562 return None 7563 7564 return self.expression(exp.ColumnConstraint(this=this, kind=constraint)) 7565 7566 return this 7567 7568 def _parse_constraint(self) -> exp.Expr | None: 7569 if not self._match(TokenType.CONSTRAINT): 7570 return self._parse_unnamed_constraint(constraints=self.SCHEMA_UNNAMED_CONSTRAINTS) 7571 7572 return self.expression( 7573 exp.Constraint(this=self._parse_id_var(), expressions=self._parse_unnamed_constraints()) 7574 ) 7575 7576 def _parse_unnamed_constraints(self) -> list[exp.Expr]: 7577 constraints = [] 7578 while True: 7579 constraint = self._parse_unnamed_constraint() or self._parse_function() 7580 if not constraint: 7581 break 7582 constraints.append(constraint) 7583 7584 return constraints 7585 7586 def _parse_unnamed_constraint(self, constraints: TEXTS_TYPE | None = None) -> exp.Expr | None: 7587 index = self._index 7588 7589 if self._match(TokenType.IDENTIFIER, advance=False) or not self._match_texts( 7590 constraints or self.CONSTRAINT_PARSERS 7591 ): 7592 return None 7593 7594 constraint_key = self._prev.text.upper() 7595 if constraint_key not in self.CONSTRAINT_PARSERS: 7596 self.raise_error(f"No parser found for schema constraint {constraint_key}.") 7597 7598 result = self.CONSTRAINT_PARSERS[constraint_key](self) 7599 if not result: 7600 self._retreat(index) 7601 7602 return result 7603 7604 def _parse_unique_key(self) -> exp.Expr | None: 7605 if ( 7606 self._curr 7607 and self._curr.token_type != TokenType.IDENTIFIER 7608 and self._curr.text.upper() in self.CONSTRAINT_PARSERS 7609 ): 7610 return None 7611 return self._parse_id_var(any_token=False) 7612 7613 def _parse_unique(self) -> exp.UniqueColumnConstraint: 7614 self._match_texts(("KEY", "INDEX")) 7615 return self.expression( 7616 exp.UniqueColumnConstraint( 7617 nulls=self._match_text_seq("NULLS", "NOT", "DISTINCT"), 7618 this=self._parse_schema(self._parse_unique_key()), 7619 index_type=self._match(TokenType.USING) and self._advance_any() and self._prev.text, 7620 on_conflict=self._parse_on_conflict(), 7621 options=self._parse_key_constraint_options(), 7622 ) 7623 ) 7624 7625 def _parse_key_constraint_options(self) -> list[str]: 7626 options = [] 7627 while True: 7628 if not self._curr: 7629 break 7630 7631 if self._match(TokenType.ON): 7632 action = None 7633 on = self._advance_any() and self._prev.text 7634 7635 if self._match_text_seq("NO", "ACTION"): 7636 action = "NO ACTION" 7637 elif self._match_text_seq("CASCADE"): 7638 action = "CASCADE" 7639 elif self._match_text_seq("RESTRICT"): 7640 action = "RESTRICT" 7641 elif self._match_pair(TokenType.SET, TokenType.NULL): 7642 action = "SET NULL" 7643 elif self._match_pair(TokenType.SET, TokenType.DEFAULT): 7644 action = "SET DEFAULT" 7645 else: 7646 self.raise_error("Invalid key constraint") 7647 7648 options.append(f"ON {on} {action}") 7649 else: 7650 var = self._parse_var_from_options( 7651 self.KEY_CONSTRAINT_OPTIONS, raise_unmatched=False 7652 ) 7653 if not var: 7654 break 7655 options.append(var.name) 7656 7657 return options 7658 7659 def _parse_references(self, match: bool = True) -> exp.Reference | None: 7660 if match and not self._match(TokenType.REFERENCES): 7661 return None 7662 7663 expressions: list | None = None 7664 this = self._parse_table(schema=True) 7665 options = self._parse_key_constraint_options() 7666 return self.expression(exp.Reference(this=this, expressions=expressions, options=options)) 7667 7668 def _parse_foreign_key(self) -> exp.ForeignKey: 7669 expressions = ( 7670 self._parse_wrapped_id_vars() 7671 if not self._match(TokenType.REFERENCES, advance=False) 7672 else None 7673 ) 7674 reference = self._parse_references() 7675 on_options = {} 7676 7677 while self._match(TokenType.ON): 7678 if not self._match_set((TokenType.DELETE, TokenType.UPDATE)): 7679 self.raise_error("Expected DELETE or UPDATE") 7680 7681 kind = self._prev.text.lower() 7682 7683 if self._match_text_seq("NO", "ACTION"): 7684 action = "NO ACTION" 7685 elif self._match(TokenType.SET): 7686 self._match_set((TokenType.NULL, TokenType.DEFAULT)) 7687 action = "SET " + self._prev.text.upper() 7688 else: 7689 self._advance() 7690 action = self._prev.text.upper() 7691 7692 on_options[kind] = action 7693 7694 return self.expression( 7695 exp.ForeignKey( 7696 expressions=expressions, 7697 reference=reference, 7698 options=self._parse_key_constraint_options(), 7699 **on_options, 7700 ) 7701 ) 7702 7703 def _parse_primary_key_part(self) -> exp.Expr | None: 7704 return self._parse_field() 7705 7706 def _parse_period_for_system_time(self) -> exp.PeriodForSystemTimeConstraint | None: 7707 if not self._match(TokenType.TIMESTAMP_SNAPSHOT): 7708 self._retreat(self._index - 1) 7709 return None 7710 7711 id_vars = self._parse_wrapped_id_vars() 7712 return self.expression( 7713 exp.PeriodForSystemTimeConstraint( 7714 this=seq_get(id_vars, 0), expression=seq_get(id_vars, 1) 7715 ) 7716 ) 7717 7718 def _parse_primary_key( 7719 self, 7720 wrapped_optional: bool = False, 7721 in_props: bool = False, 7722 named_primary_key: bool = False, 7723 ) -> exp.PrimaryKeyColumnConstraint | exp.PrimaryKey: 7724 desc = ( 7725 self._prev.token_type == TokenType.DESC 7726 if self._match_set((TokenType.ASC, TokenType.DESC)) 7727 else None 7728 ) 7729 7730 this = None 7731 if ( 7732 named_primary_key 7733 and self._curr.text.upper() not in self.CONSTRAINT_PARSERS 7734 and self._next 7735 and self._next.token_type == TokenType.L_PAREN 7736 ): 7737 this = self._parse_id_var() 7738 7739 if not in_props and not self._match(TokenType.L_PAREN, advance=False): 7740 return self.expression( 7741 exp.PrimaryKeyColumnConstraint( 7742 desc=desc, options=self._parse_key_constraint_options() 7743 ) 7744 ) 7745 7746 expressions = self._parse_wrapped_csv( 7747 self._parse_primary_key_part, optional=wrapped_optional 7748 ) 7749 7750 return self.expression( 7751 exp.PrimaryKey( 7752 this=this, 7753 expressions=expressions, 7754 include=self._parse_index_params(), 7755 options=self._parse_key_constraint_options(), 7756 ) 7757 ) 7758 7759 def _parse_bracket_key_value(self, is_map: bool = False) -> exp.Expr | None: 7760 return self._parse_slice(self._parse_alias(self._parse_disjunction(), explicit=True)) 7761 7762 def _parse_odbc_datetime_literal(self) -> exp.Expr: 7763 """ 7764 Parses a datetime column in ODBC format. We parse the column into the corresponding 7765 types, for example `{d'yyyy-mm-dd'}` will be parsed as a `Date` column, exactly the 7766 same as we did for `DATE('yyyy-mm-dd')`. 7767 7768 Reference: 7769 https://learn.microsoft.com/en-us/sql/odbc/reference/develop-app/date-time-and-timestamp-literals 7770 """ 7771 self._match(TokenType.VAR) 7772 exp_class = self.ODBC_DATETIME_LITERALS[self._prev.text.lower()] 7773 expression = self.expression(exp_class(this=self._parse_string())) 7774 if not self._match(TokenType.R_BRACE): 7775 self.raise_error("Expected }") 7776 return expression 7777 7778 def _parse_bracket(self, this: exp.Expr | None = None) -> exp.Expr | None: 7779 if not self._match_set(self.BRACKETS): 7780 return this 7781 7782 if self.MAP_KEYS_ARE_ARBITRARY_EXPRESSIONS: 7783 map_token = seq_get(self._tokens, self._index - 2) 7784 parse_map = map_token is not None and map_token.text.upper() == "MAP" 7785 else: 7786 parse_map = False 7787 7788 bracket_kind = self._prev.token_type 7789 if ( 7790 bracket_kind == TokenType.L_BRACE 7791 and self._curr 7792 and self._curr.token_type == TokenType.VAR 7793 and self._curr.text.lower() in self.ODBC_DATETIME_LITERALS 7794 ): 7795 return self._parse_odbc_datetime_literal() 7796 7797 expressions = self._parse_csv( 7798 lambda: self._parse_bracket_key_value(is_map=bracket_kind == TokenType.L_BRACE) 7799 ) 7800 7801 if bracket_kind == TokenType.L_BRACKET and not self._match(TokenType.R_BRACKET): 7802 self.raise_error("Expected ]") 7803 elif bracket_kind == TokenType.L_BRACE and not self._match(TokenType.R_BRACE): 7804 self.raise_error("Expected }") 7805 7806 # https://duckdb.org/docs/sql/data_types/struct.html#creating-structs 7807 if bracket_kind == TokenType.L_BRACE: 7808 this = self.expression( 7809 exp.Struct( 7810 expressions=self._kv_to_prop_eq(expressions=expressions, parse_map=parse_map) 7811 ) 7812 ) 7813 elif not this: 7814 this = build_array_constructor( 7815 exp.Array, args=expressions, bracket_kind=bracket_kind, dialect=self.dialect 7816 ) 7817 else: 7818 constructor_type = self.ARRAY_CONSTRUCTORS.get(this.name.upper()) 7819 if constructor_type: 7820 return build_array_constructor( 7821 constructor_type, 7822 args=expressions, 7823 bracket_kind=bracket_kind, 7824 dialect=self.dialect, 7825 ) 7826 7827 expressions = apply_index_offset( 7828 this, expressions, -self.dialect.INDEX_OFFSET, dialect=self.dialect 7829 ) 7830 this = self.expression( 7831 exp.Bracket(this=this, expressions=expressions), comments=this.pop_comments() 7832 ) 7833 7834 self._add_comments(this) 7835 return self._parse_bracket(this) 7836 7837 def _parse_slice(self, this: exp.Expr | None) -> exp.Expr | None: 7838 if not self._match(TokenType.COLON): 7839 return this 7840 7841 if self._match_pair(TokenType.DASH, TokenType.COLON, advance=False): 7842 self._advance() 7843 end: exp.Expr | None = -exp.Literal.number("1") 7844 else: 7845 end = self._parse_assignment() 7846 step = self._parse_unary() if self._match(TokenType.COLON) else None 7847 return self.expression(exp.Slice(this=this, expression=end, step=step)) 7848 7849 def _parse_case(self) -> exp.Expr | None: 7850 if self._match(TokenType.DOT, advance=False): 7851 # Avoid raising on valid expressions like case.*, supported by, e.g., spark & snowflake 7852 self._retreat(self._index - 1) 7853 return None 7854 7855 ifs = [] 7856 default = None 7857 7858 comments = self._prev_comments 7859 expression = self._parse_disjunction() 7860 7861 while self._match(TokenType.WHEN): 7862 this = self._parse_disjunction() 7863 self._match(TokenType.THEN) 7864 then = self._parse_disjunction() 7865 ifs.append(self.expression(exp.If(this=this, true=then))) 7866 7867 if self._match(TokenType.ELSE): 7868 default = self._parse_disjunction() 7869 7870 if not self._match(TokenType.END): 7871 if isinstance(default, exp.Interval) and default.this.sql().upper() == "END": 7872 default = exp.column("interval") 7873 else: 7874 self.raise_error("Expected END after CASE", self._prev) 7875 7876 return self.expression( 7877 exp.Case(this=expression, ifs=ifs, default=default), comments=comments 7878 ) 7879 7880 def _parse_if(self) -> exp.Expr | None: 7881 if self._match(TokenType.L_PAREN): 7882 args = self._parse_csv( 7883 lambda: self._parse_alias(self._parse_assignment(), explicit=True) 7884 ) 7885 this = self.validate_expression(exp.If.from_arg_list(args), args) 7886 self._match_r_paren() 7887 else: 7888 index = self._index - 1 7889 7890 if self.NO_PAREN_IF_COMMANDS and index == 0: 7891 return self._parse_as_command(self._prev) 7892 7893 condition = self._parse_disjunction() 7894 7895 if not condition: 7896 self._retreat(index) 7897 return None 7898 7899 self._match(TokenType.THEN) 7900 true = self._parse_disjunction() 7901 false = self._parse_disjunction() if self._match(TokenType.ELSE) else None 7902 self._match(TokenType.END) 7903 this = self.expression(exp.If(this=condition, true=true, false=false)) 7904 7905 return this 7906 7907 def _parse_next_value_for(self) -> exp.Expr | None: 7908 if not self._match_text_seq("VALUE", "FOR"): 7909 self._retreat(self._index - 1) 7910 return None 7911 7912 return self.expression( 7913 exp.NextValueFor( 7914 this=self._parse_column(), 7915 order=self._match(TokenType.OVER) and self._parse_wrapped(self._parse_order), 7916 ) 7917 ) 7918 7919 def _parse_extract(self) -> exp.Extract | exp.Anonymous: 7920 this = self._parse_function() or self._parse_var_or_string(upper=True) 7921 7922 if self._match(TokenType.FROM): 7923 return self.expression(exp.Extract(this=this, expression=self._parse_bitwise())) 7924 7925 if not self._match(TokenType.COMMA): 7926 self.raise_error("Expected FROM or comma after EXTRACT", self._prev) 7927 7928 return self.expression(exp.Extract(this=this, expression=self._parse_bitwise())) 7929 7930 def _parse_gap_fill(self) -> exp.GapFill: 7931 self._match(TokenType.TABLE) 7932 this = self._parse_table() 7933 7934 self._match(TokenType.COMMA) 7935 args = [this, *self._parse_csv(self._parse_lambda)] 7936 7937 gap_fill = exp.GapFill.from_arg_list(args) 7938 return self.validate_expression(gap_fill, args) 7939 7940 def _parse_char(self) -> exp.Chr: 7941 return self.expression( 7942 exp.Chr( 7943 expressions=self._parse_csv(self._parse_assignment), 7944 charset=self._match(TokenType.USING) and self._parse_charset_name(), 7945 ) 7946 ) 7947 7948 def _parse_charset_name(self) -> exp.Expr | None: 7949 """ 7950 Parse a charset name after USING or CHARACTER SET. Dialects that need to preserve quoting 7951 for specific name shapes override this. 7952 """ 7953 return self._parse_var( 7954 tokens={TokenType.BINARY, TokenType.IDENTIFIER}, 7955 ) 7956 7957 def _parse_cast(self, strict: bool, safe: bool | None = None) -> exp.Expr: 7958 this = self._parse_assignment() 7959 7960 if not self._match(TokenType.ALIAS): 7961 if self._match(TokenType.COMMA): 7962 return self.expression(exp.CastToStrType(this=this, to=self._parse_string())) 7963 7964 self.raise_error("Expected AS after CAST") 7965 7966 fmt = None 7967 to = self._parse_types(with_collation=True) 7968 7969 default = None 7970 if self._match(TokenType.DEFAULT): 7971 default = self._parse_bitwise() 7972 self._match_text_seq("ON", "CONVERSION", "ERROR") 7973 7974 if self._match_set((TokenType.FORMAT, TokenType.COMMA)): 7975 fmt_string = self._parse_wrapped(self._parse_string, optional=True) 7976 fmt = self._parse_at_time_zone(fmt_string) 7977 7978 if not to: 7979 to = exp.DType.UNKNOWN.into_expr() 7980 if to.this in exp.DataType.TEMPORAL_TYPES: 7981 this = self.expression( 7982 (exp.StrToDate if to.this == exp.DType.DATE else exp.StrToTime)( 7983 this=this, 7984 format=exp.Literal.string( 7985 format_time( 7986 fmt_string.this if fmt_string else "", 7987 self.dialect.FORMAT_MAPPING or self.dialect.TIME_MAPPING, 7988 self.dialect.FORMAT_TRIE or self.dialect.TIME_TRIE, 7989 ) 7990 ), 7991 safe=safe, 7992 ) 7993 ) 7994 7995 if isinstance(fmt, exp.AtTimeZone) and isinstance(this, exp.StrToTime): 7996 this.set("zone", fmt.args["zone"]) 7997 return this 7998 elif not to: 7999 self.raise_error("Expected TYPE after CAST") 8000 elif isinstance(to, exp.Identifier): 8001 to = exp.DataType.from_str(to.name, dialect=self.dialect, udt=True) 8002 elif to.this == exp.DType.CHAR and self._match(TokenType.CHARACTER_SET): 8003 to = exp.DType.CHARACTER_SET.into_expr(kind=self._parse_var_or_string()) 8004 8005 return self.build_cast( 8006 strict=strict, 8007 this=this, 8008 to=to, 8009 format=fmt, 8010 safe=safe, 8011 action=self._parse_var_from_options(self.CAST_ACTIONS, raise_unmatched=False), 8012 default=default, 8013 ) 8014 8015 def _parse_string_agg(self) -> exp.GroupConcat: 8016 if self._match(TokenType.DISTINCT): 8017 args: list[exp.Expr | None] = [ 8018 self.expression(exp.Distinct(expressions=[self._parse_disjunction()])) 8019 ] 8020 if self._match(TokenType.COMMA): 8021 args.extend(self._parse_csv(self._parse_disjunction)) 8022 else: 8023 args = self._parse_csv(self._parse_disjunction) # type: ignore 8024 8025 if self._match_text_seq("ON", "OVERFLOW"): 8026 # trino: LISTAGG(expression [, separator] [ON OVERFLOW overflow_behavior]) 8027 if self._match_text_seq("ERROR"): 8028 on_overflow: exp.Expr | None = exp.var("ERROR") 8029 else: 8030 self._match_text_seq("TRUNCATE") 8031 on_overflow = self.expression( 8032 exp.OverflowTruncateBehavior( 8033 this=self._parse_string(), 8034 with_count=( 8035 self._match_text_seq("WITH", "COUNT") 8036 or not self._match_text_seq("WITHOUT", "COUNT") 8037 ), 8038 ) 8039 ) 8040 else: 8041 on_overflow = None 8042 8043 index = self._index 8044 if not self._match(TokenType.R_PAREN) and args: 8045 # postgres: STRING_AGG([DISTINCT] expression, separator [ORDER BY expression1 {ASC | DESC} [, ...]]) 8046 # bigquery: STRING_AGG([DISTINCT] expression [, separator] [ORDER BY key [{ASC | DESC}] [, ... ]] [LIMIT n]) 8047 # The order is parsed through `this` as a canonicalization for WITHIN GROUPs 8048 args[0] = self._parse_limit(this=self._parse_order(this=args[0])) 8049 return self.expression(exp.GroupConcat(this=args[0], separator=seq_get(args, 1))) 8050 8051 # Checks if we can parse an order clause: WITHIN GROUP (ORDER BY <order_by_expression_list> [ASC | DESC]). 8052 # This is done "manually", instead of letting _parse_window parse it into an exp.WithinGroup node, so that 8053 # the STRING_AGG call is parsed like in MySQL / SQLite and can thus be transpiled more easily to them. 8054 if not self._match_text_seq("WITHIN", "GROUP"): 8055 self._retreat(index) 8056 return self.validate_expression(exp.GroupConcat.from_arg_list(args), args) 8057 8058 # The corresponding match_r_paren will be called in parse_function (caller) 8059 self._match_l_paren() 8060 8061 return self.expression( 8062 exp.GroupConcat( 8063 this=self._parse_order(this=seq_get(args, 0)), 8064 separator=seq_get(args, 1), 8065 on_overflow=on_overflow, 8066 ) 8067 ) 8068 8069 def _parse_convert(self, strict: bool, safe: bool | None = None) -> exp.Expr | None: 8070 this = self._parse_bitwise() 8071 8072 if self._match(TokenType.USING): 8073 to: exp.Expr | None = exp.DType.CHARACTER_SET.into_expr(kind=self._parse_charset_name()) 8074 elif self._match(TokenType.COMMA): 8075 to = self._parse_types() 8076 else: 8077 to = None 8078 8079 return self.build_cast(strict=strict, this=this, to=to, safe=safe) 8080 8081 def _parse_xml_element(self) -> exp.XMLElement: 8082 if self._match_text_seq("EVALNAME"): 8083 evalname = True 8084 this = self._parse_bitwise() 8085 else: 8086 evalname = None 8087 self._match_text_seq("NAME") 8088 this = self._parse_id_var() 8089 8090 return self.expression( 8091 exp.XMLElement( 8092 this=this, 8093 expressions=self._match(TokenType.COMMA) and self._parse_csv(self._parse_bitwise), 8094 evalname=evalname, 8095 ) 8096 ) 8097 8098 def _parse_xml_table(self) -> exp.XMLTable: 8099 namespaces = None 8100 passing = None 8101 columns = None 8102 8103 if self._match_text_seq("XMLNAMESPACES", "("): 8104 namespaces = self._parse_xml_namespace() 8105 self._match_text_seq(")", ",") 8106 8107 this = self._parse_string() 8108 8109 if self._match_text_seq("PASSING"): 8110 # The BY VALUE keywords are optional and are provided for semantic clarity 8111 self._match_text_seq("BY", "VALUE") 8112 passing = self._parse_csv(self._parse_column) 8113 8114 by_ref = self._match_text_seq("RETURNING", "SEQUENCE", "BY", "REF") 8115 8116 if self._match_text_seq("COLUMNS"): 8117 columns = self._parse_csv(self._parse_field_def) 8118 8119 return self.expression( 8120 exp.XMLTable( 8121 this=this, namespaces=namespaces, passing=passing, columns=columns, by_ref=by_ref 8122 ) 8123 ) 8124 8125 def _parse_xml_namespace(self) -> list[exp.XMLNamespace]: 8126 namespaces = [] 8127 8128 while True: 8129 if self._match(TokenType.DEFAULT): 8130 uri = self._parse_string() 8131 else: 8132 uri = self._parse_alias(self._parse_string()) 8133 namespaces.append(self.expression(exp.XMLNamespace(this=uri))) 8134 if not self._match(TokenType.COMMA): 8135 break 8136 8137 return namespaces 8138 8139 def _parse_decode(self) -> exp.Decode | exp.DecodeCase | None: 8140 args = self._parse_csv(self._parse_disjunction) 8141 8142 if len(args) < 3: 8143 return self.expression(exp.Decode(this=seq_get(args, 0), charset=seq_get(args, 1))) 8144 8145 return self.expression(exp.DecodeCase(expressions=args)) 8146 8147 def _parse_json_key_value(self) -> exp.JSONKeyValue | None: 8148 self._match_text_seq("KEY") 8149 key = self._parse_column() 8150 self._match_set(self.JSON_KEY_VALUE_SEPARATOR_TOKENS) 8151 self._match_text_seq("VALUE") 8152 value = self._parse_bitwise() 8153 8154 if not key and not value: 8155 return None 8156 return self.expression(exp.JSONKeyValue(this=key, expression=value)) 8157 8158 def _parse_format_json(self, this: exp.Expr | None) -> exp.Expr | None: 8159 if not this or not self._match_text_seq("FORMAT", "JSON"): 8160 return this 8161 8162 return self.expression(exp.FormatJson(this=this)) 8163 8164 def _parse_on_condition(self) -> exp.OnCondition | None: 8165 # MySQL uses "X ON EMPTY Y ON ERROR" (e.g. JSON_VALUE) while Oracle uses the opposite (e.g. JSON_EXISTS) 8166 if self.dialect.ON_CONDITION_EMPTY_BEFORE_ERROR: 8167 empty = self._parse_on_handling("EMPTY", *self.ON_CONDITION_TOKENS) 8168 error = self._parse_on_handling("ERROR", *self.ON_CONDITION_TOKENS) 8169 else: 8170 error = self._parse_on_handling("ERROR", *self.ON_CONDITION_TOKENS) 8171 empty = self._parse_on_handling("EMPTY", *self.ON_CONDITION_TOKENS) 8172 8173 null = self._parse_on_handling("NULL", *self.ON_CONDITION_TOKENS) 8174 8175 if not empty and not error and not null: 8176 return None 8177 8178 return self.expression(exp.OnCondition(empty=empty, error=error, null=null)) 8179 8180 def _parse_on_handling(self, on: str, *values: str) -> str | None | exp.Expr | None: 8181 # Parses the "X ON Y" or "DEFAULT <expr> ON Y syntax, e.g. NULL ON NULL (Oracle, T-SQL, MySQL) 8182 for value in values: 8183 if self._match_text_seq(value, "ON", on): 8184 return f"{value} ON {on}" 8185 8186 index = self._index 8187 if self._match(TokenType.DEFAULT): 8188 default_value = self._parse_bitwise() 8189 if self._match_text_seq("ON", on): 8190 return default_value 8191 8192 self._retreat(index) 8193 8194 return None 8195 8196 @t.overload 8197 def _parse_json_object(self, agg: t.Literal[False]) -> exp.JSONObject: ... 8198 8199 @t.overload 8200 def _parse_json_object(self, agg: t.Literal[True]) -> exp.JSONObjectAgg: ... 8201 8202 def _parse_json_object(self, agg=False): 8203 star = self._parse_star() 8204 expressions = ( 8205 [star] 8206 if star 8207 else self._parse_csv(lambda: self._parse_format_json(self._parse_json_key_value())) 8208 ) 8209 null_handling = self._parse_on_handling("NULL", "NULL", "ABSENT") 8210 8211 unique_keys = None 8212 if self._match_text_seq("WITH", "UNIQUE"): 8213 unique_keys = True 8214 elif self._match_text_seq("WITHOUT", "UNIQUE"): 8215 unique_keys = False 8216 8217 self._match_text_seq("KEYS") 8218 8219 return_type = self._match_text_seq("RETURNING") and self._parse_format_json( 8220 self._parse_type() 8221 ) 8222 encoding = self._match_text_seq("ENCODING") and self._parse_var() 8223 8224 return self.expression( 8225 (exp.JSONObjectAgg if agg else exp.JSONObject)( 8226 expressions=expressions, 8227 null_handling=null_handling, 8228 unique_keys=unique_keys, 8229 return_type=return_type, 8230 encoding=encoding, 8231 ) 8232 ) 8233 8234 # Note: this is currently incomplete; it only implements the "JSON_value_column" part 8235 def _parse_json_column_def(self) -> exp.JSONColumnDef: 8236 if not self._match_text_seq("NESTED"): 8237 this = self._parse_id_var() 8238 ordinality = self._match_pair(TokenType.FOR, TokenType.ORDINALITY) 8239 kind = self._parse_types(allow_identifiers=False) 8240 nested = None 8241 else: 8242 this = None 8243 ordinality = None 8244 kind = None 8245 nested = True 8246 8247 format_json = self._match_text_seq("FORMAT", "JSON") 8248 path = self._match_text_seq("PATH") and self._parse_string() 8249 nested_schema = nested and self._parse_json_schema() 8250 8251 return self.expression( 8252 exp.JSONColumnDef( 8253 this=this, 8254 kind=kind, 8255 path=path, 8256 nested_schema=nested_schema, 8257 ordinality=ordinality, 8258 format_json=format_json, 8259 ) 8260 ) 8261 8262 def _parse_json_schema(self) -> exp.JSONSchema: 8263 self._match_text_seq("COLUMNS") 8264 return self.expression( 8265 exp.JSONSchema( 8266 expressions=self._parse_wrapped_csv(self._parse_json_column_def, optional=True) 8267 ) 8268 ) 8269 8270 def _parse_json_table(self) -> exp.JSONTable: 8271 this = self._parse_format_json(self._parse_bitwise()) 8272 path = self._match(TokenType.COMMA) and self._parse_string() 8273 error_handling = self._parse_on_handling("ERROR", "ERROR", "NULL") 8274 empty_handling = self._parse_on_handling("EMPTY", "ERROR", "NULL") 8275 schema = self._parse_json_schema() 8276 8277 return exp.JSONTable( 8278 this=this, 8279 schema=schema, 8280 path=path, 8281 error_handling=error_handling, 8282 empty_handling=empty_handling, 8283 ) 8284 8285 def _parse_match_against(self) -> exp.MatchAgainst: 8286 if self._match_text_seq("TABLE"): 8287 # parse SingleStore MATCH(TABLE ...) syntax 8288 # https://docs.singlestore.com/cloud/reference/sql-reference/full-text-search-functions/match/ 8289 expressions = [] 8290 table = self._parse_table() 8291 if table: 8292 expressions = [table] 8293 else: 8294 expressions = self._parse_csv(self._parse_column) 8295 8296 self._match_text_seq(")", "AGAINST", "(") 8297 8298 this = self._parse_string() 8299 8300 if self._match_text_seq("IN", "NATURAL", "LANGUAGE", "MODE"): 8301 modifier = "IN NATURAL LANGUAGE MODE" 8302 if self._match_text_seq("WITH", "QUERY", "EXPANSION"): 8303 modifier = f"{modifier} WITH QUERY EXPANSION" 8304 elif self._match_text_seq("IN", "BOOLEAN", "MODE"): 8305 modifier = "IN BOOLEAN MODE" 8306 elif self._match_text_seq("WITH", "QUERY", "EXPANSION"): 8307 modifier = "WITH QUERY EXPANSION" 8308 else: 8309 modifier = None 8310 8311 return self.expression( 8312 exp.MatchAgainst(this=this, expressions=expressions, modifier=modifier) 8313 ) 8314 8315 # https://learn.microsoft.com/en-us/sql/t-sql/functions/openjson-transact-sql?view=sql-server-ver16 8316 def _parse_open_json(self) -> exp.OpenJSON: 8317 this = self._parse_bitwise() 8318 path = self._match(TokenType.COMMA) and self._parse_string() 8319 8320 def _parse_open_json_column_def() -> exp.OpenJSONColumnDef: 8321 this = self._parse_field(any_token=True) 8322 kind = self._parse_types() 8323 path = self._parse_string() 8324 as_json = self._match_pair(TokenType.ALIAS, TokenType.JSON) 8325 8326 return self.expression( 8327 exp.OpenJSONColumnDef(this=this, kind=kind, path=path, as_json=as_json) 8328 ) 8329 8330 expressions = None 8331 if self._match_pair(TokenType.R_PAREN, TokenType.WITH): 8332 self._match_l_paren() 8333 expressions = self._parse_csv(_parse_open_json_column_def) 8334 8335 return self.expression(exp.OpenJSON(this=this, path=path, expressions=expressions)) 8336 8337 def _parse_position(self, haystack_first: bool = False) -> exp.StrPosition: 8338 args = self._parse_csv(self._parse_bitwise) 8339 8340 if self._match(TokenType.IN): 8341 return self.expression( 8342 exp.StrPosition(this=self._parse_bitwise(), substr=seq_get(args, 0)) 8343 ) 8344 8345 if haystack_first: 8346 haystack = seq_get(args, 0) 8347 needle = seq_get(args, 1) 8348 else: 8349 haystack = seq_get(args, 1) 8350 needle = seq_get(args, 0) 8351 8352 return self.expression( 8353 exp.StrPosition(this=haystack, substr=needle, position=seq_get(args, 2)) 8354 ) 8355 8356 def _parse_join_hint(self, func_name: str) -> exp.JoinHint: 8357 args = self._parse_csv(self._parse_table) 8358 return exp.JoinHint(this=func_name.upper(), expressions=args) 8359 8360 def _parse_substring(self) -> exp.Substring: 8361 # Postgres supports the form: substring(string [from int] [for int]) 8362 # (despite being undocumented, the reverse order also works) 8363 # https://www.postgresql.org/docs/9.1/functions-string.html @ Table 9-6 8364 8365 args = t.cast(list[t.Optional[exp.Expr]], self._parse_csv(self._parse_bitwise)) 8366 8367 start, length = None, None 8368 8369 while self._curr: 8370 if self._match(TokenType.FROM): 8371 start = self._parse_bitwise() 8372 elif self._match(TokenType.FOR): 8373 if not start: 8374 start = exp.Literal.number(1) 8375 length = self._parse_bitwise() 8376 else: 8377 break 8378 8379 if start: 8380 args.append(start) 8381 if length: 8382 args.append(length) 8383 8384 return self.validate_expression(exp.Substring.from_arg_list(args), args) 8385 8386 def _parse_trim(self) -> exp.Trim: 8387 # https://www.w3resource.com/sql/character-functions/trim.php 8388 # https://docs.oracle.com/javadb/10.8.3.0/ref/rreftrimfunc.html 8389 8390 position = None 8391 collation = None 8392 expression = None 8393 8394 if self._match_texts(self.TRIM_TYPES): 8395 position = self._prev.text.upper() 8396 8397 this = self._parse_bitwise() 8398 if self._match_set((TokenType.FROM, TokenType.COMMA)): 8399 invert_order = self._prev.token_type == TokenType.FROM or self.TRIM_PATTERN_FIRST 8400 expression = self._parse_bitwise() 8401 8402 if invert_order: 8403 this, expression = expression, this 8404 8405 if self._match(TokenType.COLLATE): 8406 collation = self._parse_bitwise() 8407 8408 return self.expression( 8409 exp.Trim(this=this, position=position, expression=expression, collation=collation) 8410 ) 8411 8412 def _parse_window_clause(self) -> list[exp.Expr] | None: 8413 return self._parse_csv(self._parse_named_window) if self._match(TokenType.WINDOW) else None 8414 8415 def _parse_named_window(self) -> exp.Expr | None: 8416 return self._parse_window(self._parse_id_var(), alias=True) 8417 8418 def _parse_respect_or_ignore_nulls(self, this: exp.Expr | None) -> exp.Expr | None: 8419 if self._curr.token_type == TokenType.VAR: 8420 if self._match_text_seq("IGNORE", "NULLS"): 8421 return self.expression(exp.IgnoreNulls(this=this)) 8422 if self._match_text_seq("RESPECT", "NULLS"): 8423 return self.expression(exp.RespectNulls(this=this)) 8424 return this 8425 8426 def _parse_having_max(self, this: exp.Expr | None) -> exp.Expr | None: 8427 if self._match(TokenType.HAVING): 8428 self._match_texts(("MAX", "MIN")) 8429 max = self._prev.text.upper() != "MIN" 8430 return self.expression( 8431 exp.HavingMax(this=this, expression=self._parse_column(), max=max) 8432 ) 8433 8434 return this 8435 8436 def _parse_window(self, this: exp.Expr | None, alias: bool = False) -> exp.Expr | None: 8437 func = this 8438 comments = func.comments if isinstance(func, exp.Expr) else None 8439 8440 # T-SQL allows the OVER (...) syntax after WITHIN GROUP. 8441 # https://learn.microsoft.com/en-us/sql/t-sql/functions/percentile-disc-transact-sql?view=sql-server-ver16 8442 if self._match_text_seq("WITHIN", "GROUP"): 8443 order = self._parse_wrapped(self._parse_order) 8444 this = self.expression(exp.WithinGroup(this=this, expression=order)) 8445 8446 if self._match_pair(TokenType.FILTER, TokenType.L_PAREN): 8447 self._match(TokenType.WHERE) 8448 this = self.expression( 8449 exp.Filter(this=this, expression=self._parse_where(skip_where_token=True)) 8450 ) 8451 self._match_r_paren() 8452 8453 # SQL spec defines an optional [ { IGNORE | RESPECT } NULLS ] OVER 8454 # Some dialects choose to implement and some do not. 8455 # https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html 8456 8457 # There is some code above in _parse_lambda that handles 8458 # SELECT FIRST_VALUE(TABLE.COLUMN IGNORE|RESPECT NULLS) OVER ... 8459 8460 # The below changes handle 8461 # SELECT FIRST_VALUE(TABLE.COLUMN) IGNORE|RESPECT NULLS OVER ... 8462 8463 # Oracle allows both formats 8464 # (https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/img_text/first_value.html) 8465 # and Snowflake chose to do the same for familiarity 8466 # https://docs.snowflake.com/en/sql-reference/functions/first_value.html#usage-notes 8467 if isinstance(this, exp.AggFunc): 8468 ignore_respect = this.find(exp.IgnoreNulls, exp.RespectNulls) 8469 8470 if ignore_respect and ignore_respect is not this: 8471 ignore_respect.replace(ignore_respect.this) 8472 this = self.expression(ignore_respect.__class__(this=this)) 8473 8474 this = self._parse_respect_or_ignore_nulls(this) 8475 8476 # bigquery select from window x AS (partition by ...) 8477 if alias: 8478 over = None 8479 self._match(TokenType.ALIAS) 8480 elif not self._match_set(self.WINDOW_BEFORE_PAREN_TOKENS): 8481 return this 8482 else: 8483 over = self._prev.text.upper() 8484 8485 if comments and isinstance(func, exp.Expr): 8486 func.pop_comments() 8487 8488 if not self._match(TokenType.L_PAREN): 8489 return self.expression( 8490 exp.Window(this=this, alias=self._parse_id_var(False), over=over), comments=comments 8491 ) 8492 8493 window_alias = self._parse_id_var(any_token=False, tokens=self.WINDOW_ALIAS_TOKENS) 8494 8495 first: bool | None = True if self._match(TokenType.FIRST) else None 8496 if self._match_text_seq("LAST"): 8497 first = False 8498 8499 partition, order = self._parse_partition_and_order() 8500 kind = ( 8501 self._match_set((TokenType.ROWS, TokenType.RANGE)) or self._match_text_seq("GROUPS") 8502 ) and self._prev.text 8503 8504 if kind: 8505 self._match(TokenType.BETWEEN) 8506 start = self._parse_window_spec() 8507 8508 end = self._parse_window_spec() if self._match(TokenType.AND) else {} 8509 exclude = ( 8510 self._parse_var_from_options(self.WINDOW_EXCLUDE_OPTIONS) 8511 if self._match_text_seq("EXCLUDE") 8512 else None 8513 ) 8514 8515 spec = self.expression( 8516 exp.WindowSpec( 8517 kind=kind, 8518 start=start["value"], 8519 start_side=start["side"], 8520 end=end.get("value"), 8521 end_side=end.get("side"), 8522 exclude=exclude, 8523 ) 8524 ) 8525 else: 8526 spec = None 8527 8528 self._match_r_paren() 8529 8530 window = self.expression( 8531 exp.Window( 8532 this=this, 8533 partition_by=partition, 8534 order=order, 8535 spec=spec, 8536 alias=window_alias, 8537 over=over, 8538 first=first, 8539 ), 8540 comments=comments, 8541 ) 8542 8543 # This covers Oracle's FIRST/LAST syntax: aggregate KEEP (...) OVER (...) 8544 if self._match_set(self.WINDOW_BEFORE_PAREN_TOKENS, advance=False): 8545 return self._parse_window(window, alias=alias) 8546 8547 return window 8548 8549 def _parse_partition_and_order( 8550 self, 8551 ) -> tuple[list[exp.Expr], exp.Expr | None]: 8552 return self._parse_partition_by(), self._parse_order() 8553 8554 def _parse_window_spec(self) -> dict[str, str | exp.Expr | None]: 8555 self._match(TokenType.BETWEEN) 8556 8557 return { 8558 "value": ( 8559 (self._match_text_seq("UNBOUNDED") and "UNBOUNDED") 8560 or (self._match_text_seq("CURRENT", "ROW") and "CURRENT ROW") 8561 or self._parse_bitwise() 8562 ), 8563 "side": self._prev.text if self._match_texts(self.WINDOW_SIDES) else None, 8564 } 8565 8566 def _parse_alias(self, this: exp.Expr | None, explicit: bool = False) -> exp.Expr | None: 8567 # In some dialects, LIMIT and OFFSET can act as both identifiers and keywords (clauses) 8568 # so this section tries to parse the clause version and if it fails, it treats the token 8569 # as an identifier (alias) 8570 if self._can_parse_limit_or_offset(): 8571 return this 8572 8573 # WINDOW is in ID_VAR_TOKENS, so it can be consumed as an implicit alias. Detect the 8574 # named-window clause shape (`WINDOW <ident> AS (...)`) and avoid swallowing it. 8575 if self._can_parse_named_window(): 8576 return this 8577 8578 any_token = self._match(TokenType.ALIAS) 8579 comments = self._prev_comments 8580 8581 if explicit and not any_token: 8582 return this 8583 8584 if self._match(TokenType.L_PAREN): 8585 aliases = self.expression( 8586 exp.Aliases( 8587 this=this, expressions=self._parse_csv(lambda: self._parse_id_var(any_token)) 8588 ), 8589 comments=comments, 8590 ) 8591 self._match_r_paren(aliases) 8592 return aliases 8593 8594 alias = self._parse_id_var(any_token, tokens=self.ALIAS_TOKENS) or ( 8595 self.STRING_ALIASES and self._parse_string_as_identifier() 8596 ) 8597 8598 if alias: 8599 comments.extend(alias.pop_comments()) 8600 this = self.expression(exp.Alias(this=this, alias=alias), comments=comments) 8601 column = this.this 8602 8603 # Moves the comment next to the alias in `expr /* comment */ AS alias` 8604 if not this.comments and column and column.comments: 8605 this.comments = column.pop_comments() 8606 8607 return this 8608 8609 def _parse_id_var( 8610 self, 8611 any_token: bool = True, 8612 tokens: t.Collection[TokenType] | None = None, 8613 ) -> exp.Expr | None: 8614 expression = self._parse_identifier() 8615 if not expression and ( 8616 (any_token and self._advance_any()) or self._match_set(tokens or self.ID_VAR_TOKENS) 8617 ): 8618 quoted = self._prev.token_type == TokenType.STRING 8619 expression = self._identifier_expression(quoted=quoted) 8620 8621 return expression 8622 8623 def _parse_string(self) -> exp.Expr | None: 8624 if self._match_set(self.STRING_PARSERS): 8625 return self.STRING_PARSERS[self._prev.token_type](self, self._prev) 8626 return self._parse_placeholder() 8627 8628 def _parse_string_as_identifier(self) -> exp.Identifier | None: 8629 if not self._match(TokenType.STRING): 8630 return None 8631 output = exp.to_identifier(self._prev.text, quoted=True) 8632 output.update_positions(self._prev) 8633 return output 8634 8635 def _parse_number(self) -> exp.Expr | None: 8636 if self._match_set(self.NUMERIC_PARSERS): 8637 return self.NUMERIC_PARSERS[self._prev.token_type](self, self._prev) 8638 return self._parse_placeholder() 8639 8640 def _parse_identifier(self) -> exp.Expr | None: 8641 if self._match(TokenType.IDENTIFIER): 8642 return self._identifier_expression(quoted=True) 8643 return self._parse_placeholder() 8644 8645 def _parse_var( 8646 self, 8647 any_token: bool = False, 8648 tokens: t.Collection[TokenType] | None = None, 8649 upper: bool = False, 8650 ) -> exp.Expr | None: 8651 if ( 8652 (any_token and self._advance_any()) 8653 or self._match(TokenType.VAR) 8654 or (self._match_set(tokens) if tokens else False) 8655 ): 8656 return self.expression( 8657 exp.Var(this=self._prev.text.upper() if upper else self._prev.text) 8658 ) 8659 return self._parse_placeholder() 8660 8661 def _advance_any(self, ignore_reserved: bool = False) -> Token | None: 8662 if self._curr and (ignore_reserved or self._curr.token_type not in self.RESERVED_TOKENS): 8663 self._advance() 8664 return self._prev 8665 return None 8666 8667 def _parse_var_or_string(self, upper: bool = False) -> exp.Expr | None: 8668 return self._parse_string() or self._parse_var(any_token=True, upper=upper) 8669 8670 def _parse_primary_or_var(self) -> exp.Expr | None: 8671 return self._parse_primary() or self._parse_var(any_token=True) 8672 8673 def _parse_null(self) -> exp.Expr | None: 8674 if self._match_set((TokenType.NULL, TokenType.UNKNOWN)): 8675 return self.PRIMARY_PARSERS[TokenType.NULL](self, self._prev) 8676 return self._parse_placeholder() 8677 8678 def _parse_boolean(self) -> exp.Expr | None: 8679 if self._match(TokenType.TRUE): 8680 return self.PRIMARY_PARSERS[TokenType.TRUE](self, self._prev) 8681 if self._match(TokenType.FALSE): 8682 return self.PRIMARY_PARSERS[TokenType.FALSE](self, self._prev) 8683 return self._parse_placeholder() 8684 8685 def _parse_star(self) -> exp.Expr | None: 8686 if self._match(TokenType.STAR): 8687 return self.PRIMARY_PARSERS[TokenType.STAR](self, self._prev) 8688 return self._parse_placeholder() 8689 8690 def _parse_parameter(self) -> exp.Parameter: 8691 this = self._parse_identifier() or self._parse_primary_or_var() 8692 return self.expression(exp.Parameter(this=this)) 8693 8694 def _parse_placeholder(self) -> exp.Expr | None: 8695 if self._match_set(self.PLACEHOLDER_PARSERS): 8696 placeholder = self.PLACEHOLDER_PARSERS[self._prev.token_type](self) 8697 if placeholder: 8698 return placeholder 8699 self._advance(-1) 8700 return None 8701 8702 def _parse_star_op(self, *keywords: str) -> list[exp.Expr] | None: 8703 if not self._match_texts(keywords): 8704 return None 8705 if self._match(TokenType.L_PAREN, advance=False): 8706 return self._parse_wrapped_csv(self._parse_expression) 8707 8708 expression = self._parse_alias(self._parse_disjunction(), explicit=True) 8709 return [expression] if expression else None 8710 8711 def _parse_csv( 8712 self, parse_method: t.Callable[[], T | None], sep: TokenType = TokenType.COMMA 8713 ) -> list[T]: 8714 parse_result = parse_method() 8715 items = [parse_result] if parse_result is not None else [] 8716 8717 while self._match(sep): 8718 if isinstance(parse_result, exp.Expr): 8719 self._add_comments(parse_result) 8720 parse_result = parse_method() 8721 if parse_result is not None: 8722 items.append(parse_result) 8723 8724 return items 8725 8726 def _parse_wrapped_id_vars(self, optional: bool = False) -> list[exp.Expr]: 8727 return self._parse_wrapped_csv(self._parse_id_var, optional=optional) 8728 8729 def _parse_wrapped_csv( 8730 self, 8731 parse_method: t.Callable[[], T | None], 8732 sep: TokenType = TokenType.COMMA, 8733 optional: bool = False, 8734 ) -> list[T]: 8735 return self._parse_wrapped( 8736 lambda: self._parse_csv(parse_method, sep=sep), optional=optional 8737 ) 8738 8739 def _parse_wrapped(self, parse_method: t.Callable[[], T], optional: bool = False) -> T: 8740 wrapped = self._match(TokenType.L_PAREN) 8741 if not wrapped and not optional: 8742 self.raise_error("Expecting (") 8743 parse_result = parse_method() 8744 if wrapped: 8745 self._match_r_paren() 8746 return parse_result 8747 8748 def _parse_expressions(self) -> list[exp.Expr]: 8749 return self._parse_csv(self._parse_expression) 8750 8751 def _parse_select_or_expression(self, alias: bool = False) -> exp.Expr | None: 8752 return ( 8753 self._parse_set_operations( 8754 self._parse_alias(self._parse_assignment(), explicit=True) 8755 if alias 8756 else self._parse_assignment() 8757 ) 8758 or self._parse_select() 8759 ) 8760 8761 def _parse_ddl_select(self) -> exp.Expr | None: 8762 return self._parse_query_modifiers( 8763 self._parse_set_operations(self._parse_select(nested=True, parse_subquery_alias=False)) 8764 ) 8765 8766 def _parse_transaction(self) -> exp.Transaction | exp.Command: 8767 this = None 8768 if self._match_texts(self.TRANSACTION_KIND): 8769 this = self._prev.text 8770 8771 self._match_texts(("TRANSACTION", "WORK")) 8772 8773 modes = [] 8774 while True: 8775 mode = [] 8776 while self._match(TokenType.VAR) or self._match(TokenType.NOT): 8777 mode.append(self._prev.text) 8778 8779 if mode: 8780 modes.append(" ".join(mode)) 8781 if not self._match(TokenType.COMMA): 8782 break 8783 8784 return self.expression(exp.Transaction(this=this, modes=modes)) 8785 8786 def _parse_commit_or_rollback(self) -> exp.Commit | exp.Rollback: 8787 chain = None 8788 savepoint = None 8789 is_rollback = self._prev.token_type == TokenType.ROLLBACK 8790 8791 self._match_texts(("TRANSACTION", "WORK")) 8792 8793 if self._match_text_seq("TO"): 8794 self._match_text_seq("SAVEPOINT") 8795 savepoint = self._parse_id_var() 8796 8797 if self._match(TokenType.AND): 8798 chain = not self._match_text_seq("NO") 8799 self._match_text_seq("CHAIN") 8800 8801 if is_rollback: 8802 return self.expression(exp.Rollback(savepoint=savepoint)) 8803 8804 return self.expression(exp.Commit(chain=chain)) 8805 8806 def _parse_refresh(self) -> exp.Refresh | exp.Command: 8807 if self._match(TokenType.TABLE): 8808 kind = "TABLE" 8809 elif self._match_text_seq("MATERIALIZED", "VIEW"): 8810 kind = "MATERIALIZED VIEW" 8811 else: 8812 kind = "" 8813 8814 this = self._parse_string() or self._parse_table() 8815 if not kind and not isinstance(this, exp.Literal): 8816 return self._parse_as_command(self._prev) 8817 8818 return self.expression(exp.Refresh(this=this, kind=kind)) 8819 8820 def _parse_column_def_with_exists(self): 8821 start = self._index 8822 self._match(TokenType.COLUMN) 8823 8824 exists_column = self._parse_exists(not_=True) 8825 expression = self._parse_field_def() 8826 8827 if not isinstance(expression, exp.ColumnDef): 8828 self._retreat(start) 8829 return None 8830 8831 expression.set("exists", exists_column) 8832 8833 return expression 8834 8835 def _parse_add_column(self) -> exp.ColumnDef | None: 8836 if not self._prev.text.upper() == "ADD": 8837 return None 8838 8839 return self._parse_column_def_with_exists() 8840 8841 def _parse_drop_column(self) -> exp.Drop | exp.Command | None: 8842 drop = self._parse_drop() if self._match(TokenType.DROP) else None 8843 if drop and not isinstance(drop, exp.Command): 8844 drop.set("kind", drop.args.get("kind", "COLUMN")) 8845 return drop 8846 8847 def _parse_alter_drop_action(self) -> exp.Expr | None: 8848 return self._parse_drop_column() 8849 8850 # https://docs.aws.amazon.com/athena/latest/ug/alter-table-drop-partition.html 8851 def _parse_drop_partition(self, exists: bool | None = None) -> exp.DropPartition: 8852 return self.expression( 8853 exp.DropPartition(expressions=self._parse_csv(self._parse_partition), exists=exists) 8854 ) 8855 8856 def _parse_alter_table_add(self) -> list[exp.Expr]: 8857 def _parse_add_alteration() -> exp.Expr | None: 8858 self._match_text_seq("ADD") 8859 if self._match_set(self.ADD_CONSTRAINT_TOKENS, advance=False): 8860 return self.expression( 8861 exp.AddConstraint(expressions=self._parse_csv(self._parse_constraint)) 8862 ) 8863 8864 column_def = self._parse_add_column() 8865 if isinstance(column_def, exp.ColumnDef): 8866 return column_def 8867 8868 exists = self._parse_exists(not_=True) 8869 if self._match_pair(TokenType.PARTITION, TokenType.L_PAREN, advance=False): 8870 return self.expression( 8871 exp.AddPartition( 8872 exists=exists, 8873 this=self._parse_field(any_token=True), 8874 location=self._match_text_seq("LOCATION", advance=False) 8875 and self._parse_property(), 8876 ) 8877 ) 8878 8879 return None 8880 8881 if not self._match_set(self.ADD_CONSTRAINT_TOKENS, advance=False) and ( 8882 not self.dialect.ALTER_TABLE_ADD_REQUIRED_FOR_EACH_COLUMN 8883 or self._match_text_seq("COLUMNS") 8884 ): 8885 schema = self._parse_schema() 8886 8887 return ( 8888 ensure_list(schema) 8889 if schema 8890 else self._parse_csv(self._parse_column_def_with_exists) 8891 ) 8892 8893 return self._parse_csv(_parse_add_alteration) 8894 8895 def _parse_alter_table_alter(self) -> exp.Expr | None: 8896 if self._match_texts(self.ALTER_ALTER_PARSERS): 8897 return self.ALTER_ALTER_PARSERS[self._prev.text.upper()](self) 8898 8899 # Many dialects support the ALTER [COLUMN] syntax, so if there is no 8900 # keyword after ALTER we default to parsing this statement 8901 self._match(TokenType.COLUMN) 8902 column = self._parse_field(any_token=True) 8903 8904 if self._match_pair(TokenType.DROP, TokenType.DEFAULT): 8905 return self.expression(exp.AlterColumn(this=column, drop=True)) 8906 if self._match_pair(TokenType.SET, TokenType.DEFAULT): 8907 return self.expression(exp.AlterColumn(this=column, default=self._parse_disjunction())) 8908 if self._match(TokenType.COMMENT): 8909 return self.expression(exp.AlterColumn(this=column, comment=self._parse_string())) 8910 if self._match_text_seq("DROP", "NOT", "NULL"): 8911 return self.expression(exp.AlterColumn(this=column, drop=True, allow_null=True)) 8912 if self._match_text_seq("SET", "NOT", "NULL"): 8913 return self.expression(exp.AlterColumn(this=column, allow_null=False)) 8914 8915 if self._match_text_seq("SET", "VISIBLE"): 8916 return self.expression(exp.AlterColumn(this=column, visible="VISIBLE")) 8917 if self._match_text_seq("SET", "INVISIBLE"): 8918 return self.expression(exp.AlterColumn(this=column, visible="INVISIBLE")) 8919 8920 self._match_text_seq("SET", "DATA") 8921 self._match_text_seq("TYPE") 8922 return self.expression( 8923 exp.AlterColumn( 8924 this=column, 8925 dtype=self._parse_types(), 8926 collate=self._match(TokenType.COLLATE) and self._parse_term(), 8927 using=self._match(TokenType.USING) and self._parse_disjunction(), 8928 ) 8929 ) 8930 8931 def _parse_alter_diststyle(self) -> exp.AlterDistStyle: 8932 if self._match_texts(("ALL", "EVEN", "AUTO")): 8933 return self.expression(exp.AlterDistStyle(this=exp.var(self._prev.text.upper()))) 8934 8935 self._match_text_seq("KEY", "DISTKEY") 8936 return self.expression(exp.AlterDistStyle(this=self._parse_column())) 8937 8938 def _parse_alter_sortkey(self, compound: bool | None = None) -> exp.AlterSortKey: 8939 if compound: 8940 self._match_text_seq("SORTKEY") 8941 8942 if self._match(TokenType.L_PAREN, advance=False): 8943 return self.expression( 8944 exp.AlterSortKey(expressions=self._parse_wrapped_id_vars(), compound=compound) 8945 ) 8946 8947 self._match_texts(("AUTO", "NONE")) 8948 return self.expression( 8949 exp.AlterSortKey(this=exp.var(self._prev.text.upper()), compound=compound) 8950 ) 8951 8952 def _parse_alter_table_drop(self) -> list[exp.Expr]: 8953 index = self._index - 1 8954 8955 partition_exists = self._parse_exists() 8956 if self._match(TokenType.PARTITION, advance=False): 8957 return self._parse_csv(lambda: self._parse_drop_partition(exists=partition_exists)) 8958 8959 self._retreat(index) 8960 return self._parse_csv(self._parse_alter_drop_action) 8961 8962 def _parse_alter_table_rename(self) -> exp.AlterRename | exp.RenameColumn | None: 8963 if self._match(TokenType.COLUMN) or ( 8964 not self.ALTER_RENAME_REQUIRES_COLUMN and not self._match_text_seq("TO", advance=False) 8965 ): 8966 exists = self._parse_exists() 8967 old_column = self._parse_column() 8968 to = self._match_text_seq("TO") 8969 new_column = self._parse_column() 8970 8971 if old_column is None or not to or new_column is None: 8972 return None 8973 8974 return self.expression(exp.RenameColumn(this=old_column, to=new_column, exists=exists)) 8975 8976 self._match_text_seq("TO") 8977 return self.expression(exp.AlterRename(this=self._parse_table(schema=True))) 8978 8979 def _parse_alter_table_set(self) -> exp.AlterSet: 8980 alter_set = self.expression(exp.AlterSet()) 8981 8982 if self._match(TokenType.L_PAREN, advance=False) or self._match_text_seq( 8983 "TABLE", "PROPERTIES" 8984 ): 8985 alter_set.set("expressions", self._parse_wrapped_csv(self._parse_assignment)) 8986 elif self._match_text_seq("FILESTREAM_ON", advance=False): 8987 alter_set.set("expressions", [self._parse_assignment()]) 8988 elif self._match_texts(("LOGGED", "UNLOGGED")): 8989 alter_set.set("option", exp.var(self._prev.text.upper())) 8990 elif self._match_text_seq("WITHOUT") and self._match_texts(("CLUSTER", "OIDS")): 8991 alter_set.set("option", exp.var(f"WITHOUT {self._prev.text.upper()}")) 8992 elif self._match_text_seq("LOCATION"): 8993 alter_set.set("location", self._parse_field()) 8994 elif self._match_text_seq("ACCESS", "METHOD"): 8995 alter_set.set("access_method", self._parse_field()) 8996 elif self._match_text_seq("TABLESPACE"): 8997 alter_set.set("tablespace", self._parse_field()) 8998 elif self._match_text_seq("FILE", "FORMAT") or self._match_text_seq("FILEFORMAT"): 8999 alter_set.set("file_format", [self._parse_field()]) 9000 elif self._match_text_seq("STAGE_FILE_FORMAT"): 9001 alter_set.set("file_format", self._parse_wrapped_options()) 9002 elif self._match_text_seq("STAGE_COPY_OPTIONS"): 9003 alter_set.set("copy_options", self._parse_wrapped_options()) 9004 elif self._match_text_seq("TAG") or self._match_text_seq("TAGS"): 9005 alter_set.set("tag", self._parse_csv(self._parse_assignment)) 9006 else: 9007 if self._match_text_seq("SERDE"): 9008 alter_set.set("serde", self._parse_field()) 9009 9010 properties = self._parse_wrapped(self._parse_properties, optional=True) 9011 alter_set.set("expressions", [properties]) 9012 9013 return alter_set 9014 9015 def _parse_alter_session(self) -> exp.AlterSession: 9016 """Parse ALTER SESSION SET/UNSET statements.""" 9017 if self._match(TokenType.SET): 9018 expressions = self._parse_csv(lambda: self._parse_set_item_assignment()) 9019 return self.expression(exp.AlterSession(expressions=expressions, unset=False)) 9020 9021 self._match_text_seq("UNSET") 9022 expressions = self._parse_csv( 9023 lambda: self.expression(exp.SetItem(this=self._parse_id_var(any_token=True))) 9024 ) 9025 return self.expression(exp.AlterSession(expressions=expressions, unset=True)) 9026 9027 def _parse_alter(self) -> exp.Alter | exp.Command: 9028 start = self._prev 9029 9030 iceberg = self._match_text_seq("ICEBERG") 9031 9032 alter_token = self._match_set(self.ALTERABLES) and self._prev 9033 if not alter_token: 9034 return self._parse_as_command(start) 9035 if iceberg and alter_token.token_type != TokenType.TABLE: 9036 return self._parse_as_command(start) 9037 9038 exists = self._parse_exists() 9039 only = self._match_text_seq("ONLY") 9040 9041 if alter_token.token_type == TokenType.SESSION: 9042 this = None 9043 check = None 9044 cluster = None 9045 else: 9046 this = self._parse_table(schema=True, parse_partition=self.ALTER_TABLE_PARTITIONS) 9047 check = self._match_text_seq("WITH", "CHECK") 9048 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 9049 9050 if self._next: 9051 self._advance() 9052 9053 parser = self.ALTER_PARSERS.get(self._prev.text.upper()) if self._prev else None 9054 if parser: 9055 actions = ensure_list(parser(self)) 9056 not_valid = self._match_text_seq("NOT", "VALID") 9057 options = self._parse_csv(self._parse_property) 9058 cascade = self.dialect.ALTER_TABLE_SUPPORTS_CASCADE and self._match_text_seq("CASCADE") 9059 9060 if not self._curr and actions: 9061 return self.expression( 9062 exp.Alter( 9063 this=this, 9064 kind=alter_token.text.upper(), 9065 exists=exists, 9066 actions=actions, 9067 only=only, 9068 options=options, 9069 cluster=cluster, 9070 not_valid=not_valid, 9071 check=check, 9072 cascade=cascade, 9073 iceberg=iceberg, 9074 ) 9075 ) 9076 9077 return self._parse_as_command(start) 9078 9079 def _parse_analyze(self) -> exp.Analyze | exp.Command: 9080 start = self._prev 9081 # https://duckdb.org/docs/sql/statements/analyze 9082 if not self._curr: 9083 return self.expression(exp.Analyze()) 9084 9085 options = [] 9086 while self._match_texts(self.ANALYZE_STYLES): 9087 if self._prev.text.upper() == "BUFFER_USAGE_LIMIT": 9088 options.append(f"BUFFER_USAGE_LIMIT {self._parse_number()}") 9089 else: 9090 options.append(self._prev.text.upper()) 9091 9092 this: exp.Expr | None = None 9093 inner_expression: exp.Expr | None = None 9094 9095 kind = self._curr.text.upper() if self._curr else None 9096 9097 if self._match(TokenType.TABLE) or self._match(TokenType.INDEX): 9098 this = self._parse_table_parts() 9099 elif self._match_text_seq("TABLES"): 9100 if self._match_set((TokenType.FROM, TokenType.IN)): 9101 kind = f"{kind} {self._prev.text.upper()}" 9102 this = self._parse_table(schema=True, is_db_reference=True) 9103 elif self._match_text_seq("DATABASE"): 9104 this = self._parse_table(schema=True, is_db_reference=True) 9105 elif self._match_text_seq("CLUSTER"): 9106 this = self._parse_table() 9107 # Try matching inner expr keywords before fallback to parse table. 9108 elif self._match_texts(self.ANALYZE_EXPRESSION_PARSERS): 9109 kind = None 9110 inner_expression = self.ANALYZE_EXPRESSION_PARSERS[self._prev.text.upper()](self) 9111 else: 9112 # Empty kind https://prestodb.io/docs/current/sql/analyze.html 9113 kind = None 9114 this = self._parse_table_parts() 9115 9116 partition = self._try_parse(self._parse_partition) 9117 if not partition and self._match_texts(self.PARTITION_KEYWORDS): 9118 return self._parse_as_command(start) 9119 9120 # https://docs.starrocks.io/docs/sql-reference/sql-statements/cbo_stats/ANALYZE_TABLE/ 9121 if self._match_text_seq("WITH", "SYNC", "MODE") or self._match_text_seq( 9122 "WITH", "ASYNC", "MODE" 9123 ): 9124 mode = f"WITH {self._tokens[self._index - 2].text.upper()} MODE" 9125 else: 9126 mode = None 9127 9128 if self._match_texts(self.ANALYZE_EXPRESSION_PARSERS): 9129 inner_expression = self.ANALYZE_EXPRESSION_PARSERS[self._prev.text.upper()](self) 9130 9131 properties = self._parse_properties() 9132 return self.expression( 9133 exp.Analyze( 9134 kind=kind, 9135 this=this, 9136 mode=mode, 9137 partition=partition, 9138 properties=properties, 9139 expression=inner_expression, 9140 options=options, 9141 ) 9142 ) 9143 9144 # https://spark.apache.org/docs/3.5.1/sql-ref-syntax-aux-analyze-table.html 9145 def _parse_analyze_statistics(self) -> exp.AnalyzeStatistics: 9146 this = None 9147 kind = self._prev.text.upper() 9148 option = self._prev.text.upper() if self._match_text_seq("DELTA") else None 9149 expressions = [] 9150 9151 if not self._match_text_seq("STATISTICS"): 9152 self.raise_error("Expecting token STATISTICS") 9153 9154 if self._match_text_seq("NOSCAN"): 9155 this = "NOSCAN" 9156 elif self._match(TokenType.FOR): 9157 if self._match_text_seq("ALL", "COLUMNS"): 9158 this = "FOR ALL COLUMNS" 9159 if self._match_text_seq("COLUMNS"): 9160 this = "FOR COLUMNS" 9161 expressions = self._parse_csv(self._parse_column_reference) 9162 elif self._match_text_seq("SAMPLE"): 9163 sample = self._parse_number() 9164 expressions = [ 9165 self.expression( 9166 exp.AnalyzeSample( 9167 sample=sample, 9168 kind=self._prev.text.upper() if self._match(TokenType.PERCENT) else None, 9169 ) 9170 ) 9171 ] 9172 9173 return self.expression( 9174 exp.AnalyzeStatistics(kind=kind, option=option, this=this, expressions=expressions) 9175 ) 9176 9177 # https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/ANALYZE.html 9178 def _parse_analyze_validate(self) -> exp.AnalyzeValidate: 9179 kind = None 9180 this = None 9181 expression: exp.Expr | None = None 9182 if self._match_text_seq("REF", "UPDATE"): 9183 kind = "REF" 9184 this = "UPDATE" 9185 if self._match_text_seq("SET", "DANGLING", "TO", "NULL"): 9186 this = "UPDATE SET DANGLING TO NULL" 9187 elif self._match_text_seq("STRUCTURE"): 9188 kind = "STRUCTURE" 9189 if self._match_text_seq("CASCADE", "FAST"): 9190 this = "CASCADE FAST" 9191 elif self._match_text_seq("CASCADE", "COMPLETE") and self._match_texts( 9192 ("ONLINE", "OFFLINE") 9193 ): 9194 this = f"CASCADE COMPLETE {self._prev.text.upper()}" 9195 expression = self._parse_into() 9196 9197 return self.expression(exp.AnalyzeValidate(kind=kind, this=this, expression=expression)) 9198 9199 def _parse_analyze_columns(self) -> exp.AnalyzeColumns | None: 9200 this = self._prev.text.upper() 9201 if self._match_text_seq("COLUMNS"): 9202 return self.expression(exp.AnalyzeColumns(this=f"{this} {self._prev.text.upper()}")) 9203 return None 9204 9205 def _parse_analyze_delete(self) -> exp.AnalyzeDelete | None: 9206 kind = self._prev.text.upper() if self._match_text_seq("SYSTEM") else None 9207 if self._match_text_seq("STATISTICS"): 9208 return self.expression(exp.AnalyzeDelete(kind=kind)) 9209 return None 9210 9211 def _parse_analyze_list(self) -> exp.AnalyzeListChainedRows | None: 9212 if self._match_text_seq("CHAINED", "ROWS"): 9213 return self.expression(exp.AnalyzeListChainedRows(expression=self._parse_into())) 9214 return None 9215 9216 # https://dev.mysql.com/doc/refman/8.4/en/analyze-table.html 9217 def _parse_analyze_histogram(self) -> exp.AnalyzeHistogram: 9218 this = self._prev.text.upper() 9219 expression: exp.Expr | None = None 9220 expressions = [] 9221 update_options = None 9222 9223 if self._match_text_seq("HISTOGRAM", "ON"): 9224 expressions = self._parse_csv(self._parse_column_reference) 9225 with_expressions = [] 9226 while self._match(TokenType.WITH): 9227 # https://docs.starrocks.io/docs/sql-reference/sql-statements/cbo_stats/ANALYZE_TABLE/ 9228 if self._match_texts(("SYNC", "ASYNC")): 9229 if self._match_text_seq("MODE", advance=False): 9230 with_expressions.append(f"{self._prev.text.upper()} MODE") 9231 self._advance() 9232 else: 9233 buckets = self._parse_number() 9234 if self._match_text_seq("BUCKETS"): 9235 with_expressions.append(f"{buckets} BUCKETS") 9236 if with_expressions: 9237 expression = self.expression(exp.AnalyzeWith(expressions=with_expressions)) 9238 9239 if self._match_texts(("MANUAL", "AUTO")) and self._match( 9240 TokenType.UPDATE, advance=False 9241 ): 9242 update_options = self._prev.text.upper() 9243 self._advance() 9244 elif self._match_text_seq("USING", "DATA"): 9245 expression = self.expression(exp.UsingData(this=self._parse_string())) 9246 9247 return self.expression( 9248 exp.AnalyzeHistogram( 9249 this=this, 9250 expressions=expressions, 9251 expression=expression, 9252 update_options=update_options, 9253 ) 9254 ) 9255 9256 def _parse_merge(self) -> exp.Merge: 9257 self._match(TokenType.INTO) 9258 target = self._parse_table() 9259 9260 if target and self._match(TokenType.ALIAS, advance=False): 9261 target.set("alias", self._parse_table_alias()) 9262 9263 self._match(TokenType.USING) 9264 using = self._parse_table() 9265 9266 return self.expression( 9267 exp.Merge( 9268 this=target, 9269 using=using, 9270 on=self._match(TokenType.ON) and self._parse_disjunction(), 9271 using_cond=self._match(TokenType.USING) and self._parse_using_identifiers(), 9272 whens=self._parse_when_matched(), 9273 returning=self._parse_returning(), 9274 ) 9275 ) 9276 9277 def _parse_when_matched(self) -> exp.Whens: 9278 whens = [] 9279 9280 while self._match(TokenType.WHEN): 9281 matched = not self._match(TokenType.NOT) 9282 self._match_text_seq("MATCHED") 9283 source = ( 9284 False 9285 if self._match_text_seq("BY", "TARGET") 9286 else self._match_text_seq("BY", "SOURCE") 9287 ) 9288 condition = self._parse_disjunction() if self._match(TokenType.AND) else None 9289 9290 self._match(TokenType.THEN) 9291 9292 if self._match(TokenType.INSERT): 9293 this = self._parse_star() 9294 if this: 9295 then: exp.Expr | None = self.expression(exp.Insert(this=this)) 9296 else: 9297 then = self.expression( 9298 exp.Insert( 9299 this=exp.var("ROW") 9300 if self._match_text_seq("ROW") 9301 else self._parse_value(values=False), 9302 expression=self._match_text_seq("VALUES") and self._parse_value(), 9303 where=self._parse_where(), 9304 ) 9305 ) 9306 elif self._match(TokenType.UPDATE): 9307 expressions = self._parse_star() 9308 if expressions: 9309 then = self.expression(exp.Update(expressions=expressions)) 9310 else: 9311 then = self.expression( 9312 exp.Update( 9313 expressions=self._match(TokenType.SET) 9314 and self._parse_csv(self._parse_equality), 9315 where=self._parse_where(), 9316 ) 9317 ) 9318 elif self._match(TokenType.DELETE): 9319 then = self.expression(exp.Var(this=self._prev.text)) 9320 else: 9321 then = self._parse_var_from_options(self.CONFLICT_ACTIONS) 9322 9323 whens.append( 9324 self.expression( 9325 exp.When(matched=matched, source=source, condition=condition, then=then) 9326 ) 9327 ) 9328 return self.expression(exp.Whens(expressions=whens)) 9329 9330 def _parse_show(self) -> exp.Expr | None: 9331 parser = self._find_parser(self.SHOW_PARSERS, self.SHOW_TRIE) 9332 if parser: 9333 return parser(self) 9334 return self._parse_as_command(self._prev) 9335 9336 def _parse_set_item_assignment(self, kind: str | None = None) -> exp.Expr | None: 9337 index = self._index 9338 9339 if kind in ("GLOBAL", "SESSION") and self._match_text_seq("TRANSACTION"): 9340 return self._parse_set_transaction(global_=kind == "GLOBAL") 9341 9342 left = self._parse_primary() or self._parse_column() 9343 assignment_delimiter = self._match_texts(self.SET_ASSIGNMENT_DELIMITERS) 9344 9345 if not left or (self.SET_REQUIRES_ASSIGNMENT_DELIMITER and not assignment_delimiter): 9346 self._retreat(index) 9347 return None 9348 9349 right = self._parse_statement() or self._parse_id_var() 9350 if isinstance(right, (exp.Column, exp.Identifier)): 9351 right = exp.var(right.name) 9352 9353 this = self.expression(exp.EQ(this=left, expression=right)) 9354 return self.expression(exp.SetItem(this=this, kind=kind)) 9355 9356 def _parse_set_transaction(self, global_: bool = False) -> exp.Expr: 9357 self._match_text_seq("TRANSACTION") 9358 characteristics = self._parse_csv( 9359 lambda: self._parse_var_from_options(self.TRANSACTION_CHARACTERISTICS) 9360 ) 9361 return self.expression( 9362 exp.SetItem(expressions=characteristics, kind="TRANSACTION", global_=global_) 9363 ) 9364 9365 def _parse_set_item(self) -> exp.Expr | None: 9366 parser = self._find_parser(self.SET_PARSERS, self.SET_TRIE) 9367 return parser(self) if parser else self._parse_set_item_assignment(kind=None) 9368 9369 def _parse_set(self, unset: bool = False, tag: bool = False) -> exp.Set | exp.Command: 9370 index = self._index 9371 set_ = self.expression( 9372 exp.Set(expressions=self._parse_csv(self._parse_set_item), unset=unset, tag=tag) 9373 ) 9374 9375 if self._curr: 9376 self._retreat(index) 9377 return self._parse_as_command(self._prev) 9378 9379 return set_ 9380 9381 def _parse_var_from_options( 9382 self, options: OPTIONS_TYPE, raise_unmatched: bool = True 9383 ) -> exp.Var | None: 9384 start = self._curr 9385 if not start: 9386 return None 9387 9388 option = start.text.upper() 9389 continuations = ( 9390 None if start.token_type in self.TEXT_MATCH_EXCLUDED_TOKENS else options.get(option) 9391 ) 9392 9393 index = self._index 9394 self._advance() 9395 for keywords in continuations or []: 9396 if isinstance(keywords, str): 9397 keywords = (keywords,) 9398 9399 if self._match_text_seq(*keywords): 9400 option = f"{option} {' '.join(keywords)}" 9401 break 9402 else: 9403 if continuations or continuations is None: 9404 if raise_unmatched: 9405 self.raise_error(f"Unknown option {option}") 9406 9407 self._retreat(index) 9408 return None 9409 9410 return exp.var(option) 9411 9412 def _parse_as_command(self, start: Token) -> exp.Command: 9413 while self._curr: 9414 self._advance() 9415 text = self._find_sql(start, self._prev) 9416 size = len(start.text) 9417 self._warn_unsupported() 9418 return exp.Command(this=text[:size], expression=text[size:]) 9419 9420 def _parse_dict_property(self, this: str) -> exp.DictProperty: 9421 settings = [] 9422 9423 self._match_l_paren() 9424 kind = self._parse_id_var() 9425 9426 if self._match(TokenType.L_PAREN): 9427 while True: 9428 key = self._parse_id_var() 9429 value = self._parse_function() or self._parse_primary_or_var() 9430 if not key and value is None: 9431 break 9432 settings.append(self.expression(exp.DictSubProperty(this=key, value=value))) 9433 self._match(TokenType.R_PAREN) 9434 9435 self._match_r_paren() 9436 9437 return self.expression( 9438 exp.DictProperty(this=this, kind=kind.this if kind else None, settings=settings) 9439 ) 9440 9441 def _parse_dict_range(self, this: str) -> exp.DictRange: 9442 self._match_l_paren() 9443 has_min = self._match_text_seq("MIN") 9444 if has_min: 9445 min = self._parse_var() or self._parse_primary() 9446 self._match_text_seq("MAX") 9447 max = self._parse_var() or self._parse_primary() 9448 else: 9449 max = self._parse_var() or self._parse_primary() 9450 min = exp.Literal.number(0) 9451 self._match_r_paren() 9452 return self.expression(exp.DictRange(this=this, min=min, max=max)) 9453 9454 def _parse_comprehension(self, this: exp.Expr | None) -> exp.Comprehension | None: 9455 index = self._index 9456 expression = self._parse_column() 9457 position = self._match(TokenType.COMMA) and self._parse_column() 9458 9459 if not self._match(TokenType.IN): 9460 self._retreat(index - 1) 9461 return None 9462 iterator = self._parse_column() 9463 condition = self._parse_disjunction() if self._match_text_seq("IF") else None 9464 return self.expression( 9465 exp.Comprehension( 9466 this=this, 9467 expression=expression, 9468 position=position, 9469 iterator=iterator, 9470 condition=condition, 9471 ) 9472 ) 9473 9474 def _parse_heredoc(self) -> exp.Heredoc | None: 9475 if self._match(TokenType.HEREDOC_STRING): 9476 return self.expression(exp.Heredoc(this=self._prev.text)) 9477 9478 if not self._match_text_seq("$"): 9479 return None 9480 9481 tags = ["$"] 9482 tag_text = None 9483 9484 if self._is_connected(): 9485 self._advance() 9486 tags.append(self._prev.text.upper()) 9487 else: 9488 self.raise_error("No closing $ found") 9489 9490 if tags[-1] != "$": 9491 if self._is_connected() and self._match_text_seq("$"): 9492 tag_text = tags[-1] 9493 tags.append("$") 9494 else: 9495 self.raise_error("No closing $ found") 9496 9497 heredoc_start = self._curr 9498 9499 while self._curr: 9500 if self._match_text_seq(*tags, advance=False): 9501 this = self._find_sql(heredoc_start, self._prev) 9502 self._advance(len(tags)) 9503 return self.expression(exp.Heredoc(this=this, tag=tag_text)) 9504 9505 self._advance() 9506 9507 self.raise_error(f"No closing {''.join(tags)} found") 9508 return None 9509 9510 def _find_parser(self, parsers: dict[str, t.Callable], trie: dict) -> t.Callable | None: 9511 if not self._curr: 9512 return None 9513 9514 index = self._index 9515 this = [] 9516 while True: 9517 # The current token might be multiple words 9518 curr = self._curr.text.upper() 9519 key = curr.split(" ") 9520 this.append(curr) 9521 9522 self._advance() 9523 result, trie = in_trie(trie, key) 9524 if result == TrieResult.FAILED: 9525 break 9526 9527 if result == TrieResult.EXISTS: 9528 subparser = parsers[" ".join(this)] 9529 return subparser 9530 9531 self._retreat(index) 9532 return None 9533 9534 def _match_l_paren(self, expression: exp.Expr | None = None) -> None: 9535 if not self._match(TokenType.L_PAREN, expression=expression): 9536 self.raise_error("Expecting (") 9537 9538 def _match_r_paren(self, expression: exp.Expr | None = None) -> None: 9539 if not self._match(TokenType.R_PAREN, expression=expression): 9540 self.raise_error("Expecting )") 9541 9542 def _replace_lambda( 9543 self, node: exp.Expr | None, expressions: list[exp.Expr] 9544 ) -> exp.Expr | None: 9545 if not node: 9546 return node 9547 9548 lambda_types = {e.name: e.args.get("to") or False for e in expressions} 9549 9550 for column in node.find_all(exp.Column): 9551 typ = lambda_types.get(column.parts[0].name) 9552 if typ is not None: 9553 dot_or_id = column.to_dot() if column.table else column.this 9554 9555 if typ: 9556 dot_or_id = self.expression(exp.Cast(this=dot_or_id, to=typ)) 9557 9558 parent = column.parent 9559 9560 while isinstance(parent, exp.Dot): 9561 if not isinstance(parent.parent, exp.Dot): 9562 parent.replace(dot_or_id) 9563 break 9564 parent = parent.parent 9565 else: 9566 if column is node: 9567 node = dot_or_id 9568 else: 9569 column.replace(dot_or_id) 9570 return node 9571 9572 def _parse_truncate_table(self) -> exp.TruncateTable | None | exp.Expr: 9573 start = self._prev 9574 9575 # Not to be confused with TRUNCATE(number, decimals) function call 9576 if self._match(TokenType.L_PAREN): 9577 self._retreat(self._index - 2) 9578 return self._parse_function() 9579 9580 # Clickhouse supports TRUNCATE DATABASE as well 9581 is_database = self._match(TokenType.DATABASE) 9582 9583 self._match(TokenType.TABLE) 9584 9585 exists = self._parse_exists(not_=False) 9586 9587 expressions = self._parse_csv( 9588 lambda: self._parse_table(schema=True, is_db_reference=is_database) 9589 ) 9590 9591 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 9592 9593 if self._match_text_seq("RESTART", "IDENTITY"): 9594 identity = "RESTART" 9595 elif self._match_text_seq("CONTINUE", "IDENTITY"): 9596 identity = "CONTINUE" 9597 else: 9598 identity = None 9599 9600 if self._match_text_seq("CASCADE") or self._match_text_seq("RESTRICT"): 9601 option = self._prev.text 9602 else: 9603 option = None 9604 9605 partition = self._parse_partition() 9606 9607 # Fallback case 9608 if self._curr: 9609 return self._parse_as_command(start) 9610 9611 return self.expression( 9612 exp.TruncateTable( 9613 expressions=expressions, 9614 is_database=is_database, 9615 exists=exists, 9616 cluster=cluster, 9617 identity=identity, 9618 option=option, 9619 partition=partition, 9620 ) 9621 ) 9622 9623 def _parse_indexed_column(self) -> exp.Expr | None: 9624 return self._parse_ordered(self._parse_opclass) 9625 9626 def _parse_with_operator(self) -> exp.Expr | None: 9627 this = self._parse_indexed_column() 9628 9629 if not self._match(TokenType.WITH): 9630 return this 9631 9632 op = self._parse_var(any_token=True, tokens=self.RESERVED_TOKENS) 9633 9634 return self.expression(exp.WithOperator(this=this, op=op)) 9635 9636 def _parse_wrapped_options(self) -> list[exp.Expr]: 9637 self._match(TokenType.EQ) 9638 self._match(TokenType.L_PAREN) 9639 9640 opts: list[exp.Expr] = [] 9641 option: exp.Expr | list[exp.Expr] | None 9642 while self._curr and not self._match(TokenType.R_PAREN): 9643 if self._match_text_seq("FORMAT_NAME", "="): 9644 # The FORMAT_NAME can be set to an identifier for Snowflake and T-SQL 9645 option = self._parse_format_name() 9646 else: 9647 option = self._parse_property() 9648 9649 if option is None: 9650 self.raise_error("Unable to parse option") 9651 break 9652 9653 opts.extend(ensure_list(option)) 9654 9655 return opts 9656 9657 def _parse_copy_parameters(self) -> list[exp.CopyParameter]: 9658 sep = TokenType.COMMA if self.dialect.COPY_PARAMS_ARE_CSV else None 9659 9660 options = [] 9661 while self._curr and not self._match(TokenType.R_PAREN, advance=False): 9662 option = self._parse_var(any_token=True) 9663 prev = self._prev.text.upper() 9664 9665 # Different dialects might separate options and values by white space, "=" and "AS" 9666 self._match(TokenType.EQ) 9667 self._match(TokenType.ALIAS) 9668 9669 param = self.expression(exp.CopyParameter(this=option)) 9670 9671 if prev in self.COPY_INTO_VARLEN_OPTIONS and self._match( 9672 TokenType.L_PAREN, advance=False 9673 ): 9674 # Snowflake FILE_FORMAT case, Databricks COPY & FORMAT options 9675 param.set("expressions", self._parse_wrapped_options()) 9676 elif prev == "FILE_FORMAT": 9677 # T-SQL's external file format case 9678 param.set("expression", self._parse_field()) 9679 elif ( 9680 prev == "FORMAT" 9681 and self._prev.token_type == TokenType.ALIAS 9682 and self._match_texts(("AVRO", "JSON")) 9683 ): 9684 param.set("this", exp.var(f"FORMAT AS {self._prev.text.upper()}")) 9685 param.set("expression", self._parse_field()) 9686 else: 9687 param.set("expression", self._parse_unquoted_field() or self._parse_bracket()) 9688 9689 options.append(param) 9690 9691 if sep: 9692 self._match(sep) 9693 9694 return options 9695 9696 def _parse_credentials(self) -> exp.Credentials | None: 9697 expr = self.expression(exp.Credentials()) 9698 9699 if self._match_text_seq("STORAGE_INTEGRATION", "="): 9700 expr.set("storage", self._parse_field()) 9701 if self._match_text_seq("CREDENTIALS"): 9702 # Snowflake case: CREDENTIALS = (...), Redshift case: CREDENTIALS <string> 9703 creds = ( 9704 self._parse_wrapped_options() if self._match(TokenType.EQ) else self._parse_field() 9705 ) 9706 expr.set("credentials", creds) 9707 if self._match_text_seq("ENCRYPTION"): 9708 expr.set("encryption", self._parse_wrapped_options()) 9709 if self._match_text_seq("IAM_ROLE"): 9710 expr.set( 9711 "iam_role", 9712 exp.var(self._prev.text) if self._match(TokenType.DEFAULT) else self._parse_field(), 9713 ) 9714 if self._match_text_seq("REGION"): 9715 expr.set("region", self._parse_field()) 9716 9717 return expr 9718 9719 def _parse_file_location(self) -> exp.Expr | None: 9720 return self._parse_field() 9721 9722 def _parse_copy(self) -> exp.Copy | exp.Command: 9723 start = self._prev 9724 9725 self._match(TokenType.INTO) 9726 9727 this = ( 9728 self._parse_select(nested=True, parse_subquery_alias=False) 9729 if self._match(TokenType.L_PAREN, advance=False) 9730 else self._parse_table(schema=True) 9731 ) 9732 9733 kind = self._match(TokenType.FROM) or not self._match_text_seq("TO") 9734 9735 files = self._parse_csv(self._parse_file_location) 9736 if self._match(TokenType.EQ, advance=False): 9737 # Backtrack one token since we've consumed the lhs of a parameter assignment here. 9738 # This can happen for Snowflake dialect. Instead, we'd like to parse the parameter 9739 # list via `_parse_wrapped(..)` below. 9740 self._advance(-1) 9741 files = [] 9742 9743 credentials = self._parse_credentials() 9744 9745 self._match_text_seq("WITH") 9746 9747 params = self._parse_wrapped(self._parse_copy_parameters, optional=True) 9748 9749 # Fallback case 9750 if self._curr: 9751 return self._parse_as_command(start) 9752 9753 return self.expression( 9754 exp.Copy(this=this, kind=kind, credentials=credentials, files=files, params=params) 9755 ) 9756 9757 def _parse_normalize(self) -> exp.Normalize: 9758 return self.expression( 9759 exp.Normalize( 9760 this=self._parse_bitwise(), form=self._match(TokenType.COMMA) and self._parse_var() 9761 ) 9762 ) 9763 9764 def _parse_ceil_floor(self, expr_type: type[TCeilFloor]) -> TCeilFloor: 9765 args = self._parse_csv(lambda: self._parse_lambda()) 9766 9767 this = seq_get(args, 0) 9768 decimals = seq_get(args, 1) 9769 9770 return expr_type( 9771 this=this, 9772 decimals=decimals, 9773 to=self._parse_var() if self._match_text_seq("TO") else None, 9774 ) 9775 9776 def _parse_star_ops(self) -> exp.Expr | None: 9777 star_token = self._prev 9778 9779 if self._match_text_seq("COLUMNS", "(", advance=False): 9780 this = self._parse_function() 9781 if isinstance(this, exp.Columns): 9782 this.set("unpack", True) 9783 return this 9784 9785 index = self._index 9786 ilike = self._parse_string() if self._match(TokenType.ILIKE) else None 9787 if not ilike: 9788 # ILIKE without a string pattern is not a star filter, e.g. `* ILIKE (foo)` 9789 self._retreat(index) 9790 9791 return self.expression( 9792 exp.Star( 9793 ilike=ilike, 9794 except_=self._parse_star_op("EXCEPT", "EXCLUDE"), 9795 replace=self._parse_star_op("REPLACE"), 9796 rename=self._parse_star_op("RENAME"), 9797 ) 9798 ).update_positions(star_token) 9799 9800 def _parse_grant_privilege(self) -> exp.GrantPrivilege | None: 9801 privilege_parts = [] 9802 9803 # Keep consuming consecutive keywords until comma (end of this privilege) or ON 9804 # (end of privilege list) or L_PAREN (start of column list) are met 9805 while self._curr and not self._match_set(self.PRIVILEGE_FOLLOW_TOKENS, advance=False): 9806 privilege_parts.append(self._curr.text.upper()) 9807 self._advance() 9808 9809 this = exp.var(" ".join(privilege_parts)) 9810 expressions = ( 9811 self._parse_wrapped_csv(self._parse_column) 9812 if self._match(TokenType.L_PAREN, advance=False) 9813 else None 9814 ) 9815 9816 return self.expression(exp.GrantPrivilege(this=this, expressions=expressions)) 9817 9818 def _parse_grant_principal(self) -> exp.GrantPrincipal | None: 9819 kind = self._match_texts(("ROLE", "GROUP")) and self._prev.text.upper() 9820 principal = self._parse_id_var() 9821 9822 if not principal: 9823 return None 9824 9825 return self.expression(exp.GrantPrincipal(this=principal, kind=kind)) 9826 9827 def _parse_grant_revoke_common( 9828 self, 9829 ) -> tuple[list | None, str | None, exp.Expr | None]: 9830 privileges = self._parse_csv(self._parse_grant_privilege) 9831 9832 self._match(TokenType.ON) 9833 kind = self._prev.text.upper() if self._match_set(self.CREATABLES) else None 9834 9835 # Attempt to parse the securable e.g. MySQL allows names 9836 # such as "foo.*", "*.*" which are not easily parseable yet 9837 securable = self._try_parse(self._parse_table_parts) 9838 9839 return privileges, kind, securable 9840 9841 def _parse_grant(self) -> exp.Grant | exp.Command: 9842 start = self._prev 9843 9844 privileges, kind, securable = self._parse_grant_revoke_common() 9845 9846 if not securable or not self._match_text_seq("TO"): 9847 return self._parse_as_command(start) 9848 9849 principals = self._parse_csv(self._parse_grant_principal) 9850 9851 grant_option = self._match_text_seq("WITH", "GRANT", "OPTION") 9852 9853 if self._curr: 9854 return self._parse_as_command(start) 9855 9856 return self.expression( 9857 exp.Grant( 9858 privileges=privileges, 9859 kind=kind, 9860 securable=securable, 9861 principals=principals, 9862 grant_option=grant_option, 9863 ) 9864 ) 9865 9866 def _parse_revoke(self) -> exp.Revoke | exp.Command: 9867 start = self._prev 9868 9869 grant_option = self._match_text_seq("GRANT", "OPTION", "FOR") 9870 9871 privileges, kind, securable = self._parse_grant_revoke_common() 9872 9873 if not securable or not self._match_text_seq("FROM"): 9874 return self._parse_as_command(start) 9875 9876 principals = self._parse_csv(self._parse_grant_principal) 9877 9878 cascade = None 9879 if self._match_texts(("CASCADE", "RESTRICT")): 9880 cascade = self._prev.text.upper() 9881 9882 if self._curr: 9883 return self._parse_as_command(start) 9884 9885 return self.expression( 9886 exp.Revoke( 9887 privileges=privileges, 9888 kind=kind, 9889 securable=securable, 9890 principals=principals, 9891 grant_option=grant_option, 9892 cascade=cascade, 9893 ) 9894 ) 9895 9896 def _parse_overlay(self) -> exp.Overlay: 9897 def _parse_overlay_arg(text: str) -> exp.Expr | None: 9898 return ( 9899 self._parse_bitwise() 9900 if self._match(TokenType.COMMA) or self._match_text_seq(text) 9901 else None 9902 ) 9903 9904 return self.expression( 9905 exp.Overlay( 9906 this=self._parse_bitwise(), 9907 expression=_parse_overlay_arg("PLACING"), 9908 from_=_parse_overlay_arg("FROM"), 9909 for_=_parse_overlay_arg("FOR"), 9910 ) 9911 ) 9912 9913 def _parse_format_name(self) -> exp.Property: 9914 # Note: Although not specified in the docs, Snowflake does accept a string/identifier 9915 # for FILE_FORMAT = <format_name> 9916 return self.expression( 9917 exp.Property( 9918 this=exp.var("FORMAT_NAME"), value=self._parse_string() or self._parse_table_parts() 9919 ) 9920 ) 9921 9922 def _parse_distinct_arg_function(self, func: type[F], distinct_index: int = 0) -> F: 9923 is_distinct = self._match(TokenType.DISTINCT) 9924 if not is_distinct: 9925 self._match(TokenType.ALL) 9926 9927 args = [self._parse_lambda()] 9928 if self._match(TokenType.COMMA): 9929 args.extend(self._parse_function_args()) 9930 9931 target = seq_get(args, distinct_index) 9932 if is_distinct and target: 9933 args[distinct_index] = self.expression(exp.Distinct(expressions=[target])) 9934 9935 return func.from_arg_list(args) 9936 9937 def _identifier_expression( 9938 self, token: Token | None = None, quoted: bool | None = None 9939 ) -> exp.Identifier: 9940 token = token or self._prev 9941 return self.expression(exp.Identifier(this=token.text, quoted=quoted), token) 9942 9943 def _build_pipe_cte( 9944 self, 9945 query: exp.Query, 9946 expressions: list[exp.Expr], 9947 alias_cte: exp.TableAlias | None = None, 9948 ) -> exp.Select: 9949 new_cte: str | exp.TableAlias | None 9950 if alias_cte: 9951 new_cte = alias_cte 9952 else: 9953 self._pipe_cte_counter += 1 9954 new_cte = f"__tmp{self._pipe_cte_counter}" 9955 9956 with_ = query.args.get("with_") 9957 ctes = with_.pop() if with_ else None 9958 9959 new_select = exp.select(*expressions, copy=False).from_(new_cte, copy=False) 9960 if ctes: 9961 new_select.set("with_", ctes) 9962 9963 return new_select.with_(new_cte, as_=query, copy=False) 9964 9965 def _parse_pipe_syntax_select(self, query: exp.Select) -> exp.Select: 9966 select = self._parse_select(consume_pipe=False) 9967 if not select: 9968 return query 9969 9970 return self._build_pipe_cte( 9971 query=query.select(*select.expressions, append=False), expressions=[exp.Star()] 9972 ) 9973 9974 def _parse_pipe_syntax_limit(self, query: exp.Select) -> exp.Select: 9975 limit = self._parse_limit() 9976 offset = self._parse_offset() 9977 if limit: 9978 curr_limit = query.args.get("limit", limit) 9979 if curr_limit.expression.to_py() >= limit.expression.to_py(): 9980 query.limit(limit, copy=False) 9981 if offset: 9982 curr_offset = query.args.get("offset") 9983 curr_offset = curr_offset.expression.to_py() if curr_offset else 0 9984 query.offset(exp.Literal.number(curr_offset + offset.expression.to_py()), copy=False) 9985 9986 return query 9987 9988 def _parse_pipe_syntax_aggregate_fields(self) -> exp.Expr | None: 9989 this = self._parse_disjunction() 9990 if self._match_text_seq("GROUP", "AND", advance=False): 9991 return this 9992 9993 this = self._parse_alias(this) 9994 9995 if self._match_set((TokenType.ASC, TokenType.DESC), advance=False): 9996 return self._parse_ordered(lambda: this) 9997 9998 return this 9999 10000 def _parse_pipe_syntax_aggregate_group_order_by( 10001 self, query: exp.Select, group_by_exists: bool = True 10002 ) -> exp.Select: 10003 expr = self._parse_csv(self._parse_pipe_syntax_aggregate_fields) 10004 aggregates_or_groups, orders = [], [] 10005 for element in expr: 10006 if isinstance(element, exp.Ordered): 10007 this = element.this 10008 if isinstance(this, exp.Alias): 10009 element.set("this", this.args["alias"]) 10010 orders.append(element) 10011 else: 10012 this = element 10013 aggregates_or_groups.append(this) 10014 10015 if group_by_exists: 10016 query.select( 10017 *aggregates_or_groups, *query.expressions, append=False, copy=False 10018 ).group_by( 10019 *[projection.args.get("alias", projection) for projection in aggregates_or_groups], 10020 copy=False, 10021 ) 10022 else: 10023 query.select(*aggregates_or_groups, append=False, copy=False) 10024 10025 if orders: 10026 return query.order_by(*orders, append=False, copy=False) 10027 10028 return query 10029 10030 def _parse_pipe_syntax_aggregate(self, query: exp.Select) -> exp.Select: 10031 self._match_text_seq("AGGREGATE") 10032 query = self._parse_pipe_syntax_aggregate_group_order_by(query, group_by_exists=False) 10033 10034 if self._match(TokenType.GROUP_BY) or ( 10035 self._match_text_seq("GROUP", "AND") and self._match(TokenType.ORDER_BY) 10036 ): 10037 query = self._parse_pipe_syntax_aggregate_group_order_by(query) 10038 10039 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 10040 10041 def _parse_pipe_syntax_set_operator(self, query: exp.Query) -> exp.Query | None: 10042 first_setop = self.parse_set_operation(this=query) 10043 if not first_setop: 10044 return None 10045 10046 def _parse_and_unwrap_query() -> exp.Expr | None: 10047 expr = self._parse_paren() 10048 return expr.assert_is(exp.Subquery).unnest() if expr else None 10049 10050 first_setop.this.pop() 10051 10052 setops = [ 10053 first_setop.expression.pop().assert_is(exp.Subquery).unnest(), 10054 *self._parse_csv(_parse_and_unwrap_query), 10055 ] 10056 10057 query = self._build_pipe_cte(query=query, expressions=[exp.Star()]) 10058 with_ = query.args.get("with_") 10059 ctes = with_.pop() if with_ else None 10060 10061 if isinstance(first_setop, exp.Union): 10062 query = query.union(*setops, copy=False, **first_setop.args) 10063 elif isinstance(first_setop, exp.Except): 10064 query = query.except_(*setops, copy=False, **first_setop.args) 10065 else: 10066 query = query.intersect(*setops, copy=False, **first_setop.args) 10067 10068 query.set("with_", ctes) 10069 10070 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 10071 10072 def _parse_pipe_syntax_join(self, query: exp.Query) -> exp.Query | None: 10073 join = self._parse_join() 10074 if not join: 10075 return None 10076 10077 if isinstance(query, exp.Select): 10078 return query.join(join, copy=False) 10079 10080 return query 10081 10082 def _parse_pipe_syntax_pivot(self, query: exp.Select) -> exp.Select: 10083 pivots = self._parse_pivots() 10084 if not pivots: 10085 return query 10086 10087 from_ = query.args.get("from_") 10088 if from_: 10089 from_.this.set("pivots", pivots) 10090 10091 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 10092 10093 def _parse_pipe_syntax_extend(self, query: exp.Select) -> exp.Select: 10094 self._match_text_seq("EXTEND") 10095 query.select(*[exp.Star(), *self._parse_expressions()], append=False, copy=False) 10096 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 10097 10098 def _parse_pipe_syntax_tablesample(self, query: exp.Select) -> exp.Select: 10099 sample = self._parse_table_sample() 10100 10101 with_ = query.args.get("with_") 10102 if with_: 10103 with_.expressions[-1].this.set("sample", sample) 10104 else: 10105 query.set("sample", sample) 10106 10107 return query 10108 10109 def _parse_pipe_syntax_query(self, query: exp.Query) -> exp.Query | None: 10110 if isinstance(query, exp.Subquery): 10111 query = exp.select("*").from_(query, copy=False) 10112 10113 if not query.args.get("from_"): 10114 query = exp.select("*").from_(query.subquery(copy=False), copy=False) 10115 10116 while self._match(TokenType.PIPE_GT): 10117 start_index = self._index 10118 start_text = self._curr.text.upper() 10119 parser = self.PIPE_SYNTAX_TRANSFORM_PARSERS.get(start_text) 10120 if not parser: 10121 # The set operators (UNION, etc) and the JOIN operator have a few common starting 10122 # keywords, making it tricky to disambiguate them without lookahead. The approach 10123 # here is to try and parse a set operation and if that fails, then try to parse a 10124 # join operator. If that fails as well, then the operator is not supported. 10125 parsed_query = self._parse_pipe_syntax_set_operator(query) 10126 parsed_query = parsed_query or self._parse_pipe_syntax_join(query) 10127 if not parsed_query: 10128 self._retreat(start_index) 10129 self.raise_error(f"Unsupported pipe syntax operator: '{start_text}'.") 10130 break 10131 query = parsed_query 10132 else: 10133 query = parser(self, query) 10134 10135 return query 10136 10137 def _parse_declareitem(self) -> exp.DeclareItem | None: 10138 self._match_texts(("VAR", "VARIABLE")) 10139 10140 vars = self._parse_csv(self._parse_id_var) 10141 if not vars: 10142 return None 10143 10144 self._match(TokenType.ALIAS) 10145 kind = self._parse_schema() if self._match(TokenType.TABLE) else self._parse_types() 10146 default = ( 10147 self._match(TokenType.DEFAULT) or self._match(TokenType.EQ) 10148 ) and self._parse_bitwise() 10149 10150 return self.expression(exp.DeclareItem(this=vars, kind=kind, default=default)) 10151 10152 def _parse_declare(self) -> exp.Declare | exp.Command: 10153 start = self._prev 10154 replace = self._match_text_seq("OR", "REPLACE") 10155 expressions = self._try_parse(lambda: self._parse_csv(self._parse_declareitem)) 10156 10157 if not expressions or self._curr: 10158 return self._parse_as_command(start) 10159 10160 return self.expression(exp.Declare(expressions=expressions, replace=replace)) 10161 10162 def build_cast(self, strict: bool, **kwargs) -> exp.Expr: 10163 exp_class = exp.Cast if strict else exp.TryCast 10164 10165 if exp_class == exp.TryCast: 10166 kwargs["requires_string"] = self.dialect.TRY_CAST_REQUIRES_STRING 10167 10168 return self.expression(exp_class(**kwargs)) 10169 10170 def _parse_json_value(self) -> exp.JSONValue: 10171 this = self._parse_bitwise() 10172 self._match(TokenType.COMMA) 10173 path = self._parse_bitwise() 10174 10175 returning = self._match(TokenType.RETURNING) and self._parse_type() 10176 10177 return self.expression( 10178 exp.JSONValue( 10179 this=this, 10180 path=self.dialect.to_json_path(path), 10181 returning=returning, 10182 on_condition=self._parse_on_condition(), 10183 ) 10184 ) 10185 10186 def _parse_group_concat(self) -> exp.Expr | None: 10187 def concat_exprs(node: exp.Expr | None, exprs: list[exp.Expr]) -> exp.Expr: 10188 if isinstance(node, exp.Distinct) and len(node.expressions) > 1: 10189 concat_exprs = [ 10190 self.expression( 10191 exp.Concat( 10192 expressions=node.expressions, 10193 safe=True, 10194 coalesce=self.dialect.CONCAT_COALESCE, 10195 ) 10196 ) 10197 ] 10198 node.set("expressions", concat_exprs) 10199 return node 10200 if len(exprs) == 1: 10201 return exprs[0] 10202 return self.expression( 10203 exp.Concat(expressions=args, safe=True, coalesce=self.dialect.CONCAT_COALESCE) 10204 ) 10205 10206 args = self._parse_csv(self._parse_lambda) 10207 10208 if args: 10209 order = args[-1] if isinstance(args[-1], exp.Order) else None 10210 10211 if order: 10212 # Order By is the last (or only) expression in the list and has consumed the 'expr' before it, 10213 # remove 'expr' from exp.Order and add it back to args 10214 args[-1] = order.this 10215 order.set("this", concat_exprs(order.this, args)) 10216 10217 this = order or concat_exprs(args[0], args) 10218 else: 10219 this = None 10220 10221 separator = self._parse_field() if self._match(TokenType.SEPARATOR) else None 10222 10223 return self.expression(exp.GroupConcat(this=this, separator=separator)) 10224 10225 def _parse_initcap(self) -> exp.Initcap: 10226 expr = exp.Initcap.from_arg_list(self._parse_function_args()) 10227 10228 # attach dialect's default delimiters 10229 if expr.args.get("expression") is None: 10230 expr.set("expression", exp.Literal.string(self.dialect.INITCAP_DEFAULT_DELIMITER_CHARS)) 10231 10232 return expr 10233 10234 def _parse_operator(self, this: exp.Expr | None) -> exp.Expr | None: 10235 while True: 10236 if not self._match(TokenType.L_PAREN): 10237 break 10238 10239 op = "" 10240 while self._curr and not self._match(TokenType.R_PAREN): 10241 op += self._curr.text 10242 self._advance() 10243 10244 comments = self._prev_comments 10245 this = self.expression( 10246 exp.Operator(this=this, operator=op, expression=self._parse_bitwise()), 10247 comments=comments, 10248 ) 10249 10250 if not self._match(TokenType.OPERATOR): 10251 break 10252 10253 return this
50def build_var_map(args: BuilderArgs) -> exp.StarMap | exp.VarMap: 51 if len(args) == 1 and args[0].is_star: 52 return exp.StarMap(this=args[0]) 53 54 keys: list[ExpOrStr] = [] 55 values: list[ExpOrStr] = [] 56 for i in range(0, len(args), 2): 57 keys.append(args[i]) 58 values.append(args[i + 1]) 59 60 return exp.VarMap(keys=exp.array(*keys, copy=False), values=exp.array(*values, copy=False))
68def binary_range_parser( 69 expr_type: Type[exp.Expr], reverse_args: bool = False 70) -> t.Callable[[Parser, exp.Expr | None], exp.Expr | None]: 71 def _parse_binary_range(self: Parser, this: exp.Expr | None) -> exp.Expr | None: 72 expression = self._parse_bitwise() 73 if reverse_args: 74 this, expression = expression, this 75 return self._parse_escape(self.expression(expr_type(this=this, expression=expression))) 76 77 return _parse_binary_range
80def build_logarithm(args: BuilderArgs, dialect: Dialect) -> exp.Func: 81 # Default argument order is base, expression 82 this = seq_get(args, 0) 83 expression = seq_get(args, 1) 84 85 if expression: 86 if not dialect.LOG_BASE_FIRST: 87 this, expression = expression, this 88 return exp.Log(this=this, expression=expression) 89 90 return (exp.Ln if dialect.parser_class.LOG_DEFAULTS_TO_LN else exp.Log)(this=this)
110def build_extract_json_with_path( 111 expr_type: Type[E], 112) -> t.Callable[[BuilderArgs, Dialect], E]: 113 def _builder(args: BuilderArgs, dialect: Dialect) -> E: 114 expression = expr_type( 115 this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1)) 116 ) 117 if len(args) > 2 and expr_type is exp.JSONExtract: 118 expression.set("expressions", args[2:]) 119 if expr_type is exp.JSONExtractScalar: 120 expression.set("scalar_only", dialect.JSON_EXTRACT_SCALAR_SCALAR_ONLY) 121 122 return expression 123 124 return _builder
127def build_mod(args: BuilderArgs) -> exp.Mod: 128 this = seq_get(args, 0) 129 expression = seq_get(args, 1) 130 131 # Wrap the operands if they are binary nodes, e.g. MOD(a + 1, 7) -> (a + 1) % 7 132 this = exp.Paren(this=this) if isinstance(this, exp.Binary) else this 133 expression = exp.Paren(this=expression) if isinstance(expression, exp.Binary) else expression 134 135 return exp.Mod(this=this, expression=expression)
147def build_array_constructor( 148 exp_class: Type[E], args: list[t.Any], bracket_kind: TokenType, dialect: Dialect 149) -> exp.Expr: 150 array_exp = exp_class(expressions=args) 151 152 if exp_class == exp.Array and dialect.HAS_DISTINCT_ARRAY_CONSTRUCTORS: 153 array_exp.set("bracket_notation", bracket_kind == TokenType.L_BRACKET) 154 155 return array_exp
158def build_convert_timezone( 159 args: BuilderArgs, default_source_tz: str | None = None 160) -> exp.ConvertTimezone | exp.Anonymous: 161 if len(args) == 2: 162 source_tz = exp.Literal.string(default_source_tz) if default_source_tz else None 163 return exp.ConvertTimezone( 164 source_tz=source_tz, target_tz=seq_get(args, 0), timestamp=seq_get(args, 1) 165 ) 166 167 return exp.ConvertTimezone.from_arg_list(args)
170def build_trim(args: BuilderArgs, is_left: bool = True, reverse_args: bool = False) -> exp.Trim: 171 this, expression = seq_get(args, 0), seq_get(args, 1) 172 173 if expression and reverse_args: 174 this, expression = expression, this 175 176 return exp.Trim(this=this, expression=expression, position="LEADING" if is_left else "TRAILING")
193def build_array_append(args: BuilderArgs, dialect: Dialect) -> exp.ArrayAppend: 194 """ 195 Builds ArrayAppend with NULL propagation semantics based on the dialect configuration. 196 197 Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. 198 Others (DuckDB, PostgreSQL) create a new single-element array instead. 199 200 Args: 201 args: Function arguments [array, element] 202 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 203 204 Returns: 205 ArrayAppend expression with appropriate null_propagation flag 206 """ 207 return exp.ArrayAppend( 208 this=seq_get(args, 0), 209 expression=seq_get(args, 1), 210 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 211 )
Builds ArrayAppend with NULL propagation semantics based on the dialect configuration.
Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. Others (DuckDB, PostgreSQL) create a new single-element array instead.
Arguments:
- args: Function arguments [array, element]
- dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
Returns:
ArrayAppend expression with appropriate null_propagation flag
214def build_array_prepend(args: BuilderArgs, dialect: Dialect) -> exp.ArrayPrepend: 215 """ 216 Builds ArrayPrepend with NULL propagation semantics based on the dialect configuration. 217 218 Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. 219 Others (DuckDB, PostgreSQL) create a new single-element array instead. 220 221 Args: 222 args: Function arguments [array, element] 223 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 224 225 Returns: 226 ArrayPrepend expression with appropriate null_propagation flag 227 """ 228 return exp.ArrayPrepend( 229 this=seq_get(args, 0), 230 expression=seq_get(args, 1), 231 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 232 )
Builds ArrayPrepend with NULL propagation semantics based on the dialect configuration.
Some dialects (Databricks, Spark, Snowflake) return NULL when the input array is NULL. Others (DuckDB, PostgreSQL) create a new single-element array instead.
Arguments:
- args: Function arguments [array, element]
- dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
Returns:
ArrayPrepend expression with appropriate null_propagation flag
235def build_array_concat(args: BuilderArgs, dialect: Dialect) -> exp.ArrayConcat: 236 """ 237 Builds ArrayConcat with NULL propagation semantics based on the dialect configuration. 238 239 Some dialects (Redshift, Snowflake) return NULL when any input array is NULL. 240 Others (DuckDB, PostgreSQL) skip NULL arrays and continue concatenation. 241 242 Args: 243 args: Function arguments [array1, array2, ...] (variadic) 244 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 245 246 Returns: 247 ArrayConcat expression with appropriate null_propagation flag 248 """ 249 return exp.ArrayConcat( 250 this=seq_get(args, 0), 251 expressions=args[1:], 252 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 253 )
Builds ArrayConcat with NULL propagation semantics based on the dialect configuration.
Some dialects (Redshift, Snowflake) return NULL when any input array is NULL. Others (DuckDB, PostgreSQL) skip NULL arrays and continue concatenation.
Arguments:
- args: Function arguments [array1, array2, ...] (variadic)
- dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
Returns:
ArrayConcat expression with appropriate null_propagation flag
256def build_array_remove(args: BuilderArgs, dialect: Dialect) -> exp.ArrayRemove: 257 """ 258 Builds ArrayRemove with NULL propagation semantics based on the dialect configuration. 259 260 Some dialects (Snowflake) return NULL when the removal value is NULL. 261 Others (DuckDB) may return empty array due to NULL comparison semantics. 262 263 Args: 264 args: Function arguments [array, value_to_remove] 265 dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from 266 267 Returns: 268 ArrayRemove expression with appropriate null_propagation flag 269 """ 270 return exp.ArrayRemove( 271 this=seq_get(args, 0), 272 expression=seq_get(args, 1), 273 null_propagation=dialect.ARRAY_FUNCS_PROPAGATES_NULLS, 274 )
Builds ArrayRemove with NULL propagation semantics based on the dialect configuration.
Some dialects (Snowflake) return NULL when the removal value is NULL. Others (DuckDB) may return empty array due to NULL comparison semantics.
Arguments:
- args: Function arguments [array, value_to_remove]
- dialect: The dialect to read ARRAY_FUNCS_PROPAGATES_NULLS from
Returns:
ArrayRemove expression with appropriate null_propagation flag
295class Parser: 296 """ 297 Parser consumes a list of tokens produced by the Tokenizer and produces a parsed syntax tree. 298 299 Args: 300 error_level: The desired error level. 301 Default: ErrorLevel.IMMEDIATE 302 error_message_context: The amount of context to capture from a query string when displaying 303 the error message (in number of characters). 304 Default: 100 305 max_errors: Maximum number of error messages to include in a raised ParseError. 306 This is only relevant if error_level is ErrorLevel.RAISE. 307 Default: 3 308 max_nodes: Maximum number of AST nodes to prevent memory exhaustion. 309 Set to -1 (default) to disable the check. 310 """ 311 312 __slots__ = ( 313 "error_level", 314 "error_message_context", 315 "max_errors", 316 "max_nodes", 317 "dialect", 318 "sql", 319 "errors", 320 "_tokens", 321 "_index", 322 "_curr", 323 "_next", 324 "_prev", 325 "_prev_comments", 326 "_pipe_cte_counter", 327 "_chunks", 328 "_chunk_index", 329 "_tokens_size", 330 "_node_count", 331 ) 332 333 FUNCTIONS: t.ClassVar[dict[str, t.Callable]] = { 334 **{name: func.from_arg_list for name, func in exp.FUNCTION_BY_NAME.items()}, 335 **dict.fromkeys(("COALESCE", "IFNULL", "NVL"), build_coalesce), 336 "ARRAY": lambda args, dialect: exp.Array(expressions=args), 337 "ARRAYAGG": lambda args, dialect: exp.ArrayAgg( 338 this=seq_get(args, 0), nulls_excluded=dialect.ARRAY_AGG_INCLUDES_NULLS is None or None 339 ), 340 "ARRAY_AGG": lambda args, dialect: exp.ArrayAgg( 341 this=seq_get(args, 0), nulls_excluded=dialect.ARRAY_AGG_INCLUDES_NULLS is None or None 342 ), 343 "ARRAY_APPEND": build_array_append, 344 "ARRAY_CAT": build_array_concat, 345 "ARRAY_CONCAT": build_array_concat, 346 "ARRAY_INTERSECT": lambda args: exp.ArrayIntersect(expressions=args), 347 "ARRAY_INTERSECTION": lambda args: exp.ArrayIntersect(expressions=args), 348 "ARRAY_PREPEND": build_array_prepend, 349 "ARRAY_REMOVE": build_array_remove, 350 "COUNT": lambda args: exp.Count(this=seq_get(args, 0), expressions=args[1:], big_int=True), 351 "CONCAT": lambda args, dialect: exp.Concat( 352 expressions=args, 353 safe=not dialect.STRICT_STRING_CONCAT, 354 coalesce=dialect.CONCAT_COALESCE, 355 ), 356 "CONCAT_WS": lambda args, dialect: exp.ConcatWs( 357 expressions=args, 358 safe=not dialect.STRICT_STRING_CONCAT, 359 coalesce=dialect.CONCAT_WS_COALESCE, 360 ), 361 "CONVERT_TIMEZONE": build_convert_timezone, 362 "DATE_TO_DATE_STR": lambda args: exp.Cast( 363 this=seq_get(args, 0), 364 to=exp.DataType(this=exp.DType.TEXT), 365 ), 366 "GENERATE_DATE_ARRAY": lambda args: exp.GenerateDateArray( 367 start=seq_get(args, 0), 368 end=seq_get(args, 1), 369 step=seq_get(args, 2) or exp.Interval(this=exp.Literal.string(1), unit=exp.var("DAY")), 370 ), 371 "GENERATE_UUID": lambda args, dialect: exp.Uuid( 372 is_string=dialect.UUID_IS_STRING_TYPE or None 373 ), 374 "GLOB": lambda args: exp.Glob(this=seq_get(args, 1), expression=seq_get(args, 0)), 375 "GREATEST": lambda args, dialect: exp.Greatest( 376 this=seq_get(args, 0), 377 expressions=args[1:], 378 ignore_nulls=dialect.LEAST_GREATEST_IGNORES_NULLS, 379 ), 380 "LEAST": lambda args, dialect: exp.Least( 381 this=seq_get(args, 0), 382 expressions=args[1:], 383 ignore_nulls=dialect.LEAST_GREATEST_IGNORES_NULLS, 384 ), 385 "HEX": build_hex, 386 "JSON_EXTRACT": build_extract_json_with_path(exp.JSONExtract), 387 "JSON_EXTRACT_SCALAR": build_extract_json_with_path(exp.JSONExtractScalar), 388 "JSON_EXTRACT_PATH_TEXT": build_extract_json_with_path(exp.JSONExtractScalar), 389 "JSON_KEYS": lambda args, dialect: exp.JSONKeys( 390 this=seq_get(args, 0), expression=dialect.to_json_path(seq_get(args, 1)) 391 ), 392 "LIKE": build_like, 393 "LOG": build_logarithm, 394 "LOG2": lambda args: exp.Log(this=exp.Literal.number(2), expression=seq_get(args, 0)), 395 "LOG10": lambda args: exp.Log(this=exp.Literal.number(10), expression=seq_get(args, 0)), 396 "LOWER": build_lower, 397 "LPAD": lambda args: build_pad(args), 398 "LEFTPAD": lambda args: build_pad(args), 399 "LTRIM": lambda args: build_trim(args), 400 "MOD": build_mod, 401 "RIGHTPAD": lambda args: build_pad(args, is_left=False), 402 "RPAD": lambda args: build_pad(args, is_left=False), 403 "RTRIM": lambda args: build_trim(args, is_left=False), 404 "SCOPE_RESOLUTION": lambda args: ( 405 exp.ScopeResolution(expression=seq_get(args, 0)) 406 if len(args) != 2 407 else exp.ScopeResolution(this=seq_get(args, 0), expression=seq_get(args, 1)) 408 ), 409 "STRPOS": exp.StrPosition.from_arg_list, 410 "CHARINDEX": lambda args: build_locate_strposition(args), 411 "INSTR": exp.StrPosition.from_arg_list, 412 "LOCATE": lambda args: build_locate_strposition(args), 413 "TIME_TO_TIME_STR": lambda args: exp.Cast( 414 this=seq_get(args, 0), 415 to=exp.DataType(this=exp.DType.TEXT), 416 ), 417 "TO_HEX": build_hex, 418 "TS_OR_DS_TO_DATE_STR": lambda args: exp.Substring( 419 this=exp.Cast( 420 this=seq_get(args, 0), 421 to=exp.DataType(this=exp.DType.TEXT), 422 ), 423 start=exp.Literal.number(1), 424 length=exp.Literal.number(10), 425 ), 426 "UNNEST": lambda args: exp.Unnest(expressions=ensure_list(seq_get(args, 0))), 427 "UPPER": build_upper, 428 "UUID": lambda args, dialect: exp.Uuid(is_string=dialect.UUID_IS_STRING_TYPE or None), 429 "UUID_STRING": lambda args, dialect: exp.Uuid( 430 this=seq_get(args, 0), 431 name=seq_get(args, 1), 432 is_string=dialect.UUID_IS_STRING_TYPE or None, 433 ), 434 "VAR_MAP": build_var_map, 435 } 436 437 NO_PAREN_FUNCTIONS: t.ClassVar[dict] = { 438 TokenType.CURRENT_DATE: exp.CurrentDate, 439 TokenType.CURRENT_DATETIME: exp.CurrentDate, 440 TokenType.CURRENT_TIME: exp.CurrentTime, 441 TokenType.CURRENT_TIMESTAMP: exp.CurrentTimestamp, 442 TokenType.CURRENT_USER: exp.CurrentUser, 443 TokenType.CURRENT_ROLE: exp.CurrentRole, 444 } 445 446 STRUCT_TYPE_TOKENS: t.ClassVar = { 447 TokenType.NESTED, 448 TokenType.OBJECT, 449 TokenType.STRUCT, 450 TokenType.UNION, 451 } 452 453 NESTED_TYPE_TOKENS: t.ClassVar = { 454 TokenType.ARRAY, 455 TokenType.LIST, 456 TokenType.LOWCARDINALITY, 457 TokenType.MAP, 458 TokenType.NULLABLE, 459 TokenType.RANGE, 460 *STRUCT_TYPE_TOKENS, 461 } 462 463 ENUM_TYPE_TOKENS: t.ClassVar = { 464 TokenType.DYNAMIC, 465 TokenType.ENUM, 466 TokenType.ENUM8, 467 TokenType.ENUM16, 468 } 469 470 AGGREGATE_TYPE_TOKENS: t.ClassVar = { 471 TokenType.AGGREGATEFUNCTION, 472 TokenType.SIMPLEAGGREGATEFUNCTION, 473 } 474 475 TYPE_TOKENS: t.ClassVar = { 476 TokenType.BIT, 477 TokenType.BOOLEAN, 478 TokenType.TINYINT, 479 TokenType.UTINYINT, 480 TokenType.SMALLINT, 481 TokenType.USMALLINT, 482 TokenType.INT, 483 TokenType.UINT, 484 TokenType.BIGINT, 485 TokenType.UBIGINT, 486 TokenType.BIGNUM, 487 TokenType.INT128, 488 TokenType.UINT128, 489 TokenType.INT256, 490 TokenType.UINT256, 491 TokenType.MEDIUMINT, 492 TokenType.UMEDIUMINT, 493 TokenType.FIXEDSTRING, 494 TokenType.FLOAT, 495 TokenType.DOUBLE, 496 TokenType.UDOUBLE, 497 TokenType.CHAR, 498 TokenType.NCHAR, 499 TokenType.VARCHAR, 500 TokenType.NVARCHAR, 501 TokenType.BPCHAR, 502 TokenType.TEXT, 503 TokenType.MEDIUMTEXT, 504 TokenType.LONGTEXT, 505 TokenType.BLOB, 506 TokenType.MEDIUMBLOB, 507 TokenType.LONGBLOB, 508 TokenType.BINARY, 509 TokenType.VARBINARY, 510 TokenType.JSON, 511 TokenType.JSONB, 512 TokenType.INTERVAL, 513 TokenType.TINYBLOB, 514 TokenType.TINYTEXT, 515 TokenType.TIME, 516 TokenType.TIMETZ, 517 TokenType.TIME_NS, 518 TokenType.TIMESTAMP, 519 TokenType.TIMESTAMP_S, 520 TokenType.TIMESTAMP_MS, 521 TokenType.TIMESTAMP_NS, 522 TokenType.TIMESTAMPTZ, 523 TokenType.TIMESTAMPLTZ, 524 TokenType.TIMESTAMPNTZ, 525 TokenType.DATETIME, 526 TokenType.DATETIME2, 527 TokenType.DATETIME64, 528 TokenType.SMALLDATETIME, 529 TokenType.DATE, 530 TokenType.DATE32, 531 TokenType.INT4RANGE, 532 TokenType.INT4MULTIRANGE, 533 TokenType.INT8RANGE, 534 TokenType.INT8MULTIRANGE, 535 TokenType.NUMRANGE, 536 TokenType.NUMMULTIRANGE, 537 TokenType.TSRANGE, 538 TokenType.TSMULTIRANGE, 539 TokenType.TSTZRANGE, 540 TokenType.TSTZMULTIRANGE, 541 TokenType.DATERANGE, 542 TokenType.DATEMULTIRANGE, 543 TokenType.DECIMAL, 544 TokenType.DECIMAL32, 545 TokenType.DECIMAL64, 546 TokenType.DECIMAL128, 547 TokenType.DECIMAL256, 548 TokenType.DECFLOAT, 549 TokenType.UDECIMAL, 550 TokenType.BIGDECIMAL, 551 TokenType.UUID, 552 TokenType.GEOGRAPHY, 553 TokenType.GEOGRAPHYPOINT, 554 TokenType.GEOMETRY, 555 TokenType.POINT, 556 TokenType.RING, 557 TokenType.LINESTRING, 558 TokenType.MULTILINESTRING, 559 TokenType.POLYGON, 560 TokenType.MULTIPOLYGON, 561 TokenType.HLLSKETCH, 562 TokenType.HSTORE, 563 TokenType.PSEUDO_TYPE, 564 TokenType.SUPER, 565 TokenType.SERIAL, 566 TokenType.SMALLSERIAL, 567 TokenType.BIGSERIAL, 568 TokenType.XML, 569 TokenType.YEAR, 570 TokenType.USERDEFINED, 571 TokenType.MONEY, 572 TokenType.SMALLMONEY, 573 TokenType.ROWVERSION, 574 TokenType.IMAGE, 575 TokenType.VARIANT, 576 TokenType.VECTOR, 577 TokenType.VOID, 578 TokenType.OBJECT, 579 TokenType.OBJECT_IDENTIFIER, 580 TokenType.INET, 581 TokenType.IPADDRESS, 582 TokenType.IPPREFIX, 583 TokenType.IPV4, 584 TokenType.IPV6, 585 TokenType.UNKNOWN, 586 TokenType.NOTHING, 587 TokenType.NULL, 588 TokenType.NAME, 589 TokenType.TDIGEST, 590 TokenType.DYNAMIC, 591 *ENUM_TYPE_TOKENS, 592 *NESTED_TYPE_TOKENS, 593 *AGGREGATE_TYPE_TOKENS, 594 } 595 596 SIGNED_TO_UNSIGNED_TYPE_TOKEN: t.ClassVar = { 597 TokenType.BIGINT: TokenType.UBIGINT, 598 TokenType.INT: TokenType.UINT, 599 TokenType.MEDIUMINT: TokenType.UMEDIUMINT, 600 TokenType.SMALLINT: TokenType.USMALLINT, 601 TokenType.TINYINT: TokenType.UTINYINT, 602 TokenType.DECIMAL: TokenType.UDECIMAL, 603 TokenType.DOUBLE: TokenType.UDOUBLE, 604 } 605 606 SUBQUERY_PREDICATES: t.ClassVar = { 607 TokenType.ANY: exp.Any, 608 TokenType.ALL: exp.All, 609 TokenType.EXISTS: exp.Exists, 610 TokenType.SOME: exp.Any, 611 } 612 613 SUBQUERY_TOKENS: t.ClassVar = { 614 TokenType.SELECT, 615 TokenType.WITH, 616 TokenType.FROM, 617 } 618 619 RESERVED_TOKENS: t.ClassVar = { 620 *Tokenizer.SINGLE_TOKENS.values(), 621 TokenType.SELECT, 622 } - {TokenType.IDENTIFIER} 623 624 # Tokens whose text is extracted from delimited source text (e.g. quoted identifiers, 625 # string literals), so they must never be treated as keywords when matching by text 626 TEXT_MATCH_EXCLUDED_TOKENS: t.ClassVar[frozenset] = frozenset( 627 { 628 TokenType.BIT_STRING, 629 TokenType.BYTE_STRING, 630 TokenType.HEREDOC_STRING, 631 TokenType.HEX_STRING, 632 TokenType.IDENTIFIER, 633 TokenType.NATIONAL_STRING, 634 TokenType.RAW_STRING, 635 TokenType.STRING, 636 TokenType.UNICODE_STRING, 637 } 638 ) 639 640 DB_CREATABLES: t.ClassVar = { 641 TokenType.DATABASE, 642 TokenType.DICTIONARY, 643 TokenType.FILE_FORMAT, 644 TokenType.MODEL, 645 TokenType.NAMESPACE, 646 TokenType.SCHEMA, 647 TokenType.SEMANTIC_VIEW, 648 TokenType.SEQUENCE, 649 TokenType.SINK, 650 TokenType.SOURCE, 651 TokenType.STAGE, 652 TokenType.STORAGE_INTEGRATION, 653 TokenType.STREAMLIT, 654 TokenType.TABLE, 655 TokenType.TAG, 656 TokenType.VIEW, 657 TokenType.WAREHOUSE, 658 } 659 660 CREATABLES: t.ClassVar = { 661 TokenType.COLUMN, 662 TokenType.CONSTRAINT, 663 TokenType.FOREIGN_KEY, 664 TokenType.FUNCTION, 665 TokenType.INDEX, 666 TokenType.PROCEDURE, 667 TokenType.TRIGGER, 668 TokenType.TYPE, 669 *DB_CREATABLES, 670 } 671 672 TRIGGER_EVENTS: t.ClassVar = { 673 TokenType.INSERT, 674 TokenType.UPDATE, 675 TokenType.DELETE, 676 TokenType.TRUNCATE, 677 } 678 679 ALTERABLES: t.ClassVar = { 680 TokenType.INDEX, 681 TokenType.TABLE, 682 TokenType.VIEW, 683 TokenType.SESSION, 684 } 685 686 # Tokens that can represent identifiers 687 ID_VAR_TOKENS: t.ClassVar[set] = { 688 TokenType.ALL, 689 TokenType.ANALYZE, 690 TokenType.ATTACH, 691 TokenType.VAR, 692 TokenType.ANTI, 693 TokenType.APPLY, 694 TokenType.ASC, 695 TokenType.ASOF, 696 TokenType.AUTO_INCREMENT, 697 TokenType.BEGIN, 698 TokenType.BPCHAR, 699 TokenType.CACHE, 700 TokenType.CASE, 701 TokenType.COLLATE, 702 TokenType.COMMAND, 703 TokenType.COMMENT, 704 TokenType.COMMIT, 705 TokenType.CONSTRAINT, 706 TokenType.COPY, 707 TokenType.CUBE, 708 TokenType.CURRENT_SCHEMA, 709 TokenType.DEFAULT, 710 TokenType.DELETE, 711 TokenType.DESC, 712 TokenType.DESCRIBE, 713 TokenType.DETACH, 714 TokenType.DICTIONARY, 715 TokenType.DIV, 716 TokenType.END, 717 TokenType.EXECUTE, 718 TokenType.EXPORT, 719 TokenType.ESCAPE, 720 TokenType.FALSE, 721 TokenType.FIRST, 722 TokenType.FILE, 723 TokenType.FILTER, 724 TokenType.FINAL, 725 TokenType.FORMAT, 726 TokenType.FULL, 727 TokenType.GET, 728 TokenType.IDENTIFIER, 729 TokenType.INOUT, 730 TokenType.IS, 731 TokenType.ISNULL, 732 TokenType.INTERVAL, 733 TokenType.KEEP, 734 TokenType.KILL, 735 TokenType.LEFT, 736 TokenType.LIMIT, 737 TokenType.LOAD, 738 TokenType.LOCK, 739 TokenType.MATCH, 740 TokenType.MERGE, 741 TokenType.NATURAL, 742 TokenType.NEXT, 743 TokenType.OFFSET, 744 TokenType.OPERATOR, 745 TokenType.ORDINALITY, 746 TokenType.OUT, 747 TokenType.OVER, 748 TokenType.OVERLAPS, 749 TokenType.OVERWRITE, 750 TokenType.PARTITION, 751 TokenType.PERCENT, 752 TokenType.PIVOT, 753 TokenType.PROJECTION, 754 TokenType.PRAGMA, 755 TokenType.PUT, 756 TokenType.RANGE, 757 TokenType.RECURSIVE, 758 TokenType.REFERENCES, 759 TokenType.REFRESH, 760 TokenType.RENAME, 761 TokenType.REPLACE, 762 TokenType.RIGHT, 763 TokenType.ROLLUP, 764 TokenType.ROW, 765 TokenType.ROWS, 766 TokenType.SEMI, 767 TokenType.SET, 768 TokenType.SETTINGS, 769 TokenType.SHOW, 770 TokenType.STREAM, 771 TokenType.STREAMLIT, 772 TokenType.TEMPORARY, 773 TokenType.TOP, 774 TokenType.TRUE, 775 TokenType.TRUNCATE, 776 TokenType.UNIQUE, 777 TokenType.UNNEST, 778 TokenType.UNPIVOT, 779 TokenType.UPDATE, 780 TokenType.USE, 781 TokenType.VOLATILE, 782 TokenType.WINDOW, 783 TokenType.CURRENT_CATALOG, 784 TokenType.LOCALTIME, 785 TokenType.LOCALTIMESTAMP, 786 TokenType.SESSION_USER, 787 TokenType.STRAIGHT_JOIN, 788 *ALTERABLES, 789 *CREATABLES, 790 *SUBQUERY_PREDICATES, 791 *TYPE_TOKENS, 792 *NO_PAREN_FUNCTIONS, 793 } - {TokenType.UNION} 794 795 TABLE_ALIAS_TOKENS: t.ClassVar[set] = ID_VAR_TOKENS - { 796 TokenType.ANTI, 797 TokenType.ASOF, 798 TokenType.FULL, 799 TokenType.LEFT, 800 TokenType.LOCK, 801 TokenType.NATURAL, 802 TokenType.RIGHT, 803 TokenType.SEMI, 804 TokenType.WINDOW, 805 } 806 807 ALIAS_TOKENS: t.ClassVar = ID_VAR_TOKENS 808 809 COLON_PLACEHOLDER_TOKENS: t.ClassVar = ID_VAR_TOKENS 810 811 ARRAY_CONSTRUCTORS: t.ClassVar = { 812 "ARRAY": exp.Array, 813 "LIST": exp.List, 814 } 815 816 COMMENT_TABLE_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - {TokenType.IS} 817 818 UPDATE_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - {TokenType.SET} 819 820 TRIM_TYPES: t.ClassVar = {"LEADING", "TRAILING", "BOTH"} 821 822 # Tokens that indicate a simple column reference 823 IDENTIFIER_TOKENS: t.ClassVar[frozenset] = frozenset({TokenType.VAR, TokenType.IDENTIFIER}) 824 825 BRACKETS: t.ClassVar[frozenset] = frozenset({TokenType.L_BRACKET, TokenType.L_BRACE}) 826 827 # Postfix tokens that prevent the bare column fast path 828 COLUMN_POSTFIX_TOKENS: t.ClassVar[frozenset] = frozenset( 829 { 830 TokenType.L_PAREN, 831 TokenType.L_BRACKET, 832 TokenType.L_BRACE, 833 TokenType.COLON, 834 TokenType.JOIN_MARKER, 835 } 836 ) 837 838 TABLE_POSTFIX_TOKENS: t.ClassVar[frozenset] = frozenset( 839 { 840 TokenType.L_PAREN, 841 TokenType.L_BRACKET, 842 TokenType.L_BRACE, 843 TokenType.PIVOT, 844 TokenType.UNPIVOT, 845 TokenType.TABLE_SAMPLE, 846 } 847 ) 848 849 FUNC_TOKENS: t.ClassVar = { 850 TokenType.COLLATE, 851 TokenType.COMMAND, 852 TokenType.CURRENT_DATE, 853 TokenType.CURRENT_DATETIME, 854 TokenType.CURRENT_SCHEMA, 855 TokenType.CURRENT_TIMESTAMP, 856 TokenType.CURRENT_TIME, 857 TokenType.CURRENT_USER, 858 TokenType.CURRENT_CATALOG, 859 TokenType.FILTER, 860 TokenType.FIRST, 861 TokenType.FORMAT, 862 TokenType.GET, 863 TokenType.GLOB, 864 TokenType.IDENTIFIER, 865 TokenType.INDEX, 866 TokenType.ISNULL, 867 TokenType.ILIKE, 868 TokenType.INSERT, 869 TokenType.LIKE, 870 TokenType.LOCALTIME, 871 TokenType.LOCALTIMESTAMP, 872 TokenType.MERGE, 873 TokenType.NEXT, 874 TokenType.OFFSET, 875 TokenType.PRIMARY_KEY, 876 TokenType.RANGE, 877 TokenType.REPLACE, 878 TokenType.RLIKE, 879 TokenType.ROW, 880 TokenType.SESSION_USER, 881 TokenType.UNNEST, 882 TokenType.VAR, 883 TokenType.LEFT, 884 TokenType.RIGHT, 885 TokenType.SEQUENCE, 886 TokenType.DATE, 887 TokenType.DATETIME, 888 TokenType.TABLE, 889 TokenType.TIMESTAMP, 890 TokenType.TIMESTAMPTZ, 891 TokenType.TRUNCATE, 892 TokenType.UTC_DATE, 893 TokenType.UTC_TIME, 894 TokenType.UTC_TIMESTAMP, 895 TokenType.WINDOW, 896 TokenType.XOR, 897 *TYPE_TOKENS, 898 *SUBQUERY_PREDICATES, 899 } 900 901 CONJUNCTION: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 902 TokenType.AND: exp.And, 903 } 904 905 ASSIGNMENT: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 906 TokenType.COLON_EQ: exp.PropertyEQ, 907 } 908 909 DISJUNCTION: t.ClassVar[dict[TokenType, type[exp.Expr]]] = { 910 TokenType.OR: exp.Or, 911 } 912 913 EQUALITY: t.ClassVar = { 914 TokenType.EQ: exp.EQ, 915 TokenType.NEQ: exp.NEQ, 916 TokenType.NULLSAFE_EQ: exp.NullSafeEQ, 917 } 918 919 COMPARISON: t.ClassVar = { 920 TokenType.GT: exp.GT, 921 TokenType.GTE: exp.GTE, 922 TokenType.LT: exp.LT, 923 TokenType.LTE: exp.LTE, 924 } 925 926 BITWISE: t.ClassVar = { 927 TokenType.AMP: exp.BitwiseAnd, 928 TokenType.CARET: exp.BitwiseXor, 929 TokenType.PIPE: exp.BitwiseOr, 930 } 931 932 TERM: t.ClassVar = { 933 TokenType.DASH: exp.Sub, 934 TokenType.PLUS: exp.Add, 935 TokenType.MOD: exp.Mod, 936 TokenType.COLLATE: exp.Collate, 937 } 938 939 FACTOR: t.ClassVar = { 940 TokenType.DIV: exp.IntDiv, 941 TokenType.LR_ARROW: exp.Distance, 942 TokenType.LLRR_ARROW: exp.DistanceNd, 943 TokenType.SLASH: exp.Div, 944 TokenType.STAR: exp.Mul, 945 } 946 947 EXPONENT: t.ClassVar[dict[TokenType, type[exp.Expr]]] = {} 948 949 TIMES: t.ClassVar = { 950 TokenType.TIME, 951 TokenType.TIMETZ, 952 } 953 954 TIMESTAMPS: t.ClassVar = { 955 TokenType.TIMESTAMP, 956 TokenType.TIMESTAMPNTZ, 957 TokenType.TIMESTAMPTZ, 958 TokenType.TIMESTAMPLTZ, 959 *TIMES, 960 } 961 962 SET_OPERATIONS: t.ClassVar = { 963 TokenType.UNION, 964 TokenType.INTERSECT, 965 TokenType.EXCEPT, 966 } 967 968 JOIN_METHODS: t.ClassVar = { 969 TokenType.ASOF, 970 TokenType.NATURAL, 971 TokenType.POSITIONAL, 972 } 973 974 JOIN_SIDES: t.ClassVar = { 975 TokenType.LEFT, 976 TokenType.RIGHT, 977 TokenType.FULL, 978 } 979 980 JOIN_KINDS: t.ClassVar = { 981 TokenType.ANTI, 982 TokenType.CROSS, 983 TokenType.INNER, 984 TokenType.OUTER, 985 TokenType.SEMI, 986 TokenType.STRAIGHT_JOIN, 987 } 988 989 JOIN_HINTS: t.ClassVar[set[str]] = set() 990 991 # Tokens that unambiguously end a table reference on the fast path 992 TABLE_TERMINATORS: t.ClassVar[frozenset] = frozenset( 993 { 994 TokenType.COMMA, 995 TokenType.GROUP_BY, 996 TokenType.HAVING, 997 TokenType.JOIN, 998 TokenType.LIMIT, 999 TokenType.ON, 1000 TokenType.ORDER_BY, 1001 TokenType.R_PAREN, 1002 TokenType.SEMICOLON, 1003 TokenType.SENTINEL, 1004 TokenType.WHERE, 1005 *SET_OPERATIONS, 1006 *JOIN_KINDS, 1007 *JOIN_METHODS, 1008 *JOIN_SIDES, 1009 } 1010 ) 1011 1012 LAMBDAS: t.ClassVar = { 1013 TokenType.ARROW: lambda self, expressions: self.expression( 1014 exp.Lambda( 1015 this=self._replace_lambda( 1016 self._parse_disjunction(), 1017 expressions, 1018 ), 1019 expressions=expressions, 1020 ) 1021 ), 1022 TokenType.FARROW: lambda self, expressions: self.expression( 1023 exp.Kwarg( 1024 this=exp.var(expressions[0].name), 1025 expression=self._parse_disjunction() or self._parse_select(), 1026 ) 1027 ), 1028 } 1029 1030 # Whether lambda args include type annotations, e.g. TRANSFORM(arr, x INT -> x + 1) in Snowflake 1031 TYPED_LAMBDA_ARGS: t.ClassVar[bool] = False 1032 1033 LAMBDA_ARG_TERMINATORS: t.ClassVar[frozenset] = frozenset({TokenType.COMMA, TokenType.R_PAREN}) 1034 1035 COLUMN_OPERATORS: t.ClassVar = { 1036 TokenType.DOT: None, 1037 TokenType.DOTCOLON: lambda self, this, to: self.expression(exp.JSONCast(this=this, to=to)), 1038 TokenType.DCOLON: lambda self, this, to: self.build_cast( 1039 strict=self.STRICT_CAST, this=this, to=to 1040 ), 1041 TokenType.ARROW: lambda self, this, path: self.expression( 1042 exp.JSONExtract( 1043 this=this, 1044 expression=self.dialect.to_json_path(path), 1045 only_json_types=self.JSON_ARROWS_REQUIRE_JSON_TYPE, 1046 ) 1047 ), 1048 TokenType.DARROW: lambda self, this, path: self.expression( 1049 exp.JSONExtractScalar( 1050 this=this, 1051 expression=self.dialect.to_json_path(path), 1052 only_json_types=self.JSON_ARROWS_REQUIRE_JSON_TYPE, 1053 scalar_only=self.dialect.JSON_EXTRACT_SCALAR_SCALAR_ONLY, 1054 ) 1055 ), 1056 TokenType.HASH_ARROW: lambda self, this, path: self.expression( 1057 exp.JSONBExtract(this=this, expression=path) 1058 ), 1059 TokenType.DHASH_ARROW: lambda self, this, path: self.expression( 1060 exp.JSONBExtractScalar(this=this, expression=path) 1061 ), 1062 TokenType.PLACEHOLDER: lambda self, this, key: self.expression( 1063 exp.JSONBContains(this=this, expression=key) 1064 ), 1065 } 1066 1067 CAST_COLUMN_OPERATORS: t.ClassVar = { 1068 TokenType.DOTCOLON, 1069 TokenType.DCOLON, 1070 } 1071 1072 EXPRESSION_PARSERS: t.ClassVar = { 1073 exp.Cluster: lambda self: self._parse_sort(exp.Cluster, TokenType.CLUSTER_BY), 1074 exp.Column: lambda self: self._parse_column(), 1075 exp.ColumnDef: lambda self: self._parse_column_def(self._parse_column()), 1076 exp.Condition: lambda self: self._parse_disjunction(), 1077 exp.DataType: lambda self: self._parse_types(allow_identifiers=False, schema=True), 1078 exp.Expr: lambda self: self._parse_expression(), 1079 exp.From: lambda self: self._parse_from(joins=True), 1080 exp.GrantPrincipal: lambda self: self._parse_grant_principal(), 1081 exp.GrantPrivilege: lambda self: self._parse_grant_privilege(), 1082 exp.Group: lambda self: self._parse_group(), 1083 exp.Having: lambda self: self._parse_having(), 1084 exp.Hint: lambda self: self._parse_hint_body(), 1085 exp.Identifier: lambda self: self._parse_id_var(), 1086 exp.Join: lambda self: self._parse_join(), 1087 exp.Lambda: lambda self: self._parse_lambda(), 1088 exp.Lateral: lambda self: self._parse_lateral(), 1089 exp.Limit: lambda self: self._parse_limit(), 1090 exp.Offset: lambda self: self._parse_offset(), 1091 exp.Order: lambda self: self._parse_order(), 1092 exp.Ordered: lambda self: self._parse_ordered(), 1093 exp.Properties: lambda self: self._parse_properties(), 1094 exp.PartitionedByProperty: lambda self: self._parse_partitioned_by(), 1095 exp.Qualify: lambda self: self._parse_qualify(), 1096 exp.Returning: lambda self: self._parse_returning(), 1097 exp.Select: lambda self: self._parse_select(), 1098 exp.Sort: lambda self: self._parse_sort(exp.Sort, TokenType.SORT_BY), 1099 exp.Table: lambda self: self._parse_table_parts(), 1100 exp.TableAlias: lambda self: self._parse_table_alias(), 1101 exp.Tuple: lambda self: self._parse_value(values=False), 1102 exp.Whens: lambda self: self._parse_when_matched(), 1103 exp.Where: lambda self: self._parse_where(), 1104 exp.Window: lambda self: self._parse_named_window(), 1105 exp.With: lambda self: self._parse_with(), 1106 } 1107 1108 STATEMENT_PARSERS: t.ClassVar = { 1109 TokenType.ALTER: lambda self: self._parse_alter(), 1110 TokenType.ANALYZE: lambda self: self._parse_analyze(), 1111 TokenType.BEGIN: lambda self: self._parse_transaction(), 1112 TokenType.CACHE: lambda self: self._parse_cache(), 1113 TokenType.COMMENT: lambda self: self._parse_comment(), 1114 TokenType.COMMIT: lambda self: self._parse_commit_or_rollback(), 1115 TokenType.COPY: lambda self: self._parse_copy(), 1116 TokenType.CREATE: lambda self: self._parse_create(), 1117 TokenType.DELETE: lambda self: self._parse_delete(), 1118 TokenType.DESC: lambda self: self._parse_describe(), 1119 TokenType.DESCRIBE: lambda self: self._parse_describe(), 1120 TokenType.DROP: lambda self: self._parse_drop(), 1121 TokenType.GRANT: lambda self: self._parse_grant(), 1122 TokenType.REVOKE: lambda self: self._parse_revoke(), 1123 TokenType.INSERT: lambda self: self._parse_insert(), 1124 TokenType.KILL: lambda self: self._parse_kill(), 1125 TokenType.LOAD: lambda self: self._parse_load(), 1126 TokenType.MERGE: lambda self: self._parse_merge(), 1127 TokenType.PIVOT: lambda self: self._parse_simplified_pivot(), 1128 TokenType.PRAGMA: lambda self: self.expression(exp.Pragma(this=self._parse_expression())), 1129 TokenType.REFRESH: lambda self: self._parse_refresh(), 1130 TokenType.ROLLBACK: lambda self: self._parse_commit_or_rollback(), 1131 TokenType.SET: lambda self: self._parse_set(), 1132 TokenType.TRUNCATE: lambda self: self._parse_truncate_table(), 1133 TokenType.UNCACHE: lambda self: self._parse_uncache(), 1134 TokenType.UNPIVOT: lambda self: self._parse_simplified_pivot(is_unpivot=True), 1135 TokenType.UPDATE: lambda self: self._parse_update(), 1136 TokenType.USE: lambda self: self._parse_use(), 1137 TokenType.SEMICOLON: lambda self: exp.Semicolon(), 1138 } 1139 1140 UNARY_PARSERS: t.ClassVar = { 1141 TokenType.PLUS: lambda self: self._parse_unary(), # Unary + is handled as a no-op 1142 TokenType.NOT: lambda self: self.expression(exp.Not(this=self._parse_equality())), 1143 TokenType.TILDE: lambda self: self.expression(exp.BitwiseNot(this=self._parse_unary())), 1144 TokenType.DASH: lambda self: self.expression(exp.Neg(this=self._parse_unary())), 1145 TokenType.PIPE_SLASH: lambda self: self.expression(exp.Sqrt(this=self._parse_unary())), 1146 TokenType.DPIPE_SLASH: lambda self: self.expression(exp.Cbrt(this=self._parse_unary())), 1147 } 1148 1149 STRING_PARSERS: t.ClassVar = { 1150 TokenType.HEREDOC_STRING: lambda self, token: self.expression( 1151 exp.RawString(this=token.text), token 1152 ), 1153 TokenType.NATIONAL_STRING: lambda self, token: self.expression( 1154 exp.National(this=token.text), token 1155 ), 1156 TokenType.RAW_STRING: lambda self, token: self.expression( 1157 exp.RawString(this=token.text), token 1158 ), 1159 TokenType.STRING: lambda self, token: self.expression( 1160 exp.Literal(this=token.text, is_string=True), token 1161 ), 1162 TokenType.UNICODE_STRING: lambda self, token: self.expression( 1163 exp.UnicodeString( 1164 this=token.text, escape=self._match_text_seq("UESCAPE") and self._parse_string() 1165 ), 1166 token, 1167 ), 1168 } 1169 1170 NUMERIC_PARSERS: t.ClassVar = { 1171 TokenType.BIT_STRING: lambda self, token: self.expression( 1172 exp.BitString(this=token.text), token 1173 ), 1174 TokenType.BYTE_STRING: lambda self, token: self.expression( 1175 exp.ByteString( 1176 this=token.text, is_bytes=self.dialect.BYTE_STRING_IS_BYTES_TYPE or None 1177 ), 1178 token, 1179 ), 1180 TokenType.HEX_STRING: lambda self, token: self.expression( 1181 exp.HexString( 1182 this=token.text, is_integer=self.dialect.HEX_STRING_IS_INTEGER_TYPE or None 1183 ), 1184 token, 1185 ), 1186 TokenType.NUMBER: lambda self, token: self.expression( 1187 exp.Literal(this=token.text, is_string=False), token 1188 ), 1189 } 1190 1191 PRIMARY_PARSERS: t.ClassVar = { 1192 **STRING_PARSERS, 1193 **NUMERIC_PARSERS, 1194 TokenType.INTRODUCER: lambda self, token: self._parse_introducer(token), 1195 TokenType.NULL: lambda self, _: self.expression(exp.Null()), 1196 TokenType.TRUE: lambda self, _: self.expression(exp.Boolean(this=True)), 1197 TokenType.FALSE: lambda self, _: self.expression(exp.Boolean(this=False)), 1198 TokenType.SESSION_PARAMETER: lambda self, _: self._parse_session_parameter(), 1199 TokenType.STAR: lambda self, _: self._parse_star_ops(), 1200 } 1201 1202 PLACEHOLDER_PARSERS: t.ClassVar = { 1203 TokenType.PLACEHOLDER: lambda self: self.expression(exp.Placeholder()), 1204 TokenType.PARAMETER: lambda self: self._parse_parameter(), 1205 TokenType.COLON: lambda self: ( 1206 self.expression(exp.Placeholder(this=self._prev.text)) 1207 if self._match_set(self.COLON_PLACEHOLDER_TOKENS) 1208 else None 1209 ), 1210 } 1211 1212 RANGE_PARSERS: t.ClassVar = { 1213 TokenType.AT_GT: binary_range_parser(exp.ArrayContainsAll), 1214 TokenType.BETWEEN: lambda self, this: self._parse_between(this), 1215 TokenType.GLOB: binary_range_parser(exp.Glob), 1216 TokenType.ILIKE: binary_range_parser(exp.ILike), 1217 TokenType.IN: lambda self, this: self._parse_in(this), 1218 TokenType.IRLIKE: binary_range_parser(exp.RegexpILike), 1219 TokenType.IS: lambda self, this: self._parse_is(this), 1220 TokenType.LIKE: binary_range_parser(exp.Like), 1221 TokenType.LT_AT: binary_range_parser(exp.ArrayContainedBy), 1222 TokenType.OVERLAPS: binary_range_parser(exp.Overlaps), 1223 TokenType.RLIKE: binary_range_parser(exp.RegexpLike), 1224 TokenType.SIMILAR_TO: binary_range_parser(exp.SimilarTo), 1225 TokenType.FOR: lambda self, this: self._parse_comprehension(this), 1226 TokenType.QMARK_AMP: binary_range_parser(exp.JSONBContainsAllTopKeys), 1227 TokenType.QMARK_PIPE: binary_range_parser(exp.JSONBContainsAnyTopKeys), 1228 TokenType.HASH_DASH: binary_range_parser(exp.JSONBDeleteAtPath), 1229 TokenType.AT_QMARK: binary_range_parser(exp.JSONBPathExists), 1230 TokenType.ADJACENT: binary_range_parser(exp.Adjacent), 1231 TokenType.OPERATOR: lambda self, this: self._parse_operator(this), 1232 TokenType.AMP_LT: binary_range_parser(exp.ExtendsLeft), 1233 TokenType.AMP_GT: binary_range_parser(exp.ExtendsRight), 1234 } 1235 1236 PIPE_SYNTAX_TRANSFORM_PARSERS: t.ClassVar = { 1237 "AGGREGATE": lambda self, query: self._parse_pipe_syntax_aggregate(query), 1238 "AS": lambda self, query: self._build_pipe_cte( 1239 query, [exp.Star()], self._parse_table_alias() 1240 ), 1241 "DISTINCT": lambda self, query: self._advance() or query.distinct(copy=False), 1242 "EXTEND": lambda self, query: self._parse_pipe_syntax_extend(query), 1243 "LIMIT": lambda self, query: self._parse_pipe_syntax_limit(query), 1244 "ORDER BY": lambda self, query: query.order_by( 1245 self._parse_order(), append=False, copy=False 1246 ), 1247 "PIVOT": lambda self, query: self._parse_pipe_syntax_pivot(query), 1248 "SELECT": lambda self, query: self._parse_pipe_syntax_select(query), 1249 "TABLESAMPLE": lambda self, query: self._parse_pipe_syntax_tablesample(query), 1250 "UNPIVOT": lambda self, query: self._parse_pipe_syntax_pivot(query), 1251 "WHERE": lambda self, query: query.where(self._parse_where(), copy=False), 1252 } 1253 1254 PROPERTY_PARSERS: t.ClassVar[dict[str, t.Callable]] = { 1255 "ALLOWED_VALUES": lambda self: self.expression( 1256 exp.AllowedValuesProperty(expressions=self._parse_csv(self._parse_primary)) 1257 ), 1258 "ALGORITHM": lambda self: self._parse_property_assignment(exp.AlgorithmProperty), 1259 "AUTO": lambda self: self._parse_auto_property(), 1260 "AUTO_INCREMENT": lambda self: self._parse_property_assignment(exp.AutoIncrementProperty), 1261 "BACKUP": lambda self: self.expression( 1262 exp.BackupProperty(this=self._parse_var(any_token=True)) 1263 ), 1264 "BLOCKCOMPRESSION": lambda self: self._parse_blockcompression(), 1265 "CALLED": lambda self: self._parse_called_on_null_input_property(), 1266 "CHARSET": lambda self, **kwargs: self._parse_character_set(**kwargs), 1267 "CHARACTER SET": lambda self, **kwargs: self._parse_character_set(**kwargs), 1268 "CHECKSUM": lambda self: self._parse_checksum(), 1269 "CLUSTER BY": lambda self: self._parse_cluster_property(), 1270 "CLUSTERED": lambda self: self._parse_clustered_by(), 1271 "COLLATE": lambda self, **kwargs: self._parse_property_assignment( 1272 exp.CollateProperty, **kwargs 1273 ), 1274 "COMMENT": lambda self: self._parse_property_assignment(exp.SchemaCommentProperty), 1275 "CONTAINS": lambda self: self._parse_contains_property(), 1276 "COPY": lambda self: self._parse_copy_property(), 1277 "DATABLOCKSIZE": lambda self, **kwargs: self._parse_datablocksize(**kwargs), 1278 "DATA_DELETION": lambda self: self._parse_data_deletion_property(), 1279 "DEFINER": lambda self: self._parse_definer(), 1280 "DETERMINISTIC": lambda self: self.expression( 1281 exp.StabilityProperty(this=exp.Literal.string("IMMUTABLE")) 1282 ), 1283 "DISTRIBUTED": lambda self: self._parse_distributed_property(), 1284 "DUPLICATE": lambda self: self._parse_composite_key_property(exp.DuplicateKeyProperty), 1285 "DYNAMIC": lambda self: self.expression(exp.DynamicProperty()), 1286 "DISTKEY": lambda self: self._parse_distkey(), 1287 "DISTSTYLE": lambda self: self._parse_property_assignment(exp.DistStyleProperty), 1288 "EMPTY": lambda self: self.expression(exp.EmptyProperty()), 1289 "ENGINE": lambda self: self._parse_property_assignment(exp.EngineProperty), 1290 "ENVIRONMENT": lambda self: self.expression( 1291 exp.EnviromentProperty(expressions=self._parse_wrapped_csv(self._parse_assignment)) 1292 ), 1293 "HANDLER": lambda self: self._parse_property_assignment(exp.HandlerProperty), 1294 "EXECUTE": lambda self: self._parse_property_assignment(exp.ExecuteAsProperty), 1295 "EXTERNAL": lambda self: self.expression(exp.ExternalProperty()), 1296 "FALLBACK": lambda self, **kwargs: self._parse_fallback(**kwargs), 1297 "FORMAT": lambda self: self._parse_property_assignment(exp.FileFormatProperty), 1298 "FREESPACE": lambda self: self._parse_freespace(), 1299 "GLOBAL": lambda self: self.expression(exp.GlobalProperty()), 1300 "HEAP": lambda self: self.expression(exp.HeapProperty()), 1301 "ICEBERG": lambda self: self.expression(exp.IcebergProperty()), 1302 "IMMUTABLE": lambda self: self.expression( 1303 exp.StabilityProperty(this=exp.Literal.string("IMMUTABLE")) 1304 ), 1305 "INHERITS": lambda self: self.expression( 1306 exp.InheritsProperty(expressions=self._parse_wrapped_csv(self._parse_table)) 1307 ), 1308 "INPUT": lambda self: self.expression(exp.InputModelProperty(this=self._parse_schema())), 1309 "JOURNAL": lambda self, **kwargs: self._parse_journal(**kwargs), 1310 "LANGUAGE": lambda self: self._parse_property_assignment(exp.LanguageProperty), 1311 "LAYOUT": lambda self: self._parse_dict_property(this="LAYOUT"), 1312 "LIFETIME": lambda self: self._parse_dict_range(this="LIFETIME"), 1313 "LIKE": lambda self: self._parse_create_like(), 1314 "LOCATION": lambda self: self._parse_property_assignment(exp.LocationProperty), 1315 "LOCK": lambda self: self._parse_locking(), 1316 "LOCKING": lambda self: self._parse_locking(), 1317 "LOG": lambda self, **kwargs: self._parse_log(**kwargs), 1318 "MATERIALIZED": lambda self: self.expression(exp.MaterializedProperty()), 1319 "MERGEBLOCKRATIO": lambda self, **kwargs: self._parse_mergeblockratio(**kwargs), 1320 "MODIFIES": lambda self: self._parse_modifies_property(), 1321 "MULTISET": lambda self: self.expression(exp.SetProperty(multi=True)), 1322 "NO": lambda self: self._parse_no_property(), 1323 "ON": lambda self: self._parse_on_property(), 1324 "ORDER BY": lambda self: self._parse_order(skip_order_token=True), 1325 "OUTPUT": lambda self: self.expression(exp.OutputModelProperty(this=self._parse_schema())), 1326 "PARTITION": lambda self: self._parse_partitioned_of(), 1327 "PARTITION BY": lambda self: self._parse_partitioned_by(), 1328 "PARTITIONED BY": lambda self: self._parse_partitioned_by(), 1329 "PARTITIONED_BY": lambda self: self._parse_partitioned_by(), 1330 "PRIMARY KEY": lambda self: self._parse_primary_key(in_props=True), 1331 "RANGE": lambda self: self._parse_dict_range(this="RANGE"), 1332 "READS": lambda self: self._parse_reads_property(), 1333 "REMOTE": lambda self: self._parse_remote_with_connection(), 1334 "RETURNS": lambda self: self._parse_returns(), 1335 "STRICT": lambda self: self.expression(exp.StrictProperty()), 1336 "STREAMING": lambda self: self.expression(exp.StreamingTableProperty()), 1337 "ROW": lambda self: self._parse_row(), 1338 "ROW_FORMAT": lambda self: self._parse_property_assignment(exp.RowFormatProperty), 1339 "SAMPLE": lambda self: self.expression( 1340 exp.SampleProperty(this=self._match_text_seq("BY") and self._parse_bitwise()) 1341 ), 1342 "SECURE": lambda self: self.expression(exp.SecureProperty()), 1343 "SECURITY": lambda self: self._parse_sql_security(), 1344 "SQL SECURITY": lambda self: self._parse_sql_security(), 1345 "SET": lambda self: self.expression(exp.SetProperty(multi=False)), 1346 "SETTINGS": lambda self: self._parse_settings_property(), 1347 "SHARING": lambda self: self._parse_property_assignment(exp.SharingProperty), 1348 "SORTKEY": lambda self: self._parse_sortkey(), 1349 "SOURCE": lambda self: self._parse_dict_property(this="SOURCE"), 1350 "STABLE": lambda self: self.expression( 1351 exp.StabilityProperty(this=exp.Literal.string("STABLE")) 1352 ), 1353 "STORED": lambda self: self._parse_stored(), 1354 "SYSTEM_VERSIONING": lambda self: self._parse_system_versioning_property(), 1355 "TBLPROPERTIES": lambda self: self._parse_wrapped_properties(), 1356 "TEMP": lambda self: self.expression(exp.TemporaryProperty()), 1357 "TEMPORARY": lambda self: self.expression(exp.TemporaryProperty()), 1358 "TO": lambda self: self._parse_to_table(), 1359 "TRANSIENT": lambda self: self.expression(exp.TransientProperty()), 1360 "TRANSFORM": lambda self: self.expression( 1361 exp.TransformModelProperty(expressions=self._parse_wrapped_csv(self._parse_expression)) 1362 ), 1363 "TTL": lambda self: self._parse_ttl(), 1364 "USING": lambda self: self._parse_property_assignment(exp.FileFormatProperty), 1365 "UNLOGGED": lambda self: self.expression(exp.UnloggedProperty()), 1366 "VOLATILE": lambda self: self._parse_volatile_property(), 1367 "WITH": lambda self: self._parse_with_property(), 1368 } 1369 1370 CONSTRAINT_PARSERS: t.ClassVar = { 1371 "AUTOINCREMENT": lambda self: self._parse_auto_increment(), 1372 "AUTO_INCREMENT": lambda self: self._parse_auto_increment(), 1373 "CASESPECIFIC": lambda self: self.expression(exp.CaseSpecificColumnConstraint(not_=False)), 1374 "CHARACTER SET": lambda self: self.expression( 1375 exp.CharacterSetColumnConstraint(this=self._parse_var_or_string()) 1376 ), 1377 "CHECK": lambda self: self._parse_check_constraint(), 1378 "COLLATE": lambda self: self.expression( 1379 exp.CollateColumnConstraint(this=self._parse_identifier() or self._parse_column()) 1380 ), 1381 "COMMENT": lambda self: self.expression( 1382 exp.CommentColumnConstraint(this=self._parse_string()) 1383 ), 1384 "COMPRESS": lambda self: self._parse_compress(), 1385 "CLUSTERED": lambda self: self.expression( 1386 exp.ClusteredColumnConstraint(this=self._parse_wrapped_csv(self._parse_ordered)) 1387 ), 1388 "NONCLUSTERED": lambda self: self.expression( 1389 exp.NonClusteredColumnConstraint(this=self._parse_wrapped_csv(self._parse_ordered)) 1390 ), 1391 "DEFAULT": lambda self: self.expression( 1392 exp.DefaultColumnConstraint(this=self._parse_bitwise()) 1393 ), 1394 "ENCODE": lambda self: self.expression(exp.EncodeColumnConstraint(this=self._parse_var())), 1395 "EPHEMERAL": lambda self: self.expression( 1396 exp.EphemeralColumnConstraint(this=self._parse_bitwise()) 1397 ), 1398 "EXCLUDE": lambda self: self.expression( 1399 exp.ExcludeColumnConstraint(this=self._parse_index_params()) 1400 ), 1401 "FOREIGN KEY": lambda self: self._parse_foreign_key(), 1402 "FORMAT": lambda self: self.expression( 1403 exp.DateFormatColumnConstraint(this=self._parse_var_or_string()) 1404 ), 1405 "GENERATED": lambda self: self._parse_generated_as_identity(), 1406 "IDENTITY": lambda self: self._parse_auto_increment(), 1407 "INLINE": lambda self: self._parse_inline(), 1408 "LIKE": lambda self: self._parse_create_like(), 1409 "NOT": lambda self: self._parse_not_constraint(), 1410 "NULL": lambda self: self.expression(exp.NotNullColumnConstraint(allow_null=True)), 1411 "ON": lambda self: ( 1412 ( 1413 self._match(TokenType.UPDATE) 1414 and self.expression(exp.OnUpdateColumnConstraint(this=self._parse_function())) 1415 ) 1416 or self.expression(exp.OnProperty(this=self._parse_id_var())) 1417 ), 1418 "PATH": lambda self: self.expression(exp.PathColumnConstraint(this=self._parse_string())), 1419 "PERIOD": lambda self: self._parse_period_for_system_time(), 1420 "PRIMARY KEY": lambda self: self._parse_primary_key(), 1421 "REFERENCES": lambda self: self._parse_references(match=False), 1422 "TITLE": lambda self: self.expression( 1423 exp.TitleColumnConstraint(this=self._parse_var_or_string()) 1424 ), 1425 "TTL": lambda self: self.expression(exp.MergeTreeTTL(expressions=[self._parse_bitwise()])), 1426 "UNIQUE": lambda self: self._parse_unique(), 1427 "UPPERCASE": lambda self: self.expression(exp.UppercaseColumnConstraint()), 1428 "WITH": lambda self: self.expression( 1429 exp.Properties(expressions=self._parse_wrapped_properties()) 1430 ), 1431 "BUCKET": lambda self: self._parse_partitioned_by_bucket_or_truncate(), 1432 "TRUNCATE": lambda self: self._parse_partitioned_by_bucket_or_truncate(), 1433 } 1434 1435 def _parse_partitioned_by_bucket_or_truncate(self) -> exp.Expr | None: 1436 if not self._match(TokenType.L_PAREN, advance=False): 1437 # Partitioning by bucket or truncate follows the syntax: 1438 # PARTITION BY (BUCKET(..) | TRUNCATE(..)) 1439 # If we don't have parenthesis after each keyword, we should instead parse this as an identifier 1440 self._retreat(self._index - 1) 1441 return None 1442 1443 klass = ( 1444 exp.PartitionedByBucket 1445 if self._prev.text.upper() == "BUCKET" 1446 else exp.PartitionByTruncate 1447 ) 1448 1449 args = self._parse_wrapped_csv(lambda: self._parse_primary() or self._parse_column()) 1450 this, expression = seq_get(args, 0), seq_get(args, 1) 1451 1452 if isinstance(this, exp.Literal): 1453 # Check for Iceberg partition transforms (bucket / truncate) and ensure their arguments are in the right order 1454 # - For Hive, it's `bucket(<num buckets>, <col name>)` or `truncate(<num_chars>, <col_name>)` 1455 # - For Trino, it's reversed - `bucket(<col name>, <num buckets>)` or `truncate(<col_name>, <num_chars>)` 1456 # Both variants are canonicalized in the latter i.e `bucket(<col name>, <num buckets>)` 1457 # 1458 # Hive ref: https://docs.aws.amazon.com/athena/latest/ug/querying-iceberg-creating-tables.html#querying-iceberg-partitioning 1459 # Trino ref: https://docs.aws.amazon.com/athena/latest/ug/create-table-as.html#ctas-table-properties 1460 this, expression = expression, this 1461 1462 return self.expression(klass(this=this, expression=expression)) 1463 1464 ALTER_PARSERS: t.ClassVar = { 1465 "ADD": lambda self: self._parse_alter_table_add(), 1466 "AS": lambda self: self._parse_select(), 1467 "ALTER": lambda self: self._parse_alter_table_alter(), 1468 "CLUSTER BY": lambda self: self._parse_cluster_property(), 1469 "DELETE": lambda self: self.expression(exp.Delete(where=self._parse_where())), 1470 "DROP": lambda self: self._parse_alter_table_drop(), 1471 "RENAME": lambda self: self._parse_alter_table_rename(), 1472 "SET": lambda self: self._parse_alter_table_set(), 1473 "SWAP": lambda self: self.expression( 1474 exp.SwapTable(this=self._match(TokenType.WITH) and self._parse_table(schema=True)) 1475 ), 1476 } 1477 1478 ALTER_ALTER_PARSERS: t.ClassVar = { 1479 "DISTKEY": lambda self: self._parse_alter_diststyle(), 1480 "DISTSTYLE": lambda self: self._parse_alter_diststyle(), 1481 "SORTKEY": lambda self: self._parse_alter_sortkey(), 1482 "COMPOUND": lambda self: self._parse_alter_sortkey(compound=True), 1483 } 1484 1485 SCHEMA_UNNAMED_CONSTRAINTS: t.ClassVar = { 1486 "CHECK", 1487 "EXCLUDE", 1488 "FOREIGN KEY", 1489 "LIKE", 1490 "PERIOD", 1491 "PRIMARY KEY", 1492 "UNIQUE", 1493 "BUCKET", 1494 "TRUNCATE", 1495 } 1496 1497 NO_PAREN_FUNCTION_PARSERS: t.ClassVar = { 1498 "ANY": lambda self: self.expression(exp.Any(this=self._parse_bitwise())), 1499 "CASE": lambda self: self._parse_case(), 1500 "CONNECT_BY_ROOT": lambda self: self.expression( 1501 exp.ConnectByRoot(this=self._parse_column()) 1502 ), 1503 "IF": lambda self: self._parse_if(), 1504 } 1505 1506 INVALID_FUNC_NAME_TOKENS: t.ClassVar = { 1507 TokenType.IDENTIFIER, 1508 TokenType.STRING, 1509 } 1510 1511 FUNCTIONS_WITH_ALIASED_ARGS: t.ClassVar = {"STRUCT"} 1512 1513 KEY_VALUE_DEFINITIONS: t.ClassVar = (exp.Alias, exp.EQ, exp.PropertyEQ, exp.Slice) 1514 1515 FUNCTION_PARSERS: t.ClassVar[dict[str, t.Callable]] = { 1516 **{ 1517 name: lambda self: self._parse_distinct_arg_function(exp.ArgMax) 1518 for name in exp.ArgMax.sql_names() 1519 }, 1520 **{ 1521 name: lambda self: self._parse_distinct_arg_function(exp.ArgMin) 1522 for name in exp.ArgMin.sql_names() 1523 }, 1524 "CAST": lambda self: self._parse_cast(self.STRICT_CAST), 1525 "CEIL": lambda self: self._parse_ceil_floor(exp.Ceil), 1526 "CONVERT": lambda self: self._parse_convert(self.STRICT_CAST), 1527 "CHAR": lambda self: self._parse_char(), 1528 "CHR": lambda self: self._parse_char(), 1529 "DECODE": lambda self: self._parse_decode(), 1530 "EXTRACT": lambda self: self._parse_extract(), 1531 "FLOOR": lambda self: self._parse_ceil_floor(exp.Floor), 1532 "GAP_FILL": lambda self: self._parse_gap_fill(), 1533 "INITCAP": lambda self: self._parse_initcap(), 1534 "JSON_OBJECT": lambda self: self._parse_json_object(), 1535 "JSON_OBJECTAGG": lambda self: self._parse_json_object(agg=True), 1536 "JSON_TABLE": lambda self: self._parse_json_table(), 1537 "MATCH": lambda self: self._parse_match_against(), 1538 "NORMALIZE": lambda self: self._parse_normalize(), 1539 "OPENJSON": lambda self: self._parse_open_json(), 1540 "OVERLAY": lambda self: self._parse_overlay(), 1541 "POSITION": lambda self: self._parse_position(), 1542 "SAFE_CAST": lambda self: self._parse_cast(False, safe=True), 1543 "STRING_AGG": lambda self: self._parse_string_agg(), 1544 "SUBSTRING": lambda self: self._parse_substring(), 1545 "TRIM": lambda self: self._parse_trim(), 1546 "TRY_CAST": lambda self: self._parse_cast(False, safe=True), 1547 "TRY_CONVERT": lambda self: self._parse_convert(False, safe=True), 1548 "XMLELEMENT": lambda self: self._parse_xml_element(), 1549 "XMLTABLE": lambda self: self._parse_xml_table(), 1550 } 1551 1552 QUERY_MODIFIER_PARSERS: t.ClassVar = { 1553 TokenType.MATCH_RECOGNIZE: lambda self: ("match", self._parse_match_recognize()), 1554 TokenType.PREWHERE: lambda self: ("prewhere", self._parse_prewhere()), 1555 TokenType.WHERE: lambda self: ("where", self._parse_where()), 1556 TokenType.GROUP_BY: lambda self: ("group", self._parse_group()), 1557 TokenType.HAVING: lambda self: ("having", self._parse_having()), 1558 TokenType.QUALIFY: lambda self: ("qualify", self._parse_qualify()), 1559 TokenType.WINDOW: lambda self: ("windows", self._parse_window_clause()), 1560 TokenType.ORDER_BY: lambda self: ("order", self._parse_order()), 1561 TokenType.LIMIT: lambda self: ("limit", self._parse_limit()), 1562 TokenType.FETCH: lambda self: ("limit", self._parse_limit()), 1563 TokenType.OFFSET: lambda self: ("offset", self._parse_offset()), 1564 TokenType.FOR: lambda self: ("locks", self._parse_locks()), 1565 TokenType.LOCK: lambda self: ("locks", self._parse_locks()), 1566 TokenType.TABLE_SAMPLE: lambda self: ("sample", self._parse_table_sample(as_modifier=True)), 1567 TokenType.USING: lambda self: ("sample", self._parse_table_sample(as_modifier=True)), 1568 TokenType.CLUSTER_BY: lambda self: ( 1569 "cluster", 1570 self._parse_cluster(), 1571 ), 1572 TokenType.DISTRIBUTE_BY: lambda self: ( 1573 "distribute", 1574 self._parse_sort(exp.Distribute, TokenType.DISTRIBUTE_BY), 1575 ), 1576 TokenType.SORT_BY: lambda self: ("sort", self._parse_sort(exp.Sort, TokenType.SORT_BY)), 1577 TokenType.CONNECT_BY: lambda self: ("connect", self._parse_connect(skip_start_token=True)), 1578 TokenType.START_WITH: lambda self: ("connect", self._parse_connect()), 1579 } 1580 QUERY_MODIFIER_TOKENS: t.ClassVar = set(QUERY_MODIFIER_PARSERS) 1581 1582 SET_PARSERS: t.ClassVar = { 1583 "GLOBAL": lambda self: self._parse_set_item_assignment("GLOBAL"), 1584 "LOCAL": lambda self: self._parse_set_item_assignment("LOCAL"), 1585 "SESSION": lambda self: self._parse_set_item_assignment("SESSION"), 1586 "TRANSACTION": lambda self: self._parse_set_transaction(), 1587 } 1588 1589 SHOW_PARSERS: t.ClassVar[dict[str, t.Callable]] = {} 1590 1591 TYPE_LITERAL_PARSERS: t.ClassVar = { 1592 exp.DType.JSON: lambda self, this, _: self.expression(exp.ParseJSON(this=this)), 1593 } 1594 1595 TYPE_CONVERTERS: t.ClassVar[dict[exp.DType, t.Callable[[exp.DataType], exp.DataType]]] = {} 1596 1597 DDL_SELECT_TOKENS: t.ClassVar = {TokenType.SELECT, TokenType.WITH, TokenType.L_PAREN} 1598 1599 PRE_VOLATILE_TOKENS: t.ClassVar = {TokenType.CREATE, TokenType.REPLACE, TokenType.UNIQUE} 1600 1601 TRANSACTION_KIND: t.ClassVar = {"DEFERRED", "IMMEDIATE", "EXCLUSIVE"} 1602 TRANSACTION_CHARACTERISTICS: t.ClassVar[OPTIONS_TYPE] = { 1603 "ISOLATION": ( 1604 ("LEVEL", "REPEATABLE", "READ"), 1605 ("LEVEL", "READ", "COMMITTED"), 1606 ("LEVEL", "READ", "UNCOMITTED"), 1607 ("LEVEL", "SERIALIZABLE"), 1608 ), 1609 "READ": ("WRITE", "ONLY"), 1610 } 1611 1612 CONFLICT_ACTIONS: t.ClassVar[OPTIONS_TYPE] = { 1613 **dict.fromkeys(("ABORT", "FAIL", "IGNORE", "REPLACE", "ROLLBACK", "UPDATE"), tuple()), 1614 "DO": ("NOTHING", "UPDATE"), 1615 } 1616 1617 TRIGGER_TIMING: t.ClassVar[OPTIONS_TYPE] = { 1618 "INSTEAD": (("OF",),), 1619 "BEFORE": tuple(), 1620 "AFTER": tuple(), 1621 } 1622 1623 TRIGGER_DEFERRABLE: t.ClassVar[OPTIONS_TYPE] = { 1624 "NOT": (("DEFERRABLE",),), 1625 "DEFERRABLE": tuple(), 1626 } 1627 1628 CREATE_SEQUENCE: t.ClassVar[OPTIONS_TYPE] = { 1629 "SCALE": ("EXTEND", "NOEXTEND"), 1630 "SHARD": ("EXTEND", "NOEXTEND"), 1631 "NO": ("CYCLE", "CACHE", "MAXVALUE", "MINVALUE"), 1632 **dict.fromkeys( 1633 ( 1634 "SESSION", 1635 "GLOBAL", 1636 "KEEP", 1637 "NOKEEP", 1638 "ORDER", 1639 "NOORDER", 1640 "NOCACHE", 1641 "CYCLE", 1642 "NOCYCLE", 1643 "NOMINVALUE", 1644 "NOMAXVALUE", 1645 "NOSCALE", 1646 "NOSHARD", 1647 ), 1648 tuple(), 1649 ), 1650 } 1651 1652 ISOLATED_LOADING_OPTIONS: t.ClassVar[OPTIONS_TYPE] = {"FOR": ("ALL", "INSERT", "NONE")} 1653 1654 USABLES: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys( 1655 ("ROLE", "WAREHOUSE", "DATABASE", "SCHEMA", "CATALOG"), tuple() 1656 ) 1657 1658 CAST_ACTIONS: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys(("RENAME", "ADD"), ("FIELDS",)) 1659 1660 SCHEMA_BINDING_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1661 "TYPE": ("EVOLUTION",), 1662 **dict.fromkeys(("BINDING", "COMPENSATION", "EVOLUTION"), tuple()), 1663 } 1664 1665 PROCEDURE_OPTIONS: t.ClassVar[OPTIONS_TYPE] = {} 1666 1667 EXECUTE_AS_OPTIONS: t.ClassVar[OPTIONS_TYPE] = dict.fromkeys( 1668 ("CALLER", "SELF", "OWNER"), tuple() 1669 ) 1670 1671 KEY_CONSTRAINT_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1672 "NOT": ("ENFORCED",), 1673 "MATCH": ( 1674 "FULL", 1675 "PARTIAL", 1676 "SIMPLE", 1677 ), 1678 "INITIALLY": ("DEFERRED", "IMMEDIATE"), 1679 "USING": ( 1680 "BTREE", 1681 "HASH", 1682 ), 1683 **dict.fromkeys(("DEFERRABLE", "NORELY", "RELY"), tuple()), 1684 } 1685 1686 WINDOW_EXCLUDE_OPTIONS: t.ClassVar[OPTIONS_TYPE] = { 1687 "NO": ("OTHERS",), 1688 "CURRENT": ("ROW",), 1689 **dict.fromkeys(("GROUP", "TIES"), tuple()), 1690 } 1691 1692 INSERT_ALTERNATIVES: t.ClassVar = {"ABORT", "FAIL", "IGNORE", "REPLACE", "ROLLBACK"} 1693 1694 CLONE_KEYWORDS: t.ClassVar = {"CLONE", "COPY"} 1695 HISTORICAL_DATA_PREFIX: t.ClassVar = {"AT", "BEFORE", "END"} 1696 HISTORICAL_DATA_KIND: t.ClassVar = {"OFFSET", "STATEMENT", "STREAM", "TIMESTAMP", "VERSION"} 1697 1698 OPCLASS_FOLLOW_KEYWORDS: t.ClassVar = {"ASC", "DESC", "NULLS", "WITH"} 1699 1700 OPTYPE_FOLLOW_TOKENS: t.ClassVar = {TokenType.COMMA, TokenType.R_PAREN} 1701 1702 TABLE_INDEX_HINT_TOKENS: t.ClassVar = {TokenType.FORCE, TokenType.IGNORE, TokenType.USE} 1703 1704 VIEW_ATTRIBUTES: t.ClassVar = {"ENCRYPTION", "SCHEMABINDING", "VIEW_METADATA"} 1705 1706 WINDOW_ALIAS_TOKENS: t.ClassVar = ID_VAR_TOKENS - {TokenType.RANGE, TokenType.ROWS} 1707 WINDOW_BEFORE_PAREN_TOKENS: t.ClassVar = {TokenType.OVER} 1708 WINDOW_SIDES: t.ClassVar = {"FOLLOWING", "PRECEDING"} 1709 1710 JSON_KEY_VALUE_SEPARATOR_TOKENS: t.ClassVar = {TokenType.COLON, TokenType.COMMA, TokenType.IS} 1711 1712 FETCH_TOKENS: t.ClassVar = ID_VAR_TOKENS - {TokenType.ROW, TokenType.ROWS, TokenType.PERCENT} 1713 1714 ADD_CONSTRAINT_TOKENS: t.ClassVar = { 1715 TokenType.CONSTRAINT, 1716 TokenType.FOREIGN_KEY, 1717 TokenType.INDEX, 1718 TokenType.KEY, 1719 TokenType.PRIMARY_KEY, 1720 TokenType.UNIQUE, 1721 } 1722 1723 DISTINCT_TOKENS: t.ClassVar = {TokenType.DISTINCT} 1724 1725 UNNEST_OFFSET_ALIAS_TOKENS: t.ClassVar = TABLE_ALIAS_TOKENS - SET_OPERATIONS 1726 1727 SELECT_START_TOKENS: t.ClassVar = {TokenType.L_PAREN, TokenType.WITH, TokenType.SELECT} 1728 1729 COPY_INTO_VARLEN_OPTIONS: t.ClassVar = { 1730 "FILE_FORMAT", 1731 "COPY_OPTIONS", 1732 "FORMAT_OPTIONS", 1733 "CREDENTIAL", 1734 } 1735 1736 IS_JSON_PREDICATE_KIND: t.ClassVar = {"VALUE", "SCALAR", "ARRAY", "OBJECT"} 1737 1738 ODBC_DATETIME_LITERALS: t.ClassVar[dict[str, type[exp.Expr]]] = {} 1739 1740 ON_CONDITION_TOKENS: t.ClassVar = {"ERROR", "NULL", "TRUE", "FALSE", "EMPTY"} 1741 1742 PRIVILEGE_FOLLOW_TOKENS: t.ClassVar = {TokenType.ON, TokenType.COMMA, TokenType.L_PAREN} 1743 1744 # The style options for the DESCRIBE statement 1745 DESCRIBE_STYLES: t.ClassVar = {"ANALYZE", "EXTENDED", "FORMATTED", "HISTORY"} 1746 1747 SET_ASSIGNMENT_DELIMITERS: t.ClassVar = {"=", ":=", "TO"} 1748 1749 # The style options for the ANALYZE statement 1750 ANALYZE_STYLES: t.ClassVar = { 1751 "BUFFER_USAGE_LIMIT", 1752 "FULL", 1753 "LOCAL", 1754 "NO_WRITE_TO_BINLOG", 1755 "SAMPLE", 1756 "SKIP_LOCKED", 1757 "VERBOSE", 1758 } 1759 1760 ANALYZE_EXPRESSION_PARSERS: t.ClassVar = { 1761 "ALL": lambda self: self._parse_analyze_columns(), 1762 "COMPUTE": lambda self: self._parse_analyze_statistics(), 1763 "DELETE": lambda self: self._parse_analyze_delete(), 1764 "DROP": lambda self: self._parse_analyze_histogram(), 1765 "ESTIMATE": lambda self: self._parse_analyze_statistics(), 1766 "LIST": lambda self: self._parse_analyze_list(), 1767 "PREDICATE": lambda self: self._parse_analyze_columns(), 1768 "UPDATE": lambda self: self._parse_analyze_histogram(), 1769 "VALIDATE": lambda self: self._parse_analyze_validate(), 1770 } 1771 1772 PARTITION_KEYWORDS: t.ClassVar = {"PARTITION", "SUBPARTITION"} 1773 1774 AMBIGUOUS_ALIAS_TOKENS: t.ClassVar = (TokenType.LIMIT, TokenType.OFFSET) 1775 1776 OPERATION_MODIFIERS: t.ClassVar[set[str]] = set() 1777 1778 RECURSIVE_CTE_SEARCH_KIND: t.ClassVar = {"BREADTH", "DEPTH", "CYCLE"} 1779 1780 SECURITY_PROPERTY_KEYWORDS: t.ClassVar = {"DEFINER", "INVOKER", "NONE"} 1781 1782 MODIFIABLES: t.ClassVar = (exp.Query, exp.Table, exp.TableFromRows, exp.Values) 1783 1784 STRICT_CAST: t.ClassVar = True 1785 1786 PREFIXED_PIVOT_COLUMNS: t.ClassVar = False 1787 IDENTIFY_PIVOT_STRINGS: t.ClassVar = False 1788 # Controls when an aggregation's name is included in a pivoted column's name: 1789 # "agg_name_if_aliased" - only for aggregations that carry an explicit alias 1790 # "agg_name_if_aliased_or_multiple" - if aliased, or whenever there are multiple aggregations 1791 # "agg_name_if_multiple" - only when there are multiple aggregations (a lone agg is value-only) 1792 PIVOT_COLUMN_NAMING: t.ClassVar[str] = "agg_name_if_aliased" 1793 1794 LOG_DEFAULTS_TO_LN: t.ClassVar = False 1795 1796 # Whether the table sample clause expects CSV syntax 1797 TABLESAMPLE_CSV: t.ClassVar = False 1798 1799 # The default method used for table sampling 1800 DEFAULT_SAMPLING_METHOD: t.ClassVar[str | None] = None 1801 1802 # Whether the SET command needs a delimiter (e.g. "=") for assignments 1803 SET_REQUIRES_ASSIGNMENT_DELIMITER: t.ClassVar = True 1804 1805 # Whether the TRIM function expects the characters to trim as its first argument 1806 TRIM_PATTERN_FIRST: t.ClassVar = False 1807 1808 # Whether string aliases are supported `SELECT COUNT(*) 'count'` 1809 STRING_ALIASES: t.ClassVar = False 1810 1811 # Whether query modifiers such as LIMIT are attached to the UNION node (vs its right operand) 1812 MODIFIERS_ATTACHED_TO_SET_OP: t.ClassVar = True 1813 SET_OP_MODIFIERS: t.ClassVar = {"order", "limit", "offset"} 1814 1815 # Whether to parse IF statements that aren't followed by a left parenthesis as commands 1816 NO_PAREN_IF_COMMANDS: t.ClassVar = True 1817 1818 # Whether the -> and ->> operators expect documents of type JSON (e.g. Postgres) 1819 JSON_ARROWS_REQUIRE_JSON_TYPE: t.ClassVar = False 1820 1821 # Whether the `:` operator is used to extract a value from a VARIANT column 1822 COLON_IS_VARIANT_EXTRACT: t.ClassVar = False 1823 1824 # Whether a chain of colon extractions (x:y:z) is a single extraction with a merged 1825 # path (x:y.z, e.g. Snowflake) or each colon extracts from the previous result (e.g. Databricks) 1826 COLON_CHAIN_IS_SINGLE_EXTRACT: t.ClassVar = True 1827 1828 # Whether or not a VALUES keyword needs to be followed by '(' to form a VALUES clause. 1829 # If this is True and '(' is not found, the keyword will be treated as an identifier 1830 VALUES_FOLLOWED_BY_PAREN: t.ClassVar = True 1831 1832 # Whether implicit unnesting is supported, e.g. SELECT 1 FROM y.z AS z, z.a (Redshift) 1833 SUPPORTS_IMPLICIT_UNNEST: t.ClassVar = False 1834 1835 # Whether or not interval spans are supported, INTERVAL 1 YEAR TO MONTHS 1836 INTERVAL_SPANS: t.ClassVar = True 1837 1838 # Whether a PARTITION clause can follow a table reference 1839 SUPPORTS_PARTITION_SELECTION: t.ClassVar = False 1840 1841 # Whether the `name AS expr` schema/column constraint requires parentheses around `expr` 1842 WRAPPED_TRANSFORM_COLUMN_CONSTRAINT: t.ClassVar = True 1843 1844 # Whether the 'AS' keyword is optional in the CTE definition syntax 1845 OPTIONAL_ALIAS_TOKEN_CTE: t.ClassVar = True 1846 1847 # Whether renaming a column with an ALTER statement requires the presence of the COLUMN keyword 1848 ALTER_RENAME_REQUIRES_COLUMN: t.ClassVar = True 1849 1850 # Whether Alter statements are allowed to contain Partition specifications 1851 ALTER_TABLE_PARTITIONS: t.ClassVar = False 1852 1853 # Whether all join types have the same precedence, i.e., they "naturally" produce a left-deep tree. 1854 # In standard SQL, joins that use the JOIN keyword take higher precedence than comma-joins. That is 1855 # to say, JOIN operators happen before comma operators. This is not the case in some dialects, such 1856 # as BigQuery, where all joins have the same precedence. 1857 JOINS_HAVE_EQUAL_PRECEDENCE: t.ClassVar = False 1858 1859 # Whether TIMESTAMP <literal> can produce a zone-aware timestamp 1860 ZONE_AWARE_TIMESTAMP_CONSTRUCTOR: t.ClassVar = False 1861 1862 # Whether map literals support arbitrary expressions as keys. 1863 # When True, allows complex keys like arrays or literals: {[1, 2]: 3}, {1: 2} (e.g. DuckDB). 1864 # When False, keys are typically restricted to identifiers. 1865 MAP_KEYS_ARE_ARBITRARY_EXPRESSIONS: t.ClassVar = False 1866 1867 # Whether JSON_EXTRACT requires a JSON expression as the first argument, e.g this 1868 # is true for Snowflake but not for BigQuery which can also process strings 1869 JSON_EXTRACT_REQUIRES_JSON_EXPRESSION: t.ClassVar = False 1870 1871 # Dialects like Databricks support JOINS without join criteria 1872 # Adding an ON TRUE, makes transpilation semantically correct for other dialects 1873 ADD_JOIN_ON_TRUE: t.ClassVar = False 1874 1875 # Whether INTERVAL spans with literal format '\d+ hh:[mm:[ss[.ff]]]' 1876 # can omit the span unit `DAY TO MINUTE` or `DAY TO SECOND` 1877 SUPPORTS_OMITTED_INTERVAL_SPAN_UNIT: t.ClassVar = False 1878 1879 # Whether adjacent string literals like 'foo' 'bar' require a whitespace or comment between them 1880 # to be considered valid syntactically. Such expressions evaluate to the strings' concatenation. 1881 ADJACENT_STRINGS_CANNOT_BE_CONNECTED: t.ClassVar = False 1882 1883 SHOW_TRIE: t.ClassVar[dict] = new_trie(key.split(" ") for key in SHOW_PARSERS) 1884 SET_TRIE: t.ClassVar[dict] = new_trie(key.split(" ") for key in SET_PARSERS) 1885 1886 def __init__( 1887 self, 1888 error_level: ErrorLevel | None = None, 1889 error_message_context: int = 100, 1890 max_errors: int = 3, 1891 max_nodes: int = -1, 1892 dialect: DialectType = None, 1893 ): 1894 self.error_level: ErrorLevel = error_level or ErrorLevel.IMMEDIATE 1895 self.error_message_context: int = error_message_context 1896 self.max_errors: int = max_errors 1897 self.max_nodes: int = max_nodes 1898 self.dialect: t.Any = _resolve_dialect(dialect) 1899 self.sql: str = "" 1900 self.errors: list[ParseError] = [] 1901 self._tokens: list[Token] = [] 1902 self._tokens_size: i64 = 0 1903 self._index: i64 = 0 1904 self._curr: Token = SENTINEL_NONE 1905 self._next: Token = SENTINEL_NONE 1906 self._prev: Token = SENTINEL_NONE 1907 self._prev_comments: list[str] = [] 1908 self._pipe_cte_counter: int = 0 1909 self._chunks: list[list[Token]] = [] 1910 self._chunk_index: i64 = 0 1911 self._node_count: int = 0 1912 1913 def reset(self) -> None: 1914 self.sql = "" 1915 self.errors = [] 1916 self._tokens = [] 1917 self._tokens_size = 0 1918 self._index = 0 1919 self._curr = SENTINEL_NONE 1920 self._next = SENTINEL_NONE 1921 self._prev = SENTINEL_NONE 1922 self._prev_comments = [] 1923 self._pipe_cte_counter = 0 1924 self._chunks = [] 1925 self._chunk_index = 0 1926 self._node_count = 0 1927 1928 def _advance(self, times: i64 = 1) -> None: 1929 index = self._index + times 1930 self._index = index 1931 tokens = self._tokens 1932 size = self._tokens_size 1933 self._curr = tokens[index] if index < size else SENTINEL_NONE 1934 self._next = tokens[index + 1] if index + 1 < size else SENTINEL_NONE 1935 1936 if index > 0: 1937 prev = tokens[index - 1] 1938 self._prev = prev 1939 self._prev_comments = prev.comments 1940 else: 1941 self._prev = SENTINEL_NONE 1942 self._prev_comments = [] 1943 1944 def _advance_chunk(self) -> None: 1945 self._index = -1 1946 self._tokens = self._chunks[self._chunk_index] 1947 self._tokens_size = i64(len(self._tokens)) 1948 self._chunk_index += 1 1949 self._advance() 1950 1951 def _retreat(self, index: i64) -> None: 1952 if index != self._index: 1953 self._advance(index - self._index) 1954 1955 def _add_comments(self, expression: exp.Expr | None) -> None: 1956 if expression and self._prev_comments: 1957 expression.add_comments(self._prev_comments) 1958 self._prev_comments = [] 1959 1960 def _match( 1961 self, token_type: TokenType, advance: bool = True, expression: exp.Expr | None = None 1962 ) -> bool: 1963 if self._curr.token_type == token_type: 1964 if advance: 1965 self._advance() 1966 self._add_comments(expression) 1967 return True 1968 return False 1969 1970 def _match_set(self, types: t.Collection[TokenType], advance: bool = True) -> bool: 1971 if self._curr.token_type in types: 1972 if advance: 1973 self._advance() 1974 return True 1975 return False 1976 1977 def _match_pair( 1978 self, token_type_a: TokenType, token_type_b: TokenType, advance: bool = True 1979 ) -> bool: 1980 if self._curr.token_type == token_type_a and self._next.token_type == token_type_b: 1981 if advance: 1982 self._advance(2) 1983 return True 1984 return False 1985 1986 def _match_texts(self, texts: TEXTS_TYPE, advance: bool = True) -> bool: 1987 if ( 1988 self._curr.token_type not in self.TEXT_MATCH_EXCLUDED_TOKENS 1989 and self._curr.text.upper() in texts 1990 ): 1991 if advance: 1992 self._advance() 1993 return True 1994 return False 1995 1996 def _match_text_seq(self, *texts: str, advance: bool = True) -> bool: 1997 index = self._index 1998 excluded_tokens = self.TEXT_MATCH_EXCLUDED_TOKENS 1999 for text in texts: 2000 if self._curr.token_type not in excluded_tokens and self._curr.text.upper() == text: 2001 self._advance() 2002 else: 2003 self._retreat(index) 2004 return False 2005 2006 if not advance: 2007 self._retreat(index) 2008 2009 return True 2010 2011 def _is_connected(self) -> bool: 2012 prev = self._prev 2013 curr = self._curr 2014 return bool(prev and curr and prev.end + 1 == curr.start) 2015 2016 def _find_sql(self, start: Token, end: Token) -> str: 2017 return self.sql[start.start : end.end + 1] 2018 2019 def raise_error(self, message: str, token: Token = SENTINEL_NONE) -> None: 2020 token = token or self._curr or self._prev or Token.string("") 2021 formatted_sql, start_context, highlight, end_context = highlight_sql( 2022 sql=self.sql, 2023 positions=[(token.start, token.end)], 2024 context_length=self.error_message_context, 2025 ) 2026 formatted_message = f"{message}. Line {token.line}, Col: {token.col}.\n {formatted_sql}" 2027 2028 error = ParseError.new( 2029 formatted_message, 2030 description=message, 2031 line=token.line, 2032 col=token.col, 2033 start_context=start_context, 2034 highlight=highlight, 2035 end_context=end_context, 2036 ) 2037 2038 if self.error_level == ErrorLevel.IMMEDIATE: 2039 raise error 2040 2041 self.errors.append(error) 2042 2043 def validate_expression(self, expression: E, args: list | None = None) -> E: 2044 if self.max_nodes > -1: 2045 self._node_count += 1 2046 if self._node_count > self.max_nodes: 2047 self.raise_error(f"Maximum number of AST nodes ({self.max_nodes}) exceeded") 2048 if self.error_level != ErrorLevel.IGNORE: 2049 for error_message in expression.error_messages(args): 2050 self.raise_error(error_message) 2051 return expression 2052 2053 def _try_parse(self, parse_method: t.Callable[[], T], retreat: bool = False) -> T | None: 2054 index = self._index 2055 error_level = self.error_level 2056 this: T | None = None 2057 2058 self.error_level = ErrorLevel.IMMEDIATE 2059 try: 2060 this = parse_method() 2061 except ParseError: 2062 this = None 2063 finally: 2064 if not this or retreat: 2065 self._retreat(index) 2066 self.error_level = error_level 2067 2068 return this 2069 2070 def parse(self, raw_tokens: list[Token], sql: str) -> list[exp.Expr | None]: 2071 """ 2072 Parses a list of tokens and returns a list of syntax trees, one tree 2073 per parsed SQL statement. 2074 2075 Args: 2076 raw_tokens: The list of tokens. 2077 sql: The original SQL string. 2078 2079 Returns: 2080 The list of the produced syntax trees. 2081 """ 2082 return self._parse( 2083 parse_method=self.__class__._parse_statement, raw_tokens=raw_tokens, sql=sql 2084 ) 2085 2086 def parse_into( 2087 self, 2088 expression_types: exp.IntoType, 2089 raw_tokens: list[Token], 2090 sql: str | None = None, 2091 ) -> list[exp.Expr | None]: 2092 """ 2093 Parses a list of tokens into a given Expr type. If a collection of Expr 2094 types is given instead, this method will try to parse the token list into each one 2095 of them, stopping at the first for which the parsing succeeds. 2096 2097 Args: 2098 expression_types: The expression type(s) to try and parse the token list into. 2099 raw_tokens: The list of tokens. 2100 sql: The original SQL string, used to produce helpful debug messages. 2101 2102 Returns: 2103 The target Expr. 2104 """ 2105 errors = [] 2106 for expression_type in ensure_list(expression_types): 2107 parser = self.EXPRESSION_PARSERS.get(t.cast(type[exp.Expr], expression_type)) 2108 if not parser: 2109 raise TypeError(f"No parser registered for {expression_type}") 2110 2111 try: 2112 return self._parse(parser, raw_tokens, sql) 2113 except ParseError as e: 2114 e.errors[0]["into_expression"] = expression_type 2115 errors.append(e) 2116 2117 raise ParseError( 2118 f"Failed to parse '{sql or raw_tokens}' into {expression_types}", 2119 errors=merge_errors(errors), 2120 ) from errors[-1] 2121 2122 def check_errors(self) -> None: 2123 """Logs or raises any found errors, depending on the chosen error level setting.""" 2124 if self.error_level == ErrorLevel.WARN: 2125 for error in self.errors: 2126 logger.error(str(error)) 2127 elif self.error_level == ErrorLevel.RAISE and self.errors: 2128 raise ParseError( 2129 concat_messages(self.errors, self.max_errors), 2130 errors=merge_errors(self.errors), 2131 ) 2132 2133 def expression( 2134 self, 2135 instance: E, 2136 token: Token | None = None, 2137 comments: list[str] | None = None, 2138 ) -> E: 2139 if token: 2140 instance.update_positions(token) 2141 instance.add_comments(comments) if comments else self._add_comments(instance) 2142 if not instance.is_primitive: 2143 instance = self.validate_expression(instance) 2144 return instance 2145 2146 def _parse_batch_statements( 2147 self, 2148 parse_method: t.Callable[[Parser], exp.Expr | None], 2149 sep_first_statement: bool = True, 2150 ) -> list[exp.Expr | None]: 2151 expressions = [] 2152 2153 # Chunkification binds if/while statements with the first statement of the body 2154 if sep_first_statement: 2155 self._match(TokenType.BEGIN) 2156 expressions.append(parse_method(self)) 2157 2158 chunks_length = len(self._chunks) 2159 while self._chunk_index < chunks_length: 2160 self._advance_chunk() 2161 2162 if self._match(TokenType.ELSE, advance=False): 2163 return expressions 2164 2165 if expressions and not self._next and self._match(TokenType.END): 2166 expressions.append(exp.EndStatement()) 2167 continue 2168 2169 expressions.append(parse_method(self)) 2170 2171 if self._index < self._tokens_size: 2172 self.raise_error("Invalid expression / Unexpected token") 2173 2174 self.check_errors() 2175 2176 return expressions 2177 2178 def _parse( 2179 self, 2180 parse_method: t.Callable[[Parser], exp.Expr | None], 2181 raw_tokens: list[Token], 2182 sql: str | None = None, 2183 ) -> list[exp.Expr | None]: 2184 self.reset() 2185 self.sql = sql or "" 2186 2187 total = len(raw_tokens) 2188 chunks: list[list[Token]] = [[]] 2189 2190 for i, token in enumerate(raw_tokens): 2191 if token.token_type == TokenType.SEMICOLON: 2192 if token.comments: 2193 chunks.append([token]) 2194 2195 if i < total - 1: 2196 chunks.append([]) 2197 else: 2198 chunks[-1].append(token) 2199 2200 self._chunks = chunks 2201 2202 return self._parse_batch_statements(parse_method=parse_method, sep_first_statement=False) 2203 2204 def _warn_unsupported(self) -> None: 2205 if self._tokens_size <= 1: 2206 return 2207 2208 # We use _find_sql because self.sql may comprise multiple chunks, and we're only 2209 # interested in emitting a warning for the one being currently processed. 2210 sql = self._find_sql(self._tokens[0], self._tokens[-1])[: self.error_message_context] 2211 2212 logger.warning( 2213 f"'{sql}' contains unsupported syntax. Falling back to parsing as a 'Command'." 2214 ) 2215 2216 def _parse_command(self) -> exp.Command: 2217 self._warn_unsupported() 2218 comments = self._prev_comments 2219 return self.expression( 2220 exp.Command(this=self._prev.text.upper(), expression=self._parse_string()), 2221 comments=comments, 2222 ) 2223 2224 def _parse_comment(self, allow_exists: bool = True) -> exp.Expr: 2225 start = self._prev 2226 exists = self._parse_exists() if allow_exists else None 2227 2228 self._match(TokenType.ON) 2229 2230 materialized = self._match_text_seq("MATERIALIZED") 2231 kind = self._match_set(self.CREATABLES) and self._prev 2232 if not kind: 2233 return self._parse_as_command(start) 2234 2235 if kind.token_type in (TokenType.FUNCTION, TokenType.PROCEDURE): 2236 this = self._parse_user_defined_function(kind=kind.token_type) 2237 elif kind.token_type == TokenType.TABLE: 2238 this = self._parse_table(alias_tokens=self.COMMENT_TABLE_ALIAS_TOKENS) 2239 elif kind.token_type == TokenType.COLUMN: 2240 this = self._parse_column() 2241 else: 2242 this = self._parse_table_parts(schema=True) 2243 2244 self._match(TokenType.IS) 2245 2246 return self.expression( 2247 exp.Comment( 2248 this=this, 2249 kind=kind.text, 2250 expression=self._parse_string(), 2251 exists=exists, 2252 materialized=materialized, 2253 ) 2254 ) 2255 2256 def _parse_to_table( 2257 self, 2258 ) -> exp.ToTableProperty: 2259 table = self._parse_table_parts(schema=True) 2260 return self.expression(exp.ToTableProperty(this=table)) 2261 2262 # https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#mergetree-table-ttl 2263 def _parse_ttl(self) -> exp.Expr: 2264 def _parse_ttl_action() -> exp.Expr | None: 2265 this = self._parse_bitwise() 2266 2267 if self._match_text_seq("DELETE"): 2268 return self.expression(exp.MergeTreeTTLAction(this=this, delete=True)) 2269 if self._match_text_seq("RECOMPRESS"): 2270 return self.expression( 2271 exp.MergeTreeTTLAction(this=this, recompress=self._parse_bitwise()) 2272 ) 2273 if self._match_text_seq("TO", "DISK"): 2274 return self.expression( 2275 exp.MergeTreeTTLAction(this=this, to_disk=self._parse_string()) 2276 ) 2277 if self._match_text_seq("TO", "VOLUME"): 2278 return self.expression( 2279 exp.MergeTreeTTLAction(this=this, to_volume=self._parse_string()) 2280 ) 2281 2282 return this 2283 2284 expressions = self._parse_csv(_parse_ttl_action) 2285 where = self._parse_where() 2286 group = self._parse_group() 2287 2288 aggregates = None 2289 if group and self._match(TokenType.SET): 2290 aggregates = self._parse_csv(self._parse_set_item) 2291 2292 return self.expression( 2293 exp.MergeTreeTTL( 2294 expressions=expressions, where=where, group=group, aggregates=aggregates 2295 ) 2296 ) 2297 2298 def _parse_condition(self) -> exp.Expr | None: 2299 return self._parse_wrapped(parse_method=self._parse_expression, optional=True) 2300 2301 def _parse_block(self) -> exp.Block: 2302 return self.expression( 2303 exp.Block( 2304 expressions=self._parse_batch_statements( 2305 parse_method=lambda self: self._parse_statement() 2306 ) 2307 ) 2308 ) 2309 2310 def _parse_whileblock(self) -> exp.WhileBlock: 2311 return self.expression( 2312 exp.WhileBlock(this=self._parse_condition(), body=self._parse_block()) 2313 ) 2314 2315 def _parse_statement(self) -> exp.Expr | None: 2316 if not self._curr: 2317 return None 2318 2319 if self._match_set(self.STATEMENT_PARSERS): 2320 comments = self._prev_comments 2321 stmt = self.STATEMENT_PARSERS[self._prev.token_type](self) 2322 stmt.add_comments(comments, prepend=True) 2323 return stmt 2324 2325 if self._match_set(self.dialect.tokenizer_class.COMMANDS): 2326 return self._parse_command() 2327 2328 if self._match_text_seq("WHILE"): 2329 return self._parse_whileblock() 2330 2331 expression = self._parse_expression() 2332 expression = self._parse_set_operations(expression) if expression else self._parse_select() 2333 2334 if isinstance(expression, exp.Subquery) and self._match(TokenType.PIPE_GT, advance=False): 2335 expression = self._parse_pipe_syntax_query(expression) 2336 2337 return self._parse_query_modifiers(expression) 2338 2339 def _parse_drop(self, exists: bool = False) -> exp.Drop | exp.Command: 2340 start = self._prev 2341 temporary = self._match(TokenType.TEMPORARY) 2342 materialized = self._match_text_seq("MATERIALIZED") 2343 iceberg = self._match_text_seq("ICEBERG") 2344 2345 kind = self._match_set(self.CREATABLES) and self._prev.text.upper() 2346 if not kind or (iceberg and kind and kind != "TABLE"): 2347 return self._parse_as_command(start) 2348 2349 concurrently = self._match_text_seq("CONCURRENTLY") 2350 if_exists = exists or self._parse_exists() 2351 2352 if kind == "COLUMN": 2353 this = self._parse_column() 2354 else: 2355 this = self._parse_table_parts(schema=True, is_db_reference=kind == "SCHEMA") 2356 2357 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 2358 2359 if self._match(TokenType.L_PAREN, advance=False): 2360 expressions = self._parse_wrapped_csv(self._parse_types) 2361 else: 2362 expressions = None 2363 2364 cascade_or_restrict = self._match_texts(("CASCADE", "RESTRICT")) and self._prev.text.upper() 2365 2366 return self.expression( 2367 exp.Drop( 2368 exists=if_exists, 2369 this=this, 2370 expressions=expressions, 2371 kind=self.dialect.CREATABLE_KIND_MAPPING.get(kind) or kind, 2372 temporary=temporary, 2373 materialized=materialized, 2374 cascade=cascade_or_restrict == "CASCADE", 2375 restrict=cascade_or_restrict == "RESTRICT", 2376 constraints=self._match_text_seq("CONSTRAINTS"), 2377 purge=self._match_text_seq("PURGE"), 2378 cluster=cluster, 2379 concurrently=concurrently, 2380 sync=self._match_text_seq("SYNC"), 2381 iceberg=iceberg, 2382 ) 2383 ) 2384 2385 def _parse_exists(self, not_: bool = False) -> bool | None: 2386 return ( 2387 self._match_text_seq("IF") 2388 and (not not_ or self._match(TokenType.NOT)) 2389 and self._match(TokenType.EXISTS) 2390 ) 2391 2392 def _parse_create(self) -> exp.Create | exp.Command: 2393 # Note: this can't be None because we've matched a statement parser 2394 start = self._prev 2395 2396 replace = ( 2397 start.token_type == TokenType.REPLACE 2398 or self._match_pair(TokenType.OR, TokenType.REPLACE) 2399 or self._match_pair(TokenType.OR, TokenType.ALTER) 2400 ) 2401 refresh = self._match_pair(TokenType.OR, TokenType.REFRESH) 2402 2403 unique = self._match(TokenType.UNIQUE) 2404 2405 if self._match_text_seq("CLUSTERED", "COLUMNSTORE"): 2406 clustered = True 2407 elif self._match_text_seq("NONCLUSTERED", "COLUMNSTORE") or self._match_text_seq( 2408 "COLUMNSTORE" 2409 ): 2410 clustered = False 2411 else: 2412 clustered = None 2413 2414 if self._match_pair(TokenType.TABLE, TokenType.FUNCTION, advance=False): 2415 self._advance() 2416 2417 properties = None 2418 create_token = self._match_set(self.CREATABLES) and self._prev 2419 2420 if not create_token: 2421 # exp.Properties.Location.POST_CREATE 2422 properties = self._parse_properties() 2423 create_token = self._match_set(self.CREATABLES) and self._prev 2424 2425 if not properties or not create_token: 2426 return self._parse_as_command(start) 2427 2428 create_token_type = t.cast(Token, create_token).token_type 2429 2430 concurrently = self._match_text_seq("CONCURRENTLY") 2431 exists = self._parse_exists(not_=True) 2432 this = None 2433 expression: exp.Expr | None = None 2434 indexes = None 2435 no_schema_binding = None 2436 begin = None 2437 clone = None 2438 2439 def extend_props(temp_props: exp.Properties | None) -> None: 2440 nonlocal properties 2441 if properties and temp_props: 2442 properties.expressions.extend(temp_props.expressions) 2443 elif temp_props: 2444 properties = temp_props 2445 2446 if create_token_type in (TokenType.FUNCTION, TokenType.PROCEDURE): 2447 this = self._parse_user_defined_function(kind=create_token_type) 2448 2449 # exp.Properties.Location.POST_SCHEMA ("schema" here is the UDF's type signature) 2450 extend_props(self._parse_properties()) 2451 2452 expression = self._parse_heredoc() if self._match(TokenType.ALIAS) else None 2453 2454 if ( 2455 not expression 2456 and create_token_type == TokenType.FUNCTION 2457 and isinstance(this, exp.UserDefinedFunction) 2458 and this.args.get("wrapped") 2459 ): 2460 pre_table_index = self._index 2461 is_table = self._match(TokenType.TABLE) 2462 2463 expression = self._parse_expression() 2464 overload_mode = bool( 2465 expression 2466 and self._curr.token_type == TokenType.COMMA 2467 and self._next.token_type == TokenType.L_PAREN 2468 ) 2469 if not overload_mode: 2470 self._retreat(pre_table_index) 2471 is_table = False 2472 expression = None 2473 else: 2474 is_table = False 2475 overload_mode = False 2476 2477 extend_props(self._parse_function_properties()) 2478 2479 if not expression: 2480 if self._match(TokenType.COMMAND): 2481 expression = self._parse_as_command(self._prev) 2482 else: 2483 begin = self._match(TokenType.BEGIN) 2484 return_ = self._match_text_seq("RETURN") 2485 2486 if self._match(TokenType.STRING, advance=False): 2487 # Takes care of BigQuery's JavaScript UDF definitions that end in an OPTIONS property 2488 # # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_function_statement 2489 expression = self._parse_string() 2490 extend_props(self._parse_properties()) 2491 else: 2492 expression = ( 2493 self._parse_user_defined_function_expression() 2494 if create_token_type == TokenType.FUNCTION 2495 else self._parse_block() 2496 ) 2497 2498 if return_: 2499 expression = self.expression(exp.Return(this=expression)) 2500 2501 if overload_mode and expression: 2502 expression = self._parse_macro_overloads( 2503 t.cast(exp.UserDefinedFunction, this), expression, is_table 2504 ) 2505 elif create_token_type == TokenType.INDEX: 2506 # Postgres allows anonymous indexes, eg. CREATE INDEX IF NOT EXISTS ON t(c) 2507 if not self._match(TokenType.ON): 2508 index = self._parse_id_var() 2509 anonymous = False 2510 else: 2511 index = None 2512 anonymous = True 2513 2514 this = self._parse_index(index=index, anonymous=anonymous) 2515 elif ( 2516 create_token_type == TokenType.CONSTRAINT and self._match(TokenType.TRIGGER) 2517 ) or create_token_type == TokenType.TRIGGER: 2518 if is_constraint := (create_token_type == TokenType.CONSTRAINT): 2519 create_token = self._prev 2520 2521 trigger_name = self._parse_id_var() 2522 if not trigger_name: 2523 return self._parse_as_command(start) 2524 2525 timing_var = self._parse_var_from_options(self.TRIGGER_TIMING, raise_unmatched=False) 2526 timing = timing_var.this if timing_var else None 2527 if not timing: 2528 return self._parse_as_command(start) 2529 2530 events = self._parse_trigger_events() 2531 if not self._match(TokenType.ON): 2532 self.raise_error("Expected ON in trigger definition") 2533 2534 table = self._parse_table_parts() 2535 referenced_table = self._parse_table_parts() if self._match(TokenType.FROM) else None 2536 deferrable, initially = self._parse_trigger_deferrable() 2537 referencing = self._parse_trigger_referencing() 2538 for_each = self._parse_trigger_for_each() 2539 when = self._match_text_seq("WHEN") and self._parse_wrapped( 2540 self._parse_disjunction, optional=True 2541 ) 2542 execute = self._parse_trigger_execute() 2543 2544 if execute is None: 2545 return self._parse_as_command(start) 2546 2547 trigger_props = self.expression( 2548 exp.TriggerProperties( 2549 table=table, 2550 timing=timing, 2551 events=events, 2552 execute=execute, 2553 constraint=is_constraint, 2554 referenced_table=referenced_table, 2555 deferrable=deferrable, 2556 initially=initially, 2557 referencing=referencing, 2558 for_each=for_each, 2559 when=when, 2560 ) 2561 ) 2562 2563 this = trigger_name 2564 extend_props(exp.Properties(expressions=[trigger_props] if trigger_props else [])) 2565 elif create_token_type == TokenType.TYPE: 2566 this = self._parse_table_parts(schema=True) 2567 if not this or not self._match(TokenType.ALIAS): 2568 return self._parse_as_command(start) 2569 2570 if self._match(TokenType.ENUM): 2571 expression = exp.DataType( 2572 this=exp.DType.ENUM, 2573 expressions=self._parse_wrapped_csv(self._parse_string), 2574 ) 2575 elif self._match(TokenType.L_PAREN, advance=False): 2576 expression = self._parse_schema() 2577 else: 2578 return self._parse_as_command(start) 2579 elif create_token_type in self.DB_CREATABLES: 2580 table_parts = self._parse_table_parts( 2581 schema=True, is_db_reference=create_token_type == TokenType.SCHEMA 2582 ) 2583 2584 # exp.Properties.Location.POST_NAME 2585 self._match(TokenType.COMMA) 2586 extend_props(self._parse_properties(before=True)) 2587 2588 this = self._parse_schema(this=table_parts) 2589 2590 # exp.Properties.Location.POST_SCHEMA and POST_WITH 2591 extend_props(self._parse_properties()) 2592 2593 has_alias = self._match(TokenType.ALIAS) 2594 if not self._match_set(self.DDL_SELECT_TOKENS, advance=False): 2595 # exp.Properties.Location.POST_ALIAS 2596 extend_props(self._parse_properties()) 2597 2598 if create_token_type == TokenType.SEQUENCE: 2599 expression = self._parse_types() 2600 props = self._parse_properties() 2601 if props: 2602 sequence_props = exp.SequenceProperties() 2603 options = [] 2604 for prop in props: 2605 if isinstance(prop, exp.SequenceProperties): 2606 for arg, value in prop.args.items(): 2607 if arg == "options": 2608 options.extend(value) 2609 else: 2610 sequence_props.set(arg, value) 2611 prop.pop() 2612 2613 if options: 2614 sequence_props.set("options", options) 2615 2616 props.append("expressions", sequence_props) 2617 extend_props(props) 2618 else: 2619 expression = self._parse_ddl_select() 2620 2621 # Some dialects also support using a table as an alias instead of a SELECT. 2622 # Here we fallback to this as an alternative. 2623 if not expression and has_alias: 2624 expression = self._try_parse(self._parse_table_parts) 2625 2626 if create_token_type == TokenType.TABLE: 2627 # exp.Properties.Location.POST_EXPRESSION 2628 extend_props(self._parse_properties()) 2629 2630 indexes = [] 2631 while True: 2632 index = self._parse_index() 2633 2634 # exp.Properties.Location.POST_INDEX 2635 extend_props(self._parse_properties()) 2636 if not index: 2637 break 2638 else: 2639 self._match(TokenType.COMMA) 2640 indexes.append(index) 2641 elif create_token_type == TokenType.VIEW: 2642 if self._match_text_seq("WITH", "NO", "SCHEMA", "BINDING"): 2643 no_schema_binding = True 2644 elif create_token_type in (TokenType.SINK, TokenType.SOURCE): 2645 extend_props(self._parse_properties()) 2646 2647 shallow = self._match_text_seq("SHALLOW") 2648 2649 if self._match_texts(self.CLONE_KEYWORDS): 2650 copy = self._prev.text.lower() == "copy" 2651 clone = self.expression( 2652 exp.Clone(this=self._parse_table(schema=True), shallow=shallow, copy=copy) 2653 ) 2654 2655 if self._curr and not self._match_set((TokenType.R_PAREN, TokenType.COMMA), advance=False): 2656 return self._parse_as_command(start) 2657 2658 create_kind_text = create_token.text.upper() 2659 return self.expression( 2660 exp.Create( 2661 this=this, 2662 kind=self.dialect.CREATABLE_KIND_MAPPING.get(create_kind_text) or create_kind_text, 2663 replace=replace, 2664 refresh=refresh, 2665 unique=unique, 2666 expression=expression, 2667 exists=exists, 2668 properties=properties, 2669 indexes=indexes, 2670 no_schema_binding=no_schema_binding, 2671 begin=begin, 2672 clone=clone, 2673 concurrently=concurrently, 2674 clustered=clustered, 2675 ) 2676 ) 2677 2678 def _parse_sequence_properties(self) -> exp.SequenceProperties | None: 2679 seq = exp.SequenceProperties() 2680 2681 options = [] 2682 index = self._index 2683 2684 while self._curr: 2685 self._match(TokenType.COMMA) 2686 if self._match_text_seq("INCREMENT"): 2687 self._match_text_seq("BY") 2688 self._match_text_seq("=") 2689 seq.set("increment", self._parse_term()) 2690 elif self._match_text_seq("MINVALUE"): 2691 seq.set("minvalue", self._parse_term()) 2692 elif self._match_text_seq("MAXVALUE"): 2693 seq.set("maxvalue", self._parse_term()) 2694 elif self._match(TokenType.START_WITH) or self._match_text_seq("START"): 2695 self._match_text_seq("=") 2696 seq.set("start", self._parse_term()) 2697 elif self._match_text_seq("CACHE"): 2698 # T-SQL allows empty CACHE which is initialized dynamically 2699 seq.set("cache", self._parse_number() or True) 2700 elif self._match_text_seq("OWNED", "BY"): 2701 # "OWNED BY NONE" is the default 2702 seq.set("owned", None if self._match_text_seq("NONE") else self._parse_column()) 2703 else: 2704 opt = self._parse_var_from_options(self.CREATE_SEQUENCE, raise_unmatched=False) 2705 if opt: 2706 options.append(opt) 2707 else: 2708 break 2709 2710 seq.set("options", options if options else None) 2711 return None if self._index == index else seq 2712 2713 def _parse_trigger_events(self) -> list[exp.TriggerEvent]: 2714 events = [] 2715 2716 while True: 2717 event_type = self._match_set(self.TRIGGER_EVENTS) and self._prev.text.upper() 2718 2719 if not event_type: 2720 self.raise_error("Expected trigger event (INSERT, UPDATE, DELETE, TRUNCATE)") 2721 2722 columns = ( 2723 self._parse_csv(self._parse_column) 2724 if event_type == "UPDATE" and self._match_text_seq("OF") 2725 else None 2726 ) 2727 2728 events.append(self.expression(exp.TriggerEvent(this=event_type, columns=columns))) 2729 2730 if not self._match(TokenType.OR): 2731 break 2732 2733 return events 2734 2735 def _parse_trigger_deferrable( 2736 self, 2737 ) -> tuple[str | None, str | None]: 2738 deferrable_var = self._parse_var_from_options( 2739 self.TRIGGER_DEFERRABLE, raise_unmatched=False 2740 ) 2741 deferrable = deferrable_var.this if deferrable_var else None 2742 2743 initially = None 2744 if deferrable and self._match_text_seq("INITIALLY"): 2745 initially = ( 2746 self._prev.text.upper() if self._match_texts(("IMMEDIATE", "DEFERRED")) else None 2747 ) 2748 2749 return deferrable, initially 2750 2751 def _parse_trigger_referencing_clause(self, keyword: str) -> exp.Expr | None: 2752 if not self._match_text_seq(keyword): 2753 return None 2754 if not self._match_text_seq("TABLE"): 2755 self.raise_error(f"Expected TABLE after {keyword} in REFERENCING clause") 2756 self._match_text_seq("AS") 2757 return self._parse_id_var() 2758 2759 def _parse_trigger_referencing(self) -> exp.TriggerReferencing | None: 2760 if not self._match_text_seq("REFERENCING"): 2761 return None 2762 2763 old_alias = None 2764 new_alias = None 2765 2766 while True: 2767 if alias := self._parse_trigger_referencing_clause("OLD"): 2768 if old_alias is not None: 2769 self.raise_error("Duplicate OLD clause in REFERENCING") 2770 old_alias = alias 2771 elif alias := self._parse_trigger_referencing_clause("NEW"): 2772 if new_alias is not None: 2773 self.raise_error("Duplicate NEW clause in REFERENCING") 2774 new_alias = alias 2775 else: 2776 break 2777 2778 if old_alias is None and new_alias is None: 2779 self.raise_error("REFERENCING clause requires at least OLD TABLE or NEW TABLE") 2780 2781 return self.expression(exp.TriggerReferencing(old=old_alias, new=new_alias)) 2782 2783 def _parse_trigger_for_each(self) -> str | None: 2784 if not self._match_text_seq("FOR", "EACH"): 2785 return None 2786 2787 return self._prev.text.upper() if self._match_texts(("ROW", "STATEMENT")) else None 2788 2789 def _parse_trigger_execute(self) -> exp.TriggerExecute | None: 2790 if not self._match(TokenType.EXECUTE): 2791 return None 2792 2793 if not self._match_set((TokenType.FUNCTION, TokenType.PROCEDURE)): 2794 self.raise_error("Expected FUNCTION or PROCEDURE after EXECUTE") 2795 2796 func_call = self._parse_column() 2797 return self.expression(exp.TriggerExecute(this=func_call)) 2798 2799 def _parse_property_before(self) -> exp.Expr | list[exp.Expr] | None: 2800 # only used for teradata currently 2801 self._match(TokenType.COMMA) 2802 2803 kwargs = { 2804 "no": self._match_text_seq("NO"), 2805 "dual": self._match_text_seq("DUAL"), 2806 "before": self._match_text_seq("BEFORE"), 2807 "default": self._match_text_seq("DEFAULT"), 2808 "local": (self._match_text_seq("LOCAL") and "LOCAL") 2809 or (self._match_text_seq("NOT", "LOCAL") and "NOT LOCAL"), 2810 "after": self._match_text_seq("AFTER"), 2811 "minimum": self._match_texts(("MIN", "MINIMUM")), 2812 "maximum": self._match_texts(("MAX", "MAXIMUM")), 2813 } 2814 2815 if self._match_texts(self.PROPERTY_PARSERS): 2816 parser = self.PROPERTY_PARSERS[self._prev.text.upper()] 2817 try: 2818 return parser(self, **{k: v for k, v in kwargs.items() if v}) 2819 except TypeError: 2820 self.raise_error(f"Cannot parse property '{self._prev.text}'") 2821 2822 return None 2823 2824 def _parse_wrapped_properties(self) -> list[exp.Expr | list[exp.Expr]]: 2825 return self._parse_wrapped_csv(self._parse_property) 2826 2827 def _parse_property(self) -> exp.Expr | list[exp.Expr] | None: 2828 if self._match_texts(self.PROPERTY_PARSERS): 2829 return self.PROPERTY_PARSERS[self._prev.text.upper()](self) 2830 2831 if self._match(TokenType.DEFAULT) and self._match_texts(self.PROPERTY_PARSERS): 2832 return self.PROPERTY_PARSERS[self._prev.text.upper()](self, default=True) 2833 2834 if self._match_text_seq("COMPOUND", "SORTKEY"): 2835 return self._parse_sortkey(compound=True) 2836 2837 if self._match_text_seq("PARAMETER", "STYLE", "PANDAS"): 2838 return self.expression(exp.ParameterStyleProperty(this="PANDAS")) 2839 2840 index = self._index 2841 2842 seq_props = self._parse_sequence_properties() 2843 if seq_props: 2844 return seq_props 2845 2846 self._retreat(index) 2847 return self._parse_key_value_property() 2848 2849 def _parse_key_value_property( 2850 self, parse_value: t.Callable[[], exp.Expr | None] | None = None 2851 ) -> exp.Property | None: 2852 index = self._index 2853 key = self._parse_column() 2854 2855 if not self._match(TokenType.EQ): 2856 self._retreat(index) 2857 return None 2858 2859 # Transform the key to exp.Dot if it's dotted identifiers wrapped in exp.Column or to exp.Var otherwise 2860 if isinstance(key, exp.Column): 2861 key = key.to_dot() if len(key.parts) > 1 else exp.var(key.name) 2862 2863 value = ( 2864 parse_value() 2865 if parse_value 2866 else self._parse_bitwise() or self._parse_var(any_token=True) 2867 ) 2868 2869 # Transform the value to exp.Var if it was parsed as exp.Column(exp.Identifier()) 2870 if isinstance(value, exp.Column): 2871 value = exp.var(value.name) 2872 2873 return self.expression(exp.Property(this=key, value=value)) 2874 2875 def _parse_stored(self) -> exp.FileFormatProperty | exp.StorageHandlerProperty: 2876 if self._match_text_seq("BY"): 2877 return self.expression(exp.StorageHandlerProperty(this=self._parse_var_or_string())) 2878 2879 self._match(TokenType.ALIAS) 2880 input_format = self._parse_string() if self._match_text_seq("INPUTFORMAT") else None 2881 output_format = self._parse_string() if self._match_text_seq("OUTPUTFORMAT") else None 2882 2883 return self.expression( 2884 exp.FileFormatProperty( 2885 this=( 2886 self.expression( 2887 exp.InputOutputFormat( 2888 input_format=input_format, output_format=output_format 2889 ) 2890 ) 2891 if input_format or output_format 2892 else self._parse_var_or_string() or self._parse_number() or self._parse_id_var() 2893 ), 2894 hive_format=True, 2895 ) 2896 ) 2897 2898 def _parse_unquoted_field(self) -> exp.Expr | None: 2899 field = self._parse_field() 2900 if isinstance(field, exp.Identifier) and not field.quoted: 2901 field = exp.var(field) 2902 2903 return field 2904 2905 def _parse_property_assignment(self, exp_class: type[E], **kwargs: t.Any) -> E: 2906 self._match(TokenType.EQ) 2907 self._match(TokenType.ALIAS) 2908 2909 return self.expression(exp_class(this=self._parse_unquoted_field(), **kwargs)) 2910 2911 def _parse_properties(self, before: bool | None = None) -> exp.Properties | None: 2912 properties = [] 2913 while True: 2914 if before: 2915 prop = self._parse_property_before() 2916 else: 2917 prop = self._parse_property() 2918 if not prop: 2919 break 2920 for p in ensure_list(prop): 2921 properties.append(p) 2922 2923 if properties: 2924 return self.expression(exp.Properties(expressions=properties)) 2925 2926 return None 2927 2928 def _parse_fallback(self, no: bool = False) -> exp.FallbackProperty: 2929 return self.expression( 2930 exp.FallbackProperty(no=no, protection=self._match_text_seq("PROTECTION")) 2931 ) 2932 2933 def _parse_sql_security(self) -> exp.SqlSecurityProperty: 2934 return self.expression( 2935 exp.SqlSecurityProperty( 2936 this=self._match_texts(self.SECURITY_PROPERTY_KEYWORDS) and self._prev.text.upper() 2937 ) 2938 ) 2939 2940 def _parse_settings_property(self) -> exp.SettingsProperty: 2941 return self.expression( 2942 exp.SettingsProperty(expressions=self._parse_csv(self._parse_assignment)) 2943 ) 2944 2945 def _parse_called_on_null_input_property(self) -> exp.CalledOnNullInputProperty | None: 2946 if not self._match_text_seq("ON", "NULL", "INPUT"): 2947 self._retreat(self._index - 1) 2948 return None 2949 2950 return self.expression(exp.CalledOnNullInputProperty()) 2951 2952 def _parse_volatile_property(self) -> exp.VolatileProperty | exp.StabilityProperty: 2953 if self._index >= 2: 2954 pre_volatile_token = self._tokens[self._index - 2] 2955 else: 2956 pre_volatile_token = None 2957 2958 if pre_volatile_token and pre_volatile_token.token_type in self.PRE_VOLATILE_TOKENS: 2959 return exp.VolatileProperty() 2960 2961 return self.expression(exp.StabilityProperty(this=exp.Literal.string("VOLATILE"))) 2962 2963 def _parse_retention_period(self) -> exp.Var: 2964 # Parse TSQL's HISTORY_RETENTION_PERIOD: {INFINITE | <number> DAY | DAYS | MONTH ...} 2965 number = self._parse_number() 2966 number_str = f"{number} " if number else "" 2967 unit = self._parse_var(any_token=True) 2968 return exp.var(f"{number_str}{unit}") 2969 2970 def _parse_system_versioning_property( 2971 self, with_: bool = False 2972 ) -> exp.WithSystemVersioningProperty: 2973 self._match(TokenType.EQ) 2974 prop = self.expression(exp.WithSystemVersioningProperty(on=True, with_=with_)) 2975 2976 if self._match_text_seq("OFF"): 2977 prop.set("on", False) 2978 return prop 2979 2980 self._match(TokenType.ON) 2981 if self._match(TokenType.L_PAREN): 2982 while self._curr and not self._match(TokenType.R_PAREN): 2983 if self._match_text_seq("HISTORY_TABLE", "="): 2984 prop.set("this", self._parse_table_parts()) 2985 elif self._match_text_seq("DATA_CONSISTENCY_CHECK", "="): 2986 prop.set("data_consistency", self._advance_any() and self._prev.text.upper()) 2987 elif self._match_text_seq("HISTORY_RETENTION_PERIOD", "="): 2988 prop.set("retention_period", self._parse_retention_period()) 2989 2990 self._match(TokenType.COMMA) 2991 2992 return prop 2993 2994 def _parse_data_deletion_property(self) -> exp.DataDeletionProperty: 2995 self._match(TokenType.EQ) 2996 on = self._match_text_seq("ON") or not self._match_text_seq("OFF") 2997 prop = self.expression(exp.DataDeletionProperty(on=on)) 2998 2999 if self._match(TokenType.L_PAREN): 3000 while self._curr and not self._match(TokenType.R_PAREN): 3001 if self._match_text_seq("FILTER_COLUMN", "="): 3002 prop.set("filter_column", self._parse_column()) 3003 elif self._match_text_seq("RETENTION_PERIOD", "="): 3004 prop.set("retention_period", self._parse_retention_period()) 3005 3006 self._match(TokenType.COMMA) 3007 3008 return prop 3009 3010 def _parse_distributed_property(self) -> exp.DistributedByProperty: 3011 kind = "HASH" 3012 expressions: list[exp.Expr] | None = None 3013 if self._match_text_seq("BY", "HASH"): 3014 expressions = self._parse_wrapped_csv(self._parse_id_var) 3015 elif self._match_text_seq("BY", "RANDOM"): 3016 kind = "RANDOM" 3017 3018 # If the BUCKETS keyword is not present, the number of buckets is AUTO 3019 buckets: exp.Expr | None = None 3020 if self._match_text_seq("BUCKETS") and not self._match_text_seq("AUTO"): 3021 buckets = self._parse_number() 3022 3023 return self.expression( 3024 exp.DistributedByProperty( 3025 expressions=expressions, kind=kind, buckets=buckets, order=self._parse_order() 3026 ) 3027 ) 3028 3029 def _parse_composite_key_property(self, expr_type: type[E]) -> E: 3030 self._match_text_seq("KEY") 3031 expressions = self._parse_wrapped_id_vars() 3032 return self.expression(expr_type(expressions=expressions)) 3033 3034 def _parse_with_property(self) -> exp.Expr | None | list[exp.Expr]: 3035 if self._match_text_seq("(", "SYSTEM_VERSIONING"): 3036 prop = self._parse_system_versioning_property(with_=True) 3037 self._match_r_paren() 3038 return prop 3039 3040 if self._match(TokenType.L_PAREN, advance=False): 3041 result: list[exp.Expr] = [] 3042 for i in self._parse_wrapped_properties(): 3043 result.extend(i) if isinstance(i, list) else result.append(i) 3044 return result 3045 3046 if self._match_text_seq("JOURNAL"): 3047 return self._parse_withjournaltable() 3048 3049 if self._match_texts(self.VIEW_ATTRIBUTES): 3050 return self.expression(exp.ViewAttributeProperty(this=self._prev.text.upper())) 3051 3052 if self._match_text_seq("DATA"): 3053 return self._parse_withdata(no=False) 3054 elif self._match_text_seq("NO", "DATA"): 3055 return self._parse_withdata(no=True) 3056 3057 if self._match(TokenType.SERDE_PROPERTIES, advance=False): 3058 return self._parse_serde_properties(with_=True) 3059 3060 if self._match(TokenType.SCHEMA): 3061 return self.expression( 3062 exp.WithSchemaBindingProperty( 3063 this=self._parse_var_from_options(self.SCHEMA_BINDING_OPTIONS) 3064 ) 3065 ) 3066 3067 if self._match_texts(self.PROCEDURE_OPTIONS, advance=False): 3068 return self.expression( 3069 exp.WithProcedureOptions(expressions=self._parse_csv(self._parse_procedure_option)) 3070 ) 3071 3072 if not self._next: 3073 return None 3074 3075 return self._parse_withisolatedloading() 3076 3077 def _parse_procedure_option(self) -> exp.Expr | None: 3078 if self._match_text_seq("EXECUTE", "AS"): 3079 return self.expression( 3080 exp.ExecuteAsProperty( 3081 this=self._parse_var_from_options( 3082 self.EXECUTE_AS_OPTIONS, raise_unmatched=False 3083 ) 3084 or self._parse_string() 3085 ) 3086 ) 3087 3088 return self._parse_var_from_options(self.PROCEDURE_OPTIONS) 3089 3090 # https://dev.mysql.com/doc/refman/8.0/en/create-view.html 3091 def _parse_definer(self) -> exp.DefinerProperty | None: 3092 self._match(TokenType.EQ) 3093 3094 user = self._parse_id_var() 3095 self._match(TokenType.PARAMETER) 3096 host = self._parse_id_var() or (self._match(TokenType.MOD) and self._prev.text) 3097 3098 if not user or not host: 3099 return None 3100 3101 return exp.DefinerProperty(this=f"{user}@{host}") 3102 3103 def _parse_withjournaltable(self) -> exp.WithJournalTableProperty: 3104 self._match(TokenType.TABLE) 3105 self._match(TokenType.EQ) 3106 return self.expression(exp.WithJournalTableProperty(this=self._parse_table_parts())) 3107 3108 def _parse_log(self, no: bool = False) -> exp.LogProperty: 3109 return self.expression(exp.LogProperty(no=no)) 3110 3111 def _parse_journal(self, **kwargs) -> exp.JournalProperty: 3112 return self.expression(exp.JournalProperty(**kwargs)) 3113 3114 def _parse_checksum(self) -> exp.ChecksumProperty: 3115 self._match(TokenType.EQ) 3116 3117 on = None 3118 if self._match(TokenType.ON): 3119 on = True 3120 elif self._match_text_seq("OFF"): 3121 on = False 3122 3123 return self.expression(exp.ChecksumProperty(on=on, default=self._match(TokenType.DEFAULT))) 3124 3125 def _parse_cluster(self) -> exp.Cluster: 3126 self._match(TokenType.CLUSTER_BY) 3127 return self.expression( 3128 exp.Cluster( 3129 expressions=self._parse_csv(self._parse_column), 3130 ) 3131 ) 3132 3133 def _parse_cluster_property(self) -> exp.ClusterProperty: 3134 return self.expression( 3135 exp.ClusterProperty( 3136 expressions=self._parse_wrapped_csv(self._parse_column), 3137 ) 3138 ) 3139 3140 def _parse_clustered_by(self) -> exp.ClusteredByProperty: 3141 self._match_text_seq("BY") 3142 3143 self._match_l_paren() 3144 expressions = self._parse_csv(self._parse_column) 3145 self._match_r_paren() 3146 3147 if self._match_text_seq("SORTED", "BY"): 3148 self._match_l_paren() 3149 sorted_by = self._parse_csv(self._parse_ordered) 3150 self._match_r_paren() 3151 else: 3152 sorted_by = None 3153 3154 self._match(TokenType.INTO) 3155 buckets = self._parse_number() 3156 self._match_text_seq("BUCKETS") 3157 3158 return self.expression( 3159 exp.ClusteredByProperty(expressions=expressions, sorted_by=sorted_by, buckets=buckets) 3160 ) 3161 3162 def _parse_copy_property(self) -> exp.CopyGrantsProperty | None: 3163 if not self._match_text_seq("GRANTS"): 3164 self._retreat(self._index - 1) 3165 return None 3166 3167 return self.expression(exp.CopyGrantsProperty()) 3168 3169 def _parse_freespace(self) -> exp.FreespaceProperty: 3170 self._match(TokenType.EQ) 3171 return self.expression( 3172 exp.FreespaceProperty(this=self._parse_number(), percent=self._match(TokenType.PERCENT)) 3173 ) 3174 3175 def _parse_mergeblockratio( 3176 self, no: bool = False, default: bool = False 3177 ) -> exp.MergeBlockRatioProperty: 3178 if self._match(TokenType.EQ): 3179 return self.expression( 3180 exp.MergeBlockRatioProperty( 3181 this=self._parse_number(), percent=self._match(TokenType.PERCENT) 3182 ) 3183 ) 3184 3185 return self.expression(exp.MergeBlockRatioProperty(no=no, default=default)) 3186 3187 def _parse_datablocksize( 3188 self, 3189 default: bool | None = None, 3190 minimum: bool | None = None, 3191 maximum: bool | None = None, 3192 ) -> exp.DataBlocksizeProperty: 3193 self._match(TokenType.EQ) 3194 size = self._parse_number() 3195 3196 units = None 3197 if self._match_texts(("BYTES", "KBYTES", "KILOBYTES")): 3198 units = self._prev.text 3199 3200 return self.expression( 3201 exp.DataBlocksizeProperty( 3202 size=size, units=units, default=default, minimum=minimum, maximum=maximum 3203 ) 3204 ) 3205 3206 def _parse_blockcompression(self) -> exp.BlockCompressionProperty: 3207 self._match(TokenType.EQ) 3208 always = self._match_text_seq("ALWAYS") 3209 manual = self._match_text_seq("MANUAL") 3210 never = self._match_text_seq("NEVER") 3211 default = self._match_text_seq("DEFAULT") 3212 3213 autotemp = None 3214 if self._match_text_seq("AUTOTEMP"): 3215 autotemp = self._parse_schema() 3216 3217 return self.expression( 3218 exp.BlockCompressionProperty( 3219 always=always, manual=manual, never=never, default=default, autotemp=autotemp 3220 ) 3221 ) 3222 3223 def _parse_withisolatedloading(self) -> exp.IsolatedLoadingProperty | None: 3224 index = self._index 3225 no = self._match_text_seq("NO") 3226 concurrent = self._match_text_seq("CONCURRENT") 3227 3228 if not self._match_text_seq("ISOLATED", "LOADING"): 3229 self._retreat(index) 3230 return None 3231 3232 target = self._parse_var_from_options(self.ISOLATED_LOADING_OPTIONS, raise_unmatched=False) 3233 return self.expression( 3234 exp.IsolatedLoadingProperty(no=no, concurrent=concurrent, target=target) 3235 ) 3236 3237 def _parse_locking(self) -> exp.LockingProperty: 3238 if self._match(TokenType.TABLE): 3239 kind = "TABLE" 3240 elif self._match(TokenType.VIEW): 3241 kind = "VIEW" 3242 elif self._match(TokenType.ROW): 3243 kind = "ROW" 3244 elif self._match_text_seq("DATABASE"): 3245 kind = "DATABASE" 3246 else: 3247 kind = None 3248 3249 if kind in ("DATABASE", "TABLE", "VIEW"): 3250 this = self._parse_table_parts() 3251 else: 3252 this = None 3253 3254 if self._match(TokenType.FOR): 3255 for_or_in = "FOR" 3256 elif self._match(TokenType.IN): 3257 for_or_in = "IN" 3258 else: 3259 for_or_in = None 3260 3261 if self._match_text_seq("ACCESS"): 3262 lock_type = "ACCESS" 3263 elif self._match_texts(("EXCL", "EXCLUSIVE")): 3264 lock_type = "EXCLUSIVE" 3265 elif self._match_text_seq("SHARE"): 3266 lock_type = "SHARE" 3267 elif self._match_text_seq("READ"): 3268 lock_type = "READ" 3269 elif self._match_text_seq("WRITE"): 3270 lock_type = "WRITE" 3271 elif self._match_text_seq("CHECKSUM"): 3272 lock_type = "CHECKSUM" 3273 else: 3274 lock_type = None 3275 3276 override = self._match_text_seq("OVERRIDE") 3277 3278 return self.expression( 3279 exp.LockingProperty( 3280 this=this, kind=kind, for_or_in=for_or_in, lock_type=lock_type, override=override 3281 ) 3282 ) 3283 3284 def _parse_partition_by(self) -> list[exp.Expr]: 3285 if self._match(TokenType.PARTITION_BY): 3286 return self._parse_csv(self._parse_disjunction) 3287 return [] 3288 3289 def _parse_partition_bound_spec(self) -> exp.PartitionBoundSpec: 3290 def _parse_partition_bound_expr() -> exp.Expr | None: 3291 if self._match_text_seq("MINVALUE"): 3292 return exp.var("MINVALUE") 3293 if self._match_text_seq("MAXVALUE"): 3294 return exp.var("MAXVALUE") 3295 return self._parse_bitwise() 3296 3297 this: exp.Expr | list[exp.Expr] | None = None 3298 expression = None 3299 from_expressions = None 3300 to_expressions = None 3301 3302 if self._match(TokenType.IN): 3303 this = self._parse_wrapped_csv(self._parse_bitwise) 3304 elif self._match(TokenType.FROM): 3305 from_expressions = self._parse_wrapped_csv(_parse_partition_bound_expr) 3306 self._match_text_seq("TO") 3307 to_expressions = self._parse_wrapped_csv(_parse_partition_bound_expr) 3308 elif self._match_text_seq("WITH", "(", "MODULUS"): 3309 this = self._parse_number() 3310 self._match_text_seq(",", "REMAINDER") 3311 expression = self._parse_number() 3312 self._match_r_paren() 3313 else: 3314 self.raise_error("Failed to parse partition bound spec.") 3315 3316 return self.expression( 3317 exp.PartitionBoundSpec( 3318 this=this, 3319 expression=expression, 3320 from_expressions=from_expressions, 3321 to_expressions=to_expressions, 3322 ) 3323 ) 3324 3325 # https://www.postgresql.org/docs/current/sql-createtable.html 3326 def _parse_partitioned_of(self) -> exp.PartitionedOfProperty | None: 3327 if not self._match_text_seq("OF"): 3328 self._retreat(self._index - 1) 3329 return None 3330 3331 this = self._parse_table(schema=True) 3332 3333 if self._match(TokenType.DEFAULT): 3334 expression: exp.Var | exp.PartitionBoundSpec = exp.var("DEFAULT") 3335 elif self._match_text_seq("FOR", "VALUES"): 3336 expression = self._parse_partition_bound_spec() 3337 else: 3338 self.raise_error("Expecting either DEFAULT or FOR VALUES clause.") 3339 3340 return self.expression(exp.PartitionedOfProperty(this=this, expression=expression)) 3341 3342 def _parse_partitioned_by(self) -> exp.PartitionedByProperty: 3343 self._match(TokenType.EQ) 3344 return self.expression( 3345 exp.PartitionedByProperty( 3346 this=self._parse_schema() or self._parse_bracket(self._parse_field()) 3347 ) 3348 ) 3349 3350 def _parse_withdata(self, no: bool = False) -> exp.WithDataProperty: 3351 if self._match_text_seq("AND", "STATISTICS"): 3352 statistics = True 3353 elif self._match_text_seq("AND", "NO", "STATISTICS"): 3354 statistics = False 3355 else: 3356 statistics = None 3357 3358 return self.expression(exp.WithDataProperty(no=no, statistics=statistics)) 3359 3360 def _parse_contains_property(self) -> exp.SqlReadWriteProperty | None: 3361 if self._match_text_seq("SQL"): 3362 return self.expression(exp.SqlReadWriteProperty(this="CONTAINS SQL")) 3363 return None 3364 3365 def _parse_modifies_property(self) -> exp.SqlReadWriteProperty | None: 3366 if self._match_text_seq("SQL", "DATA"): 3367 return self.expression(exp.SqlReadWriteProperty(this="MODIFIES SQL DATA")) 3368 return None 3369 3370 def _parse_no_property(self) -> exp.Expr | None: 3371 if self._match_text_seq("PRIMARY", "INDEX"): 3372 return exp.NoPrimaryIndexProperty() 3373 if self._match_text_seq("SQL"): 3374 return self.expression(exp.SqlReadWriteProperty(this="NO SQL")) 3375 return None 3376 3377 def _parse_on_property(self) -> exp.Expr | None: 3378 if self._match_text_seq("COMMIT", "PRESERVE", "ROWS"): 3379 return exp.OnCommitProperty() 3380 if self._match_text_seq("COMMIT", "DELETE", "ROWS"): 3381 return exp.OnCommitProperty(delete=True) 3382 return self.expression(exp.OnProperty(this=self._parse_schema(self._parse_id_var()))) 3383 3384 def _parse_reads_property(self) -> exp.SqlReadWriteProperty | None: 3385 if self._match_text_seq("SQL", "DATA"): 3386 return self.expression(exp.SqlReadWriteProperty(this="READS SQL DATA")) 3387 return None 3388 3389 def _parse_distkey(self) -> exp.DistKeyProperty: 3390 return self.expression(exp.DistKeyProperty(this=self._parse_wrapped(self._parse_id_var))) 3391 3392 def _parse_create_like(self) -> exp.LikeProperty | None: 3393 table = self._parse_table(schema=True) 3394 3395 options = [] 3396 while self._match_texts(("INCLUDING", "EXCLUDING")): 3397 this = self._prev.text.upper() 3398 3399 id_var = self._parse_id_var() 3400 if not id_var: 3401 return None 3402 3403 options.append( 3404 self.expression(exp.Property(this=this, value=exp.var(id_var.this.upper()))) 3405 ) 3406 3407 return self.expression(exp.LikeProperty(this=table, expressions=options)) 3408 3409 def _parse_sortkey(self, compound: bool = False) -> exp.SortKeyProperty: 3410 return self.expression( 3411 exp.SortKeyProperty(this=self._parse_wrapped_id_vars(), compound=compound) 3412 ) 3413 3414 def _parse_character_set(self, default: bool = False) -> exp.CharacterSetProperty: 3415 self._match(TokenType.EQ) 3416 return self.expression( 3417 exp.CharacterSetProperty(this=self._parse_var_or_string(), default=default) 3418 ) 3419 3420 def _parse_remote_with_connection(self) -> exp.RemoteWithConnectionModelProperty: 3421 self._match_text_seq("WITH", "CONNECTION") 3422 return self.expression( 3423 exp.RemoteWithConnectionModelProperty(this=self._parse_table_parts()) 3424 ) 3425 3426 def _parse_returns(self) -> exp.ReturnsProperty: 3427 value: exp.Expr | None 3428 null = None 3429 is_table = self._match(TokenType.TABLE) 3430 3431 if is_table: 3432 if self._match(TokenType.LT): 3433 value = self.expression( 3434 exp.Schema(this="TABLE", expressions=self._parse_csv(self._parse_struct_types)) 3435 ) 3436 if not self._match(TokenType.GT): 3437 self.raise_error("Expecting >") 3438 else: 3439 value = self._parse_schema(exp.var("TABLE")) 3440 elif self._match_text_seq("NULL", "ON", "NULL", "INPUT"): 3441 null = True 3442 value = None 3443 else: 3444 value = self._parse_types() 3445 3446 return self.expression(exp.ReturnsProperty(this=value, is_table=is_table, null=null)) 3447 3448 def _parse_describe(self) -> exp.Describe: 3449 kind = self._prev.text if self._match_set(self.CREATABLES) else None 3450 style: str | None = ( 3451 self._prev.text.upper() if self._match_texts(self.DESCRIBE_STYLES) else None 3452 ) 3453 if self._match(TokenType.DOT): 3454 style = None 3455 self._retreat(self._index - 2) 3456 3457 format = self._parse_property() if self._match(TokenType.FORMAT, advance=False) else None 3458 3459 if self._match_set(self.STATEMENT_PARSERS, advance=False): 3460 this = self._parse_statement() 3461 else: 3462 this = self._parse_table(schema=True) 3463 3464 properties = self._parse_properties() 3465 expressions = properties.expressions if properties else None 3466 partition = self._parse_partition() 3467 return self.expression( 3468 exp.Describe( 3469 this=this, 3470 style=style, 3471 kind=kind, 3472 expressions=expressions, 3473 partition=partition, 3474 format=format, 3475 as_json=self._match_text_seq("AS", "JSON"), 3476 ) 3477 ) 3478 3479 def _parse_multitable_inserts(self, comments: list[str] | None) -> exp.MultitableInserts: 3480 kind = self._prev.text.upper() 3481 expressions = [] 3482 3483 def parse_conditional_insert() -> exp.ConditionalInsert | None: 3484 if self._match(TokenType.WHEN): 3485 expression = self._parse_disjunction() 3486 self._match(TokenType.THEN) 3487 else: 3488 expression = None 3489 3490 else_ = self._match(TokenType.ELSE) 3491 3492 if not self._match(TokenType.INTO): 3493 return None 3494 3495 return self.expression( 3496 exp.ConditionalInsert( 3497 this=self.expression( 3498 exp.Insert( 3499 this=self._parse_table(schema=True), 3500 expression=self._parse_derived_table_values(), 3501 ) 3502 ), 3503 expression=expression, 3504 else_=else_, 3505 ) 3506 ) 3507 3508 expression = parse_conditional_insert() 3509 while expression is not None: 3510 expressions.append(expression) 3511 expression = parse_conditional_insert() 3512 3513 return self.expression( 3514 exp.MultitableInserts(kind=kind, expressions=expressions, source=self._parse_table()), 3515 comments=comments, 3516 ) 3517 3518 def _parse_insert(self) -> exp.Insert | exp.MultitableInserts: 3519 comments: list[str] = [] 3520 hint = self._parse_hint() 3521 overwrite = self._match(TokenType.OVERWRITE) 3522 ignore = self._match(TokenType.IGNORE) 3523 local = self._match_text_seq("LOCAL") 3524 alternative = None 3525 is_function = None 3526 3527 if self._match_text_seq("DIRECTORY"): 3528 this: exp.Expr | None = self.expression( 3529 exp.Directory( 3530 this=self._parse_var_or_string(), 3531 local=local, 3532 row_format=self._parse_row_format(match_row=True), 3533 ) 3534 ) 3535 else: 3536 if self._match_set((TokenType.FIRST, TokenType.ALL)): 3537 comments += ensure_list(self._prev_comments) 3538 return self._parse_multitable_inserts(comments) 3539 3540 if self._match(TokenType.OR): 3541 alternative = self._match_texts(self.INSERT_ALTERNATIVES) and self._prev.text 3542 3543 self._match(TokenType.INTO) 3544 comments += ensure_list(self._prev_comments) 3545 self._match(TokenType.TABLE) 3546 is_function = self._match(TokenType.FUNCTION) 3547 3548 this = self._parse_function() if is_function else self._parse_insert_table() 3549 3550 # MySQL's INSERT ... SET is normalized into the INSERT ... (cols) VALUES (vals) variant 3551 set_values = None 3552 if self._match(TokenType.SET): 3553 columns = [] 3554 values = [] 3555 3556 def _parse_set_assignment() -> exp.Expr | None: 3557 target = self._parse_column() 3558 if isinstance(target, exp.Column) and self._match(TokenType.EQ): 3559 if self.dialect.SUPPORTS_VALUES_DEFAULT and self._match(TokenType.DEFAULT): 3560 value: exp.Expr | None = exp.var(self._prev.text.upper()) 3561 else: 3562 value = self._parse_disjunction() 3563 3564 if value: 3565 columns.append(target.this) 3566 values.append(value) 3567 return value 3568 3569 self.raise_error("Expected column assignment in INSERT ... SET") 3570 return None 3571 3572 self._parse_csv(_parse_set_assignment) 3573 3574 this = self.expression(exp.Schema(this=this, expressions=columns)) 3575 set_values = self.expression( 3576 exp.Values( 3577 expressions=[exp.Tuple(expressions=values)], 3578 alias=self._parse_table_alias(), 3579 ) 3580 ) 3581 3582 returning = self._parse_returning() # TSQL allows RETURNING before source 3583 3584 return self.expression( 3585 exp.Insert( 3586 hint=hint, 3587 is_function=is_function, 3588 this=this, 3589 stored=self._match_text_seq("STORED") and self._parse_stored(), 3590 by_name=self._match_text_seq("BY", "NAME"), 3591 exists=self._parse_exists(), 3592 where=self._match_pair(TokenType.REPLACE, TokenType.WHERE) 3593 and self._parse_disjunction(), 3594 partition=self._match(TokenType.PARTITION_BY) and self._parse_partitioned_by(), 3595 settings=self._match_text_seq("SETTINGS") and self._parse_settings_property(), 3596 default=self._match_text_seq("DEFAULT", "VALUES"), 3597 expression=set_values 3598 or self._parse_derived_table_values() 3599 or self._parse_ddl_select(), 3600 conflict=self._parse_on_conflict(), 3601 returning=returning or self._parse_returning(), 3602 overwrite=overwrite, 3603 alternative=alternative, 3604 ignore=ignore, 3605 source=self._match(TokenType.TABLE) and self._parse_table(), 3606 ), 3607 comments=comments, 3608 ) 3609 3610 def _parse_insert_table(self) -> exp.Expr | None: 3611 this = self._parse_table(schema=True, parse_partition=True) 3612 if isinstance(this, exp.Table) and self._match(TokenType.ALIAS, advance=False): 3613 this.set("alias", self._parse_table_alias()) 3614 return this 3615 3616 def _parse_kill(self) -> exp.Kill: 3617 kind = exp.var(self._prev.text) if self._match_texts(("CONNECTION", "QUERY")) else None 3618 3619 return self.expression(exp.Kill(this=self._parse_primary(), kind=kind)) 3620 3621 def _parse_on_conflict(self) -> exp.OnConflict | None: 3622 conflict = self._match_text_seq("ON", "CONFLICT") 3623 duplicate = self._match_text_seq("ON", "DUPLICATE", "KEY") 3624 3625 if not conflict and not duplicate: 3626 return None 3627 3628 conflict_keys = None 3629 constraint = None 3630 3631 if conflict: 3632 if self._match_text_seq("ON", "CONSTRAINT"): 3633 constraint = self._parse_id_var() 3634 elif self._match(TokenType.L_PAREN): 3635 conflict_keys = self._parse_csv(self._parse_indexed_column) 3636 self._match_r_paren() 3637 3638 index_predicate = self._parse_where() 3639 3640 action = self._parse_var_from_options(self.CONFLICT_ACTIONS) 3641 if self._prev.token_type == TokenType.UPDATE: 3642 self._match(TokenType.SET) 3643 expressions = self._parse_csv(self._parse_equality) 3644 else: 3645 expressions = None 3646 3647 return self.expression( 3648 exp.OnConflict( 3649 duplicate=duplicate, 3650 expressions=expressions, 3651 action=action, 3652 conflict_keys=conflict_keys, 3653 index_predicate=index_predicate, 3654 constraint=constraint, 3655 where=self._parse_where(), 3656 ) 3657 ) 3658 3659 def _parse_returning(self) -> exp.Returning | None: 3660 if not self._match(TokenType.RETURNING): 3661 return None 3662 return self.expression( 3663 exp.Returning( 3664 expressions=self._parse_csv(self._parse_expression), 3665 into=self._match(TokenType.INTO) and self._parse_table_part(), 3666 ) 3667 ) 3668 3669 def _parse_row(self) -> exp.RowFormatSerdeProperty | exp.RowFormatDelimitedProperty | None: 3670 if not self._match(TokenType.FORMAT): 3671 return None 3672 return self._parse_row_format() 3673 3674 def _parse_serde_properties(self, with_: bool = False) -> exp.SerdeProperties | None: 3675 index = self._index 3676 with_ = with_ or self._match_text_seq("WITH") 3677 3678 if not self._match(TokenType.SERDE_PROPERTIES): 3679 self._retreat(index) 3680 return None 3681 return self.expression( 3682 exp.SerdeProperties(expressions=self._parse_wrapped_properties(), with_=with_) 3683 ) 3684 3685 def _parse_row_format( 3686 self, match_row: bool = False 3687 ) -> exp.RowFormatSerdeProperty | exp.RowFormatDelimitedProperty | None: 3688 if match_row and not self._match_pair(TokenType.ROW, TokenType.FORMAT): 3689 return None 3690 3691 if self._match_text_seq("SERDE"): 3692 this = self._parse_string() 3693 3694 serde_properties = self._parse_serde_properties() 3695 3696 return self.expression( 3697 exp.RowFormatSerdeProperty(this=this, serde_properties=serde_properties) 3698 ) 3699 3700 self._match_text_seq("DELIMITED") 3701 3702 kwargs = {} 3703 3704 if self._match_text_seq("FIELDS", "TERMINATED", "BY"): 3705 kwargs["fields"] = self._parse_string() 3706 if self._match_text_seq("ESCAPED", "BY"): 3707 kwargs["escaped"] = self._parse_string() 3708 if self._match_text_seq("COLLECTION", "ITEMS", "TERMINATED", "BY"): 3709 kwargs["collection_items"] = self._parse_string() 3710 if self._match_text_seq("MAP", "KEYS", "TERMINATED", "BY"): 3711 kwargs["map_keys"] = self._parse_string() 3712 if self._match_text_seq("LINES", "TERMINATED", "BY"): 3713 kwargs["lines"] = self._parse_string() 3714 if self._match_text_seq("NULL", "DEFINED", "AS"): 3715 kwargs["null"] = self._parse_string() 3716 3717 return self.expression(exp.RowFormatDelimitedProperty(**kwargs)) # type: ignore 3718 3719 def _parse_load(self) -> exp.LoadData | exp.Command: 3720 if self._match_text_seq("DATA"): 3721 local = self._match_text_seq("LOCAL") 3722 self._match_text_seq("INPATH") 3723 inpath = self._parse_string() 3724 overwrite = self._match(TokenType.OVERWRITE) 3725 temp: bool | None = None 3726 if self._match(TokenType.INTO): 3727 temp = self._match(TokenType.TEMPORARY) 3728 self._match(TokenType.TABLE) 3729 3730 return self.expression( 3731 exp.LoadData( 3732 this=self._parse_table(schema=True), 3733 local=local, 3734 overwrite=overwrite, 3735 temp=temp, 3736 inpath=inpath, 3737 files=self._match_text_seq("FROM", "FILES") 3738 and exp.Properties(expressions=self._parse_wrapped_properties()), 3739 partition=self._parse_partition(), 3740 input_format=self._match_text_seq("INPUTFORMAT") and self._parse_string(), 3741 serde=self._match_text_seq("SERDE") and self._parse_string(), 3742 ) 3743 ) 3744 return self._parse_as_command(self._prev) 3745 3746 def _parse_delete(self) -> exp.Delete: 3747 hint = self._parse_hint() 3748 3749 # This handles MySQL's "Multiple-Table Syntax" 3750 # https://dev.mysql.com/doc/refman/8.0/en/delete.html 3751 tables = None 3752 if not self._match(TokenType.FROM, advance=False): 3753 tables = self._parse_csv(self._parse_table) or None 3754 3755 returning = self._parse_returning() 3756 3757 return self.expression( 3758 exp.Delete( 3759 hint=hint, 3760 tables=tables, 3761 this=self._match(TokenType.FROM) and self._parse_table(joins=True), 3762 using=self._match(TokenType.USING) 3763 and self._parse_csv(lambda: self._parse_table(joins=True)), 3764 cluster=self._match(TokenType.ON) and self._parse_on_property(), 3765 where=self._parse_where(), 3766 returning=returning or self._parse_returning(), 3767 order=self._parse_order(), 3768 limit=self._parse_limit(), 3769 ) 3770 ) 3771 3772 def _parse_update(self) -> exp.Update: 3773 hint = self._parse_hint() 3774 kwargs: dict[str, object] = { 3775 "hint": hint, 3776 "this": self._parse_table(joins=True, alias_tokens=self.UPDATE_ALIAS_TOKENS), 3777 } 3778 while self._curr: 3779 if self._match(TokenType.SET): 3780 kwargs["expressions"] = self._parse_csv(self._parse_equality) 3781 elif self._match(TokenType.RETURNING, advance=False): 3782 kwargs["returning"] = self._parse_returning() 3783 elif self._match(TokenType.FROM, advance=False): 3784 from_ = self._parse_from(joins=True) 3785 table = from_.this if from_ else None 3786 if isinstance(table, exp.Subquery) and self._match(TokenType.JOIN, advance=False): 3787 table.set("joins", list(self._parse_joins()) or None) 3788 3789 kwargs["from_"] = from_ 3790 elif self._match(TokenType.WHERE, advance=False): 3791 kwargs["where"] = self._parse_where() 3792 elif self._match(TokenType.ORDER_BY, advance=False): 3793 kwargs["order"] = self._parse_order() 3794 elif self._match(TokenType.LIMIT, advance=False): 3795 kwargs["limit"] = self._parse_limit() 3796 else: 3797 break 3798 3799 return self.expression(exp.Update(**kwargs)) 3800 3801 def _parse_use(self) -> exp.Use: 3802 return self.expression( 3803 exp.Use( 3804 kind=self._parse_var_from_options(self.USABLES, raise_unmatched=False), 3805 this=self._parse_table(schema=False), 3806 ) 3807 ) 3808 3809 def _parse_uncache(self) -> exp.Uncache: 3810 if not self._match(TokenType.TABLE): 3811 self.raise_error("Expecting TABLE after UNCACHE") 3812 3813 return self.expression( 3814 exp.Uncache(exists=self._parse_exists(), this=self._parse_table(schema=True)) 3815 ) 3816 3817 def _parse_cache(self) -> exp.Cache: 3818 lazy = self._match_text_seq("LAZY") 3819 self._match(TokenType.TABLE) 3820 table = self._parse_table(schema=True) 3821 3822 options = [] 3823 if self._match_text_seq("OPTIONS"): 3824 self._match_l_paren() 3825 k = self._parse_string() 3826 self._match(TokenType.EQ) 3827 v = self._parse_string() 3828 options = [k, v] 3829 self._match_r_paren() 3830 3831 self._match(TokenType.ALIAS) 3832 return self.expression( 3833 exp.Cache( 3834 this=table, lazy=lazy, options=options, expression=self._parse_select(nested=True) 3835 ) 3836 ) 3837 3838 def _parse_partition(self) -> exp.Partition | None: 3839 if not self._match_texts(self.PARTITION_KEYWORDS): 3840 return None 3841 3842 return self.expression( 3843 exp.Partition( 3844 subpartition=self._prev.text.upper() == "SUBPARTITION", 3845 expressions=self._parse_wrapped_csv(self._parse_disjunction), 3846 ) 3847 ) 3848 3849 def _parse_value(self, values: bool = True) -> exp.Tuple | None: 3850 def _parse_value_expression() -> exp.Expr | None: 3851 if self.dialect.SUPPORTS_VALUES_DEFAULT and self._match(TokenType.DEFAULT): 3852 return exp.var(self._prev.text.upper()) 3853 return self._parse_expression() 3854 3855 if self._match(TokenType.L_PAREN): 3856 expressions = self._parse_csv(_parse_value_expression) 3857 self._match_r_paren() 3858 return self.expression(exp.Tuple(expressions=expressions)) 3859 3860 # In some dialects we can have VALUES 1, 2 which results in 1 column & 2 rows. 3861 expression = self._parse_expression() 3862 if expression: 3863 return self.expression(exp.Tuple(expressions=[expression])) 3864 return None 3865 3866 def _parse_projections( 3867 self, 3868 ) -> tuple[list[exp.Expr], list[exp.Expr] | None]: 3869 return self._parse_expressions(), None 3870 3871 def _parse_wrapped_select(self, table: bool = False) -> exp.Expr | None: 3872 if self._match_set((TokenType.PIVOT, TokenType.UNPIVOT)): 3873 this: exp.Expr | None = self._parse_simplified_pivot( 3874 is_unpivot=self._prev.token_type == TokenType.UNPIVOT 3875 ) 3876 elif self._match(TokenType.FROM): 3877 from_ = self._parse_from(joins=True, skip_from_token=True, consume_pipe=True) 3878 # Support parentheses for duckdb FROM-first syntax 3879 select = self._parse_select(from_=from_) 3880 if select: 3881 if not select.args.get("from_"): 3882 select.set("from_", from_) 3883 this = select 3884 else: 3885 this = exp.select("*").from_(t.cast(exp.From, from_)) 3886 this = self._parse_query_modifiers(self._parse_set_operations(this)) 3887 else: 3888 this = ( 3889 self._parse_table(consume_pipe=True) 3890 if table 3891 else self._parse_select(nested=True, parse_set_operation=False) 3892 ) 3893 3894 # Transform exp.Values into a exp.Table to pass through parse_query_modifiers 3895 # in case a modifier (e.g. join) is following 3896 if table and isinstance(this, exp.Values) and this.alias: 3897 alias = this.args["alias"].pop() 3898 this = exp.Table(this=this, alias=alias) 3899 3900 this = self._parse_query_modifiers(self._parse_set_operations(this)) 3901 3902 return this 3903 3904 def _parse_select( 3905 self, 3906 nested: bool = False, 3907 table: bool = False, 3908 parse_subquery_alias: bool = True, 3909 parse_set_operation: bool = True, 3910 consume_pipe: bool = True, 3911 from_: exp.From | None = None, 3912 ) -> exp.Expr | None: 3913 query = self._parse_select_query( 3914 nested=nested, 3915 table=table, 3916 parse_subquery_alias=parse_subquery_alias, 3917 parse_set_operation=parse_set_operation, 3918 ) 3919 3920 if consume_pipe and self._match(TokenType.PIPE_GT, advance=False): 3921 if not query and from_: 3922 query = exp.select("*").from_(from_) 3923 if isinstance(query, exp.Query): 3924 query = self._parse_pipe_syntax_query(query) 3925 query = query.subquery(copy=False) if query and table else query 3926 3927 return query 3928 3929 def _parse_select_query( 3930 self, 3931 nested: bool = False, 3932 table: bool = False, 3933 parse_subquery_alias: bool = True, 3934 parse_set_operation: bool = True, 3935 ) -> exp.Expr | None: 3936 cte = self._parse_with() 3937 3938 if cte: 3939 this = self._parse_statement() 3940 3941 if not this: 3942 self.raise_error("Failed to parse any statement following CTE") 3943 return cte 3944 3945 while isinstance(this, exp.Subquery) and this.is_wrapper: 3946 this = this.this 3947 3948 assert this is not None 3949 if "with_" in this.arg_types: 3950 if inner_cte := this.args.get("with_"): 3951 cte.set("expressions", cte.expressions + inner_cte.expressions) 3952 if inner_cte.args.get("recursive"): 3953 cte.set("recursive", True) 3954 this.set("with_", cte) 3955 else: 3956 self.raise_error(f"{this.key} does not support CTE") 3957 this = cte 3958 3959 return this 3960 3961 # duckdb supports leading with FROM x 3962 from_ = ( 3963 self._parse_from(joins=True, consume_pipe=True) 3964 if self._match(TokenType.FROM, advance=False) 3965 else None 3966 ) 3967 3968 if self._match(TokenType.SELECT): 3969 comments = self._prev_comments 3970 3971 hint = self._parse_hint() 3972 3973 if self._next and not self._next.token_type == TokenType.DOT: 3974 all_ = self._match(TokenType.ALL) 3975 matched_distinct = self._match_set(self.DISTINCT_TOKENS) 3976 else: 3977 all_, matched_distinct = None, False 3978 3979 kind = ( 3980 self._prev.text.upper() 3981 if self._match(TokenType.ALIAS) and self._match_texts(("STRUCT", "VALUE")) 3982 else None 3983 ) 3984 3985 distinct: exp.Expr | None = ( 3986 self.expression( 3987 exp.Distinct( 3988 on=self._parse_value(values=False) if self._match(TokenType.ON) else None 3989 ) 3990 ) 3991 if matched_distinct 3992 else None 3993 ) 3994 3995 operation_modifiers = [] 3996 while self._curr and self._match_texts(self.OPERATION_MODIFIERS): 3997 operation_modifiers.append(exp.var(self._prev.text.upper())) 3998 3999 limit = self._parse_limit(top=True) 4000 4001 # Some dialects (e.g. Redshift, T-SQL) allow SELECT TOP N DISTINCT ... 4002 if limit and not matched_distinct and not all_: 4003 matched_distinct = self._match_set(self.DISTINCT_TOKENS) 4004 if matched_distinct: 4005 distinct = self.expression( 4006 exp.Distinct( 4007 on=self._parse_value(values=False) 4008 if self._match(TokenType.ON) 4009 else None 4010 ) 4011 ) 4012 else: 4013 all_ = self._match(TokenType.ALL) 4014 4015 if all_ and distinct: 4016 self.raise_error("Cannot specify both ALL and DISTINCT after SELECT") 4017 4018 projections, exclude = self._parse_projections() 4019 4020 this = self.expression( 4021 exp.Select( 4022 kind=kind, 4023 hint=hint, 4024 distinct=distinct, 4025 expressions=projections, 4026 limit=limit, 4027 exclude=exclude, 4028 operation_modifiers=operation_modifiers or None, 4029 ) 4030 ) 4031 this.comments = comments 4032 4033 into = self._parse_into() 4034 if into: 4035 this.set("into", into) 4036 4037 if not from_: 4038 from_ = self._parse_from() 4039 4040 if from_: 4041 this.set("from_", from_) 4042 4043 this = self._parse_query_modifiers(this) 4044 elif (table or nested) and self._match(TokenType.L_PAREN): 4045 comments = self._prev_comments 4046 this = self._parse_wrapped_select(table=table) 4047 4048 if this: 4049 this.add_comments(comments, prepend=True) 4050 4051 # We return early here so that the UNION isn't attached to the subquery by the 4052 # following call to _parse_set_operations, but instead becomes the parent node 4053 self._match_r_paren() 4054 return self._parse_subquery(this, parse_alias=parse_subquery_alias) 4055 elif self._match(TokenType.VALUES, advance=False): 4056 this = self._parse_derived_table_values() 4057 elif from_: 4058 this = exp.select("*").from_(from_.this, copy=False) 4059 this = self._parse_query_modifiers(this) 4060 elif self._match(TokenType.SUMMARIZE): 4061 table = self._match(TokenType.TABLE) 4062 this = self._parse_select() or self._parse_string() or self._parse_table() 4063 return self.expression(exp.Summarize(this=this, table=table)) 4064 elif self._match(TokenType.DESCRIBE): 4065 this = self._parse_describe() 4066 else: 4067 this = None 4068 4069 return self._parse_set_operations(this) if parse_set_operation else this 4070 4071 def _parse_recursive_with_search(self) -> exp.RecursiveWithSearch | None: 4072 self._match_text_seq("SEARCH") 4073 4074 kind = self._match_texts(self.RECURSIVE_CTE_SEARCH_KIND) and self._prev.text.upper() 4075 4076 if not kind: 4077 return None 4078 4079 self._match_text_seq("FIRST", "BY") 4080 4081 return self.expression( 4082 exp.RecursiveWithSearch( 4083 kind=kind, 4084 this=self._parse_id_var(), 4085 expression=self._match_text_seq("SET") and self._parse_id_var(), 4086 using=self._match_text_seq("USING") and self._parse_id_var(), 4087 ) 4088 ) 4089 4090 def _parse_with(self, skip_with_token: bool = False) -> exp.With | None: 4091 if not skip_with_token and not self._match(TokenType.WITH): 4092 return None 4093 4094 comments = self._prev_comments 4095 recursive = self._match(TokenType.RECURSIVE) 4096 4097 last_comments = None 4098 expressions = [] 4099 while True: 4100 cte = self._parse_cte() 4101 if isinstance(cte, exp.CTE): 4102 expressions.append(cte) 4103 if last_comments: 4104 cte.add_comments(last_comments) 4105 4106 if not self._match(TokenType.COMMA) and not self._match(TokenType.WITH): 4107 break 4108 else: 4109 self._match(TokenType.WITH) 4110 4111 last_comments = self._prev_comments 4112 4113 return self.expression( 4114 exp.With( 4115 expressions=expressions, 4116 recursive=recursive or None, 4117 search=self._parse_recursive_with_search(), 4118 ), 4119 comments=comments, 4120 ) 4121 4122 def _parse_cte(self) -> exp.CTE | None: 4123 index = self._index 4124 4125 alias = self._parse_table_alias(self.ID_VAR_TOKENS) 4126 if not alias or not alias.this: 4127 self.raise_error("Expected CTE to have alias") 4128 4129 key_expressions = ( 4130 self._parse_wrapped_id_vars() if self._match_text_seq("USING", "KEY") else None 4131 ) 4132 4133 if not self._match(TokenType.ALIAS) and not self.OPTIONAL_ALIAS_TOKEN_CTE: 4134 self._retreat(index) 4135 return None 4136 4137 comments = self._prev_comments 4138 4139 if self._match_text_seq("NOT", "MATERIALIZED"): 4140 materialized = False 4141 elif self._match_text_seq("MATERIALIZED"): 4142 materialized = True 4143 else: 4144 materialized = None 4145 4146 cte = self.expression( 4147 exp.CTE( 4148 this=self._parse_wrapped(self._parse_statement), 4149 alias=alias, 4150 materialized=materialized, 4151 key_expressions=key_expressions, 4152 ), 4153 comments=comments, 4154 ) 4155 4156 values = cte.this 4157 if isinstance(values, exp.Values): 4158 cte.set("this", self._values_to_select(values)) 4159 4160 return cte 4161 4162 def _values_to_select(self, values: exp.Values) -> exp.Select: 4163 if values.alias: 4164 return exp.select("*").from_(values) 4165 return exp.select("*").from_(exp.alias_(values, "_values", table=True)) 4166 4167 def _parse_table_alias( 4168 self, alias_tokens: t.Collection[TokenType] | None = None 4169 ) -> exp.TableAlias | None: 4170 # In some dialects, LIMIT and OFFSET can act as both identifiers and keywords (clauses) 4171 # so this section tries to parse the clause version and if it fails, it treats the token 4172 # as an identifier (alias) 4173 if self._can_parse_limit_or_offset(): 4174 return None 4175 4176 any_token = self._match(TokenType.ALIAS) 4177 alias = ( 4178 self._parse_id_var(any_token=any_token, tokens=alias_tokens or self.TABLE_ALIAS_TOKENS) 4179 or self._parse_string_as_identifier() 4180 ) 4181 4182 index = self._index 4183 if self._match(TokenType.L_PAREN): 4184 columns = self._parse_csv(self._parse_function_parameter) 4185 self._match_r_paren() if columns else self._retreat(index) 4186 else: 4187 columns = None 4188 4189 if not alias and not columns: 4190 return None 4191 4192 table_alias = self.expression(exp.TableAlias(this=alias, columns=columns)) 4193 4194 # We bubble up comments from the Identifier to the TableAlias 4195 if isinstance(alias, exp.Identifier): 4196 table_alias.add_comments(alias.pop_comments()) 4197 4198 return table_alias 4199 4200 def _parse_subquery( 4201 self, this: exp.Expr | None, parse_alias: bool = True 4202 ) -> exp.Subquery | None: 4203 if not this: 4204 return None 4205 4206 return self.expression( 4207 exp.Subquery( 4208 this=this, 4209 pivots=self._parse_pivots(), 4210 alias=self._parse_table_alias() if parse_alias else None, 4211 sample=self._parse_table_sample(), 4212 ) 4213 ) 4214 4215 def _implicit_unnests_to_explicit(self, this: E) -> E: 4216 from sqlglot.optimizer.normalize_identifiers import normalize_identifiers as _norm 4217 4218 refs = {_norm(this.args["from_"].this.copy(), dialect=self.dialect).alias_or_name} 4219 for i, join in enumerate(this.args.get("joins") or []): 4220 table = join.this 4221 normalized_table = table.copy() 4222 normalized_table.meta["maybe_column"] = True 4223 normalized_table = _norm(normalized_table, dialect=self.dialect) 4224 4225 if isinstance(table, exp.Table) and not join.args.get("on"): 4226 if len(normalized_table.parts) > 1 and normalized_table.parts[0].name in refs: 4227 table_as_column = table.to_column() 4228 unnest = exp.Unnest(expressions=[table_as_column]) 4229 4230 # Table.to_column creates a parent Alias node that we want to convert to 4231 # a TableAlias and attach to the Unnest, so it matches the parser's output 4232 if isinstance(table.args.get("alias"), exp.TableAlias): 4233 table_as_column.replace(table_as_column.this) 4234 exp.alias_(unnest, None, table=[table.args["alias"].this], copy=False) 4235 4236 table.replace(unnest) 4237 4238 refs.add(normalized_table.alias_or_name) 4239 4240 return this 4241 4242 @t.overload 4243 def _parse_query_modifiers(self, this: E) -> E: ... 4244 4245 @t.overload 4246 def _parse_query_modifiers(self, this: None) -> None: ... 4247 4248 def _parse_query_modifiers(self, this): 4249 if isinstance(this, self.MODIFIABLES): 4250 for join in self._parse_joins(): 4251 this.append("joins", join) 4252 for lateral in iter(self._parse_lateral, None): 4253 this.append("laterals", lateral) 4254 4255 while True: 4256 if self._match_set(self.QUERY_MODIFIER_PARSERS, advance=False): 4257 modifier_token = self._curr 4258 parser = self.QUERY_MODIFIER_PARSERS[modifier_token.token_type] 4259 key, expression = parser(self) 4260 4261 if expression: 4262 if this.args.get(key): 4263 self.raise_error( 4264 f"Found multiple '{modifier_token.text.upper()}' clauses", 4265 token=modifier_token, 4266 ) 4267 4268 this.set(key, expression) 4269 if key == "limit": 4270 offset = expression.args.get("offset") 4271 expression.set("offset", None) 4272 4273 if offset: 4274 offset = exp.Offset(expression=offset) 4275 this.set("offset", offset) 4276 4277 limit_by_expressions = expression.expressions 4278 expression.set("expressions", None) 4279 offset.set("expressions", limit_by_expressions) 4280 continue 4281 break 4282 4283 if self.SUPPORTS_IMPLICIT_UNNEST and this and this.args.get("from_"): 4284 this = self._implicit_unnests_to_explicit(this) 4285 4286 return this 4287 4288 def _parse_hint_fallback_to_string(self) -> exp.Hint | None: 4289 start = self._curr 4290 while self._curr: 4291 self._advance() 4292 4293 end = self._tokens[self._index - 1] 4294 return exp.Hint(expressions=[self._find_sql(start, end)]) 4295 4296 def _parse_hint_function_call(self) -> exp.Expr | None: 4297 return self._parse_function_call() 4298 4299 def _parse_hint_body(self) -> exp.Hint | None: 4300 start_index = self._index 4301 should_fallback_to_string = False 4302 4303 hints = [] 4304 try: 4305 for hint in iter( 4306 lambda: self._parse_csv( 4307 lambda: self._parse_hint_function_call() or self._parse_var(upper=True), 4308 ), 4309 [], 4310 ): 4311 hints.extend(hint) 4312 except ParseError: 4313 should_fallback_to_string = True 4314 4315 if should_fallback_to_string or self._curr: 4316 self._retreat(start_index) 4317 return self._parse_hint_fallback_to_string() 4318 4319 return self.expression(exp.Hint(expressions=hints)) 4320 4321 def _parse_hint(self) -> exp.Hint | None: 4322 if self._match(TokenType.HINT) and self._prev_comments: 4323 return exp.maybe_parse(self._prev_comments[0], into=exp.Hint, dialect=self.dialect) 4324 4325 return None 4326 4327 def _parse_into(self) -> exp.Into | None: 4328 if not self._match(TokenType.INTO): 4329 return None 4330 4331 temp = self._match(TokenType.TEMPORARY) 4332 unlogged = self._match_text_seq("UNLOGGED") 4333 self._match(TokenType.TABLE) 4334 4335 return self.expression( 4336 exp.Into(this=self._parse_table(schema=True), temporary=temp, unlogged=unlogged) 4337 ) 4338 4339 def _parse_from( 4340 self, 4341 joins: bool = False, 4342 skip_from_token: bool = False, 4343 consume_pipe: bool = False, 4344 ) -> exp.From | None: 4345 if not skip_from_token and not self._match(TokenType.FROM): 4346 return None 4347 4348 comments = self._prev_comments 4349 return self.expression( 4350 exp.From(this=self._parse_table(joins=joins, consume_pipe=consume_pipe)), 4351 comments=comments, 4352 ) 4353 4354 def _parse_match_recognize_measure(self) -> exp.MatchRecognizeMeasure: 4355 return self.expression( 4356 exp.MatchRecognizeMeasure( 4357 window_frame=self._match_texts(("FINAL", "RUNNING")) and self._prev.text.upper(), 4358 this=self._parse_expression(), 4359 ) 4360 ) 4361 4362 def _parse_match_recognize(self) -> exp.MatchRecognize | None: 4363 if not self._match(TokenType.MATCH_RECOGNIZE): 4364 return None 4365 4366 self._match_l_paren() 4367 4368 partition = self._parse_partition_by() 4369 order = self._parse_order() 4370 4371 measures = ( 4372 self._parse_csv(self._parse_match_recognize_measure) 4373 if self._match_text_seq("MEASURES") 4374 else None 4375 ) 4376 4377 if self._match_text_seq("ONE", "ROW", "PER", "MATCH"): 4378 rows = exp.var("ONE ROW PER MATCH") 4379 elif self._match_text_seq("ALL", "ROWS", "PER", "MATCH"): 4380 text = "ALL ROWS PER MATCH" 4381 if self._match_text_seq("SHOW", "EMPTY", "MATCHES"): 4382 text += " SHOW EMPTY MATCHES" 4383 elif self._match_text_seq("OMIT", "EMPTY", "MATCHES"): 4384 text += " OMIT EMPTY MATCHES" 4385 elif self._match_text_seq("WITH", "UNMATCHED", "ROWS"): 4386 text += " WITH UNMATCHED ROWS" 4387 rows = exp.var(text) 4388 else: 4389 rows = None 4390 4391 if self._match_text_seq("AFTER", "MATCH", "SKIP"): 4392 text = "AFTER MATCH SKIP" 4393 if self._match_text_seq("PAST", "LAST", "ROW"): 4394 text += " PAST LAST ROW" 4395 elif self._match_text_seq("TO", "NEXT", "ROW"): 4396 text += " TO NEXT ROW" 4397 elif self._match_text_seq("TO", "FIRST"): 4398 text += f" TO FIRST {self._advance_any().text}" # type: ignore 4399 elif self._match_text_seq("TO", "LAST"): 4400 text += f" TO LAST {self._advance_any().text}" # type: ignore 4401 after = exp.var(text) 4402 else: 4403 after = None 4404 4405 if self._match_text_seq("PATTERN"): 4406 self._match_l_paren() 4407 4408 if not self._curr: 4409 self.raise_error("Expecting )", self._curr) 4410 4411 paren = 1 4412 start = self._curr 4413 4414 while self._curr and paren > 0: 4415 if self._curr.token_type == TokenType.L_PAREN: 4416 paren += 1 4417 if self._curr.token_type == TokenType.R_PAREN: 4418 paren -= 1 4419 4420 end = self._prev 4421 self._advance() 4422 4423 if paren > 0: 4424 self.raise_error("Expecting )", self._curr) 4425 4426 pattern = exp.var(self._find_sql(start, end)) 4427 else: 4428 pattern = None 4429 4430 define = ( 4431 self._parse_csv(self._parse_name_as_expression) 4432 if self._match_text_seq("DEFINE") 4433 else None 4434 ) 4435 4436 self._match_r_paren() 4437 4438 return self.expression( 4439 exp.MatchRecognize( 4440 partition_by=partition, 4441 order=order, 4442 measures=measures, 4443 rows=rows, 4444 after=after, 4445 pattern=pattern, 4446 define=define, 4447 alias=self._parse_table_alias(), 4448 ) 4449 ) 4450 4451 def _parse_lateral(self) -> exp.Lateral | None: 4452 cross_apply: bool | None = None 4453 if self._match_pair(TokenType.CROSS, TokenType.APPLY): 4454 cross_apply = True 4455 elif self._match_pair(TokenType.OUTER, TokenType.APPLY): 4456 cross_apply = False 4457 4458 if cross_apply is not None: 4459 this = self._parse_select(table=True) 4460 view = None 4461 outer = None 4462 elif self._match(TokenType.LATERAL): 4463 this = self._parse_select(table=True) 4464 view = self._match(TokenType.VIEW) 4465 outer = self._match(TokenType.OUTER) 4466 else: 4467 return None 4468 4469 if not this: 4470 this = ( 4471 self._parse_unnest() 4472 or self._parse_function() 4473 or self._parse_id_var(any_token=False) 4474 ) 4475 4476 while self._match(TokenType.DOT): 4477 this = exp.Dot( 4478 this=this, 4479 expression=self._parse_function() or self._parse_id_var(any_token=False), 4480 ) 4481 4482 ordinality: bool | None = None 4483 4484 if view: 4485 table = self._parse_id_var(any_token=False) 4486 columns = self._parse_csv(self._parse_id_var) if self._match(TokenType.ALIAS) else [] 4487 table_alias: exp.TableAlias | None = self.expression( 4488 exp.TableAlias(this=table, columns=columns) 4489 ) 4490 elif isinstance(this, (exp.Subquery, exp.Unnest)) and this.alias: 4491 # We move the alias from the lateral's child node to the lateral itself 4492 table_alias = this.args["alias"].pop() 4493 else: 4494 ordinality = self._match_pair(TokenType.WITH, TokenType.ORDINALITY) 4495 table_alias = self._parse_table_alias() 4496 4497 return self.expression( 4498 exp.Lateral( 4499 this=this, 4500 view=view, 4501 outer=outer, 4502 alias=table_alias, 4503 cross_apply=cross_apply, 4504 ordinality=ordinality, 4505 ) 4506 ) 4507 4508 def _parse_stream(self) -> exp.Stream | None: 4509 index = self._index 4510 if self._match(TokenType.STREAM): 4511 if this := self._try_parse(self._parse_table): 4512 return self.expression(exp.Stream(this=this)) 4513 self._retreat(index) 4514 return None 4515 4516 def _parse_join_parts( 4517 self, 4518 ) -> tuple[Token | None, Token | None, Token | None]: 4519 return ( 4520 self._prev if self._match_set(self.JOIN_METHODS) else None, 4521 self._prev if self._match_set(self.JOIN_SIDES) else None, 4522 self._prev if self._match_set(self.JOIN_KINDS) else None, 4523 ) 4524 4525 def _parse_using_identifiers(self) -> list[exp.Expr]: 4526 def _parse_column_as_identifier() -> exp.Expr | None: 4527 this = self._parse_column() 4528 if isinstance(this, exp.Column): 4529 return this.this 4530 return this 4531 4532 return self._parse_wrapped_csv(_parse_column_as_identifier, optional=True) 4533 4534 def _parse_join( 4535 self, 4536 skip_join_token: bool = False, 4537 parse_bracket: bool = False, 4538 alias_tokens: t.Collection[TokenType] | None = None, 4539 ) -> exp.Join | None: 4540 if self._match(TokenType.COMMA): 4541 table = self._try_parse(lambda: self._parse_table(alias_tokens=alias_tokens)) 4542 cross_join = self.expression(exp.Join(this=table)) if table else None 4543 4544 if cross_join and self.JOINS_HAVE_EQUAL_PRECEDENCE: 4545 cross_join.set("kind", "CROSS") 4546 4547 return cross_join 4548 4549 index = self._index 4550 method, side, kind = self._parse_join_parts() 4551 directed = self._match_text_seq("DIRECTED") 4552 hint = self._prev.text if self._match_texts(self.JOIN_HINTS) else None 4553 join = self._match(TokenType.JOIN) or (kind and kind.token_type == TokenType.STRAIGHT_JOIN) 4554 join_comments = self._prev_comments 4555 4556 if not skip_join_token and not join: 4557 self._retreat(index) 4558 kind = None 4559 method = None 4560 side = None 4561 4562 outer_apply = self._match_pair(TokenType.OUTER, TokenType.APPLY, False) 4563 cross_apply = self._match_pair(TokenType.CROSS, TokenType.APPLY, False) 4564 4565 if not skip_join_token and not join and not outer_apply and not cross_apply: 4566 return None 4567 4568 kwargs: dict[str, t.Any] = { 4569 "this": self._parse_table(parse_bracket=parse_bracket, alias_tokens=alias_tokens) 4570 } 4571 if kind and kind.token_type == TokenType.ARRAY and self._match(TokenType.COMMA): 4572 kwargs["expressions"] = self._parse_csv( 4573 lambda: self._parse_table(parse_bracket=parse_bracket, alias_tokens=alias_tokens) 4574 ) 4575 4576 if method: 4577 kwargs["method"] = method.text.upper() 4578 if side: 4579 kwargs["side"] = side.text.upper() 4580 if kind: 4581 kwargs["kind"] = kind.text.upper() 4582 if hint: 4583 kwargs["hint"] = hint 4584 4585 if self._match(TokenType.MATCH_CONDITION): 4586 kwargs["match_condition"] = self._parse_wrapped(self._parse_comparison) 4587 4588 if self._match(TokenType.ON): 4589 kwargs["on"] = self._parse_disjunction() 4590 elif self._match(TokenType.USING): 4591 kwargs["using"] = self._parse_using_identifiers() 4592 elif ( 4593 not method 4594 and not (outer_apply or cross_apply) 4595 and not isinstance(kwargs["this"], exp.Unnest) 4596 and not (kind and kind.token_type in (TokenType.CROSS, TokenType.ARRAY)) 4597 ): 4598 index = self._index 4599 joins: list | None = list(self._parse_joins(alias_tokens=alias_tokens)) 4600 4601 if joins and self._match(TokenType.ON): 4602 kwargs["on"] = self._parse_disjunction() 4603 elif joins and self._match(TokenType.USING): 4604 kwargs["using"] = self._parse_using_identifiers() 4605 else: 4606 joins = None 4607 self._retreat(index) 4608 4609 kwargs["this"].set("joins", joins if joins else None) 4610 4611 kwargs["pivots"] = self._parse_pivots() 4612 4613 comments = [c for token in (method, side, kind) if token for c in token.comments] 4614 comments = (join_comments or []) + comments 4615 4616 if ( 4617 self.ADD_JOIN_ON_TRUE 4618 and not kwargs.get("on") 4619 and not kwargs.get("using") 4620 and not kwargs.get("method") 4621 and kwargs.get("kind") in (None, "INNER", "OUTER") 4622 ): 4623 kwargs["on"] = exp.true() 4624 4625 if directed: 4626 kwargs["directed"] = directed 4627 4628 return self.expression(exp.Join(**kwargs), comments=comments) 4629 4630 def _parse_opclass(self) -> exp.Expr | None: 4631 this = self._parse_disjunction() 4632 4633 if self._match_texts(self.OPCLASS_FOLLOW_KEYWORDS, advance=False): 4634 return this 4635 4636 if not self._match_set(self.OPTYPE_FOLLOW_TOKENS, advance=False): 4637 return self.expression(exp.Opclass(this=this, expression=self._parse_table_parts())) 4638 4639 return this 4640 4641 def _parse_index_params(self) -> exp.IndexParameters: 4642 using = self._parse_var(any_token=True) if self._match(TokenType.USING) else None 4643 4644 if self._match(TokenType.L_PAREN, advance=False): 4645 columns = self._parse_wrapped_csv(self._parse_with_operator) 4646 else: 4647 columns = None 4648 4649 include = self._parse_wrapped_id_vars() if self._match_text_seq("INCLUDE") else None 4650 partition_by = self._parse_partition_by() 4651 with_storage = self._match(TokenType.WITH) and self._parse_wrapped_properties() 4652 tablespace = ( 4653 self._parse_var(any_token=True) 4654 if self._match_text_seq("USING", "INDEX", "TABLESPACE") 4655 else None 4656 ) 4657 where = self._parse_where() 4658 4659 on = self._parse_field() if self._match(TokenType.ON) else None 4660 4661 return self.expression( 4662 exp.IndexParameters( 4663 using=using, 4664 columns=columns, 4665 include=include, 4666 partition_by=partition_by, 4667 where=where, 4668 with_storage=with_storage, 4669 tablespace=tablespace, 4670 on=on, 4671 ) 4672 ) 4673 4674 def _parse_index( 4675 self, index: exp.Expr | None = None, anonymous: bool = False 4676 ) -> exp.Index | None: 4677 if index or anonymous: 4678 unique = None 4679 primary = None 4680 amp = None 4681 4682 self._match(TokenType.ON) 4683 self._match(TokenType.TABLE) # hive 4684 table = self._parse_table_parts(schema=True) 4685 else: 4686 unique = self._match(TokenType.UNIQUE) 4687 primary = self._match_text_seq("PRIMARY") 4688 amp = self._match_text_seq("AMP") 4689 4690 if not self._match(TokenType.INDEX): 4691 return None 4692 4693 index = self._parse_id_var() 4694 table = None 4695 4696 params = self._parse_index_params() 4697 4698 return self.expression( 4699 exp.Index( 4700 this=index, table=table, unique=unique, primary=primary, amp=amp, params=params 4701 ) 4702 ) 4703 4704 def _parse_table_hints(self) -> list[exp.Expr] | None: 4705 hints: list[exp.Expr] = [] 4706 if self._match_pair(TokenType.WITH, TokenType.L_PAREN): 4707 # https://learn.microsoft.com/en-us/sql/t-sql/queries/hints-transact-sql-table?view=sql-server-ver16 4708 hints.append( 4709 self.expression( 4710 exp.WithTableHint( 4711 expressions=self._parse_csv( 4712 lambda: self._parse_function() or self._parse_var(any_token=True) 4713 ) 4714 ) 4715 ) 4716 ) 4717 self._match_r_paren() 4718 else: 4719 # https://dev.mysql.com/doc/refman/8.0/en/index-hints.html 4720 while self._match_set(self.TABLE_INDEX_HINT_TOKENS): 4721 hint = exp.IndexTableHint(this=self._prev.text.upper()) 4722 4723 self._match_set((TokenType.INDEX, TokenType.KEY)) 4724 if self._match(TokenType.FOR): 4725 hint.set("target", self._advance_any() and self._prev.text.upper()) 4726 4727 hint.set("expressions", self._parse_wrapped_id_vars()) 4728 hints.append(hint) 4729 4730 return hints or None 4731 4732 def _parse_table_part(self, schema: bool = False) -> exp.Expr | None: 4733 return ( 4734 (not schema and self._parse_function(optional_parens=False)) 4735 or self._parse_id_var(any_token=False) 4736 or self._parse_string_as_identifier() 4737 or self._parse_placeholder() 4738 ) 4739 4740 def _parse_table_parts_fast(self) -> exp.Table | None: 4741 index = self._index 4742 parts: list[exp.Identifier] | None = None 4743 all_comments: list[str] | None = None 4744 4745 while self._match_set(self.IDENTIFIER_TOKENS): 4746 token = self._prev 4747 comments = self._prev_comments 4748 4749 has_dot = self._match(TokenType.DOT) 4750 curr_tt = self._curr.token_type 4751 4752 if not has_dot: 4753 if curr_tt in self.TABLE_POSTFIX_TOKENS: 4754 self._retreat(index) 4755 return None 4756 elif curr_tt not in self.IDENTIFIER_TOKENS: 4757 self._retreat(index) 4758 return None 4759 4760 if parts is None: 4761 parts = [] 4762 4763 if comments: 4764 if all_comments is None: 4765 all_comments = [] 4766 all_comments.extend(comments) 4767 self._prev_comments = [] 4768 4769 parts.append( 4770 self.expression( 4771 exp.Identifier( 4772 this=token.text, quoted=token.token_type == TokenType.IDENTIFIER 4773 ), 4774 token, 4775 ) 4776 ) 4777 4778 if not has_dot: 4779 break 4780 4781 if parts is None: 4782 return None 4783 4784 n = len(parts) 4785 4786 if n == 1: 4787 table: exp.Table = exp.Table(this=parts[0]) 4788 elif n == 2: 4789 table = exp.Table(this=parts[1], db=parts[0]) 4790 elif n >= 3: 4791 this: exp.Identifier | exp.Dot = parts[2] 4792 for i in range(3, n): 4793 this = exp.Dot(this=this, expression=parts[i]) 4794 4795 table = exp.Table(this=this, db=parts[1], catalog=parts[0]) 4796 4797 if table is None: 4798 self._retreat(index) 4799 elif all_comments: 4800 table.add_comments(all_comments) 4801 return table 4802 4803 def _parse_table_parts( 4804 self, 4805 schema: bool = False, 4806 is_db_reference: bool = False, 4807 wildcard: bool = False, 4808 fast: bool = False, 4809 ) -> exp.Table | exp.Dot | None: 4810 if fast: 4811 return self._parse_table_parts_fast() 4812 4813 catalog: exp.Expr | str | None = None 4814 db: exp.Expr | str | None = None 4815 table: exp.Expr | str | None = self._parse_table_part(schema=schema) 4816 4817 while self._match(TokenType.DOT): 4818 if catalog: 4819 # This allows nesting the table in arbitrarily many dot expressions if needed 4820 table = self.expression( 4821 exp.Dot(this=table, expression=self._parse_table_part(schema=schema)) 4822 ) 4823 else: 4824 catalog = db 4825 db = table 4826 # "" used for tsql FROM a..b case 4827 table = self._parse_table_part(schema=schema) or "" 4828 4829 if ( 4830 wildcard 4831 and self._is_connected() 4832 and (isinstance(table, exp.Identifier) or not table) 4833 and self._match(TokenType.STAR) 4834 ): 4835 if isinstance(table, exp.Identifier): 4836 table.args["this"] += "*" 4837 else: 4838 table = exp.Identifier(this="*") 4839 4840 if is_db_reference: 4841 catalog = db 4842 db = table 4843 table = None 4844 4845 if not table and not is_db_reference: 4846 self.raise_error(f"Expected table name but got {self._curr}") 4847 if not db and is_db_reference: 4848 self.raise_error(f"Expected database name but got {self._curr}") 4849 4850 table = self.expression(exp.Table(this=table, db=db, catalog=catalog)) 4851 4852 # Bubble up comments from identifier parts to the Table 4853 comments = [] 4854 for part in table.parts: 4855 if part_comments := part.pop_comments(): 4856 comments.extend(part_comments) 4857 if comments: 4858 table.add_comments(comments) 4859 4860 changes = self._parse_changes() 4861 if changes: 4862 table.set("changes", changes) 4863 4864 at_before = self._parse_historical_data() 4865 if at_before: 4866 table.set("when", at_before) 4867 4868 pivots = self._parse_pivots() 4869 if pivots: 4870 table.set("pivots", pivots) 4871 4872 return table 4873 4874 def _parse_table( 4875 self, 4876 schema: bool = False, 4877 joins: bool = False, 4878 alias_tokens: t.Collection[TokenType] | None = None, 4879 parse_bracket: bool = False, 4880 is_db_reference: bool = False, 4881 parse_partition: bool = False, 4882 consume_pipe: bool = False, 4883 ) -> exp.Expr | None: 4884 if not schema and not is_db_reference and not consume_pipe and not joins: 4885 index = self._index 4886 table = self._parse_table_parts(fast=True) 4887 4888 if table is not None: 4889 curr_tt = self._curr.token_type 4890 next_tt = self._next.token_type 4891 4892 fast_terminators = self.TABLE_TERMINATORS 4893 4894 # only return the table if we're sure there are no other operators 4895 # MATCH_CONDITION is a special case because it accepts any alias before it like LIMIT 4896 if curr_tt in fast_terminators and next_tt != TokenType.MATCH_CONDITION: 4897 return table 4898 4899 postfix_tokens = self.TABLE_POSTFIX_TOKENS 4900 4901 if curr_tt not in postfix_tokens and next_tt not in postfix_tokens: 4902 if alias := self._parse_table_alias( 4903 alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS 4904 ): 4905 table.set("alias", alias) 4906 4907 if self._curr.token_type in fast_terminators: 4908 return table 4909 4910 self._retreat(index) 4911 4912 if stream := self._parse_stream(): 4913 return stream 4914 4915 if lateral := self._parse_lateral(): 4916 return lateral 4917 4918 if unnest := self._parse_unnest(): 4919 return unnest 4920 4921 if values := self._parse_derived_table_values(): 4922 return values 4923 4924 if subquery := self._parse_select(table=True, consume_pipe=consume_pipe): 4925 if not subquery.args.get("pivots"): 4926 subquery.set("pivots", self._parse_pivots()) 4927 if joins: 4928 for join in self._parse_joins(): 4929 subquery.append("joins", join) 4930 return subquery 4931 4932 bracket = parse_bracket and self._parse_bracket(None) 4933 bracket = self.expression(exp.Table(this=bracket)) if bracket else None 4934 4935 rows_from_tables = ( 4936 self._parse_wrapped_csv(self._parse_table) 4937 if self._match_text_seq("ROWS", "FROM") 4938 else None 4939 ) 4940 rows_from = ( 4941 self.expression(exp.Table(rows_from=rows_from_tables)) if rows_from_tables else None 4942 ) 4943 4944 only = self._match(TokenType.ONLY) 4945 4946 this = t.cast( 4947 exp.Expr, 4948 bracket 4949 or rows_from 4950 or self._parse_bracket( 4951 self._parse_table_parts(schema=schema, is_db_reference=is_db_reference) 4952 ), 4953 ) 4954 4955 if only: 4956 this.set("only", only) 4957 4958 # Postgres supports a wildcard (table) suffix operator, which is a no-op in this context 4959 self._match(TokenType.STAR) 4960 4961 parse_partition = parse_partition or self.SUPPORTS_PARTITION_SELECTION 4962 if parse_partition and self._match(TokenType.PARTITION, advance=False): 4963 this.set("partition", self._parse_partition()) 4964 4965 if schema: 4966 return self._parse_schema(this=this) 4967 4968 if self.dialect.ALIAS_POST_VERSION: 4969 this.set("version", self._parse_version()) 4970 4971 if self.dialect.ALIAS_POST_TABLESAMPLE: 4972 this.set("sample", self._parse_table_sample()) 4973 4974 alias = self._parse_table_alias(alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS) 4975 if alias: 4976 this.set("alias", alias) 4977 4978 # DuckDB requires the time-travel clause to come after the alias, e.g. 4979 # SELECT * FROM t AS a AT (VERSION => 1) 4980 if isinstance(this, exp.Table) and not this.args.get("when"): 4981 this.set("when", self._parse_historical_data()) 4982 4983 if self._match(TokenType.INDEXED_BY): 4984 this.set("indexed", self._parse_table_parts()) 4985 elif self._match_text_seq("NOT", "INDEXED"): 4986 this.set("indexed", False) 4987 4988 if isinstance(this, exp.Table) and self._match_text_seq("AT"): 4989 return self.expression( 4990 exp.AtIndex(this=this.to_column(copy=False), expression=self._parse_id_var()) 4991 ) 4992 4993 this.set("hints", self._parse_table_hints()) 4994 4995 if not this.args.get("pivots"): 4996 this.set("pivots", self._parse_pivots()) 4997 4998 if not self.dialect.ALIAS_POST_TABLESAMPLE: 4999 this.set("sample", self._parse_table_sample()) 5000 5001 if not self.dialect.ALIAS_POST_VERSION: 5002 this.set("version", self._parse_version()) 5003 5004 if joins: 5005 for join in self._parse_joins(alias_tokens=alias_tokens): 5006 this.append("joins", join) 5007 5008 if self._match_pair(TokenType.WITH, TokenType.ORDINALITY): 5009 this.set("ordinality", True) 5010 this.set("alias", self._parse_table_alias()) 5011 5012 return this 5013 5014 def _parse_version(self) -> exp.Version | None: 5015 if self._match(TokenType.TIMESTAMP_SNAPSHOT): 5016 this = "TIMESTAMP" 5017 elif self._match(TokenType.VERSION_SNAPSHOT): 5018 this = "VERSION" 5019 else: 5020 return None 5021 5022 if self._match_set((TokenType.FROM, TokenType.BETWEEN)): 5023 kind = self._prev.text.upper() 5024 start = self._parse_bitwise() 5025 self._match_texts(("TO", "AND")) 5026 end = self._parse_bitwise() 5027 expression: exp.Expr | None = self.expression(exp.Tuple(expressions=[start, end])) 5028 elif self._match_text_seq("CONTAINED", "IN"): 5029 kind = "CONTAINED IN" 5030 expression = self.expression( 5031 exp.Tuple(expressions=self._parse_wrapped_csv(self._parse_bitwise)) 5032 ) 5033 elif self._match(TokenType.ALL): 5034 kind = "ALL" 5035 expression = None 5036 else: 5037 self._match_text_seq("AS", "OF") 5038 kind = "AS OF" 5039 expression = self._parse_type() 5040 5041 return self.expression(exp.Version(this=this, expression=expression, kind=kind)) 5042 5043 def _parse_historical_data(self) -> exp.HistoricalData | None: 5044 # https://docs.snowflake.com/en/sql-reference/constructs/at-before 5045 index = self._index 5046 historical_data = None 5047 if self._match_texts(self.HISTORICAL_DATA_PREFIX): 5048 this = self._prev.text.upper() 5049 kind = ( 5050 self._match(TokenType.L_PAREN) 5051 and self._match_texts(self.HISTORICAL_DATA_KIND) 5052 and self._prev.text.upper() 5053 ) 5054 expression = self._match(TokenType.FARROW) and self._parse_bitwise() 5055 5056 if expression: 5057 self._match_r_paren() 5058 historical_data = self.expression( 5059 exp.HistoricalData(this=this, kind=kind, expression=expression) 5060 ) 5061 else: 5062 self._retreat(index) 5063 5064 return historical_data 5065 5066 def _parse_changes(self) -> exp.Changes | None: 5067 if not self._match_text_seq("CHANGES", "(", "INFORMATION", "=>"): 5068 return None 5069 5070 information = self._parse_var(any_token=True) 5071 self._match_r_paren() 5072 5073 return self.expression( 5074 exp.Changes( 5075 information=information, 5076 at_before=self._parse_historical_data(), 5077 end=self._parse_historical_data(), 5078 ) 5079 ) 5080 5081 def _parse_unnest(self, with_alias: bool = True) -> exp.Unnest | None: 5082 if not self._match_pair(TokenType.UNNEST, TokenType.L_PAREN, advance=False): 5083 return None 5084 5085 self._advance() 5086 5087 expressions = self._parse_wrapped_csv(self._parse_equality) 5088 offset: bool | exp.Expr = self._match_pair(TokenType.WITH, TokenType.ORDINALITY) 5089 5090 alias = self._parse_table_alias() if with_alias else None 5091 5092 if alias: 5093 if self.dialect.UNNEST_COLUMN_ONLY: 5094 if alias.args.get("columns"): 5095 self.raise_error("Unexpected extra column alias in unnest.") 5096 5097 alias.set("columns", [alias.this]) 5098 alias.set("this", None) 5099 5100 columns = alias.args.get("columns") or [] 5101 if offset and len(expressions) < len(columns): 5102 offset = columns.pop() 5103 5104 if not offset and self._match_pair(TokenType.WITH, TokenType.OFFSET): 5105 self._match(TokenType.ALIAS) 5106 offset = self._parse_id_var( 5107 any_token=False, tokens=self.UNNEST_OFFSET_ALIAS_TOKENS 5108 ) or exp.to_identifier("offset") 5109 5110 return self.expression(exp.Unnest(expressions=expressions, alias=alias, offset=offset)) 5111 5112 def _parse_derived_table_values(self) -> exp.Values | None: 5113 is_derived = self._match_pair(TokenType.L_PAREN, TokenType.VALUES) 5114 if not is_derived and not ( 5115 # ClickHouse's `FORMAT Values` is equivalent to `VALUES` 5116 self._match_text_seq("VALUES") or self._match_text_seq("FORMAT", "VALUES") 5117 ): 5118 return None 5119 5120 expressions = self._parse_csv(self._parse_value) 5121 alias = self._parse_table_alias() 5122 5123 if is_derived: 5124 self._match_r_paren() 5125 5126 return self.expression( 5127 exp.Values(expressions=expressions, alias=alias or self._parse_table_alias()) 5128 ) 5129 5130 def _parse_table_sample(self, as_modifier: bool = False) -> exp.TableSample | None: 5131 if not self._match(TokenType.TABLE_SAMPLE) and not ( 5132 as_modifier and self._match_text_seq("USING", "SAMPLE") 5133 ): 5134 return None 5135 5136 bucket_numerator = None 5137 bucket_denominator = None 5138 bucket_field = None 5139 percent = None 5140 size = None 5141 seed = None 5142 5143 method = self._parse_var(tokens=(TokenType.ROW,), upper=True) 5144 matched_l_paren = self._match(TokenType.L_PAREN) 5145 5146 if self.TABLESAMPLE_CSV: 5147 num = None 5148 expressions = self._parse_csv(self._parse_primary) 5149 else: 5150 expressions = None 5151 num = ( 5152 self._parse_factor() 5153 if self._match(TokenType.NUMBER, advance=False) 5154 else self._parse_primary() or self._parse_placeholder() 5155 ) 5156 5157 if self._match_text_seq("BUCKET"): 5158 bucket_numerator = self._parse_number() 5159 self._match_text_seq("OUT", "OF") 5160 bucket_denominator = bucket_denominator = self._parse_number() 5161 self._match(TokenType.ON) 5162 bucket_field = self._parse_field() 5163 elif self._match_set((TokenType.PERCENT, TokenType.MOD)): 5164 percent = num 5165 elif self._match(TokenType.ROWS) or not self.dialect.TABLESAMPLE_SIZE_IS_PERCENT: 5166 size = num 5167 else: 5168 percent = num 5169 5170 if matched_l_paren: 5171 self._match_r_paren() 5172 5173 if self._match(TokenType.L_PAREN): 5174 method = self._parse_var(upper=True) 5175 seed = self._match(TokenType.COMMA) and self._parse_number() 5176 self._match_r_paren() 5177 elif self._match_texts(("SEED", "REPEATABLE")): 5178 seed = self._parse_wrapped(self._parse_number) 5179 5180 if not method and self.DEFAULT_SAMPLING_METHOD: 5181 method = exp.var(self.DEFAULT_SAMPLING_METHOD) 5182 5183 return self.expression( 5184 exp.TableSample( 5185 expressions=expressions, 5186 method=method, 5187 bucket_numerator=bucket_numerator, 5188 bucket_denominator=bucket_denominator, 5189 bucket_field=bucket_field, 5190 percent=percent, 5191 size=size, 5192 seed=seed, 5193 ) 5194 ) 5195 5196 def _parse_pivots(self) -> list[exp.Pivot] | None: 5197 if self._curr.token_type not in (TokenType.PIVOT, TokenType.UNPIVOT): 5198 return None 5199 return list(iter(self._parse_pivot, None)) or None 5200 5201 def _parse_joins( 5202 self, alias_tokens: t.Collection[TokenType] | None = None 5203 ) -> t.Iterator[exp.Join]: 5204 return iter(lambda: self._parse_join(alias_tokens=alias_tokens), None) 5205 5206 def _parse_unpivot_columns(self) -> exp.UnpivotColumns | None: 5207 if not self._match(TokenType.INTO): 5208 return None 5209 5210 return self.expression( 5211 exp.UnpivotColumns( 5212 this=self._match_text_seq("NAME") and self._parse_column(), 5213 expressions=self._match_text_seq("VALUE") and self._parse_csv(self._parse_column), 5214 ) 5215 ) 5216 5217 # https://duckdb.org/docs/sql/statements/pivot 5218 def _parse_simplified_pivot(self, is_unpivot: bool | None = None) -> exp.Pivot: 5219 def _parse_on() -> exp.Expr | None: 5220 this = self._parse_bitwise() 5221 5222 if self._match(TokenType.IN): 5223 # PIVOT ... ON col IN (row_val1, row_val2) 5224 return self._parse_in(this) 5225 if self._match(TokenType.ALIAS, advance=False): 5226 # UNPIVOT ... ON (col1, col2, col3) AS row_val 5227 return self._parse_alias(this) 5228 5229 return this 5230 5231 this = self._parse_table() 5232 expressions = self._match(TokenType.ON) and self._parse_csv(_parse_on) 5233 into = self._parse_unpivot_columns() 5234 using = self._match(TokenType.USING) and self._parse_csv( 5235 lambda: self._parse_alias(self._parse_column()) 5236 ) 5237 group = self._parse_group() 5238 5239 return self.expression( 5240 exp.Pivot( 5241 this=this, 5242 expressions=expressions, 5243 using=using, 5244 group=group, 5245 unpivot=is_unpivot, 5246 into=into, 5247 ) 5248 ) 5249 5250 def _parse_pivot_in(self) -> exp.In: 5251 def _parse_aliased_expression() -> exp.Expr | None: 5252 this = self._parse_select_or_expression() 5253 5254 self._match(TokenType.ALIAS) 5255 alias = self._parse_bitwise() 5256 if alias: 5257 if isinstance(alias, exp.Column) and not alias.db: 5258 alias = alias.this 5259 return self.expression(exp.PivotAlias(this=this, alias=alias)) 5260 5261 return this 5262 5263 value = self._parse_column() 5264 5265 if not self._match(TokenType.IN): 5266 self.raise_error("Expecting IN") 5267 5268 if self._match(TokenType.L_PAREN): 5269 if self._match(TokenType.ANY): 5270 exprs: list[exp.Expr] = ensure_list(exp.PivotAny(this=self._parse_order())) 5271 else: 5272 exprs = self._parse_csv(_parse_aliased_expression) 5273 self._match_r_paren() 5274 return self.expression(exp.In(this=value, expressions=exprs)) 5275 5276 return self.expression(exp.In(this=value, field=self._parse_id_var())) 5277 5278 def _parse_pivot_aggregation(self) -> exp.Expr | None: 5279 func = self._parse_function() 5280 if not func: 5281 if self._prev.token_type == TokenType.COMMA: 5282 return None 5283 self.raise_error("Expecting an aggregation function in PIVOT") 5284 5285 return self._parse_alias(func) 5286 5287 def _parse_pivot(self) -> exp.Pivot | None: 5288 index = self._index 5289 include_nulls = None 5290 5291 if self._match(TokenType.PIVOT): 5292 unpivot = False 5293 elif self._match(TokenType.UNPIVOT): 5294 unpivot = True 5295 5296 # https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-qry-select-unpivot.html#syntax 5297 if self._match_text_seq("INCLUDE", "NULLS"): 5298 include_nulls = True 5299 elif self._match_text_seq("EXCLUDE", "NULLS"): 5300 include_nulls = False 5301 else: 5302 return None 5303 5304 expressions = [] 5305 5306 if not self._match(TokenType.L_PAREN): 5307 self._retreat(index) 5308 return None 5309 5310 if unpivot: 5311 expressions = self._parse_csv(self._parse_column) 5312 else: 5313 expressions = self._parse_csv(self._parse_pivot_aggregation) 5314 5315 if not expressions: 5316 self.raise_error("Failed to parse PIVOT's aggregation list") 5317 5318 if not self._match(TokenType.FOR): 5319 self.raise_error("Expecting FOR") 5320 5321 fields = [] 5322 while True: 5323 field = self._try_parse(self._parse_pivot_in) 5324 if not field: 5325 break 5326 fields.append(field) 5327 5328 default_on_null = self._match_text_seq("DEFAULT", "ON", "NULL") and self._parse_wrapped( 5329 self._parse_bitwise 5330 ) 5331 5332 group = self._parse_group() 5333 5334 self._match_r_paren() 5335 5336 pivot = self.expression( 5337 exp.Pivot( 5338 expressions=expressions, 5339 fields=fields, 5340 unpivot=unpivot, 5341 include_nulls=include_nulls, 5342 default_on_null=default_on_null, 5343 group=group, 5344 ) 5345 ) 5346 5347 if unpivot: 5348 pivot.set("expressions", [_unpivot_target(e) for e in pivot.expressions]) 5349 for pivot_field in pivot.fields: 5350 if isinstance(pivot_field, exp.In): 5351 pivot_field.set("this", _unpivot_target(pivot_field.this)) 5352 5353 if not self._match_set((TokenType.PIVOT, TokenType.UNPIVOT), advance=False): 5354 pivot.set("alias", self._parse_table_alias()) 5355 5356 if not unpivot: 5357 names = self._pivot_column_names(t.cast(list[exp.Expr], expressions)) 5358 5359 columns: list[exp.Expr] = [] 5360 all_fields = [] 5361 for pivot_field in pivot.fields: 5362 pivot_field_expressions = pivot_field.expressions 5363 5364 # The `PivotAny` expression corresponds to `ANY ORDER BY <column>`; we can't infer in this case. 5365 if isinstance(seq_get(pivot_field_expressions, 0), exp.PivotAny): 5366 continue 5367 5368 all_fields.append( 5369 [ 5370 fld.sql() if self.IDENTIFY_PIVOT_STRINGS else fld.alias_or_name 5371 for fld in pivot_field_expressions 5372 ] 5373 ) 5374 5375 if all_fields: 5376 if names: 5377 all_fields.append(names) 5378 5379 # Generate all possible combinations of the pivot columns 5380 # e.g PIVOT(sum(...) as total FOR year IN (2000, 2010) FOR country IN ('NL', 'US')) 5381 # generates the product between [[2000, 2010], ['NL', 'US'], ['total']] 5382 for fld_parts_tuple in itertools.product(*all_fields): 5383 fld_parts = list(fld_parts_tuple) 5384 5385 if names and self.PREFIXED_PIVOT_COLUMNS: 5386 # Move the "name" to the front of the list 5387 fld_parts.insert(0, fld_parts.pop(-1)) 5388 5389 columns.append(exp.to_identifier("_".join(fld_parts))) 5390 5391 pivot.set("columns", columns) 5392 pivot.set("identify_pivot_strings", self.IDENTIFY_PIVOT_STRINGS) 5393 pivot.set("prefixed_pivot_columns", self.PREFIXED_PIVOT_COLUMNS) 5394 pivot.set("pivot_column_naming", self.PIVOT_COLUMN_NAMING) 5395 5396 return pivot 5397 5398 def _pivot_column_names(self, aggregations: list[exp.Expr]) -> list[str]: 5399 return [agg.alias for agg in aggregations if agg.alias] 5400 5401 def _parse_prewhere(self, skip_where_token: bool = False) -> exp.PreWhere | None: 5402 if not skip_where_token and not self._match(TokenType.PREWHERE): 5403 return None 5404 5405 comments = self._prev_comments 5406 return self.expression( 5407 exp.PreWhere(this=self._parse_disjunction()), 5408 comments=comments, 5409 ) 5410 5411 def _parse_where(self, skip_where_token: bool = False) -> exp.Where | None: 5412 if not skip_where_token and not self._match(TokenType.WHERE): 5413 return None 5414 5415 comments = self._prev_comments 5416 return self.expression( 5417 exp.Where(this=self._parse_disjunction()), 5418 comments=comments, 5419 ) 5420 5421 def _parse_group(self, skip_group_by_token: bool = False) -> exp.Group | None: 5422 if not skip_group_by_token and not self._match(TokenType.GROUP_BY): 5423 return None 5424 comments = self._prev_comments 5425 5426 elements: dict[str, t.Any] = defaultdict(list) 5427 5428 if self._match(TokenType.ALL): 5429 elements["all"] = True 5430 elif self._match(TokenType.DISTINCT): 5431 elements["all"] = False 5432 5433 if self._match_set(self.QUERY_MODIFIER_TOKENS, advance=False): 5434 return self.expression(exp.Group(**elements), comments=comments) # type: ignore 5435 5436 while True: 5437 index = self._index 5438 5439 elements["expressions"].extend( 5440 self._parse_csv( 5441 lambda: ( 5442 None 5443 if self._match_set((TokenType.CUBE, TokenType.ROLLUP), advance=False) 5444 else self._parse_disjunction() 5445 ) 5446 ) 5447 ) 5448 5449 before_with_index = self._index 5450 with_prefix = self._match(TokenType.WITH) 5451 5452 if cube_or_rollup := self._parse_cube_or_rollup(with_prefix=with_prefix): 5453 key = "rollup" if isinstance(cube_or_rollup, exp.Rollup) else "cube" 5454 elements[key].append(cube_or_rollup) 5455 elif grouping_sets := self._parse_grouping_sets(): 5456 elements["grouping_sets"].append(grouping_sets) 5457 elif self._match_text_seq("TOTALS"): 5458 elements["totals"] = True # type: ignore 5459 5460 if before_with_index <= self._index <= before_with_index + 1: 5461 self._retreat(before_with_index) 5462 break 5463 5464 if index == self._index: 5465 break 5466 5467 return self.expression(exp.Group(**elements), comments=comments) # type: ignore 5468 5469 def _parse_cube_or_rollup(self, with_prefix: bool = False) -> exp.Cube | exp.Rollup | None: 5470 if self._match(TokenType.CUBE): 5471 kind: type[exp.Cube | exp.Rollup] = exp.Cube 5472 elif self._match(TokenType.ROLLUP): 5473 kind = exp.Rollup 5474 else: 5475 return None 5476 5477 return self.expression( 5478 kind(expressions=[] if with_prefix else self._parse_wrapped_csv(self._parse_bitwise)) 5479 ) 5480 5481 def _parse_grouping_sets(self) -> exp.GroupingSets | None: 5482 if self._match(TokenType.GROUPING_SETS): 5483 return self.expression( 5484 exp.GroupingSets(expressions=self._parse_wrapped_csv(self._parse_grouping_set)) 5485 ) 5486 return None 5487 5488 def _parse_grouping_set(self) -> exp.Expr | None: 5489 return self._parse_grouping_sets() or self._parse_cube_or_rollup() or self._parse_bitwise() 5490 5491 def _parse_having(self, skip_having_token: bool = False) -> exp.Having | None: 5492 if not skip_having_token and not self._match(TokenType.HAVING): 5493 return None 5494 comments = self._prev_comments 5495 return self.expression( 5496 exp.Having(this=self._parse_disjunction()), 5497 comments=comments, 5498 ) 5499 5500 def _parse_qualify(self) -> exp.Qualify | None: 5501 if not self._match(TokenType.QUALIFY): 5502 return None 5503 return self.expression(exp.Qualify(this=self._parse_disjunction())) 5504 5505 def _parse_connect_with_prior(self) -> exp.Expr | None: 5506 self.NO_PAREN_FUNCTION_PARSERS["PRIOR"] = lambda self: self.expression( 5507 exp.Prior(this=self._parse_bitwise()) 5508 ) 5509 connect = self._parse_disjunction() 5510 self.NO_PAREN_FUNCTION_PARSERS.pop("PRIOR") 5511 return connect 5512 5513 def _parse_connect(self, skip_start_token: bool = False) -> exp.Connect | None: 5514 if skip_start_token: 5515 start = None 5516 elif self._match(TokenType.START_WITH): 5517 start = self._parse_disjunction() 5518 else: 5519 return None 5520 5521 self._match(TokenType.CONNECT_BY) 5522 nocycle = self._match_text_seq("NOCYCLE") 5523 connect = self._parse_connect_with_prior() 5524 5525 if not start and self._match(TokenType.START_WITH): 5526 start = self._parse_disjunction() 5527 5528 return self.expression(exp.Connect(start=start, connect=connect, nocycle=nocycle)) 5529 5530 def _parse_name_as_expression(self) -> exp.Expr | None: 5531 this = self._parse_id_var(any_token=True) 5532 if self._match(TokenType.ALIAS): 5533 this = self.expression(exp.Alias(alias=this, this=self._parse_disjunction())) 5534 return this 5535 5536 def _parse_interpolate(self) -> list[exp.Expr] | None: 5537 if self._match_text_seq("INTERPOLATE"): 5538 return self._parse_wrapped_csv(self._parse_name_as_expression) 5539 return None 5540 5541 def _parse_order( 5542 self, this: exp.Expr | None = None, skip_order_token: bool = False 5543 ) -> exp.Expr | None: 5544 siblings = None 5545 if not skip_order_token and not self._match(TokenType.ORDER_BY): 5546 if not self._match(TokenType.ORDER_SIBLINGS_BY): 5547 return this 5548 5549 siblings = True 5550 5551 comments = self._prev_comments 5552 return self.expression( 5553 exp.Order( 5554 this=this, 5555 expressions=self._parse_csv(self._parse_ordered), 5556 siblings=siblings, 5557 ), 5558 comments=comments, 5559 ) 5560 5561 def _parse_sort(self, exp_class: type[E], token: TokenType) -> E | None: 5562 if not self._match(token): 5563 return None 5564 return self.expression(exp_class(expressions=self._parse_csv(self._parse_ordered))) 5565 5566 def _parse_ordered( 5567 self, parse_method: t.Callable[[], exp.Expr | None] | None = None 5568 ) -> exp.Ordered | None: 5569 this = parse_method() if parse_method else self._parse_disjunction() 5570 if not this: 5571 return None 5572 5573 if this.name.upper() == "ALL" and self.dialect.SUPPORTS_ORDER_BY_ALL: 5574 this = exp.var("ALL") 5575 5576 asc = self._match(TokenType.ASC) 5577 desc: bool | None = True if self._match(TokenType.DESC) else (False if asc else None) 5578 5579 is_nulls_first = self._match_text_seq("NULLS", "FIRST") 5580 is_nulls_last = self._match_text_seq("NULLS", "LAST") 5581 5582 nulls_first = is_nulls_first or False 5583 explicitly_null_ordered = is_nulls_first or is_nulls_last 5584 5585 if ( 5586 not explicitly_null_ordered 5587 and ( 5588 (not desc and self.dialect.NULL_ORDERING == "nulls_are_small") 5589 or (desc and self.dialect.NULL_ORDERING != "nulls_are_small") 5590 ) 5591 and self.dialect.NULL_ORDERING != "nulls_are_last" 5592 ): 5593 nulls_first = True 5594 5595 if self._match_text_seq("WITH", "FILL"): 5596 with_fill = self.expression( 5597 exp.WithFill( 5598 from_=self._match(TokenType.FROM) and self._parse_bitwise(), 5599 to=self._match_text_seq("TO") and self._parse_bitwise(), 5600 step=self._match_text_seq("STEP") and self._parse_bitwise(), 5601 interpolate=self._parse_interpolate(), 5602 ) 5603 ) 5604 else: 5605 with_fill = None 5606 5607 return self.expression( 5608 exp.Ordered(this=this, desc=desc, nulls_first=nulls_first, with_fill=with_fill) 5609 ) 5610 5611 def _parse_limit_options(self) -> exp.LimitOptions | None: 5612 percent = self._match_set((TokenType.PERCENT, TokenType.MOD)) 5613 rows = self._match_set((TokenType.ROW, TokenType.ROWS)) 5614 self._match_text_seq("ONLY") 5615 with_ties = self._match_text_seq("WITH", "TIES") 5616 5617 if not (percent or rows or with_ties): 5618 return None 5619 5620 return self.expression(exp.LimitOptions(percent=percent, rows=rows, with_ties=with_ties)) 5621 5622 def _parse_limit( 5623 self, 5624 this: exp.Expr | None = None, 5625 top: bool = False, 5626 skip_limit_token: bool = False, 5627 ) -> exp.Expr | None: 5628 if skip_limit_token or self._match(TokenType.TOP if top else TokenType.LIMIT): 5629 comments = self._prev_comments 5630 if top: 5631 limit_paren = self._match(TokenType.L_PAREN) 5632 expression = ( 5633 self._parse_term() or self._parse_select() 5634 if limit_paren 5635 else self._parse_number() 5636 ) 5637 5638 if limit_paren: 5639 self._match_r_paren() 5640 5641 else: 5642 if self.dialect.SUPPORTS_LIMIT_ALL and self._match(TokenType.ALL): 5643 return this 5644 5645 # Parsing LIMIT x% (i.e x PERCENT) as a term leads to an error, since 5646 # we try to build an exp.Mod expr. For that matter, we backtrack and instead 5647 # consume the factor plus parse the percentage separately 5648 index = self._index 5649 expression = self._try_parse(self._parse_term) 5650 if isinstance(expression, exp.Mod): 5651 self._retreat(index) 5652 expression = self._parse_factor() 5653 elif not expression: 5654 expression = self._parse_factor() 5655 limit_options = self._parse_limit_options() 5656 5657 if self._match(TokenType.COMMA): 5658 offset = expression 5659 expression = self._parse_term() 5660 else: 5661 offset = None 5662 5663 limit_exp = self.expression( 5664 exp.Limit( 5665 this=this, 5666 expression=expression, 5667 offset=offset, 5668 limit_options=limit_options, 5669 expressions=self._parse_limit_by(), 5670 ), 5671 comments=comments, 5672 ) 5673 5674 return limit_exp 5675 5676 if self._match(TokenType.FETCH): 5677 direction = ( 5678 self._prev.text.upper() 5679 if self._match_set((TokenType.FIRST, TokenType.NEXT)) 5680 else "FIRST" 5681 ) 5682 5683 count = self._parse_field(tokens=self.FETCH_TOKENS) 5684 5685 return self.expression( 5686 exp.Fetch( 5687 direction=direction, count=count, limit_options=self._parse_limit_options() 5688 ) 5689 ) 5690 5691 return this 5692 5693 def _parse_offset(self, this: exp.Expr | None = None) -> exp.Expr | None: 5694 if not self._match(TokenType.OFFSET): 5695 return this 5696 5697 count = self._parse_term() 5698 self._match_set((TokenType.ROW, TokenType.ROWS)) 5699 5700 return self.expression( 5701 exp.Offset(this=this, expression=count, expressions=self._parse_limit_by()) 5702 ) 5703 5704 def _can_parse_limit_or_offset(self) -> bool: 5705 if not self._match_set(self.AMBIGUOUS_ALIAS_TOKENS, advance=False): 5706 return False 5707 5708 index = self._index 5709 result = bool( 5710 self._try_parse(self._parse_limit, retreat=True) 5711 or self._try_parse(self._parse_offset, retreat=True) 5712 ) 5713 self._retreat(index) 5714 5715 # MATCH_CONDITION (...) is a special construct that should not be consumed by limit/offset 5716 if self._next.token_type == TokenType.MATCH_CONDITION: 5717 result = False 5718 5719 return result 5720 5721 def _can_parse_named_window(self) -> bool: 5722 # `WINDOW` is in ID_VAR_TOKENS so it could be mistakenly consumed as an implicit alias. 5723 # Refuse only when the following tokens look like a named-window clause: `WINDOW <id> AS (`. 5724 if not self._match(TokenType.WINDOW, advance=False): 5725 return False 5726 5727 name = self._tokens[self._index + 1] if self._index + 1 < len(self._tokens) else None 5728 if name is None or name.token_type not in self.ID_VAR_TOKENS: 5729 return False 5730 5731 alias_tok = self._tokens[self._index + 2] if self._index + 2 < len(self._tokens) else None 5732 if alias_tok is None or alias_tok.token_type != TokenType.ALIAS: 5733 return False 5734 5735 body = self._tokens[self._index + 3] if self._index + 3 < len(self._tokens) else None 5736 return body is not None and body.token_type == TokenType.L_PAREN 5737 5738 def _parse_limit_by(self) -> list[exp.Expr] | None: 5739 return self._parse_csv(self._parse_bitwise) if self._match_text_seq("BY") else None 5740 5741 def _parse_locks(self) -> list[exp.Lock]: 5742 locks = [] 5743 while True: 5744 update, key = None, None 5745 if self._match_text_seq("FOR", "UPDATE"): 5746 update = True 5747 elif self._match_text_seq("FOR", "SHARE") or self._match_text_seq( 5748 "LOCK", "IN", "SHARE", "MODE" 5749 ): 5750 update = False 5751 elif self._match_text_seq("FOR", "KEY", "SHARE"): 5752 update, key = False, True 5753 elif self._match_text_seq("FOR", "NO", "KEY", "UPDATE"): 5754 update, key = True, True 5755 else: 5756 break 5757 5758 expressions = None 5759 if self._match_text_seq("OF"): 5760 expressions = self._parse_csv(lambda: self._parse_table(schema=True)) 5761 5762 wait: bool | exp.Expr | None = None 5763 if self._match_text_seq("NOWAIT"): 5764 wait = True 5765 elif self._match_text_seq("WAIT"): 5766 wait = self._parse_primary() 5767 elif self._match_text_seq("SKIP", "LOCKED"): 5768 wait = False 5769 5770 locks.append( 5771 self.expression( 5772 exp.Lock(update=update, expressions=expressions, wait=wait, key=key) 5773 ) 5774 ) 5775 5776 return locks 5777 5778 def parse_set_operation( 5779 self, this: exp.Expr | None, consume_pipe: bool = False 5780 ) -> exp.Expr | None: 5781 start = self._index 5782 _, side_token, kind_token = self._parse_join_parts() 5783 5784 side = side_token.text if side_token else None 5785 kind = kind_token.text if kind_token else None 5786 5787 if not self._match_set(self.SET_OPERATIONS): 5788 self._retreat(start) 5789 return None 5790 5791 token_type = self._prev.token_type 5792 5793 if token_type == TokenType.UNION: 5794 operation: type[exp.SetOperation] = exp.Union 5795 elif token_type == TokenType.EXCEPT: 5796 operation = exp.Except 5797 else: 5798 operation = exp.Intersect 5799 5800 comments = self._prev.comments 5801 5802 if self._match(TokenType.DISTINCT): 5803 distinct: bool | None = True 5804 elif self._match(TokenType.ALL): 5805 distinct = False 5806 else: 5807 distinct = self.dialect.SET_OP_DISTINCT_BY_DEFAULT[operation] 5808 if distinct is None: 5809 self.raise_error(f"Expected DISTINCT or ALL for {operation.__name__}") 5810 5811 by_name = ( 5812 self._match_text_seq("BY", "NAME") 5813 or self._match_text_seq("STRICT", "CORRESPONDING") 5814 or None 5815 ) 5816 if self._match_text_seq("CORRESPONDING"): 5817 by_name = True 5818 if not side and not kind: 5819 kind = "INNER" 5820 5821 on_column_list = None 5822 if by_name and self._match_texts(("ON", "BY")): 5823 on_column_list = self._parse_wrapped_csv(self._parse_column) 5824 5825 expression = self._parse_select( 5826 nested=True, parse_set_operation=False, consume_pipe=consume_pipe 5827 ) 5828 5829 # Wrap VALUES operands in selects, both for consistency with the CTE canonicalization 5830 # in _parse_cte and so that alias pushdown can reach into set operation branches 5831 if isinstance(this, exp.Values): 5832 this = self._values_to_select(this) 5833 if isinstance(expression, exp.Values): 5834 expression = self._values_to_select(expression) 5835 5836 return self.expression( 5837 operation( 5838 this=this, 5839 distinct=distinct, 5840 by_name=by_name, 5841 expression=expression, 5842 side=side, 5843 kind=kind, 5844 on=on_column_list, 5845 ), 5846 comments=comments, 5847 ) 5848 5849 def _parse_set_operations(self, this: exp.Expr | None) -> exp.Expr | None: 5850 while this: 5851 setop = self.parse_set_operation(this) 5852 if not setop: 5853 break 5854 this = setop 5855 5856 if isinstance(this, exp.SetOperation) and self.MODIFIERS_ATTACHED_TO_SET_OP: 5857 expression = this.expression 5858 5859 if expression: 5860 for arg in self.SET_OP_MODIFIERS: 5861 expr = expression.args.get(arg) 5862 if expr: 5863 this.set(arg, expr.pop()) 5864 5865 return this 5866 5867 def _parse_expression(self) -> exp.Expr | None: 5868 return self._parse_alias(self._parse_assignment()) 5869 5870 def _parse_assignment(self) -> exp.Expr | None: 5871 this = self._parse_disjunction() 5872 if not this and self._next.token_type in self.ASSIGNMENT: 5873 # This allows us to parse <non-identifier token> := <expr> 5874 this = exp.column( 5875 t.cast(str, self._advance_any(ignore_reserved=True) and self._prev.text) 5876 ) 5877 5878 while self._match_set(self.ASSIGNMENT): 5879 if isinstance(this, exp.Column) and len(this.parts) == 1: 5880 this = this.this 5881 5882 comments = self._prev_comments 5883 this = self.expression( 5884 self.ASSIGNMENT[self._prev.token_type]( 5885 this=this, expression=self._parse_assignment() 5886 ), 5887 comments=comments, 5888 ) 5889 5890 return this 5891 5892 def _parse_disjunction(self) -> exp.Expr | None: 5893 this = self._parse_conjunction() 5894 while self._match_set(self.DISJUNCTION): 5895 comments = self._prev_comments 5896 this = self.expression( 5897 self.DISJUNCTION[self._prev.token_type]( 5898 this=this, expression=self._parse_conjunction() 5899 ), 5900 comments=comments, 5901 ) 5902 return this 5903 5904 def _parse_conjunction(self) -> exp.Expr | None: 5905 this = self._parse_equality() 5906 while self._match_set(self.CONJUNCTION): 5907 comments = self._prev_comments 5908 this = self.expression( 5909 self.CONJUNCTION[self._prev.token_type]( 5910 this=this, expression=self._parse_equality() 5911 ), 5912 comments=comments, 5913 ) 5914 return this 5915 5916 def _parse_equality(self) -> exp.Expr | None: 5917 this = self._parse_comparison() 5918 while self._match_set(self.EQUALITY): 5919 comments = self._prev_comments 5920 this = self.expression( 5921 self.EQUALITY[self._prev.token_type]( 5922 this=this, expression=self._parse_comparison() 5923 ), 5924 comments=comments, 5925 ) 5926 return this 5927 5928 def _parse_comparison(self) -> exp.Expr | None: 5929 this = self._parse_range() 5930 while self._match_set(self.COMPARISON): 5931 comments = self._prev_comments 5932 this = self.expression( 5933 self.COMPARISON[self._prev.token_type](this=this, expression=self._parse_range()), 5934 comments=comments, 5935 ) 5936 return this 5937 5938 def _parse_range(self, this: exp.Expr | None = None) -> exp.Expr | None: 5939 this = this or self._parse_bitwise() 5940 5941 while True: 5942 negate = self._match(TokenType.NOT) 5943 if self._match_set(self.RANGE_PARSERS): 5944 expression = self.RANGE_PARSERS[self._prev.token_type](self, this) 5945 if not expression: 5946 return this 5947 5948 this = expression 5949 elif self._match(TokenType.ISNULL) or (negate and self._match(TokenType.NULL)): 5950 this = self.expression(exp.Is(this=this, expression=exp.Null())) 5951 elif self._match(TokenType.NOTNULL): 5952 # Postgres supports ISNULL and NOTNULL for conditions. 5953 # https://blog.andreiavram.ro/postgresql-null-composite-type/ 5954 if self.dialect.NORMALIZE_NOT_NULL: 5955 this = self.expression(exp.Is(this=this, expression=exp.Null())) 5956 this = self.expression(exp.Not(this=this)) 5957 else: 5958 this = self.expression(exp.Is(this=this, expression=exp.Null(), negate=True)) 5959 else: 5960 if negate: 5961 self._retreat(self._index - 1) 5962 break 5963 5964 if negate: 5965 this = self._negate_range(this) 5966 if self._curr and ( 5967 self._curr.token_type == TokenType.NOT 5968 or self._curr.token_type in self.RANGE_PARSERS 5969 ): 5970 this = self.expression(exp.Paren(this=this)) 5971 5972 return this 5973 5974 def _negate_range(self, this: exp.Expr | None = None) -> exp.Expr | None: 5975 if not this: 5976 return this 5977 5978 expression = this.this if isinstance(this, exp.Escape) else this 5979 if isinstance(expression, (exp.Like, exp.ILike)): 5980 expression.set("negate", True) 5981 return this 5982 5983 return self.expression(exp.Not(this=this)) 5984 5985 def _parse_is(self, this: exp.Expr | None) -> exp.Expr | None: 5986 index = self._index - 1 5987 negate = self._match(TokenType.NOT) 5988 5989 if self._match_text_seq("DISTINCT", "FROM"): 5990 klass = exp.NullSafeEQ if negate else exp.NullSafeNEQ 5991 return self.expression(klass(this=this, expression=self._parse_bitwise())) 5992 5993 if self._match(TokenType.JSON): 5994 kind = self._match_texts(self.IS_JSON_PREDICATE_KIND) and self._prev.text.upper() 5995 5996 if self._match_text_seq("WITH"): 5997 _with = True 5998 elif self._match_text_seq("WITHOUT"): 5999 _with = False 6000 else: 6001 _with = None 6002 6003 unique = self._match(TokenType.UNIQUE) 6004 self._match_text_seq("KEYS") 6005 expression: exp.Expr | None = self.expression( 6006 exp.JSON(this=kind, with_=_with, unique=unique) 6007 ) 6008 else: 6009 expression = self._parse_null() or self._parse_bitwise() 6010 if not expression: 6011 self._retreat(index) 6012 return None 6013 6014 if negate and isinstance(expression, exp.Null) and not self.dialect.NORMALIZE_NOT_NULL: 6015 this = self.expression(exp.Is(this=this, expression=expression, negate=True)) 6016 else: 6017 this = self.expression(exp.Is(this=this, expression=expression)) 6018 this = self.expression(exp.Not(this=this)) if negate else this 6019 6020 return self._parse_column_ops(this) 6021 6022 def _parse_in(self, this: exp.Expr | None, alias: bool = False) -> exp.In: 6023 unnest = self._parse_unnest(with_alias=False) 6024 if unnest: 6025 this = self.expression(exp.In(this=this, unnest=unnest)) 6026 elif self._match_set((TokenType.L_PAREN, TokenType.L_BRACKET)): 6027 matched_l_paren = self._prev.token_type == TokenType.L_PAREN 6028 expressions = self._parse_csv(lambda: self._parse_select_or_expression(alias=alias)) 6029 6030 if len(expressions) == 1 and isinstance(query := expressions[0], exp.Query): 6031 this = self.expression( 6032 exp.In(this=this, query=self._parse_query_modifiers(query).subquery(copy=False)) 6033 ) 6034 else: 6035 this = self.expression(exp.In(this=this, expressions=expressions)) 6036 6037 if matched_l_paren: 6038 self._match_r_paren(this) 6039 elif not self._match(TokenType.R_BRACKET, expression=this): 6040 self.raise_error("Expecting ]") 6041 else: 6042 this = self.expression(exp.In(this=this, field=self._parse_column())) 6043 6044 return this 6045 6046 def _parse_between(self, this: exp.Expr | None) -> exp.Between: 6047 symmetric = None 6048 if self._match_text_seq("SYMMETRIC"): 6049 symmetric = True 6050 elif self._match_text_seq("ASYMMETRIC"): 6051 symmetric = False 6052 6053 low = self._parse_bitwise() 6054 self._match(TokenType.AND) 6055 high = self._parse_bitwise() 6056 6057 return self.expression(exp.Between(this=this, low=low, high=high, symmetric=symmetric)) 6058 6059 def _parse_escape(self, this: exp.Expr | None) -> exp.Expr | None: 6060 if not self._match(TokenType.ESCAPE): 6061 return this 6062 return self.expression( 6063 exp.Escape(this=this, expression=self._parse_string() or self._parse_null()) 6064 ) 6065 6066 def _parse_interval_span(self, this: exp.Expr) -> exp.Interval: 6067 # handle day-time format interval span with omitted units: 6068 # INTERVAL '<number days> hh[:][mm[:ss[.ff]]]' <maybe `unit TO unit`> 6069 interval_span_units_omitted = None 6070 if ( 6071 this 6072 and this.is_string 6073 and self.SUPPORTS_OMITTED_INTERVAL_SPAN_UNIT 6074 and exp.INTERVAL_DAY_TIME_RE.match(this.name) 6075 ): 6076 index = self._index 6077 6078 # Var "TO" Var 6079 first_unit = self._parse_var(any_token=True, upper=True) 6080 second_unit = None 6081 if first_unit and self._match_text_seq("TO"): 6082 second_unit = self._parse_var(any_token=True, upper=True) 6083 6084 interval_span_units_omitted = not (first_unit and second_unit) 6085 6086 self._retreat(index) 6087 6088 if interval_span_units_omitted: 6089 unit = None 6090 else: 6091 unit = self._parse_function() 6092 if not unit and ( 6093 self._curr.token_type == TokenType.VAR 6094 or self._curr.text.upper() in self.dialect.VALID_INTERVAL_UNITS 6095 ): 6096 unit = self._parse_var(any_token=True, upper=True) 6097 6098 # Most dialects support, e.g., the form INTERVAL '5' day, thus we try to parse 6099 # each INTERVAL expression into this canonical form so it's easy to transpile 6100 if this and this.is_number: 6101 this = exp.Literal.string(this.to_py()) 6102 elif this and this.is_string: 6103 parts = exp.INTERVAL_STRING_RE.findall(this.name) 6104 if parts and unit: 6105 # Unconsume the eagerly-parsed unit, since the real unit was part of the string 6106 unit = None 6107 self._retreat(self._index - 1) 6108 6109 if len(parts) == 1: 6110 this = exp.Literal.string(parts[0][0]) 6111 unit = self.expression(exp.Var(this=parts[0][1].upper())) 6112 6113 if self.INTERVAL_SPANS and self._match_text_seq("TO"): 6114 unit = self.expression( 6115 exp.IntervalSpan( 6116 this=unit, 6117 expression=self._parse_function() 6118 or self._parse_var(any_token=True, upper=True), 6119 ) 6120 ) 6121 6122 return self.expression(exp.Interval(this=this, unit=unit)) 6123 6124 def _parse_interval(self, require_interval: bool = True) -> exp.Add | exp.Interval | None: 6125 index = self._index 6126 6127 if not self._match(TokenType.INTERVAL) and require_interval: 6128 return None 6129 6130 if self._match(TokenType.STRING, advance=False): 6131 this = self._parse_primary() 6132 else: 6133 this = self._parse_term() 6134 6135 if not this or ( 6136 isinstance(this, exp.Column) 6137 and not this.table 6138 and not this.this.quoted 6139 and self._curr 6140 and self._curr.text.upper() not in self.dialect.VALID_INTERVAL_UNITS 6141 ): 6142 self._retreat(index) 6143 return None 6144 6145 interval = self._parse_interval_span(this) 6146 6147 index = self._index 6148 self._match(TokenType.PLUS) 6149 6150 # Convert INTERVAL 'val_1' unit_1 [+] ... [+] 'val_n' unit_n into a sum of intervals 6151 if self._match_set((TokenType.STRING, TokenType.NUMBER), advance=False): 6152 return self.expression(exp.Add(this=interval, expression=self._parse_interval(False))) 6153 6154 self._retreat(index) 6155 return interval 6156 6157 def _parse_bitwise(self) -> exp.Expr | None: 6158 this = self._parse_term() 6159 6160 while True: 6161 if self._match_set(self.BITWISE): 6162 this = self.expression( 6163 self.BITWISE[self._prev.token_type](this=this, expression=self._parse_term()) 6164 ) 6165 elif self.dialect.DPIPE_IS_STRING_CONCAT and self._match(TokenType.DPIPE): 6166 this = self.expression( 6167 exp.DPipe( 6168 this=this, 6169 expression=self._parse_term(), 6170 safe=not self.dialect.STRICT_STRING_CONCAT, 6171 ) 6172 ) 6173 elif self._match(TokenType.DQMARK): 6174 this = self.expression( 6175 exp.Coalesce(this=this, expressions=ensure_list(self._parse_term())) 6176 ) 6177 elif self._match_pair(TokenType.LT, TokenType.LT): 6178 this = self.expression( 6179 exp.BitwiseLeftShift(this=this, expression=self._parse_term()) 6180 ) 6181 elif self._match_pair(TokenType.GT, TokenType.GT): 6182 this = self.expression( 6183 exp.BitwiseRightShift(this=this, expression=self._parse_term()) 6184 ) 6185 else: 6186 break 6187 6188 return this 6189 6190 def _parse_term(self) -> exp.Expr | None: 6191 this = self._parse_factor() 6192 6193 while self._match_set(self.TERM): 6194 klass = self.TERM[self._prev.token_type] 6195 comments = self._prev_comments 6196 expression = self._parse_factor() 6197 6198 this = self.expression(klass(this=this, expression=expression), comments=comments) 6199 6200 if isinstance(this, exp.Collate): 6201 expr = this.expression 6202 6203 # Preserve collations such as pg_catalog."default" (Postgres) as columns, otherwise 6204 # fallback to Identifier / Var 6205 if isinstance(expr, exp.Column) and len(expr.parts) == 1: 6206 ident = expr.this 6207 if isinstance(ident, exp.Identifier): 6208 this.set("expression", ident if ident.quoted else exp.var(ident.name)) 6209 6210 return this 6211 6212 def _parse_factor(self) -> exp.Expr | None: 6213 parse_method = self._parse_exponent if self.EXPONENT else self._parse_unary 6214 this = self._parse_at_time_zone(parse_method()) 6215 6216 while self._match_set(self.FACTOR): 6217 klass = self.FACTOR[self._prev.token_type] 6218 comments = self._prev_comments 6219 expression = parse_method() 6220 6221 if not expression and klass is exp.IntDiv and self._prev.text.isalpha(): 6222 self._retreat(self._index - 1) 6223 return this 6224 6225 this = self.expression(klass(this=this, expression=expression), comments=comments) 6226 6227 if isinstance(this, exp.Div): 6228 this.set("typed", self.dialect.TYPED_DIVISION) 6229 this.set("safe", self.dialect.SAFE_DIVISION) 6230 6231 return this 6232 6233 def _parse_exponent(self) -> exp.Expr | None: 6234 this = self._parse_unary() 6235 while self._match_set(self.EXPONENT): 6236 comments = self._prev_comments 6237 this = self.expression( 6238 self.EXPONENT[self._prev.token_type](this=this, expression=self._parse_unary()), 6239 comments=comments, 6240 ) 6241 return this 6242 6243 def _parse_unary(self) -> exp.Expr | None: 6244 if self._match_set(self.UNARY_PARSERS): 6245 return self.UNARY_PARSERS[self._prev.token_type](self) 6246 return self._parse_type() 6247 6248 def _parse_type( 6249 self, parse_interval: bool = True, fallback_to_identifier: bool = False 6250 ) -> exp.Expr | None: 6251 if not fallback_to_identifier and (atom := self._parse_atom()) is not None: 6252 return atom 6253 6254 if interval := parse_interval and self._parse_interval(): 6255 return self._parse_column_ops(interval) 6256 6257 index = self._index 6258 data_type = self._parse_types(check_func=True, allow_identifiers=False) 6259 6260 # parse_types() returns a Cast if we parsed BQ's inline constructor <type>(<values>) e.g. 6261 # STRUCT<a INT, b STRING>(1, 'foo'), which is canonicalized to CAST(<values> AS <type>) 6262 if isinstance(data_type, exp.Cast): 6263 # This constructor can contain ops directly after it, for instance struct unnesting: 6264 # STRUCT<a INT, b STRING>(1, 'foo').* --> CAST(STRUCT(1, 'foo') AS STRUCT<a iNT, b STRING).* 6265 return self._parse_column_ops(data_type) 6266 6267 if data_type: 6268 index2 = self._index 6269 this = self._parse_primary() 6270 6271 if isinstance(this, exp.Literal): 6272 literal = this.name 6273 this = self._parse_column_ops(this) 6274 6275 parser = self.TYPE_LITERAL_PARSERS.get(data_type.this) 6276 if parser: 6277 return parser(self, this, data_type) 6278 6279 if ( 6280 self.ZONE_AWARE_TIMESTAMP_CONSTRUCTOR 6281 and data_type.is_type(exp.DType.TIMESTAMP) 6282 and TIME_ZONE_RE.search(literal) 6283 ): 6284 data_type = exp.DType.TIMESTAMPTZ.into_expr() 6285 6286 return self.expression(exp.Cast(this=this, to=data_type)) 6287 6288 # The expressions arg gets set by the parser when we have something like DECIMAL(38, 0) 6289 # in the input SQL. In that case, we'll produce these tokens: DECIMAL ( 38 , 0 ) 6290 # 6291 # If the index difference here is greater than 1, that means the parser itself must have 6292 # consumed additional tokens such as the DECIMAL scale and precision in the above example. 6293 # 6294 # If it's not greater than 1, then it must be 1, because we've consumed at least the type 6295 # keyword, meaning that the expressions arg of the DataType must have gotten set by a 6296 # callable in the TYPE_CONVERTERS mapping. For example, Snowflake converts DECIMAL to 6297 # DECIMAL(38, 0)) in order to facilitate the data type's transpilation. 6298 # 6299 # In these cases, we don't really want to return the converted type, but instead retreat 6300 # and try to parse a Column or Identifier in the section below. 6301 if data_type.expressions and index2 - index > 1: 6302 self._retreat(index2) 6303 return self._parse_column_ops(data_type) 6304 6305 self._retreat(index) 6306 6307 if fallback_to_identifier: 6308 return self._parse_id_var() 6309 6310 return self._parse_column() 6311 6312 def _parse_type_size(self) -> exp.DataTypeParam | None: 6313 this = self._parse_type() 6314 if not this: 6315 return None 6316 6317 if isinstance(this, exp.Column) and not this.table: 6318 this = exp.var(this.name.upper()) 6319 6320 return self.expression( 6321 exp.DataTypeParam(this=this, expression=self._parse_var(any_token=True)) 6322 ) 6323 6324 def _parse_user_defined_type(self, identifier: exp.Identifier) -> exp.Expr | None: 6325 type_name = identifier.name 6326 6327 while self._match(TokenType.DOT): 6328 type_name = f"{type_name}.{self._advance_any() and self._prev.text}" 6329 6330 return exp.DataType.from_str(type_name, dialect=self.dialect, udt=True) 6331 6332 def _parse_types( 6333 self, 6334 check_func: bool = False, 6335 schema: bool = False, 6336 allow_identifiers: bool = True, 6337 with_collation: bool = False, 6338 ) -> exp.Expr | None: 6339 index = self._index 6340 this: exp.Expr | None = None 6341 6342 if self._match_set(self.TYPE_TOKENS): 6343 type_token = self._prev.token_type 6344 else: 6345 type_token = None 6346 identifier = allow_identifiers and self._parse_id_var( 6347 any_token=False, tokens=(TokenType.VAR,) 6348 ) 6349 if isinstance(identifier, exp.Identifier): 6350 try: 6351 tokens = self.dialect.tokenize(identifier.name) 6352 except TokenError: 6353 tokens = None 6354 6355 if tokens and (type_token := tokens[0].token_type) in self.TYPE_TOKENS: 6356 if len(tokens) > 1: 6357 return exp.DataType.from_str(identifier.name, dialect=self.dialect) 6358 elif self.dialect.SUPPORTS_USER_DEFINED_TYPES: 6359 this = self._parse_user_defined_type(identifier) 6360 else: 6361 self._retreat(self._index - 1) 6362 return None 6363 else: 6364 return None 6365 6366 if type_token == TokenType.PSEUDO_TYPE: 6367 return self.expression(exp.PseudoType(this=self._prev.text.upper())) 6368 6369 if type_token == TokenType.OBJECT_IDENTIFIER: 6370 return self.expression(exp.ObjectIdentifier(this=self._prev.text.upper())) 6371 6372 # https://materialize.com/docs/sql/types/map/ 6373 if type_token == TokenType.MAP and self._match(TokenType.L_BRACKET): 6374 key_type = self._parse_types( 6375 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6376 ) 6377 if not self._match(TokenType.FARROW): 6378 self._retreat(index) 6379 return None 6380 6381 value_type = self._parse_types( 6382 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6383 ) 6384 if not self._match(TokenType.R_BRACKET): 6385 self._retreat(index) 6386 return None 6387 6388 return exp.DataType( 6389 this=exp.DType.MAP, 6390 expressions=[key_type, value_type], 6391 nested=True, 6392 ) 6393 6394 nested = type_token in self.NESTED_TYPE_TOKENS 6395 is_struct = type_token in self.STRUCT_TYPE_TOKENS 6396 is_aggregate = type_token in self.AGGREGATE_TYPE_TOKENS 6397 expressions = None 6398 maybe_func = False 6399 6400 if self._match(TokenType.L_PAREN): 6401 if is_struct: 6402 expressions = self._parse_csv(lambda: self._parse_struct_types(type_required=True)) 6403 elif nested: 6404 expressions = self._parse_csv( 6405 lambda: self._parse_types( 6406 check_func=check_func, schema=schema, allow_identifiers=allow_identifiers 6407 ) 6408 ) 6409 if type_token == TokenType.NULLABLE and len(expressions) == 1: 6410 this = expressions[0] 6411 this.set("nullable", True) 6412 self._match_r_paren() 6413 return this 6414 elif type_token in self.ENUM_TYPE_TOKENS: 6415 expressions = self._parse_csv(self._parse_equality) 6416 elif type_token == TokenType.JSON: 6417 # ClickHouse JSON type supports arguments: JSON(col Type, SKIP col, param=value) 6418 # https://clickhouse.com/docs/sql-reference/data-types/newjson 6419 expressions = self._parse_csv(self._parse_json_type_arg) 6420 elif is_aggregate: 6421 func_or_ident = self._parse_function(anonymous=True) or self._parse_id_var( 6422 any_token=False, tokens=(TokenType.VAR, TokenType.ANY) 6423 ) 6424 if not func_or_ident: 6425 return None 6426 expressions = [func_or_ident] 6427 if self._match(TokenType.COMMA): 6428 expressions.extend( 6429 self._parse_csv( 6430 lambda: self._parse_types( 6431 check_func=check_func, 6432 schema=schema, 6433 allow_identifiers=allow_identifiers, 6434 ) 6435 ) 6436 ) 6437 else: 6438 expressions = self._parse_csv(self._parse_type_size) 6439 6440 # https://docs.snowflake.com/en/sql-reference/data-types-vector 6441 if type_token == TokenType.VECTOR and len(expressions) == 2: 6442 expressions = self._parse_vector_expressions(expressions) 6443 6444 if not self._match(TokenType.R_PAREN): 6445 self._retreat(index) 6446 return None 6447 6448 maybe_func = True 6449 6450 values: list[exp.Expr] | None = None 6451 6452 if nested and self._match(TokenType.LT): 6453 if is_struct: 6454 expressions = self._parse_csv(lambda: self._parse_struct_types(type_required=True)) 6455 else: 6456 expressions = self._parse_csv( 6457 lambda: self._parse_types( 6458 check_func=check_func, 6459 schema=schema, 6460 allow_identifiers=allow_identifiers, 6461 with_collation=True, 6462 ) 6463 ) 6464 6465 if not self._match(TokenType.GT): 6466 self.raise_error("Expecting >") 6467 6468 if self._match_set((TokenType.L_BRACKET, TokenType.L_PAREN)): 6469 values = self._parse_csv(self._parse_disjunction) 6470 if not values and is_struct: 6471 values = None 6472 self._retreat(self._index - 1) 6473 else: 6474 self._match_set((TokenType.R_BRACKET, TokenType.R_PAREN)) 6475 6476 if type_token in self.TIMESTAMPS: 6477 if self._match_text_seq("WITH", "TIME", "ZONE"): 6478 maybe_func = False 6479 tz_type = exp.DType.TIMETZ if type_token in self.TIMES else exp.DType.TIMESTAMPTZ 6480 this = exp.DataType(this=tz_type, expressions=expressions) 6481 elif self._match_text_seq("WITH", "LOCAL", "TIME", "ZONE"): 6482 maybe_func = False 6483 this = exp.DataType(this=exp.DType.TIMESTAMPLTZ, expressions=expressions) 6484 elif self._match_text_seq("WITHOUT", "TIME", "ZONE"): 6485 maybe_func = False 6486 elif type_token == TokenType.INTERVAL: 6487 if self._curr.text.upper() in self.dialect.VALID_INTERVAL_UNITS: 6488 unit = self._parse_var(upper=True) 6489 if self._match_text_seq("TO"): 6490 unit = exp.IntervalSpan(this=unit, expression=self._parse_var(upper=True)) 6491 6492 this = self.expression(exp.DataType(this=self.expression(exp.Interval(unit=unit)))) 6493 else: 6494 this = self.expression(exp.DataType(this=exp.DType.INTERVAL)) 6495 elif type_token == TokenType.VOID: 6496 this = exp.DataType(this=exp.DType.NULL) 6497 6498 if maybe_func and check_func: 6499 index2 = self._index 6500 peek = self._parse_string() 6501 6502 if not peek: 6503 self._retreat(index) 6504 return None 6505 6506 self._retreat(index2) 6507 6508 if not this: 6509 assert type_token is not None 6510 if self._match_text_seq("UNSIGNED"): 6511 unsigned_type_token = self.SIGNED_TO_UNSIGNED_TYPE_TOKEN.get(type_token) 6512 if not unsigned_type_token: 6513 self.raise_error(f"Cannot convert {type_token.name} to unsigned.") 6514 6515 type_token = unsigned_type_token or type_token 6516 6517 # NULLABLE without parentheses can be a column (Presto/Trino) 6518 if type_token == TokenType.NULLABLE and not expressions: 6519 self._retreat(index) 6520 return None 6521 6522 this = exp.DataType( 6523 this=exp.DType[type_token.name], 6524 expressions=expressions, 6525 nested=nested, 6526 ) 6527 6528 # Empty arrays/structs are allowed 6529 if values is not None: 6530 cls = exp.Struct if is_struct else exp.Array 6531 this = exp.cast(cls(expressions=values), this, copy=False) 6532 6533 elif expressions: 6534 this.set("expressions", expressions) 6535 6536 # https://materialize.com/docs/sql/types/list/#type-name 6537 while self._match(TokenType.LIST): 6538 this = exp.DataType(this=exp.DType.LIST, expressions=[this], nested=True) 6539 6540 index = self._index 6541 6542 # Postgres supports the INT ARRAY[3] syntax as a synonym for INT[3] 6543 matched_array = self._match(TokenType.ARRAY) 6544 6545 while self._curr: 6546 datatype_token = self._prev.token_type 6547 matched_l_bracket = self._match(TokenType.L_BRACKET) 6548 6549 if (not matched_l_bracket and not matched_array) or ( 6550 datatype_token == TokenType.ARRAY and self._match(TokenType.R_BRACKET) 6551 ): 6552 # Postgres allows casting empty arrays such as ARRAY[]::INT[], 6553 # not to be confused with the fixed size array parsing 6554 break 6555 6556 matched_array = False 6557 values = self._parse_csv(self._parse_disjunction) or None 6558 if ( 6559 values 6560 and not schema 6561 and ( 6562 not self.dialect.SUPPORTS_FIXED_SIZE_ARRAYS 6563 or datatype_token == TokenType.ARRAY 6564 or not self._match(TokenType.R_BRACKET, advance=False) 6565 ) 6566 ): 6567 # Retreating here means that we should not parse the following values as part of the data type, e.g. in DuckDB 6568 # ARRAY[1] should retreat and instead be parsed into exp.Array in contrast to INT[x][y] which denotes a fixed-size array data type 6569 self._retreat(index) 6570 break 6571 6572 this = exp.DataType( 6573 this=exp.DType.ARRAY, expressions=[this], values=values, nested=True 6574 ) 6575 self._match(TokenType.R_BRACKET) 6576 6577 if self.TYPE_CONVERTERS and isinstance(this.this, exp.DType): 6578 converter = self.TYPE_CONVERTERS.get(this.this) 6579 if converter: 6580 this = converter(t.cast(exp.DataType, this)) 6581 6582 if with_collation and isinstance(this, exp.DataType) and self._match(TokenType.COLLATE): 6583 this.set("collate", self._parse_identifier() or self._parse_column()) 6584 6585 return this 6586 6587 def _parse_json_type_arg(self) -> exp.Expr | None: 6588 """Parse a single argument to ClickHouse's JSON type.""" 6589 6590 # SKIP col or SKIP REGEXP 'pattern' 6591 if self._match_text_seq("SKIP"): 6592 regexp = self._match(TokenType.RLIKE) 6593 arg = self._parse_column() 6594 if isinstance(arg, exp.Column): 6595 arg = arg.to_dot() 6596 return self.expression(exp.SkipJSONColumn(regexp=regexp, expression=arg)) 6597 6598 param_or_col = self._parse_column() 6599 if not isinstance(param_or_col, exp.Column): 6600 return None 6601 6602 # Parameter: name=value (e.g., max_dynamic_paths=2) 6603 if len(param_or_col.parts) == 1 and self._match(TokenType.EQ): 6604 param = param_or_col.name 6605 value = self._parse_primary() 6606 return self.expression(exp.EQ(this=exp.var(param), expression=value)) 6607 6608 # Column type hint: col_name Type 6609 col = param_or_col.to_dot() 6610 kind = self._parse_types(check_func=False, allow_identifiers=False) 6611 return self.expression(exp.ColumnDef(this=col, kind=kind)) 6612 6613 def _parse_vector_expressions(self, expressions: list[exp.Expr]) -> list[exp.Expr]: 6614 return [exp.DataType.from_str(expressions[0].name, dialect=self.dialect), *expressions[1:]] 6615 6616 def _parse_struct_types(self, type_required: bool = False) -> exp.Expr | None: 6617 index = self._index 6618 6619 if ( 6620 self._curr 6621 and self._next 6622 and self._curr.token_type in self.TYPE_TOKENS 6623 and self._next.token_type in self.TYPE_TOKENS 6624 ): 6625 # Takes care of special cases like `STRUCT<list ARRAY<...>>` where the identifier is also a 6626 # type token. Without this, the list will be parsed as a type and we'll eventually crash 6627 this = self._parse_id_var() 6628 else: 6629 this = ( 6630 self._parse_type(parse_interval=False, fallback_to_identifier=True) 6631 or self._parse_id_var() 6632 ) 6633 6634 self._match(TokenType.COLON) 6635 6636 if ( 6637 type_required 6638 and not isinstance(this, exp.DataType) 6639 and not self._match_set(self.TYPE_TOKENS, advance=False) 6640 ): 6641 self._retreat(index) 6642 return self._parse_types() 6643 6644 return self._parse_column_def(this) 6645 6646 def _parse_at_time_zone(self, this: exp.Expr | None) -> exp.Expr | None: 6647 if not self._match_text_seq("AT", "TIME", "ZONE"): 6648 return this 6649 return self._parse_at_time_zone( 6650 self.expression(exp.AtTimeZone(this=this, zone=self._parse_unary())) 6651 ) 6652 6653 def _parse_atom(self) -> exp.Expr | None: 6654 if ( 6655 self._curr.token_type in self.IDENTIFIER_TOKENS 6656 and (column := self._parse_column()) is not None 6657 ): 6658 return column 6659 6660 token = self._curr 6661 token_type = token.token_type 6662 6663 if not (primary_parser := self.PRIMARY_PARSERS.get(token_type)): 6664 return None 6665 6666 next_type = self._next.token_type 6667 6668 if ( 6669 next_type in self.COLUMN_OPERATORS 6670 or next_type in self.COLUMN_POSTFIX_TOKENS 6671 or (token_type == TokenType.STRING and next_type == TokenType.STRING) 6672 ): 6673 return None 6674 6675 self._advance() 6676 return primary_parser(self, token) 6677 6678 def _parse_column(self) -> exp.Expr | None: 6679 column: exp.Expr | None = self._parse_column_parts_fast() 6680 if column is None: 6681 this = self._parse_column_reference() 6682 if not this: 6683 this = self._parse_bracket(this) 6684 column = self._parse_column_ops(this) if this else this 6685 6686 if column: 6687 if self.dialect.SUPPORTS_COLUMN_JOIN_MARKS: 6688 column.set("join_mark", self._match(TokenType.JOIN_MARKER)) 6689 if self.COLON_IS_VARIANT_EXTRACT: 6690 column = self._parse_colon_as_variant_extract(column) 6691 6692 return column 6693 6694 def _parse_column_parts_fast(self) -> exp.Column | exp.Dot | None: 6695 """Fast path for simple column and dot references (a, a.b, ...). 6696 6697 Greedily consumes VAR/IDENTIFIER tokens separated by DOTs, then checks 6698 that nothing complex follows. If it does, retreats and returns None so 6699 the slow path can handle it. For >4 parts, wraps in exp.Dot nodes. 6700 """ 6701 index = self._index 6702 parts: list[exp.Identifier] | None = None 6703 all_comments: list[str] | None = None 6704 6705 while self._match_set(self.IDENTIFIER_TOKENS): 6706 token = self._prev 6707 comments = self._prev_comments 6708 6709 if parts is None and token.text.upper() in self.NO_PAREN_FUNCTION_PARSERS: 6710 self._retreat(index) 6711 return None 6712 6713 has_dot = self._match(TokenType.DOT) 6714 curr_tt = self._curr.token_type 6715 6716 if not has_dot: 6717 if curr_tt in self.COLUMN_OPERATORS or curr_tt in self.COLUMN_POSTFIX_TOKENS: 6718 self._retreat(index) 6719 return None 6720 elif curr_tt not in self.IDENTIFIER_TOKENS: 6721 self._retreat(index) 6722 return None 6723 6724 if parts is None: 6725 parts = [] 6726 6727 if comments: 6728 if all_comments is None: 6729 all_comments = [] 6730 all_comments.extend(comments) 6731 self._prev_comments = [] 6732 6733 parts.append( 6734 self.expression( 6735 exp.Identifier( 6736 this=token.text, quoted=token.token_type == TokenType.IDENTIFIER 6737 ), 6738 token, 6739 ) 6740 ) 6741 6742 if not has_dot: 6743 break 6744 6745 if parts is None: 6746 return None 6747 6748 n = len(parts) 6749 6750 if n == 1: 6751 column: exp.Column | exp.Dot = exp.Column(this=parts[0]) 6752 elif n == 2: 6753 column = exp.Column(this=parts[1], table=parts[0]) 6754 elif n == 3: 6755 column = exp.Column(this=parts[2], table=parts[1], db=parts[0]) 6756 else: 6757 column = exp.Column(this=parts[3], table=parts[2], db=parts[1], catalog=parts[0]) 6758 6759 for i in range(4, n): 6760 column = exp.Dot(this=column, expression=parts[i]) 6761 6762 if all_comments: 6763 column.add_comments(all_comments) 6764 6765 return column 6766 6767 def _parse_column_reference(self) -> exp.Expr | None: 6768 this = self._parse_field() 6769 if ( 6770 not this 6771 and self._match(TokenType.VALUES, advance=False) 6772 and self.VALUES_FOLLOWED_BY_PAREN 6773 and (not self._next or self._next.token_type != TokenType.L_PAREN) 6774 ): 6775 this = self._parse_id_var() 6776 6777 if isinstance(this, exp.Identifier): 6778 # We bubble up comments from the Identifier to the Column 6779 this = self.expression(exp.Column(this=this), comments=this.pop_comments()) 6780 6781 return this 6782 6783 def _build_json_extract( 6784 self, 6785 this: exp.Expr | None, 6786 path_parts: list[exp.JSONPathPart], 6787 ) -> tuple[exp.Expr | None, list[exp.JSONPathPart]]: 6788 if len(path_parts) > 1: 6789 this = self.expression( 6790 exp.JSONExtract( 6791 this=this, 6792 expression=exp.JSONPath(expressions=path_parts), 6793 variant_extract=True, 6794 requires_json=self.JSON_EXTRACT_REQUIRES_JSON_EXPRESSION, 6795 ) 6796 ) 6797 path_parts = [exp.JSONPathRoot()] 6798 6799 return this, path_parts 6800 6801 def _parse_colon_as_variant_extract(self, this: exp.Expr | None) -> exp.Expr | None: 6802 path_parts: list[exp.JSONPathPart] = [exp.JSONPathRoot()] 6803 6804 while self._match(TokenType.COLON): 6805 if not self.COLON_CHAIN_IS_SINGLE_EXTRACT: 6806 this, path_parts = self._build_json_extract(this, path_parts) 6807 6808 key = self._parse_id_var(any_token=True, tokens=(TokenType.SELECT,)) 6809 6810 if key: 6811 quoted = isinstance(key, exp.Identifier) and key.quoted 6812 path_parts.append(exp.JSONPathKey(this=key.name, quoted=quoted)) 6813 6814 while True: 6815 if self._match(TokenType.DOT): 6816 next_key = self._parse_id_var(any_token=True, tokens=(TokenType.SELECT,)) 6817 6818 if next_key: 6819 quoted = isinstance(next_key, exp.Identifier) and next_key.quoted 6820 path_parts.append(exp.JSONPathKey(this=next_key.name, quoted=quoted)) 6821 elif self._match(TokenType.L_BRACKET): 6822 bracket_expr = self._parse_bracket_key_value() 6823 6824 if not self._match(TokenType.R_BRACKET): 6825 self.raise_error("Expected ]") 6826 6827 if bracket_expr: 6828 if bracket_expr.is_string: 6829 path_parts.append(exp.JSONPathKey(this=bracket_expr.name, quoted=True)) 6830 elif bracket_expr.is_star: 6831 path_parts.append(exp.JSONPathSubscript(this=exp.JSONPathWildcard())) 6832 elif bracket_expr.is_number: 6833 path_parts.append(exp.JSONPathSubscript(this=bracket_expr.to_py())) 6834 else: 6835 this, path_parts = self._build_json_extract(this, path_parts) 6836 6837 this = self.expression( 6838 exp.Bracket( 6839 this=this, expressions=[bracket_expr], json_access=True 6840 ), 6841 ) 6842 6843 elif self._match(TokenType.DCOLON): 6844 this, path_parts = self._build_json_extract(this, path_parts) 6845 6846 cast_type = self._parse_types() 6847 if cast_type: 6848 this = self.expression(exp.Cast(this=this, to=cast_type)) 6849 else: 6850 self.raise_error("Expected type after '::'") 6851 else: 6852 break 6853 6854 this, _ = self._build_json_extract(this, path_parts) 6855 6856 return this 6857 6858 def _parse_dcolon(self) -> exp.Expr | None: 6859 return self._parse_types() 6860 6861 def _parse_column_ops(self, this: exp.Expr | None) -> exp.Expr | None: 6862 while self._curr.token_type in self.BRACKETS: 6863 this = self._parse_bracket(this) 6864 6865 column_operators = self.COLUMN_OPERATORS 6866 cast_column_operators = self.CAST_COLUMN_OPERATORS 6867 while self._curr: 6868 op_token = self._curr.token_type 6869 6870 if op_token not in column_operators: 6871 break 6872 op = column_operators[op_token] 6873 self._advance() 6874 6875 if op_token in cast_column_operators: 6876 field = self._parse_dcolon() 6877 if not field: 6878 self.raise_error("Expected type") 6879 elif op and self._curr: 6880 field = self._parse_column_reference() or self._parse_bitwise() 6881 if isinstance(field, exp.Column) and self._match(TokenType.DOT, advance=False): 6882 field = self._parse_column_ops(field) 6883 else: 6884 dot = self._is_connected() and self._prev.token_type == TokenType.DOT 6885 field = self._parse_field(any_token=True, anonymous_func=True) 6886 6887 # In t.true, t.null we should produce an Identifier node 6888 if dot and isinstance(field, (exp.Null, exp.Boolean)): 6889 field = self.expression( 6890 exp.Identifier(this=self._prev.text), 6891 comments=field.comments, 6892 ) 6893 6894 # Function calls can be qualified, e.g., x.y.FOO() 6895 # This converts the final AST to a series of Dots leading to the function call 6896 # https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-reference#function_call_rules 6897 if isinstance(field, (exp.Func, exp.Window)) and this: 6898 this = this.transform( 6899 lambda n: n.to_dot(include_dots=False) if isinstance(n, exp.Column) else n 6900 ) 6901 6902 if op: 6903 this = op(self, this, field) 6904 elif isinstance(this, exp.Column) and not this.args.get("catalog"): 6905 this = self.expression( 6906 exp.Column( 6907 this=field, 6908 table=this.this, 6909 db=this.args.get("table"), 6910 catalog=this.args.get("db"), 6911 ), 6912 comments=this.comments, 6913 ) 6914 elif isinstance(field, exp.Window): 6915 # Move the exp.Dot's to the window's function 6916 window_func = self.expression(exp.Dot(this=this, expression=field.this)) 6917 field.set("this", window_func) 6918 this = field 6919 else: 6920 this = self.expression(exp.Dot(this=this, expression=field)) 6921 6922 if field and field.comments: 6923 t.cast(exp.Expr, this).add_comments(field.pop_comments()) 6924 6925 this = self._parse_bracket(this) 6926 6927 return this 6928 6929 def _parse_paren(self) -> exp.Expr | None: 6930 if not self._match(TokenType.L_PAREN): 6931 return None 6932 6933 comments = self._prev_comments 6934 query = self._parse_select() 6935 6936 if query: 6937 expressions = [query] 6938 else: 6939 expressions = self._parse_expressions() 6940 6941 this = seq_get(expressions, 0) 6942 6943 if not this and self._match(TokenType.R_PAREN, advance=False): 6944 this = self.expression(exp.Tuple()) 6945 elif len(expressions) > 1 or self._prev.token_type == TokenType.COMMA: 6946 this = self.expression(exp.Tuple(expressions=expressions)) 6947 elif isinstance(this, exp.UNWRAPPED_QUERIES): 6948 this = self._parse_subquery(this=this, parse_alias=False) 6949 elif isinstance(this, (exp.Subquery, exp.Values)): 6950 this = self._parse_subquery( 6951 this=self._parse_query_modifiers(self._parse_set_operations(this)), 6952 parse_alias=False, 6953 ) 6954 else: 6955 this = self.expression(exp.Paren(this=this)) 6956 6957 if this: 6958 this.add_comments(comments) 6959 6960 self._match_r_paren(expression=this) 6961 6962 if isinstance(this, exp.Paren) and isinstance(this.this, exp.AggFunc): 6963 return self._parse_window(this) 6964 6965 return this 6966 6967 def _parse_primary(self) -> exp.Expr | None: 6968 if self._match_set(self.PRIMARY_PARSERS): 6969 token_type = self._prev.token_type 6970 primary = self.PRIMARY_PARSERS[token_type](self, self._prev) 6971 6972 if token_type == TokenType.STRING: 6973 expressions = [primary] 6974 while self._match(TokenType.STRING, advance=False): 6975 if self._is_connected() and self.ADJACENT_STRINGS_CANNOT_BE_CONNECTED: 6976 self.raise_error( 6977 "Adjacent string literals need to be separated by whitespace or comments" 6978 ) 6979 6980 self._advance() 6981 expressions.append(exp.Literal.string(self._prev.text)) 6982 6983 if len(expressions) > 1: 6984 return self.expression( 6985 exp.Concat(expressions=expressions, coalesce=self.dialect.CONCAT_COALESCE) 6986 ) 6987 6988 return primary 6989 6990 if self._match_pair(TokenType.DOT, TokenType.NUMBER): 6991 return exp.Literal.number(f"0.{self._prev.text}") 6992 6993 return self._parse_paren() 6994 6995 def _parse_field( 6996 self, 6997 any_token: bool = False, 6998 tokens: t.Collection[TokenType] | None = None, 6999 anonymous_func: bool = False, 7000 ) -> exp.Expr | None: 7001 if anonymous_func: 7002 field = ( 7003 self._parse_function(anonymous=anonymous_func, any_token=any_token) 7004 or self._parse_primary() 7005 ) 7006 else: 7007 field = self._parse_primary() or self._parse_function( 7008 anonymous=anonymous_func, any_token=any_token 7009 ) 7010 return field or self._parse_id_var(any_token=any_token, tokens=tokens) 7011 7012 def _parse_function( 7013 self, 7014 functions: dict[str, t.Callable] | None = None, 7015 anonymous: bool = False, 7016 optional_parens: bool = True, 7017 any_token: bool = False, 7018 ) -> exp.Expr | None: 7019 # This allows us to also parse {fn <function>} syntax (Snowflake, MySQL support this) 7020 # See: https://community.snowflake.com/s/article/SQL-Escape-Sequences 7021 fn_syntax = False 7022 if ( 7023 self._match(TokenType.L_BRACE, advance=False) 7024 and self._next 7025 and self._next.text.upper() == "FN" 7026 ): 7027 self._advance(2) 7028 fn_syntax = True 7029 7030 func = self._parse_function_call( 7031 functions=functions, 7032 anonymous=anonymous, 7033 optional_parens=optional_parens, 7034 any_token=any_token, 7035 ) 7036 7037 if fn_syntax: 7038 self._match(TokenType.R_BRACE) 7039 7040 return func 7041 7042 def _parse_function_args(self, alias: bool = False) -> list[exp.Expr]: 7043 return self._parse_csv(lambda: self._parse_lambda(alias=alias)) 7044 7045 def _parse_function_call( 7046 self, 7047 functions: dict[str, t.Callable] | None = None, 7048 anonymous: bool = False, 7049 optional_parens: bool = True, 7050 any_token: bool = False, 7051 ) -> exp.Expr | None: 7052 if not self._curr: 7053 return None 7054 7055 comments = self._curr.comments 7056 prev = self._prev 7057 token = self._curr 7058 token_type = self._curr.token_type 7059 this: str | exp.Expr = self._curr.text 7060 upper = self._curr.text.upper() 7061 7062 after_dot = prev.token_type == TokenType.DOT 7063 parser = self.NO_PAREN_FUNCTION_PARSERS.get(upper) 7064 if ( 7065 optional_parens 7066 and parser 7067 and token_type not in self.INVALID_FUNC_NAME_TOKENS 7068 and not after_dot 7069 ): 7070 self._advance() 7071 return self._parse_window(parser(self)) 7072 7073 if self._next.token_type != TokenType.L_PAREN: 7074 if optional_parens and token_type in self.NO_PAREN_FUNCTIONS and not after_dot: 7075 self._advance() 7076 return self.expression(self.NO_PAREN_FUNCTIONS[token_type]()) 7077 7078 return None 7079 7080 if any_token: 7081 if token_type in self.RESERVED_TOKENS: 7082 return None 7083 elif token_type not in self.FUNC_TOKENS: 7084 return None 7085 7086 self._advance(2) 7087 7088 parser = self.FUNCTION_PARSERS.get(upper) 7089 if parser and not anonymous: 7090 result = parser(self) 7091 else: 7092 subquery_predicate = self.SUBQUERY_PREDICATES.get(token_type) 7093 7094 if subquery_predicate: 7095 expr = None 7096 if self._curr.token_type in self.SUBQUERY_TOKENS: 7097 expr = self._parse_select() 7098 self._match_r_paren() 7099 elif prev and prev.token_type in (TokenType.LIKE, TokenType.ILIKE): 7100 # Backtrack one token since we've consumed the L_PAREN here. Instead, we'd like 7101 # to parse "LIKE [ANY | ALL] (...)" as a whole into an exp.Tuple or exp.Paren 7102 self._advance(-1) 7103 expr = self._parse_bitwise() 7104 7105 if expr: 7106 return self.expression(subquery_predicate(this=expr), comments=comments) 7107 7108 if functions is None: 7109 functions = self.FUNCTIONS 7110 7111 function = functions.get(upper) 7112 known_function = function and not anonymous 7113 7114 alias = not known_function or upper in self.FUNCTIONS_WITH_ALIASED_ARGS 7115 args = self._parse_function_args(alias) 7116 7117 post_func_comments = self._curr.comments if self._curr else None 7118 if known_function and post_func_comments: 7119 # If the user-inputted comment "/* sqlglot.anonymous */" is following the function 7120 # call we'll construct it as exp.Anonymous, even if it's "known" 7121 if any( 7122 comment.lstrip().startswith(exp.SQLGLOT_ANONYMOUS) 7123 for comment in post_func_comments 7124 ): 7125 known_function = False 7126 7127 if alias and known_function: 7128 args = self._kv_to_prop_eq(args) 7129 7130 if known_function: 7131 func_builder = t.cast(t.Callable, function) 7132 7133 # mypyc compiled functions don't have __code__, so we use 7134 # try/except to check if func_builder accepts 'dialect'. 7135 try: 7136 func = func_builder(args) 7137 except TypeError: 7138 func = func_builder(args, dialect=self.dialect) 7139 7140 func = self.validate_expression(func, args) 7141 if self.dialect.PRESERVE_ORIGINAL_NAMES: 7142 func.meta["name"] = this 7143 7144 result = func 7145 else: 7146 if token_type == TokenType.IDENTIFIER: 7147 this = exp.Identifier(this=this, quoted=True).update_positions(token) 7148 7149 result = self.expression(exp.Anonymous(this=this, expressions=args)) 7150 7151 result = result.update_positions(token) 7152 7153 if isinstance(result, exp.Expr): 7154 result.add_comments(comments) 7155 7156 if parser: 7157 self._match(TokenType.R_PAREN, expression=result) 7158 else: 7159 self._match_r_paren(result) 7160 return self._parse_window(result) 7161 7162 def _to_prop_eq(self, expression: exp.Expr, index: int) -> exp.Expr: 7163 return expression 7164 7165 def _kv_to_prop_eq( 7166 self, expressions: list[exp.Expr], parse_map: bool = False 7167 ) -> list[exp.Expr]: 7168 transformed = [] 7169 7170 for index, e in enumerate(expressions): 7171 if isinstance(e, self.KEY_VALUE_DEFINITIONS): 7172 if isinstance(e, exp.Alias): 7173 e = self.expression(exp.PropertyEQ(this=e.args.get("alias"), expression=e.this)) 7174 7175 if not isinstance(e, exp.PropertyEQ): 7176 e = self.expression( 7177 exp.PropertyEQ( 7178 this=e.this if parse_map else exp.to_identifier(e.this.name), 7179 expression=e.expression, 7180 ) 7181 ) 7182 7183 if isinstance(e.this, exp.Column): 7184 e.this.replace(e.this.this) 7185 else: 7186 e = self._to_prop_eq(e, index) 7187 7188 transformed.append(e) 7189 7190 return transformed 7191 7192 def _parse_function_properties(self) -> exp.Properties | None: 7193 # Skip the generic `key = value` fallback in _parse_property since this 7194 # runs post-AS where a function body like `name = expr` can be misread 7195 # as a property. 7196 properties = [] 7197 while True: 7198 if self._match_texts(self.PROPERTY_PARSERS): 7199 prop = self.PROPERTY_PARSERS[self._prev.text.upper()](self) 7200 elif self._match(TokenType.DEFAULT) and self._match_texts(self.PROPERTY_PARSERS): 7201 prop = self.PROPERTY_PARSERS[self._prev.text.upper()](self, default=True) 7202 else: 7203 break 7204 for p in ensure_list(prop): 7205 properties.append(p) 7206 7207 return self.expression(exp.Properties(expressions=properties)) if properties else None 7208 7209 def _parse_user_defined_function_expression(self) -> exp.Expr | None: 7210 return self._parse_statement() 7211 7212 def _parse_function_parameter(self) -> exp.Expr | None: 7213 return self._parse_column_def(this=self._parse_id_var(), computed_column=False) 7214 7215 def _parse_user_defined_function(self, kind: TokenType | None = None) -> exp.Expr | None: 7216 this = self._parse_table_parts(schema=True) 7217 7218 if not self._match(TokenType.L_PAREN): 7219 return this 7220 7221 expressions = self._parse_csv(self._parse_function_parameter) 7222 self._match_r_paren() 7223 return self.expression( 7224 exp.UserDefinedFunction(this=this, expressions=expressions, wrapped=True) 7225 ) 7226 7227 def _parse_macro_overloads( 7228 self, 7229 this: exp.UserDefinedFunction, 7230 first_body: exp.Expr, 7231 first_is_table: bool = False, 7232 ) -> exp.MacroOverloads: 7233 overloads = [ 7234 self.expression( 7235 exp.MacroOverload( 7236 this=first_body, 7237 expressions=this.expressions or None, 7238 is_table=first_is_table, 7239 ) 7240 ) 7241 ] 7242 this.set("expressions", None) 7243 this.set("wrapped", False) 7244 7245 while self._match(TokenType.COMMA): 7246 if not self._match(TokenType.L_PAREN): 7247 break 7248 7249 params = self._parse_csv(self._parse_function_parameter) 7250 self._match_r_paren() 7251 7252 if not self._match(TokenType.ALIAS): 7253 break 7254 7255 is_table = self._match(TokenType.TABLE) 7256 body = self._parse_expression() 7257 macro = exp.MacroOverload(this=body, expressions=params, is_table=is_table) 7258 overloads.append(self.expression(macro)) 7259 7260 return self.expression(exp.MacroOverloads(expressions=overloads)) 7261 7262 def _parse_introducer(self, token: Token) -> exp.Introducer | exp.Identifier: 7263 literal = self._parse_primary() 7264 if literal: 7265 return self.expression(exp.Introducer(this=token.text, expression=literal), token) 7266 7267 return self._identifier_expression(token) 7268 7269 def _parse_session_parameter(self) -> exp.SessionParameter: 7270 kind = None 7271 this = self._parse_id_var() or self._parse_primary() 7272 7273 if this and self._match(TokenType.DOT): 7274 kind = this.name 7275 this = self._parse_var() or self._parse_primary() 7276 7277 return self.expression(exp.SessionParameter(this=this, kind=kind)) 7278 7279 def _parse_lambda_arg(self) -> exp.Expr | None: 7280 return self._parse_id_var() 7281 7282 def _parse_lambda(self, alias: bool = False) -> exp.Expr | None: 7283 next_token_type = self._next.token_type 7284 7285 # Fast path: simple atom (column, literal, null, bool) followed by , or ) 7286 if ( 7287 next_token_type in self.LAMBDA_ARG_TERMINATORS 7288 and (atom := self._parse_atom()) is not None 7289 ): 7290 return atom 7291 7292 index = self._index 7293 7294 if self._match(TokenType.L_PAREN): 7295 expressions = t.cast( 7296 list[t.Optional[exp.Expr]], self._parse_csv(self._parse_lambda_arg) 7297 ) 7298 7299 if not self._match(TokenType.R_PAREN): 7300 self._retreat(index) 7301 elif self._match_set(self.LAMBDAS): 7302 return self.LAMBDAS[self._prev.token_type](self, expressions) 7303 else: 7304 self._retreat(index) 7305 elif self.TYPED_LAMBDA_ARGS or next_token_type in self.LAMBDAS: 7306 expressions = [self._parse_lambda_arg()] 7307 7308 if self._match_set(self.LAMBDAS): 7309 return self.LAMBDAS[self._prev.token_type](self, expressions) 7310 7311 self._retreat(index) 7312 7313 this: exp.Expr | None 7314 7315 if self._match(TokenType.DISTINCT): 7316 this = self.expression( 7317 exp.Distinct(expressions=self._parse_csv(self._parse_disjunction)) 7318 ) 7319 else: 7320 self._match(TokenType.ALL) # ALL is the default/no-op aggregate modifier (SQL-92) 7321 this = self._parse_select_or_expression(alias=alias) 7322 7323 return self._parse_limit( 7324 self._parse_respect_or_ignore_nulls( 7325 self._parse_order(self._parse_having_max(self._parse_respect_or_ignore_nulls(this))) 7326 ) 7327 ) 7328 7329 def _parse_schema(self, this: exp.Expr | None = None) -> exp.Expr | None: 7330 index = self._index 7331 if not self._match(TokenType.L_PAREN): 7332 return this 7333 7334 # Disambiguate between schema and subquery/CTE, e.g. in INSERT INTO table (<expr>), 7335 # expr can be of both types 7336 if self._match_set(self.SELECT_START_TOKENS): 7337 self._retreat(index) 7338 return this 7339 args = self._parse_csv(lambda: self._parse_constraint() or self._parse_field_def()) 7340 self._match_r_paren() 7341 return self.expression(exp.Schema(this=this, expressions=args)) 7342 7343 def _parse_field_def(self) -> exp.Expr | None: 7344 return self._parse_column_def(self._parse_field(any_token=True)) 7345 7346 def _parse_column_def( 7347 self, this: exp.Expr | None, computed_column: bool = True 7348 ) -> exp.Expr | None: 7349 # column defs are not really columns, they're identifiers 7350 if isinstance(this, exp.Column): 7351 this = this.this 7352 7353 if not computed_column: 7354 self._match(TokenType.ALIAS) 7355 7356 kind = self._parse_types(schema=True) 7357 7358 if self._match_text_seq("FOR", "ORDINALITY"): 7359 return self.expression(exp.ColumnDef(this=this, ordinality=True)) 7360 7361 constraints: list[exp.Expr] = [] 7362 7363 if (not kind and self._match(TokenType.ALIAS)) or self._match_texts( 7364 ("ALIAS", "MATERIALIZED") 7365 ): 7366 persisted = self._prev.text.upper() == "MATERIALIZED" 7367 constraint_kind = exp.ComputedColumnConstraint( 7368 this=self._parse_disjunction(), 7369 persisted=persisted or self._match_text_seq("PERSISTED"), 7370 data_type=exp.Var(this="AUTO") 7371 if self._match_text_seq("AUTO") 7372 else self._parse_types(), 7373 not_null=self._match_pair(TokenType.NOT, TokenType.NULL), 7374 ) 7375 constraints.append(self.expression(exp.ColumnConstraint(kind=constraint_kind))) 7376 elif not kind and self._match_set({TokenType.IN, TokenType.OUT}, advance=False): 7377 in_out_constraint = self.expression( 7378 exp.InOutColumnConstraint( 7379 input_=self._match(TokenType.IN), output=self._match(TokenType.OUT) 7380 ) 7381 ) 7382 constraints.append(in_out_constraint) 7383 kind = self._parse_types() 7384 elif ( 7385 kind 7386 and self._match(TokenType.ALIAS, advance=False) 7387 and ( 7388 not self.WRAPPED_TRANSFORM_COLUMN_CONSTRAINT 7389 or self._next.token_type == TokenType.L_PAREN 7390 ) 7391 ): 7392 self._advance() 7393 constraints.append( 7394 self.expression( 7395 exp.ColumnConstraint( 7396 kind=exp.ComputedColumnConstraint( 7397 this=self._parse_disjunction(), 7398 persisted=self._match_texts(("STORED", "VIRTUAL")) 7399 and self._prev.text.upper() == "STORED", 7400 ) 7401 ) 7402 ) 7403 ) 7404 7405 while True: 7406 constraint = self._parse_column_constraint() 7407 if not constraint: 7408 break 7409 constraints.append(constraint) 7410 7411 if not kind and not constraints: 7412 return this 7413 7414 position = None 7415 if self._match_texts(("FIRST", "AFTER")): 7416 pos = self._prev.text 7417 position = self.expression(exp.ColumnPosition(this=self._parse_column(), position=pos)) 7418 7419 return self.expression( 7420 exp.ColumnDef(this=this, kind=kind, constraints=constraints, position=position) 7421 ) 7422 7423 def _parse_auto_increment( 7424 self, 7425 ) -> exp.GeneratedAsIdentityColumnConstraint | exp.AutoIncrementColumnConstraint: 7426 start = None 7427 increment = None 7428 order = None 7429 7430 if self._match(TokenType.L_PAREN, advance=False): 7431 args = self._parse_wrapped_csv(self._parse_bitwise) 7432 start = seq_get(args, 0) 7433 increment = seq_get(args, 1) 7434 7435 # The remaining parts form an unordered bag and any of them can be omitted, in which 7436 # case the engine falls back to its own default, so they're parsed independently. 7437 while True: 7438 if self._match_text_seq("START"): 7439 start = self._parse_bitwise() 7440 elif self._match_text_seq("INCREMENT"): 7441 increment = self._parse_bitwise() 7442 elif self._match_text_seq("ORDER"): 7443 order = True 7444 elif self._match_text_seq("NOORDER"): 7445 order = False 7446 else: 7447 break 7448 7449 if start or increment or order is not None: 7450 return exp.GeneratedAsIdentityColumnConstraint( 7451 start=start, increment=increment, this=False, order=order 7452 ) 7453 7454 return exp.AutoIncrementColumnConstraint() 7455 7456 def _parse_check_constraint(self) -> exp.CheckColumnConstraint | None: 7457 if not self._match(TokenType.L_PAREN, advance=False): 7458 return None 7459 7460 return self.expression( 7461 exp.CheckColumnConstraint( 7462 this=self._parse_wrapped(self._parse_assignment), 7463 enforced=self._match_text_seq("ENFORCED"), 7464 ) 7465 ) 7466 7467 def _parse_auto_property(self) -> exp.AutoRefreshProperty | None: 7468 if not self._match_text_seq("REFRESH"): 7469 self._retreat(self._index - 1) 7470 return None 7471 return self.expression(exp.AutoRefreshProperty(this=self._parse_var(upper=True))) 7472 7473 def _parse_compress(self) -> exp.CompressColumnConstraint: 7474 if self._match(TokenType.L_PAREN, advance=False): 7475 return self.expression( 7476 exp.CompressColumnConstraint(this=self._parse_wrapped_csv(self._parse_bitwise)) 7477 ) 7478 7479 return self.expression(exp.CompressColumnConstraint(this=self._parse_bitwise())) 7480 7481 def _parse_generated_as_identity( 7482 self, 7483 ) -> ( 7484 exp.GeneratedAsIdentityColumnConstraint 7485 | exp.ComputedColumnConstraint 7486 | exp.GeneratedAsRowColumnConstraint 7487 ): 7488 if self._match_text_seq("BY", "DEFAULT"): 7489 on_null = self._match_pair(TokenType.ON, TokenType.NULL) 7490 this = self.expression( 7491 exp.GeneratedAsIdentityColumnConstraint(this=False, on_null=on_null) 7492 ) 7493 else: 7494 self._match_text_seq("ALWAYS") 7495 this = self.expression(exp.GeneratedAsIdentityColumnConstraint(this=True)) 7496 7497 self._match(TokenType.ALIAS) 7498 7499 if self._match_text_seq("ROW"): 7500 start = self._match_text_seq("START") 7501 if not start: 7502 self._match(TokenType.END) 7503 hidden = self._match_text_seq("HIDDEN") 7504 return self.expression(exp.GeneratedAsRowColumnConstraint(start=start, hidden=hidden)) 7505 7506 identity = self._match_text_seq("IDENTITY") 7507 7508 if self._match(TokenType.L_PAREN): 7509 if self._match(TokenType.START_WITH): 7510 this.set("start", self._parse_bitwise()) 7511 if self._match_text_seq("INCREMENT", "BY"): 7512 this.set("increment", self._parse_bitwise()) 7513 if self._match_text_seq("MINVALUE"): 7514 this.set("minvalue", self._parse_bitwise()) 7515 if self._match_text_seq("MAXVALUE"): 7516 this.set("maxvalue", self._parse_bitwise()) 7517 7518 if self._match_text_seq("CYCLE"): 7519 this.set("cycle", True) 7520 elif self._match_text_seq("NO", "CYCLE"): 7521 this.set("cycle", False) 7522 7523 if not identity: 7524 this.set("expression", self._parse_range()) 7525 elif not this.args.get("start") and self._match(TokenType.NUMBER, advance=False): 7526 args = self._parse_csv(self._parse_bitwise) 7527 this.set("start", seq_get(args, 0)) 7528 this.set("increment", seq_get(args, 1)) 7529 7530 self._match_r_paren() 7531 7532 return this 7533 7534 def _parse_inline(self) -> exp.InlineLengthColumnConstraint: 7535 self._match_text_seq("LENGTH") 7536 return self.expression(exp.InlineLengthColumnConstraint(this=self._parse_bitwise())) 7537 7538 def _parse_not_constraint(self) -> exp.Expr | None: 7539 if self._match_text_seq("NULL"): 7540 return self.expression(exp.NotNullColumnConstraint()) 7541 if self._match_text_seq("CASESPECIFIC"): 7542 return self.expression(exp.CaseSpecificColumnConstraint(not_=True)) 7543 if self._match_text_seq("FOR", "REPLICATION"): 7544 return self.expression(exp.NotForReplicationColumnConstraint()) 7545 7546 # Unconsume the `NOT` token 7547 self._retreat(self._index - 1) 7548 return None 7549 7550 def _parse_column_constraint(self) -> exp.Expr | None: 7551 this = self._parse_id_var() if self._match(TokenType.CONSTRAINT) else None 7552 7553 procedure_option_follows = ( 7554 self._match(TokenType.WITH, advance=False) 7555 and self._next 7556 and self._next.text.upper() in self.PROCEDURE_OPTIONS 7557 ) 7558 7559 if not procedure_option_follows and self._match_texts(self.CONSTRAINT_PARSERS): 7560 constraint = self.CONSTRAINT_PARSERS[self._prev.text.upper()](self) 7561 if not constraint: 7562 self._retreat(self._index - 1) 7563 return None 7564 7565 return self.expression(exp.ColumnConstraint(this=this, kind=constraint)) 7566 7567 return this 7568 7569 def _parse_constraint(self) -> exp.Expr | None: 7570 if not self._match(TokenType.CONSTRAINT): 7571 return self._parse_unnamed_constraint(constraints=self.SCHEMA_UNNAMED_CONSTRAINTS) 7572 7573 return self.expression( 7574 exp.Constraint(this=self._parse_id_var(), expressions=self._parse_unnamed_constraints()) 7575 ) 7576 7577 def _parse_unnamed_constraints(self) -> list[exp.Expr]: 7578 constraints = [] 7579 while True: 7580 constraint = self._parse_unnamed_constraint() or self._parse_function() 7581 if not constraint: 7582 break 7583 constraints.append(constraint) 7584 7585 return constraints 7586 7587 def _parse_unnamed_constraint(self, constraints: TEXTS_TYPE | None = None) -> exp.Expr | None: 7588 index = self._index 7589 7590 if self._match(TokenType.IDENTIFIER, advance=False) or not self._match_texts( 7591 constraints or self.CONSTRAINT_PARSERS 7592 ): 7593 return None 7594 7595 constraint_key = self._prev.text.upper() 7596 if constraint_key not in self.CONSTRAINT_PARSERS: 7597 self.raise_error(f"No parser found for schema constraint {constraint_key}.") 7598 7599 result = self.CONSTRAINT_PARSERS[constraint_key](self) 7600 if not result: 7601 self._retreat(index) 7602 7603 return result 7604 7605 def _parse_unique_key(self) -> exp.Expr | None: 7606 if ( 7607 self._curr 7608 and self._curr.token_type != TokenType.IDENTIFIER 7609 and self._curr.text.upper() in self.CONSTRAINT_PARSERS 7610 ): 7611 return None 7612 return self._parse_id_var(any_token=False) 7613 7614 def _parse_unique(self) -> exp.UniqueColumnConstraint: 7615 self._match_texts(("KEY", "INDEX")) 7616 return self.expression( 7617 exp.UniqueColumnConstraint( 7618 nulls=self._match_text_seq("NULLS", "NOT", "DISTINCT"), 7619 this=self._parse_schema(self._parse_unique_key()), 7620 index_type=self._match(TokenType.USING) and self._advance_any() and self._prev.text, 7621 on_conflict=self._parse_on_conflict(), 7622 options=self._parse_key_constraint_options(), 7623 ) 7624 ) 7625 7626 def _parse_key_constraint_options(self) -> list[str]: 7627 options = [] 7628 while True: 7629 if not self._curr: 7630 break 7631 7632 if self._match(TokenType.ON): 7633 action = None 7634 on = self._advance_any() and self._prev.text 7635 7636 if self._match_text_seq("NO", "ACTION"): 7637 action = "NO ACTION" 7638 elif self._match_text_seq("CASCADE"): 7639 action = "CASCADE" 7640 elif self._match_text_seq("RESTRICT"): 7641 action = "RESTRICT" 7642 elif self._match_pair(TokenType.SET, TokenType.NULL): 7643 action = "SET NULL" 7644 elif self._match_pair(TokenType.SET, TokenType.DEFAULT): 7645 action = "SET DEFAULT" 7646 else: 7647 self.raise_error("Invalid key constraint") 7648 7649 options.append(f"ON {on} {action}") 7650 else: 7651 var = self._parse_var_from_options( 7652 self.KEY_CONSTRAINT_OPTIONS, raise_unmatched=False 7653 ) 7654 if not var: 7655 break 7656 options.append(var.name) 7657 7658 return options 7659 7660 def _parse_references(self, match: bool = True) -> exp.Reference | None: 7661 if match and not self._match(TokenType.REFERENCES): 7662 return None 7663 7664 expressions: list | None = None 7665 this = self._parse_table(schema=True) 7666 options = self._parse_key_constraint_options() 7667 return self.expression(exp.Reference(this=this, expressions=expressions, options=options)) 7668 7669 def _parse_foreign_key(self) -> exp.ForeignKey: 7670 expressions = ( 7671 self._parse_wrapped_id_vars() 7672 if not self._match(TokenType.REFERENCES, advance=False) 7673 else None 7674 ) 7675 reference = self._parse_references() 7676 on_options = {} 7677 7678 while self._match(TokenType.ON): 7679 if not self._match_set((TokenType.DELETE, TokenType.UPDATE)): 7680 self.raise_error("Expected DELETE or UPDATE") 7681 7682 kind = self._prev.text.lower() 7683 7684 if self._match_text_seq("NO", "ACTION"): 7685 action = "NO ACTION" 7686 elif self._match(TokenType.SET): 7687 self._match_set((TokenType.NULL, TokenType.DEFAULT)) 7688 action = "SET " + self._prev.text.upper() 7689 else: 7690 self._advance() 7691 action = self._prev.text.upper() 7692 7693 on_options[kind] = action 7694 7695 return self.expression( 7696 exp.ForeignKey( 7697 expressions=expressions, 7698 reference=reference, 7699 options=self._parse_key_constraint_options(), 7700 **on_options, 7701 ) 7702 ) 7703 7704 def _parse_primary_key_part(self) -> exp.Expr | None: 7705 return self._parse_field() 7706 7707 def _parse_period_for_system_time(self) -> exp.PeriodForSystemTimeConstraint | None: 7708 if not self._match(TokenType.TIMESTAMP_SNAPSHOT): 7709 self._retreat(self._index - 1) 7710 return None 7711 7712 id_vars = self._parse_wrapped_id_vars() 7713 return self.expression( 7714 exp.PeriodForSystemTimeConstraint( 7715 this=seq_get(id_vars, 0), expression=seq_get(id_vars, 1) 7716 ) 7717 ) 7718 7719 def _parse_primary_key( 7720 self, 7721 wrapped_optional: bool = False, 7722 in_props: bool = False, 7723 named_primary_key: bool = False, 7724 ) -> exp.PrimaryKeyColumnConstraint | exp.PrimaryKey: 7725 desc = ( 7726 self._prev.token_type == TokenType.DESC 7727 if self._match_set((TokenType.ASC, TokenType.DESC)) 7728 else None 7729 ) 7730 7731 this = None 7732 if ( 7733 named_primary_key 7734 and self._curr.text.upper() not in self.CONSTRAINT_PARSERS 7735 and self._next 7736 and self._next.token_type == TokenType.L_PAREN 7737 ): 7738 this = self._parse_id_var() 7739 7740 if not in_props and not self._match(TokenType.L_PAREN, advance=False): 7741 return self.expression( 7742 exp.PrimaryKeyColumnConstraint( 7743 desc=desc, options=self._parse_key_constraint_options() 7744 ) 7745 ) 7746 7747 expressions = self._parse_wrapped_csv( 7748 self._parse_primary_key_part, optional=wrapped_optional 7749 ) 7750 7751 return self.expression( 7752 exp.PrimaryKey( 7753 this=this, 7754 expressions=expressions, 7755 include=self._parse_index_params(), 7756 options=self._parse_key_constraint_options(), 7757 ) 7758 ) 7759 7760 def _parse_bracket_key_value(self, is_map: bool = False) -> exp.Expr | None: 7761 return self._parse_slice(self._parse_alias(self._parse_disjunction(), explicit=True)) 7762 7763 def _parse_odbc_datetime_literal(self) -> exp.Expr: 7764 """ 7765 Parses a datetime column in ODBC format. We parse the column into the corresponding 7766 types, for example `{d'yyyy-mm-dd'}` will be parsed as a `Date` column, exactly the 7767 same as we did for `DATE('yyyy-mm-dd')`. 7768 7769 Reference: 7770 https://learn.microsoft.com/en-us/sql/odbc/reference/develop-app/date-time-and-timestamp-literals 7771 """ 7772 self._match(TokenType.VAR) 7773 exp_class = self.ODBC_DATETIME_LITERALS[self._prev.text.lower()] 7774 expression = self.expression(exp_class(this=self._parse_string())) 7775 if not self._match(TokenType.R_BRACE): 7776 self.raise_error("Expected }") 7777 return expression 7778 7779 def _parse_bracket(self, this: exp.Expr | None = None) -> exp.Expr | None: 7780 if not self._match_set(self.BRACKETS): 7781 return this 7782 7783 if self.MAP_KEYS_ARE_ARBITRARY_EXPRESSIONS: 7784 map_token = seq_get(self._tokens, self._index - 2) 7785 parse_map = map_token is not None and map_token.text.upper() == "MAP" 7786 else: 7787 parse_map = False 7788 7789 bracket_kind = self._prev.token_type 7790 if ( 7791 bracket_kind == TokenType.L_BRACE 7792 and self._curr 7793 and self._curr.token_type == TokenType.VAR 7794 and self._curr.text.lower() in self.ODBC_DATETIME_LITERALS 7795 ): 7796 return self._parse_odbc_datetime_literal() 7797 7798 expressions = self._parse_csv( 7799 lambda: self._parse_bracket_key_value(is_map=bracket_kind == TokenType.L_BRACE) 7800 ) 7801 7802 if bracket_kind == TokenType.L_BRACKET and not self._match(TokenType.R_BRACKET): 7803 self.raise_error("Expected ]") 7804 elif bracket_kind == TokenType.L_BRACE and not self._match(TokenType.R_BRACE): 7805 self.raise_error("Expected }") 7806 7807 # https://duckdb.org/docs/sql/data_types/struct.html#creating-structs 7808 if bracket_kind == TokenType.L_BRACE: 7809 this = self.expression( 7810 exp.Struct( 7811 expressions=self._kv_to_prop_eq(expressions=expressions, parse_map=parse_map) 7812 ) 7813 ) 7814 elif not this: 7815 this = build_array_constructor( 7816 exp.Array, args=expressions, bracket_kind=bracket_kind, dialect=self.dialect 7817 ) 7818 else: 7819 constructor_type = self.ARRAY_CONSTRUCTORS.get(this.name.upper()) 7820 if constructor_type: 7821 return build_array_constructor( 7822 constructor_type, 7823 args=expressions, 7824 bracket_kind=bracket_kind, 7825 dialect=self.dialect, 7826 ) 7827 7828 expressions = apply_index_offset( 7829 this, expressions, -self.dialect.INDEX_OFFSET, dialect=self.dialect 7830 ) 7831 this = self.expression( 7832 exp.Bracket(this=this, expressions=expressions), comments=this.pop_comments() 7833 ) 7834 7835 self._add_comments(this) 7836 return self._parse_bracket(this) 7837 7838 def _parse_slice(self, this: exp.Expr | None) -> exp.Expr | None: 7839 if not self._match(TokenType.COLON): 7840 return this 7841 7842 if self._match_pair(TokenType.DASH, TokenType.COLON, advance=False): 7843 self._advance() 7844 end: exp.Expr | None = -exp.Literal.number("1") 7845 else: 7846 end = self._parse_assignment() 7847 step = self._parse_unary() if self._match(TokenType.COLON) else None 7848 return self.expression(exp.Slice(this=this, expression=end, step=step)) 7849 7850 def _parse_case(self) -> exp.Expr | None: 7851 if self._match(TokenType.DOT, advance=False): 7852 # Avoid raising on valid expressions like case.*, supported by, e.g., spark & snowflake 7853 self._retreat(self._index - 1) 7854 return None 7855 7856 ifs = [] 7857 default = None 7858 7859 comments = self._prev_comments 7860 expression = self._parse_disjunction() 7861 7862 while self._match(TokenType.WHEN): 7863 this = self._parse_disjunction() 7864 self._match(TokenType.THEN) 7865 then = self._parse_disjunction() 7866 ifs.append(self.expression(exp.If(this=this, true=then))) 7867 7868 if self._match(TokenType.ELSE): 7869 default = self._parse_disjunction() 7870 7871 if not self._match(TokenType.END): 7872 if isinstance(default, exp.Interval) and default.this.sql().upper() == "END": 7873 default = exp.column("interval") 7874 else: 7875 self.raise_error("Expected END after CASE", self._prev) 7876 7877 return self.expression( 7878 exp.Case(this=expression, ifs=ifs, default=default), comments=comments 7879 ) 7880 7881 def _parse_if(self) -> exp.Expr | None: 7882 if self._match(TokenType.L_PAREN): 7883 args = self._parse_csv( 7884 lambda: self._parse_alias(self._parse_assignment(), explicit=True) 7885 ) 7886 this = self.validate_expression(exp.If.from_arg_list(args), args) 7887 self._match_r_paren() 7888 else: 7889 index = self._index - 1 7890 7891 if self.NO_PAREN_IF_COMMANDS and index == 0: 7892 return self._parse_as_command(self._prev) 7893 7894 condition = self._parse_disjunction() 7895 7896 if not condition: 7897 self._retreat(index) 7898 return None 7899 7900 self._match(TokenType.THEN) 7901 true = self._parse_disjunction() 7902 false = self._parse_disjunction() if self._match(TokenType.ELSE) else None 7903 self._match(TokenType.END) 7904 this = self.expression(exp.If(this=condition, true=true, false=false)) 7905 7906 return this 7907 7908 def _parse_next_value_for(self) -> exp.Expr | None: 7909 if not self._match_text_seq("VALUE", "FOR"): 7910 self._retreat(self._index - 1) 7911 return None 7912 7913 return self.expression( 7914 exp.NextValueFor( 7915 this=self._parse_column(), 7916 order=self._match(TokenType.OVER) and self._parse_wrapped(self._parse_order), 7917 ) 7918 ) 7919 7920 def _parse_extract(self) -> exp.Extract | exp.Anonymous: 7921 this = self._parse_function() or self._parse_var_or_string(upper=True) 7922 7923 if self._match(TokenType.FROM): 7924 return self.expression(exp.Extract(this=this, expression=self._parse_bitwise())) 7925 7926 if not self._match(TokenType.COMMA): 7927 self.raise_error("Expected FROM or comma after EXTRACT", self._prev) 7928 7929 return self.expression(exp.Extract(this=this, expression=self._parse_bitwise())) 7930 7931 def _parse_gap_fill(self) -> exp.GapFill: 7932 self._match(TokenType.TABLE) 7933 this = self._parse_table() 7934 7935 self._match(TokenType.COMMA) 7936 args = [this, *self._parse_csv(self._parse_lambda)] 7937 7938 gap_fill = exp.GapFill.from_arg_list(args) 7939 return self.validate_expression(gap_fill, args) 7940 7941 def _parse_char(self) -> exp.Chr: 7942 return self.expression( 7943 exp.Chr( 7944 expressions=self._parse_csv(self._parse_assignment), 7945 charset=self._match(TokenType.USING) and self._parse_charset_name(), 7946 ) 7947 ) 7948 7949 def _parse_charset_name(self) -> exp.Expr | None: 7950 """ 7951 Parse a charset name after USING or CHARACTER SET. Dialects that need to preserve quoting 7952 for specific name shapes override this. 7953 """ 7954 return self._parse_var( 7955 tokens={TokenType.BINARY, TokenType.IDENTIFIER}, 7956 ) 7957 7958 def _parse_cast(self, strict: bool, safe: bool | None = None) -> exp.Expr: 7959 this = self._parse_assignment() 7960 7961 if not self._match(TokenType.ALIAS): 7962 if self._match(TokenType.COMMA): 7963 return self.expression(exp.CastToStrType(this=this, to=self._parse_string())) 7964 7965 self.raise_error("Expected AS after CAST") 7966 7967 fmt = None 7968 to = self._parse_types(with_collation=True) 7969 7970 default = None 7971 if self._match(TokenType.DEFAULT): 7972 default = self._parse_bitwise() 7973 self._match_text_seq("ON", "CONVERSION", "ERROR") 7974 7975 if self._match_set((TokenType.FORMAT, TokenType.COMMA)): 7976 fmt_string = self._parse_wrapped(self._parse_string, optional=True) 7977 fmt = self._parse_at_time_zone(fmt_string) 7978 7979 if not to: 7980 to = exp.DType.UNKNOWN.into_expr() 7981 if to.this in exp.DataType.TEMPORAL_TYPES: 7982 this = self.expression( 7983 (exp.StrToDate if to.this == exp.DType.DATE else exp.StrToTime)( 7984 this=this, 7985 format=exp.Literal.string( 7986 format_time( 7987 fmt_string.this if fmt_string else "", 7988 self.dialect.FORMAT_MAPPING or self.dialect.TIME_MAPPING, 7989 self.dialect.FORMAT_TRIE or self.dialect.TIME_TRIE, 7990 ) 7991 ), 7992 safe=safe, 7993 ) 7994 ) 7995 7996 if isinstance(fmt, exp.AtTimeZone) and isinstance(this, exp.StrToTime): 7997 this.set("zone", fmt.args["zone"]) 7998 return this 7999 elif not to: 8000 self.raise_error("Expected TYPE after CAST") 8001 elif isinstance(to, exp.Identifier): 8002 to = exp.DataType.from_str(to.name, dialect=self.dialect, udt=True) 8003 elif to.this == exp.DType.CHAR and self._match(TokenType.CHARACTER_SET): 8004 to = exp.DType.CHARACTER_SET.into_expr(kind=self._parse_var_or_string()) 8005 8006 return self.build_cast( 8007 strict=strict, 8008 this=this, 8009 to=to, 8010 format=fmt, 8011 safe=safe, 8012 action=self._parse_var_from_options(self.CAST_ACTIONS, raise_unmatched=False), 8013 default=default, 8014 ) 8015 8016 def _parse_string_agg(self) -> exp.GroupConcat: 8017 if self._match(TokenType.DISTINCT): 8018 args: list[exp.Expr | None] = [ 8019 self.expression(exp.Distinct(expressions=[self._parse_disjunction()])) 8020 ] 8021 if self._match(TokenType.COMMA): 8022 args.extend(self._parse_csv(self._parse_disjunction)) 8023 else: 8024 args = self._parse_csv(self._parse_disjunction) # type: ignore 8025 8026 if self._match_text_seq("ON", "OVERFLOW"): 8027 # trino: LISTAGG(expression [, separator] [ON OVERFLOW overflow_behavior]) 8028 if self._match_text_seq("ERROR"): 8029 on_overflow: exp.Expr | None = exp.var("ERROR") 8030 else: 8031 self._match_text_seq("TRUNCATE") 8032 on_overflow = self.expression( 8033 exp.OverflowTruncateBehavior( 8034 this=self._parse_string(), 8035 with_count=( 8036 self._match_text_seq("WITH", "COUNT") 8037 or not self._match_text_seq("WITHOUT", "COUNT") 8038 ), 8039 ) 8040 ) 8041 else: 8042 on_overflow = None 8043 8044 index = self._index 8045 if not self._match(TokenType.R_PAREN) and args: 8046 # postgres: STRING_AGG([DISTINCT] expression, separator [ORDER BY expression1 {ASC | DESC} [, ...]]) 8047 # bigquery: STRING_AGG([DISTINCT] expression [, separator] [ORDER BY key [{ASC | DESC}] [, ... ]] [LIMIT n]) 8048 # The order is parsed through `this` as a canonicalization for WITHIN GROUPs 8049 args[0] = self._parse_limit(this=self._parse_order(this=args[0])) 8050 return self.expression(exp.GroupConcat(this=args[0], separator=seq_get(args, 1))) 8051 8052 # Checks if we can parse an order clause: WITHIN GROUP (ORDER BY <order_by_expression_list> [ASC | DESC]). 8053 # This is done "manually", instead of letting _parse_window parse it into an exp.WithinGroup node, so that 8054 # the STRING_AGG call is parsed like in MySQL / SQLite and can thus be transpiled more easily to them. 8055 if not self._match_text_seq("WITHIN", "GROUP"): 8056 self._retreat(index) 8057 return self.validate_expression(exp.GroupConcat.from_arg_list(args), args) 8058 8059 # The corresponding match_r_paren will be called in parse_function (caller) 8060 self._match_l_paren() 8061 8062 return self.expression( 8063 exp.GroupConcat( 8064 this=self._parse_order(this=seq_get(args, 0)), 8065 separator=seq_get(args, 1), 8066 on_overflow=on_overflow, 8067 ) 8068 ) 8069 8070 def _parse_convert(self, strict: bool, safe: bool | None = None) -> exp.Expr | None: 8071 this = self._parse_bitwise() 8072 8073 if self._match(TokenType.USING): 8074 to: exp.Expr | None = exp.DType.CHARACTER_SET.into_expr(kind=self._parse_charset_name()) 8075 elif self._match(TokenType.COMMA): 8076 to = self._parse_types() 8077 else: 8078 to = None 8079 8080 return self.build_cast(strict=strict, this=this, to=to, safe=safe) 8081 8082 def _parse_xml_element(self) -> exp.XMLElement: 8083 if self._match_text_seq("EVALNAME"): 8084 evalname = True 8085 this = self._parse_bitwise() 8086 else: 8087 evalname = None 8088 self._match_text_seq("NAME") 8089 this = self._parse_id_var() 8090 8091 return self.expression( 8092 exp.XMLElement( 8093 this=this, 8094 expressions=self._match(TokenType.COMMA) and self._parse_csv(self._parse_bitwise), 8095 evalname=evalname, 8096 ) 8097 ) 8098 8099 def _parse_xml_table(self) -> exp.XMLTable: 8100 namespaces = None 8101 passing = None 8102 columns = None 8103 8104 if self._match_text_seq("XMLNAMESPACES", "("): 8105 namespaces = self._parse_xml_namespace() 8106 self._match_text_seq(")", ",") 8107 8108 this = self._parse_string() 8109 8110 if self._match_text_seq("PASSING"): 8111 # The BY VALUE keywords are optional and are provided for semantic clarity 8112 self._match_text_seq("BY", "VALUE") 8113 passing = self._parse_csv(self._parse_column) 8114 8115 by_ref = self._match_text_seq("RETURNING", "SEQUENCE", "BY", "REF") 8116 8117 if self._match_text_seq("COLUMNS"): 8118 columns = self._parse_csv(self._parse_field_def) 8119 8120 return self.expression( 8121 exp.XMLTable( 8122 this=this, namespaces=namespaces, passing=passing, columns=columns, by_ref=by_ref 8123 ) 8124 ) 8125 8126 def _parse_xml_namespace(self) -> list[exp.XMLNamespace]: 8127 namespaces = [] 8128 8129 while True: 8130 if self._match(TokenType.DEFAULT): 8131 uri = self._parse_string() 8132 else: 8133 uri = self._parse_alias(self._parse_string()) 8134 namespaces.append(self.expression(exp.XMLNamespace(this=uri))) 8135 if not self._match(TokenType.COMMA): 8136 break 8137 8138 return namespaces 8139 8140 def _parse_decode(self) -> exp.Decode | exp.DecodeCase | None: 8141 args = self._parse_csv(self._parse_disjunction) 8142 8143 if len(args) < 3: 8144 return self.expression(exp.Decode(this=seq_get(args, 0), charset=seq_get(args, 1))) 8145 8146 return self.expression(exp.DecodeCase(expressions=args)) 8147 8148 def _parse_json_key_value(self) -> exp.JSONKeyValue | None: 8149 self._match_text_seq("KEY") 8150 key = self._parse_column() 8151 self._match_set(self.JSON_KEY_VALUE_SEPARATOR_TOKENS) 8152 self._match_text_seq("VALUE") 8153 value = self._parse_bitwise() 8154 8155 if not key and not value: 8156 return None 8157 return self.expression(exp.JSONKeyValue(this=key, expression=value)) 8158 8159 def _parse_format_json(self, this: exp.Expr | None) -> exp.Expr | None: 8160 if not this or not self._match_text_seq("FORMAT", "JSON"): 8161 return this 8162 8163 return self.expression(exp.FormatJson(this=this)) 8164 8165 def _parse_on_condition(self) -> exp.OnCondition | None: 8166 # MySQL uses "X ON EMPTY Y ON ERROR" (e.g. JSON_VALUE) while Oracle uses the opposite (e.g. JSON_EXISTS) 8167 if self.dialect.ON_CONDITION_EMPTY_BEFORE_ERROR: 8168 empty = self._parse_on_handling("EMPTY", *self.ON_CONDITION_TOKENS) 8169 error = self._parse_on_handling("ERROR", *self.ON_CONDITION_TOKENS) 8170 else: 8171 error = self._parse_on_handling("ERROR", *self.ON_CONDITION_TOKENS) 8172 empty = self._parse_on_handling("EMPTY", *self.ON_CONDITION_TOKENS) 8173 8174 null = self._parse_on_handling("NULL", *self.ON_CONDITION_TOKENS) 8175 8176 if not empty and not error and not null: 8177 return None 8178 8179 return self.expression(exp.OnCondition(empty=empty, error=error, null=null)) 8180 8181 def _parse_on_handling(self, on: str, *values: str) -> str | None | exp.Expr | None: 8182 # Parses the "X ON Y" or "DEFAULT <expr> ON Y syntax, e.g. NULL ON NULL (Oracle, T-SQL, MySQL) 8183 for value in values: 8184 if self._match_text_seq(value, "ON", on): 8185 return f"{value} ON {on}" 8186 8187 index = self._index 8188 if self._match(TokenType.DEFAULT): 8189 default_value = self._parse_bitwise() 8190 if self._match_text_seq("ON", on): 8191 return default_value 8192 8193 self._retreat(index) 8194 8195 return None 8196 8197 @t.overload 8198 def _parse_json_object(self, agg: t.Literal[False]) -> exp.JSONObject: ... 8199 8200 @t.overload 8201 def _parse_json_object(self, agg: t.Literal[True]) -> exp.JSONObjectAgg: ... 8202 8203 def _parse_json_object(self, agg=False): 8204 star = self._parse_star() 8205 expressions = ( 8206 [star] 8207 if star 8208 else self._parse_csv(lambda: self._parse_format_json(self._parse_json_key_value())) 8209 ) 8210 null_handling = self._parse_on_handling("NULL", "NULL", "ABSENT") 8211 8212 unique_keys = None 8213 if self._match_text_seq("WITH", "UNIQUE"): 8214 unique_keys = True 8215 elif self._match_text_seq("WITHOUT", "UNIQUE"): 8216 unique_keys = False 8217 8218 self._match_text_seq("KEYS") 8219 8220 return_type = self._match_text_seq("RETURNING") and self._parse_format_json( 8221 self._parse_type() 8222 ) 8223 encoding = self._match_text_seq("ENCODING") and self._parse_var() 8224 8225 return self.expression( 8226 (exp.JSONObjectAgg if agg else exp.JSONObject)( 8227 expressions=expressions, 8228 null_handling=null_handling, 8229 unique_keys=unique_keys, 8230 return_type=return_type, 8231 encoding=encoding, 8232 ) 8233 ) 8234 8235 # Note: this is currently incomplete; it only implements the "JSON_value_column" part 8236 def _parse_json_column_def(self) -> exp.JSONColumnDef: 8237 if not self._match_text_seq("NESTED"): 8238 this = self._parse_id_var() 8239 ordinality = self._match_pair(TokenType.FOR, TokenType.ORDINALITY) 8240 kind = self._parse_types(allow_identifiers=False) 8241 nested = None 8242 else: 8243 this = None 8244 ordinality = None 8245 kind = None 8246 nested = True 8247 8248 format_json = self._match_text_seq("FORMAT", "JSON") 8249 path = self._match_text_seq("PATH") and self._parse_string() 8250 nested_schema = nested and self._parse_json_schema() 8251 8252 return self.expression( 8253 exp.JSONColumnDef( 8254 this=this, 8255 kind=kind, 8256 path=path, 8257 nested_schema=nested_schema, 8258 ordinality=ordinality, 8259 format_json=format_json, 8260 ) 8261 ) 8262 8263 def _parse_json_schema(self) -> exp.JSONSchema: 8264 self._match_text_seq("COLUMNS") 8265 return self.expression( 8266 exp.JSONSchema( 8267 expressions=self._parse_wrapped_csv(self._parse_json_column_def, optional=True) 8268 ) 8269 ) 8270 8271 def _parse_json_table(self) -> exp.JSONTable: 8272 this = self._parse_format_json(self._parse_bitwise()) 8273 path = self._match(TokenType.COMMA) and self._parse_string() 8274 error_handling = self._parse_on_handling("ERROR", "ERROR", "NULL") 8275 empty_handling = self._parse_on_handling("EMPTY", "ERROR", "NULL") 8276 schema = self._parse_json_schema() 8277 8278 return exp.JSONTable( 8279 this=this, 8280 schema=schema, 8281 path=path, 8282 error_handling=error_handling, 8283 empty_handling=empty_handling, 8284 ) 8285 8286 def _parse_match_against(self) -> exp.MatchAgainst: 8287 if self._match_text_seq("TABLE"): 8288 # parse SingleStore MATCH(TABLE ...) syntax 8289 # https://docs.singlestore.com/cloud/reference/sql-reference/full-text-search-functions/match/ 8290 expressions = [] 8291 table = self._parse_table() 8292 if table: 8293 expressions = [table] 8294 else: 8295 expressions = self._parse_csv(self._parse_column) 8296 8297 self._match_text_seq(")", "AGAINST", "(") 8298 8299 this = self._parse_string() 8300 8301 if self._match_text_seq("IN", "NATURAL", "LANGUAGE", "MODE"): 8302 modifier = "IN NATURAL LANGUAGE MODE" 8303 if self._match_text_seq("WITH", "QUERY", "EXPANSION"): 8304 modifier = f"{modifier} WITH QUERY EXPANSION" 8305 elif self._match_text_seq("IN", "BOOLEAN", "MODE"): 8306 modifier = "IN BOOLEAN MODE" 8307 elif self._match_text_seq("WITH", "QUERY", "EXPANSION"): 8308 modifier = "WITH QUERY EXPANSION" 8309 else: 8310 modifier = None 8311 8312 return self.expression( 8313 exp.MatchAgainst(this=this, expressions=expressions, modifier=modifier) 8314 ) 8315 8316 # https://learn.microsoft.com/en-us/sql/t-sql/functions/openjson-transact-sql?view=sql-server-ver16 8317 def _parse_open_json(self) -> exp.OpenJSON: 8318 this = self._parse_bitwise() 8319 path = self._match(TokenType.COMMA) and self._parse_string() 8320 8321 def _parse_open_json_column_def() -> exp.OpenJSONColumnDef: 8322 this = self._parse_field(any_token=True) 8323 kind = self._parse_types() 8324 path = self._parse_string() 8325 as_json = self._match_pair(TokenType.ALIAS, TokenType.JSON) 8326 8327 return self.expression( 8328 exp.OpenJSONColumnDef(this=this, kind=kind, path=path, as_json=as_json) 8329 ) 8330 8331 expressions = None 8332 if self._match_pair(TokenType.R_PAREN, TokenType.WITH): 8333 self._match_l_paren() 8334 expressions = self._parse_csv(_parse_open_json_column_def) 8335 8336 return self.expression(exp.OpenJSON(this=this, path=path, expressions=expressions)) 8337 8338 def _parse_position(self, haystack_first: bool = False) -> exp.StrPosition: 8339 args = self._parse_csv(self._parse_bitwise) 8340 8341 if self._match(TokenType.IN): 8342 return self.expression( 8343 exp.StrPosition(this=self._parse_bitwise(), substr=seq_get(args, 0)) 8344 ) 8345 8346 if haystack_first: 8347 haystack = seq_get(args, 0) 8348 needle = seq_get(args, 1) 8349 else: 8350 haystack = seq_get(args, 1) 8351 needle = seq_get(args, 0) 8352 8353 return self.expression( 8354 exp.StrPosition(this=haystack, substr=needle, position=seq_get(args, 2)) 8355 ) 8356 8357 def _parse_join_hint(self, func_name: str) -> exp.JoinHint: 8358 args = self._parse_csv(self._parse_table) 8359 return exp.JoinHint(this=func_name.upper(), expressions=args) 8360 8361 def _parse_substring(self) -> exp.Substring: 8362 # Postgres supports the form: substring(string [from int] [for int]) 8363 # (despite being undocumented, the reverse order also works) 8364 # https://www.postgresql.org/docs/9.1/functions-string.html @ Table 9-6 8365 8366 args = t.cast(list[t.Optional[exp.Expr]], self._parse_csv(self._parse_bitwise)) 8367 8368 start, length = None, None 8369 8370 while self._curr: 8371 if self._match(TokenType.FROM): 8372 start = self._parse_bitwise() 8373 elif self._match(TokenType.FOR): 8374 if not start: 8375 start = exp.Literal.number(1) 8376 length = self._parse_bitwise() 8377 else: 8378 break 8379 8380 if start: 8381 args.append(start) 8382 if length: 8383 args.append(length) 8384 8385 return self.validate_expression(exp.Substring.from_arg_list(args), args) 8386 8387 def _parse_trim(self) -> exp.Trim: 8388 # https://www.w3resource.com/sql/character-functions/trim.php 8389 # https://docs.oracle.com/javadb/10.8.3.0/ref/rreftrimfunc.html 8390 8391 position = None 8392 collation = None 8393 expression = None 8394 8395 if self._match_texts(self.TRIM_TYPES): 8396 position = self._prev.text.upper() 8397 8398 this = self._parse_bitwise() 8399 if self._match_set((TokenType.FROM, TokenType.COMMA)): 8400 invert_order = self._prev.token_type == TokenType.FROM or self.TRIM_PATTERN_FIRST 8401 expression = self._parse_bitwise() 8402 8403 if invert_order: 8404 this, expression = expression, this 8405 8406 if self._match(TokenType.COLLATE): 8407 collation = self._parse_bitwise() 8408 8409 return self.expression( 8410 exp.Trim(this=this, position=position, expression=expression, collation=collation) 8411 ) 8412 8413 def _parse_window_clause(self) -> list[exp.Expr] | None: 8414 return self._parse_csv(self._parse_named_window) if self._match(TokenType.WINDOW) else None 8415 8416 def _parse_named_window(self) -> exp.Expr | None: 8417 return self._parse_window(self._parse_id_var(), alias=True) 8418 8419 def _parse_respect_or_ignore_nulls(self, this: exp.Expr | None) -> exp.Expr | None: 8420 if self._curr.token_type == TokenType.VAR: 8421 if self._match_text_seq("IGNORE", "NULLS"): 8422 return self.expression(exp.IgnoreNulls(this=this)) 8423 if self._match_text_seq("RESPECT", "NULLS"): 8424 return self.expression(exp.RespectNulls(this=this)) 8425 return this 8426 8427 def _parse_having_max(self, this: exp.Expr | None) -> exp.Expr | None: 8428 if self._match(TokenType.HAVING): 8429 self._match_texts(("MAX", "MIN")) 8430 max = self._prev.text.upper() != "MIN" 8431 return self.expression( 8432 exp.HavingMax(this=this, expression=self._parse_column(), max=max) 8433 ) 8434 8435 return this 8436 8437 def _parse_window(self, this: exp.Expr | None, alias: bool = False) -> exp.Expr | None: 8438 func = this 8439 comments = func.comments if isinstance(func, exp.Expr) else None 8440 8441 # T-SQL allows the OVER (...) syntax after WITHIN GROUP. 8442 # https://learn.microsoft.com/en-us/sql/t-sql/functions/percentile-disc-transact-sql?view=sql-server-ver16 8443 if self._match_text_seq("WITHIN", "GROUP"): 8444 order = self._parse_wrapped(self._parse_order) 8445 this = self.expression(exp.WithinGroup(this=this, expression=order)) 8446 8447 if self._match_pair(TokenType.FILTER, TokenType.L_PAREN): 8448 self._match(TokenType.WHERE) 8449 this = self.expression( 8450 exp.Filter(this=this, expression=self._parse_where(skip_where_token=True)) 8451 ) 8452 self._match_r_paren() 8453 8454 # SQL spec defines an optional [ { IGNORE | RESPECT } NULLS ] OVER 8455 # Some dialects choose to implement and some do not. 8456 # https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html 8457 8458 # There is some code above in _parse_lambda that handles 8459 # SELECT FIRST_VALUE(TABLE.COLUMN IGNORE|RESPECT NULLS) OVER ... 8460 8461 # The below changes handle 8462 # SELECT FIRST_VALUE(TABLE.COLUMN) IGNORE|RESPECT NULLS OVER ... 8463 8464 # Oracle allows both formats 8465 # (https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/img_text/first_value.html) 8466 # and Snowflake chose to do the same for familiarity 8467 # https://docs.snowflake.com/en/sql-reference/functions/first_value.html#usage-notes 8468 if isinstance(this, exp.AggFunc): 8469 ignore_respect = this.find(exp.IgnoreNulls, exp.RespectNulls) 8470 8471 if ignore_respect and ignore_respect is not this: 8472 ignore_respect.replace(ignore_respect.this) 8473 this = self.expression(ignore_respect.__class__(this=this)) 8474 8475 this = self._parse_respect_or_ignore_nulls(this) 8476 8477 # bigquery select from window x AS (partition by ...) 8478 if alias: 8479 over = None 8480 self._match(TokenType.ALIAS) 8481 elif not self._match_set(self.WINDOW_BEFORE_PAREN_TOKENS): 8482 return this 8483 else: 8484 over = self._prev.text.upper() 8485 8486 if comments and isinstance(func, exp.Expr): 8487 func.pop_comments() 8488 8489 if not self._match(TokenType.L_PAREN): 8490 return self.expression( 8491 exp.Window(this=this, alias=self._parse_id_var(False), over=over), comments=comments 8492 ) 8493 8494 window_alias = self._parse_id_var(any_token=False, tokens=self.WINDOW_ALIAS_TOKENS) 8495 8496 first: bool | None = True if self._match(TokenType.FIRST) else None 8497 if self._match_text_seq("LAST"): 8498 first = False 8499 8500 partition, order = self._parse_partition_and_order() 8501 kind = ( 8502 self._match_set((TokenType.ROWS, TokenType.RANGE)) or self._match_text_seq("GROUPS") 8503 ) and self._prev.text 8504 8505 if kind: 8506 self._match(TokenType.BETWEEN) 8507 start = self._parse_window_spec() 8508 8509 end = self._parse_window_spec() if self._match(TokenType.AND) else {} 8510 exclude = ( 8511 self._parse_var_from_options(self.WINDOW_EXCLUDE_OPTIONS) 8512 if self._match_text_seq("EXCLUDE") 8513 else None 8514 ) 8515 8516 spec = self.expression( 8517 exp.WindowSpec( 8518 kind=kind, 8519 start=start["value"], 8520 start_side=start["side"], 8521 end=end.get("value"), 8522 end_side=end.get("side"), 8523 exclude=exclude, 8524 ) 8525 ) 8526 else: 8527 spec = None 8528 8529 self._match_r_paren() 8530 8531 window = self.expression( 8532 exp.Window( 8533 this=this, 8534 partition_by=partition, 8535 order=order, 8536 spec=spec, 8537 alias=window_alias, 8538 over=over, 8539 first=first, 8540 ), 8541 comments=comments, 8542 ) 8543 8544 # This covers Oracle's FIRST/LAST syntax: aggregate KEEP (...) OVER (...) 8545 if self._match_set(self.WINDOW_BEFORE_PAREN_TOKENS, advance=False): 8546 return self._parse_window(window, alias=alias) 8547 8548 return window 8549 8550 def _parse_partition_and_order( 8551 self, 8552 ) -> tuple[list[exp.Expr], exp.Expr | None]: 8553 return self._parse_partition_by(), self._parse_order() 8554 8555 def _parse_window_spec(self) -> dict[str, str | exp.Expr | None]: 8556 self._match(TokenType.BETWEEN) 8557 8558 return { 8559 "value": ( 8560 (self._match_text_seq("UNBOUNDED") and "UNBOUNDED") 8561 or (self._match_text_seq("CURRENT", "ROW") and "CURRENT ROW") 8562 or self._parse_bitwise() 8563 ), 8564 "side": self._prev.text if self._match_texts(self.WINDOW_SIDES) else None, 8565 } 8566 8567 def _parse_alias(self, this: exp.Expr | None, explicit: bool = False) -> exp.Expr | None: 8568 # In some dialects, LIMIT and OFFSET can act as both identifiers and keywords (clauses) 8569 # so this section tries to parse the clause version and if it fails, it treats the token 8570 # as an identifier (alias) 8571 if self._can_parse_limit_or_offset(): 8572 return this 8573 8574 # WINDOW is in ID_VAR_TOKENS, so it can be consumed as an implicit alias. Detect the 8575 # named-window clause shape (`WINDOW <ident> AS (...)`) and avoid swallowing it. 8576 if self._can_parse_named_window(): 8577 return this 8578 8579 any_token = self._match(TokenType.ALIAS) 8580 comments = self._prev_comments 8581 8582 if explicit and not any_token: 8583 return this 8584 8585 if self._match(TokenType.L_PAREN): 8586 aliases = self.expression( 8587 exp.Aliases( 8588 this=this, expressions=self._parse_csv(lambda: self._parse_id_var(any_token)) 8589 ), 8590 comments=comments, 8591 ) 8592 self._match_r_paren(aliases) 8593 return aliases 8594 8595 alias = self._parse_id_var(any_token, tokens=self.ALIAS_TOKENS) or ( 8596 self.STRING_ALIASES and self._parse_string_as_identifier() 8597 ) 8598 8599 if alias: 8600 comments.extend(alias.pop_comments()) 8601 this = self.expression(exp.Alias(this=this, alias=alias), comments=comments) 8602 column = this.this 8603 8604 # Moves the comment next to the alias in `expr /* comment */ AS alias` 8605 if not this.comments and column and column.comments: 8606 this.comments = column.pop_comments() 8607 8608 return this 8609 8610 def _parse_id_var( 8611 self, 8612 any_token: bool = True, 8613 tokens: t.Collection[TokenType] | None = None, 8614 ) -> exp.Expr | None: 8615 expression = self._parse_identifier() 8616 if not expression and ( 8617 (any_token and self._advance_any()) or self._match_set(tokens or self.ID_VAR_TOKENS) 8618 ): 8619 quoted = self._prev.token_type == TokenType.STRING 8620 expression = self._identifier_expression(quoted=quoted) 8621 8622 return expression 8623 8624 def _parse_string(self) -> exp.Expr | None: 8625 if self._match_set(self.STRING_PARSERS): 8626 return self.STRING_PARSERS[self._prev.token_type](self, self._prev) 8627 return self._parse_placeholder() 8628 8629 def _parse_string_as_identifier(self) -> exp.Identifier | None: 8630 if not self._match(TokenType.STRING): 8631 return None 8632 output = exp.to_identifier(self._prev.text, quoted=True) 8633 output.update_positions(self._prev) 8634 return output 8635 8636 def _parse_number(self) -> exp.Expr | None: 8637 if self._match_set(self.NUMERIC_PARSERS): 8638 return self.NUMERIC_PARSERS[self._prev.token_type](self, self._prev) 8639 return self._parse_placeholder() 8640 8641 def _parse_identifier(self) -> exp.Expr | None: 8642 if self._match(TokenType.IDENTIFIER): 8643 return self._identifier_expression(quoted=True) 8644 return self._parse_placeholder() 8645 8646 def _parse_var( 8647 self, 8648 any_token: bool = False, 8649 tokens: t.Collection[TokenType] | None = None, 8650 upper: bool = False, 8651 ) -> exp.Expr | None: 8652 if ( 8653 (any_token and self._advance_any()) 8654 or self._match(TokenType.VAR) 8655 or (self._match_set(tokens) if tokens else False) 8656 ): 8657 return self.expression( 8658 exp.Var(this=self._prev.text.upper() if upper else self._prev.text) 8659 ) 8660 return self._parse_placeholder() 8661 8662 def _advance_any(self, ignore_reserved: bool = False) -> Token | None: 8663 if self._curr and (ignore_reserved or self._curr.token_type not in self.RESERVED_TOKENS): 8664 self._advance() 8665 return self._prev 8666 return None 8667 8668 def _parse_var_or_string(self, upper: bool = False) -> exp.Expr | None: 8669 return self._parse_string() or self._parse_var(any_token=True, upper=upper) 8670 8671 def _parse_primary_or_var(self) -> exp.Expr | None: 8672 return self._parse_primary() or self._parse_var(any_token=True) 8673 8674 def _parse_null(self) -> exp.Expr | None: 8675 if self._match_set((TokenType.NULL, TokenType.UNKNOWN)): 8676 return self.PRIMARY_PARSERS[TokenType.NULL](self, self._prev) 8677 return self._parse_placeholder() 8678 8679 def _parse_boolean(self) -> exp.Expr | None: 8680 if self._match(TokenType.TRUE): 8681 return self.PRIMARY_PARSERS[TokenType.TRUE](self, self._prev) 8682 if self._match(TokenType.FALSE): 8683 return self.PRIMARY_PARSERS[TokenType.FALSE](self, self._prev) 8684 return self._parse_placeholder() 8685 8686 def _parse_star(self) -> exp.Expr | None: 8687 if self._match(TokenType.STAR): 8688 return self.PRIMARY_PARSERS[TokenType.STAR](self, self._prev) 8689 return self._parse_placeholder() 8690 8691 def _parse_parameter(self) -> exp.Parameter: 8692 this = self._parse_identifier() or self._parse_primary_or_var() 8693 return self.expression(exp.Parameter(this=this)) 8694 8695 def _parse_placeholder(self) -> exp.Expr | None: 8696 if self._match_set(self.PLACEHOLDER_PARSERS): 8697 placeholder = self.PLACEHOLDER_PARSERS[self._prev.token_type](self) 8698 if placeholder: 8699 return placeholder 8700 self._advance(-1) 8701 return None 8702 8703 def _parse_star_op(self, *keywords: str) -> list[exp.Expr] | None: 8704 if not self._match_texts(keywords): 8705 return None 8706 if self._match(TokenType.L_PAREN, advance=False): 8707 return self._parse_wrapped_csv(self._parse_expression) 8708 8709 expression = self._parse_alias(self._parse_disjunction(), explicit=True) 8710 return [expression] if expression else None 8711 8712 def _parse_csv( 8713 self, parse_method: t.Callable[[], T | None], sep: TokenType = TokenType.COMMA 8714 ) -> list[T]: 8715 parse_result = parse_method() 8716 items = [parse_result] if parse_result is not None else [] 8717 8718 while self._match(sep): 8719 if isinstance(parse_result, exp.Expr): 8720 self._add_comments(parse_result) 8721 parse_result = parse_method() 8722 if parse_result is not None: 8723 items.append(parse_result) 8724 8725 return items 8726 8727 def _parse_wrapped_id_vars(self, optional: bool = False) -> list[exp.Expr]: 8728 return self._parse_wrapped_csv(self._parse_id_var, optional=optional) 8729 8730 def _parse_wrapped_csv( 8731 self, 8732 parse_method: t.Callable[[], T | None], 8733 sep: TokenType = TokenType.COMMA, 8734 optional: bool = False, 8735 ) -> list[T]: 8736 return self._parse_wrapped( 8737 lambda: self._parse_csv(parse_method, sep=sep), optional=optional 8738 ) 8739 8740 def _parse_wrapped(self, parse_method: t.Callable[[], T], optional: bool = False) -> T: 8741 wrapped = self._match(TokenType.L_PAREN) 8742 if not wrapped and not optional: 8743 self.raise_error("Expecting (") 8744 parse_result = parse_method() 8745 if wrapped: 8746 self._match_r_paren() 8747 return parse_result 8748 8749 def _parse_expressions(self) -> list[exp.Expr]: 8750 return self._parse_csv(self._parse_expression) 8751 8752 def _parse_select_or_expression(self, alias: bool = False) -> exp.Expr | None: 8753 return ( 8754 self._parse_set_operations( 8755 self._parse_alias(self._parse_assignment(), explicit=True) 8756 if alias 8757 else self._parse_assignment() 8758 ) 8759 or self._parse_select() 8760 ) 8761 8762 def _parse_ddl_select(self) -> exp.Expr | None: 8763 return self._parse_query_modifiers( 8764 self._parse_set_operations(self._parse_select(nested=True, parse_subquery_alias=False)) 8765 ) 8766 8767 def _parse_transaction(self) -> exp.Transaction | exp.Command: 8768 this = None 8769 if self._match_texts(self.TRANSACTION_KIND): 8770 this = self._prev.text 8771 8772 self._match_texts(("TRANSACTION", "WORK")) 8773 8774 modes = [] 8775 while True: 8776 mode = [] 8777 while self._match(TokenType.VAR) or self._match(TokenType.NOT): 8778 mode.append(self._prev.text) 8779 8780 if mode: 8781 modes.append(" ".join(mode)) 8782 if not self._match(TokenType.COMMA): 8783 break 8784 8785 return self.expression(exp.Transaction(this=this, modes=modes)) 8786 8787 def _parse_commit_or_rollback(self) -> exp.Commit | exp.Rollback: 8788 chain = None 8789 savepoint = None 8790 is_rollback = self._prev.token_type == TokenType.ROLLBACK 8791 8792 self._match_texts(("TRANSACTION", "WORK")) 8793 8794 if self._match_text_seq("TO"): 8795 self._match_text_seq("SAVEPOINT") 8796 savepoint = self._parse_id_var() 8797 8798 if self._match(TokenType.AND): 8799 chain = not self._match_text_seq("NO") 8800 self._match_text_seq("CHAIN") 8801 8802 if is_rollback: 8803 return self.expression(exp.Rollback(savepoint=savepoint)) 8804 8805 return self.expression(exp.Commit(chain=chain)) 8806 8807 def _parse_refresh(self) -> exp.Refresh | exp.Command: 8808 if self._match(TokenType.TABLE): 8809 kind = "TABLE" 8810 elif self._match_text_seq("MATERIALIZED", "VIEW"): 8811 kind = "MATERIALIZED VIEW" 8812 else: 8813 kind = "" 8814 8815 this = self._parse_string() or self._parse_table() 8816 if not kind and not isinstance(this, exp.Literal): 8817 return self._parse_as_command(self._prev) 8818 8819 return self.expression(exp.Refresh(this=this, kind=kind)) 8820 8821 def _parse_column_def_with_exists(self): 8822 start = self._index 8823 self._match(TokenType.COLUMN) 8824 8825 exists_column = self._parse_exists(not_=True) 8826 expression = self._parse_field_def() 8827 8828 if not isinstance(expression, exp.ColumnDef): 8829 self._retreat(start) 8830 return None 8831 8832 expression.set("exists", exists_column) 8833 8834 return expression 8835 8836 def _parse_add_column(self) -> exp.ColumnDef | None: 8837 if not self._prev.text.upper() == "ADD": 8838 return None 8839 8840 return self._parse_column_def_with_exists() 8841 8842 def _parse_drop_column(self) -> exp.Drop | exp.Command | None: 8843 drop = self._parse_drop() if self._match(TokenType.DROP) else None 8844 if drop and not isinstance(drop, exp.Command): 8845 drop.set("kind", drop.args.get("kind", "COLUMN")) 8846 return drop 8847 8848 def _parse_alter_drop_action(self) -> exp.Expr | None: 8849 return self._parse_drop_column() 8850 8851 # https://docs.aws.amazon.com/athena/latest/ug/alter-table-drop-partition.html 8852 def _parse_drop_partition(self, exists: bool | None = None) -> exp.DropPartition: 8853 return self.expression( 8854 exp.DropPartition(expressions=self._parse_csv(self._parse_partition), exists=exists) 8855 ) 8856 8857 def _parse_alter_table_add(self) -> list[exp.Expr]: 8858 def _parse_add_alteration() -> exp.Expr | None: 8859 self._match_text_seq("ADD") 8860 if self._match_set(self.ADD_CONSTRAINT_TOKENS, advance=False): 8861 return self.expression( 8862 exp.AddConstraint(expressions=self._parse_csv(self._parse_constraint)) 8863 ) 8864 8865 column_def = self._parse_add_column() 8866 if isinstance(column_def, exp.ColumnDef): 8867 return column_def 8868 8869 exists = self._parse_exists(not_=True) 8870 if self._match_pair(TokenType.PARTITION, TokenType.L_PAREN, advance=False): 8871 return self.expression( 8872 exp.AddPartition( 8873 exists=exists, 8874 this=self._parse_field(any_token=True), 8875 location=self._match_text_seq("LOCATION", advance=False) 8876 and self._parse_property(), 8877 ) 8878 ) 8879 8880 return None 8881 8882 if not self._match_set(self.ADD_CONSTRAINT_TOKENS, advance=False) and ( 8883 not self.dialect.ALTER_TABLE_ADD_REQUIRED_FOR_EACH_COLUMN 8884 or self._match_text_seq("COLUMNS") 8885 ): 8886 schema = self._parse_schema() 8887 8888 return ( 8889 ensure_list(schema) 8890 if schema 8891 else self._parse_csv(self._parse_column_def_with_exists) 8892 ) 8893 8894 return self._parse_csv(_parse_add_alteration) 8895 8896 def _parse_alter_table_alter(self) -> exp.Expr | None: 8897 if self._match_texts(self.ALTER_ALTER_PARSERS): 8898 return self.ALTER_ALTER_PARSERS[self._prev.text.upper()](self) 8899 8900 # Many dialects support the ALTER [COLUMN] syntax, so if there is no 8901 # keyword after ALTER we default to parsing this statement 8902 self._match(TokenType.COLUMN) 8903 column = self._parse_field(any_token=True) 8904 8905 if self._match_pair(TokenType.DROP, TokenType.DEFAULT): 8906 return self.expression(exp.AlterColumn(this=column, drop=True)) 8907 if self._match_pair(TokenType.SET, TokenType.DEFAULT): 8908 return self.expression(exp.AlterColumn(this=column, default=self._parse_disjunction())) 8909 if self._match(TokenType.COMMENT): 8910 return self.expression(exp.AlterColumn(this=column, comment=self._parse_string())) 8911 if self._match_text_seq("DROP", "NOT", "NULL"): 8912 return self.expression(exp.AlterColumn(this=column, drop=True, allow_null=True)) 8913 if self._match_text_seq("SET", "NOT", "NULL"): 8914 return self.expression(exp.AlterColumn(this=column, allow_null=False)) 8915 8916 if self._match_text_seq("SET", "VISIBLE"): 8917 return self.expression(exp.AlterColumn(this=column, visible="VISIBLE")) 8918 if self._match_text_seq("SET", "INVISIBLE"): 8919 return self.expression(exp.AlterColumn(this=column, visible="INVISIBLE")) 8920 8921 self._match_text_seq("SET", "DATA") 8922 self._match_text_seq("TYPE") 8923 return self.expression( 8924 exp.AlterColumn( 8925 this=column, 8926 dtype=self._parse_types(), 8927 collate=self._match(TokenType.COLLATE) and self._parse_term(), 8928 using=self._match(TokenType.USING) and self._parse_disjunction(), 8929 ) 8930 ) 8931 8932 def _parse_alter_diststyle(self) -> exp.AlterDistStyle: 8933 if self._match_texts(("ALL", "EVEN", "AUTO")): 8934 return self.expression(exp.AlterDistStyle(this=exp.var(self._prev.text.upper()))) 8935 8936 self._match_text_seq("KEY", "DISTKEY") 8937 return self.expression(exp.AlterDistStyle(this=self._parse_column())) 8938 8939 def _parse_alter_sortkey(self, compound: bool | None = None) -> exp.AlterSortKey: 8940 if compound: 8941 self._match_text_seq("SORTKEY") 8942 8943 if self._match(TokenType.L_PAREN, advance=False): 8944 return self.expression( 8945 exp.AlterSortKey(expressions=self._parse_wrapped_id_vars(), compound=compound) 8946 ) 8947 8948 self._match_texts(("AUTO", "NONE")) 8949 return self.expression( 8950 exp.AlterSortKey(this=exp.var(self._prev.text.upper()), compound=compound) 8951 ) 8952 8953 def _parse_alter_table_drop(self) -> list[exp.Expr]: 8954 index = self._index - 1 8955 8956 partition_exists = self._parse_exists() 8957 if self._match(TokenType.PARTITION, advance=False): 8958 return self._parse_csv(lambda: self._parse_drop_partition(exists=partition_exists)) 8959 8960 self._retreat(index) 8961 return self._parse_csv(self._parse_alter_drop_action) 8962 8963 def _parse_alter_table_rename(self) -> exp.AlterRename | exp.RenameColumn | None: 8964 if self._match(TokenType.COLUMN) or ( 8965 not self.ALTER_RENAME_REQUIRES_COLUMN and not self._match_text_seq("TO", advance=False) 8966 ): 8967 exists = self._parse_exists() 8968 old_column = self._parse_column() 8969 to = self._match_text_seq("TO") 8970 new_column = self._parse_column() 8971 8972 if old_column is None or not to or new_column is None: 8973 return None 8974 8975 return self.expression(exp.RenameColumn(this=old_column, to=new_column, exists=exists)) 8976 8977 self._match_text_seq("TO") 8978 return self.expression(exp.AlterRename(this=self._parse_table(schema=True))) 8979 8980 def _parse_alter_table_set(self) -> exp.AlterSet: 8981 alter_set = self.expression(exp.AlterSet()) 8982 8983 if self._match(TokenType.L_PAREN, advance=False) or self._match_text_seq( 8984 "TABLE", "PROPERTIES" 8985 ): 8986 alter_set.set("expressions", self._parse_wrapped_csv(self._parse_assignment)) 8987 elif self._match_text_seq("FILESTREAM_ON", advance=False): 8988 alter_set.set("expressions", [self._parse_assignment()]) 8989 elif self._match_texts(("LOGGED", "UNLOGGED")): 8990 alter_set.set("option", exp.var(self._prev.text.upper())) 8991 elif self._match_text_seq("WITHOUT") and self._match_texts(("CLUSTER", "OIDS")): 8992 alter_set.set("option", exp.var(f"WITHOUT {self._prev.text.upper()}")) 8993 elif self._match_text_seq("LOCATION"): 8994 alter_set.set("location", self._parse_field()) 8995 elif self._match_text_seq("ACCESS", "METHOD"): 8996 alter_set.set("access_method", self._parse_field()) 8997 elif self._match_text_seq("TABLESPACE"): 8998 alter_set.set("tablespace", self._parse_field()) 8999 elif self._match_text_seq("FILE", "FORMAT") or self._match_text_seq("FILEFORMAT"): 9000 alter_set.set("file_format", [self._parse_field()]) 9001 elif self._match_text_seq("STAGE_FILE_FORMAT"): 9002 alter_set.set("file_format", self._parse_wrapped_options()) 9003 elif self._match_text_seq("STAGE_COPY_OPTIONS"): 9004 alter_set.set("copy_options", self._parse_wrapped_options()) 9005 elif self._match_text_seq("TAG") or self._match_text_seq("TAGS"): 9006 alter_set.set("tag", self._parse_csv(self._parse_assignment)) 9007 else: 9008 if self._match_text_seq("SERDE"): 9009 alter_set.set("serde", self._parse_field()) 9010 9011 properties = self._parse_wrapped(self._parse_properties, optional=True) 9012 alter_set.set("expressions", [properties]) 9013 9014 return alter_set 9015 9016 def _parse_alter_session(self) -> exp.AlterSession: 9017 """Parse ALTER SESSION SET/UNSET statements.""" 9018 if self._match(TokenType.SET): 9019 expressions = self._parse_csv(lambda: self._parse_set_item_assignment()) 9020 return self.expression(exp.AlterSession(expressions=expressions, unset=False)) 9021 9022 self._match_text_seq("UNSET") 9023 expressions = self._parse_csv( 9024 lambda: self.expression(exp.SetItem(this=self._parse_id_var(any_token=True))) 9025 ) 9026 return self.expression(exp.AlterSession(expressions=expressions, unset=True)) 9027 9028 def _parse_alter(self) -> exp.Alter | exp.Command: 9029 start = self._prev 9030 9031 iceberg = self._match_text_seq("ICEBERG") 9032 9033 alter_token = self._match_set(self.ALTERABLES) and self._prev 9034 if not alter_token: 9035 return self._parse_as_command(start) 9036 if iceberg and alter_token.token_type != TokenType.TABLE: 9037 return self._parse_as_command(start) 9038 9039 exists = self._parse_exists() 9040 only = self._match_text_seq("ONLY") 9041 9042 if alter_token.token_type == TokenType.SESSION: 9043 this = None 9044 check = None 9045 cluster = None 9046 else: 9047 this = self._parse_table(schema=True, parse_partition=self.ALTER_TABLE_PARTITIONS) 9048 check = self._match_text_seq("WITH", "CHECK") 9049 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 9050 9051 if self._next: 9052 self._advance() 9053 9054 parser = self.ALTER_PARSERS.get(self._prev.text.upper()) if self._prev else None 9055 if parser: 9056 actions = ensure_list(parser(self)) 9057 not_valid = self._match_text_seq("NOT", "VALID") 9058 options = self._parse_csv(self._parse_property) 9059 cascade = self.dialect.ALTER_TABLE_SUPPORTS_CASCADE and self._match_text_seq("CASCADE") 9060 9061 if not self._curr and actions: 9062 return self.expression( 9063 exp.Alter( 9064 this=this, 9065 kind=alter_token.text.upper(), 9066 exists=exists, 9067 actions=actions, 9068 only=only, 9069 options=options, 9070 cluster=cluster, 9071 not_valid=not_valid, 9072 check=check, 9073 cascade=cascade, 9074 iceberg=iceberg, 9075 ) 9076 ) 9077 9078 return self._parse_as_command(start) 9079 9080 def _parse_analyze(self) -> exp.Analyze | exp.Command: 9081 start = self._prev 9082 # https://duckdb.org/docs/sql/statements/analyze 9083 if not self._curr: 9084 return self.expression(exp.Analyze()) 9085 9086 options = [] 9087 while self._match_texts(self.ANALYZE_STYLES): 9088 if self._prev.text.upper() == "BUFFER_USAGE_LIMIT": 9089 options.append(f"BUFFER_USAGE_LIMIT {self._parse_number()}") 9090 else: 9091 options.append(self._prev.text.upper()) 9092 9093 this: exp.Expr | None = None 9094 inner_expression: exp.Expr | None = None 9095 9096 kind = self._curr.text.upper() if self._curr else None 9097 9098 if self._match(TokenType.TABLE) or self._match(TokenType.INDEX): 9099 this = self._parse_table_parts() 9100 elif self._match_text_seq("TABLES"): 9101 if self._match_set((TokenType.FROM, TokenType.IN)): 9102 kind = f"{kind} {self._prev.text.upper()}" 9103 this = self._parse_table(schema=True, is_db_reference=True) 9104 elif self._match_text_seq("DATABASE"): 9105 this = self._parse_table(schema=True, is_db_reference=True) 9106 elif self._match_text_seq("CLUSTER"): 9107 this = self._parse_table() 9108 # Try matching inner expr keywords before fallback to parse table. 9109 elif self._match_texts(self.ANALYZE_EXPRESSION_PARSERS): 9110 kind = None 9111 inner_expression = self.ANALYZE_EXPRESSION_PARSERS[self._prev.text.upper()](self) 9112 else: 9113 # Empty kind https://prestodb.io/docs/current/sql/analyze.html 9114 kind = None 9115 this = self._parse_table_parts() 9116 9117 partition = self._try_parse(self._parse_partition) 9118 if not partition and self._match_texts(self.PARTITION_KEYWORDS): 9119 return self._parse_as_command(start) 9120 9121 # https://docs.starrocks.io/docs/sql-reference/sql-statements/cbo_stats/ANALYZE_TABLE/ 9122 if self._match_text_seq("WITH", "SYNC", "MODE") or self._match_text_seq( 9123 "WITH", "ASYNC", "MODE" 9124 ): 9125 mode = f"WITH {self._tokens[self._index - 2].text.upper()} MODE" 9126 else: 9127 mode = None 9128 9129 if self._match_texts(self.ANALYZE_EXPRESSION_PARSERS): 9130 inner_expression = self.ANALYZE_EXPRESSION_PARSERS[self._prev.text.upper()](self) 9131 9132 properties = self._parse_properties() 9133 return self.expression( 9134 exp.Analyze( 9135 kind=kind, 9136 this=this, 9137 mode=mode, 9138 partition=partition, 9139 properties=properties, 9140 expression=inner_expression, 9141 options=options, 9142 ) 9143 ) 9144 9145 # https://spark.apache.org/docs/3.5.1/sql-ref-syntax-aux-analyze-table.html 9146 def _parse_analyze_statistics(self) -> exp.AnalyzeStatistics: 9147 this = None 9148 kind = self._prev.text.upper() 9149 option = self._prev.text.upper() if self._match_text_seq("DELTA") else None 9150 expressions = [] 9151 9152 if not self._match_text_seq("STATISTICS"): 9153 self.raise_error("Expecting token STATISTICS") 9154 9155 if self._match_text_seq("NOSCAN"): 9156 this = "NOSCAN" 9157 elif self._match(TokenType.FOR): 9158 if self._match_text_seq("ALL", "COLUMNS"): 9159 this = "FOR ALL COLUMNS" 9160 if self._match_text_seq("COLUMNS"): 9161 this = "FOR COLUMNS" 9162 expressions = self._parse_csv(self._parse_column_reference) 9163 elif self._match_text_seq("SAMPLE"): 9164 sample = self._parse_number() 9165 expressions = [ 9166 self.expression( 9167 exp.AnalyzeSample( 9168 sample=sample, 9169 kind=self._prev.text.upper() if self._match(TokenType.PERCENT) else None, 9170 ) 9171 ) 9172 ] 9173 9174 return self.expression( 9175 exp.AnalyzeStatistics(kind=kind, option=option, this=this, expressions=expressions) 9176 ) 9177 9178 # https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/ANALYZE.html 9179 def _parse_analyze_validate(self) -> exp.AnalyzeValidate: 9180 kind = None 9181 this = None 9182 expression: exp.Expr | None = None 9183 if self._match_text_seq("REF", "UPDATE"): 9184 kind = "REF" 9185 this = "UPDATE" 9186 if self._match_text_seq("SET", "DANGLING", "TO", "NULL"): 9187 this = "UPDATE SET DANGLING TO NULL" 9188 elif self._match_text_seq("STRUCTURE"): 9189 kind = "STRUCTURE" 9190 if self._match_text_seq("CASCADE", "FAST"): 9191 this = "CASCADE FAST" 9192 elif self._match_text_seq("CASCADE", "COMPLETE") and self._match_texts( 9193 ("ONLINE", "OFFLINE") 9194 ): 9195 this = f"CASCADE COMPLETE {self._prev.text.upper()}" 9196 expression = self._parse_into() 9197 9198 return self.expression(exp.AnalyzeValidate(kind=kind, this=this, expression=expression)) 9199 9200 def _parse_analyze_columns(self) -> exp.AnalyzeColumns | None: 9201 this = self._prev.text.upper() 9202 if self._match_text_seq("COLUMNS"): 9203 return self.expression(exp.AnalyzeColumns(this=f"{this} {self._prev.text.upper()}")) 9204 return None 9205 9206 def _parse_analyze_delete(self) -> exp.AnalyzeDelete | None: 9207 kind = self._prev.text.upper() if self._match_text_seq("SYSTEM") else None 9208 if self._match_text_seq("STATISTICS"): 9209 return self.expression(exp.AnalyzeDelete(kind=kind)) 9210 return None 9211 9212 def _parse_analyze_list(self) -> exp.AnalyzeListChainedRows | None: 9213 if self._match_text_seq("CHAINED", "ROWS"): 9214 return self.expression(exp.AnalyzeListChainedRows(expression=self._parse_into())) 9215 return None 9216 9217 # https://dev.mysql.com/doc/refman/8.4/en/analyze-table.html 9218 def _parse_analyze_histogram(self) -> exp.AnalyzeHistogram: 9219 this = self._prev.text.upper() 9220 expression: exp.Expr | None = None 9221 expressions = [] 9222 update_options = None 9223 9224 if self._match_text_seq("HISTOGRAM", "ON"): 9225 expressions = self._parse_csv(self._parse_column_reference) 9226 with_expressions = [] 9227 while self._match(TokenType.WITH): 9228 # https://docs.starrocks.io/docs/sql-reference/sql-statements/cbo_stats/ANALYZE_TABLE/ 9229 if self._match_texts(("SYNC", "ASYNC")): 9230 if self._match_text_seq("MODE", advance=False): 9231 with_expressions.append(f"{self._prev.text.upper()} MODE") 9232 self._advance() 9233 else: 9234 buckets = self._parse_number() 9235 if self._match_text_seq("BUCKETS"): 9236 with_expressions.append(f"{buckets} BUCKETS") 9237 if with_expressions: 9238 expression = self.expression(exp.AnalyzeWith(expressions=with_expressions)) 9239 9240 if self._match_texts(("MANUAL", "AUTO")) and self._match( 9241 TokenType.UPDATE, advance=False 9242 ): 9243 update_options = self._prev.text.upper() 9244 self._advance() 9245 elif self._match_text_seq("USING", "DATA"): 9246 expression = self.expression(exp.UsingData(this=self._parse_string())) 9247 9248 return self.expression( 9249 exp.AnalyzeHistogram( 9250 this=this, 9251 expressions=expressions, 9252 expression=expression, 9253 update_options=update_options, 9254 ) 9255 ) 9256 9257 def _parse_merge(self) -> exp.Merge: 9258 self._match(TokenType.INTO) 9259 target = self._parse_table() 9260 9261 if target and self._match(TokenType.ALIAS, advance=False): 9262 target.set("alias", self._parse_table_alias()) 9263 9264 self._match(TokenType.USING) 9265 using = self._parse_table() 9266 9267 return self.expression( 9268 exp.Merge( 9269 this=target, 9270 using=using, 9271 on=self._match(TokenType.ON) and self._parse_disjunction(), 9272 using_cond=self._match(TokenType.USING) and self._parse_using_identifiers(), 9273 whens=self._parse_when_matched(), 9274 returning=self._parse_returning(), 9275 ) 9276 ) 9277 9278 def _parse_when_matched(self) -> exp.Whens: 9279 whens = [] 9280 9281 while self._match(TokenType.WHEN): 9282 matched = not self._match(TokenType.NOT) 9283 self._match_text_seq("MATCHED") 9284 source = ( 9285 False 9286 if self._match_text_seq("BY", "TARGET") 9287 else self._match_text_seq("BY", "SOURCE") 9288 ) 9289 condition = self._parse_disjunction() if self._match(TokenType.AND) else None 9290 9291 self._match(TokenType.THEN) 9292 9293 if self._match(TokenType.INSERT): 9294 this = self._parse_star() 9295 if this: 9296 then: exp.Expr | None = self.expression(exp.Insert(this=this)) 9297 else: 9298 then = self.expression( 9299 exp.Insert( 9300 this=exp.var("ROW") 9301 if self._match_text_seq("ROW") 9302 else self._parse_value(values=False), 9303 expression=self._match_text_seq("VALUES") and self._parse_value(), 9304 where=self._parse_where(), 9305 ) 9306 ) 9307 elif self._match(TokenType.UPDATE): 9308 expressions = self._parse_star() 9309 if expressions: 9310 then = self.expression(exp.Update(expressions=expressions)) 9311 else: 9312 then = self.expression( 9313 exp.Update( 9314 expressions=self._match(TokenType.SET) 9315 and self._parse_csv(self._parse_equality), 9316 where=self._parse_where(), 9317 ) 9318 ) 9319 elif self._match(TokenType.DELETE): 9320 then = self.expression(exp.Var(this=self._prev.text)) 9321 else: 9322 then = self._parse_var_from_options(self.CONFLICT_ACTIONS) 9323 9324 whens.append( 9325 self.expression( 9326 exp.When(matched=matched, source=source, condition=condition, then=then) 9327 ) 9328 ) 9329 return self.expression(exp.Whens(expressions=whens)) 9330 9331 def _parse_show(self) -> exp.Expr | None: 9332 parser = self._find_parser(self.SHOW_PARSERS, self.SHOW_TRIE) 9333 if parser: 9334 return parser(self) 9335 return self._parse_as_command(self._prev) 9336 9337 def _parse_set_item_assignment(self, kind: str | None = None) -> exp.Expr | None: 9338 index = self._index 9339 9340 if kind in ("GLOBAL", "SESSION") and self._match_text_seq("TRANSACTION"): 9341 return self._parse_set_transaction(global_=kind == "GLOBAL") 9342 9343 left = self._parse_primary() or self._parse_column() 9344 assignment_delimiter = self._match_texts(self.SET_ASSIGNMENT_DELIMITERS) 9345 9346 if not left or (self.SET_REQUIRES_ASSIGNMENT_DELIMITER and not assignment_delimiter): 9347 self._retreat(index) 9348 return None 9349 9350 right = self._parse_statement() or self._parse_id_var() 9351 if isinstance(right, (exp.Column, exp.Identifier)): 9352 right = exp.var(right.name) 9353 9354 this = self.expression(exp.EQ(this=left, expression=right)) 9355 return self.expression(exp.SetItem(this=this, kind=kind)) 9356 9357 def _parse_set_transaction(self, global_: bool = False) -> exp.Expr: 9358 self._match_text_seq("TRANSACTION") 9359 characteristics = self._parse_csv( 9360 lambda: self._parse_var_from_options(self.TRANSACTION_CHARACTERISTICS) 9361 ) 9362 return self.expression( 9363 exp.SetItem(expressions=characteristics, kind="TRANSACTION", global_=global_) 9364 ) 9365 9366 def _parse_set_item(self) -> exp.Expr | None: 9367 parser = self._find_parser(self.SET_PARSERS, self.SET_TRIE) 9368 return parser(self) if parser else self._parse_set_item_assignment(kind=None) 9369 9370 def _parse_set(self, unset: bool = False, tag: bool = False) -> exp.Set | exp.Command: 9371 index = self._index 9372 set_ = self.expression( 9373 exp.Set(expressions=self._parse_csv(self._parse_set_item), unset=unset, tag=tag) 9374 ) 9375 9376 if self._curr: 9377 self._retreat(index) 9378 return self._parse_as_command(self._prev) 9379 9380 return set_ 9381 9382 def _parse_var_from_options( 9383 self, options: OPTIONS_TYPE, raise_unmatched: bool = True 9384 ) -> exp.Var | None: 9385 start = self._curr 9386 if not start: 9387 return None 9388 9389 option = start.text.upper() 9390 continuations = ( 9391 None if start.token_type in self.TEXT_MATCH_EXCLUDED_TOKENS else options.get(option) 9392 ) 9393 9394 index = self._index 9395 self._advance() 9396 for keywords in continuations or []: 9397 if isinstance(keywords, str): 9398 keywords = (keywords,) 9399 9400 if self._match_text_seq(*keywords): 9401 option = f"{option} {' '.join(keywords)}" 9402 break 9403 else: 9404 if continuations or continuations is None: 9405 if raise_unmatched: 9406 self.raise_error(f"Unknown option {option}") 9407 9408 self._retreat(index) 9409 return None 9410 9411 return exp.var(option) 9412 9413 def _parse_as_command(self, start: Token) -> exp.Command: 9414 while self._curr: 9415 self._advance() 9416 text = self._find_sql(start, self._prev) 9417 size = len(start.text) 9418 self._warn_unsupported() 9419 return exp.Command(this=text[:size], expression=text[size:]) 9420 9421 def _parse_dict_property(self, this: str) -> exp.DictProperty: 9422 settings = [] 9423 9424 self._match_l_paren() 9425 kind = self._parse_id_var() 9426 9427 if self._match(TokenType.L_PAREN): 9428 while True: 9429 key = self._parse_id_var() 9430 value = self._parse_function() or self._parse_primary_or_var() 9431 if not key and value is None: 9432 break 9433 settings.append(self.expression(exp.DictSubProperty(this=key, value=value))) 9434 self._match(TokenType.R_PAREN) 9435 9436 self._match_r_paren() 9437 9438 return self.expression( 9439 exp.DictProperty(this=this, kind=kind.this if kind else None, settings=settings) 9440 ) 9441 9442 def _parse_dict_range(self, this: str) -> exp.DictRange: 9443 self._match_l_paren() 9444 has_min = self._match_text_seq("MIN") 9445 if has_min: 9446 min = self._parse_var() or self._parse_primary() 9447 self._match_text_seq("MAX") 9448 max = self._parse_var() or self._parse_primary() 9449 else: 9450 max = self._parse_var() or self._parse_primary() 9451 min = exp.Literal.number(0) 9452 self._match_r_paren() 9453 return self.expression(exp.DictRange(this=this, min=min, max=max)) 9454 9455 def _parse_comprehension(self, this: exp.Expr | None) -> exp.Comprehension | None: 9456 index = self._index 9457 expression = self._parse_column() 9458 position = self._match(TokenType.COMMA) and self._parse_column() 9459 9460 if not self._match(TokenType.IN): 9461 self._retreat(index - 1) 9462 return None 9463 iterator = self._parse_column() 9464 condition = self._parse_disjunction() if self._match_text_seq("IF") else None 9465 return self.expression( 9466 exp.Comprehension( 9467 this=this, 9468 expression=expression, 9469 position=position, 9470 iterator=iterator, 9471 condition=condition, 9472 ) 9473 ) 9474 9475 def _parse_heredoc(self) -> exp.Heredoc | None: 9476 if self._match(TokenType.HEREDOC_STRING): 9477 return self.expression(exp.Heredoc(this=self._prev.text)) 9478 9479 if not self._match_text_seq("$"): 9480 return None 9481 9482 tags = ["$"] 9483 tag_text = None 9484 9485 if self._is_connected(): 9486 self._advance() 9487 tags.append(self._prev.text.upper()) 9488 else: 9489 self.raise_error("No closing $ found") 9490 9491 if tags[-1] != "$": 9492 if self._is_connected() and self._match_text_seq("$"): 9493 tag_text = tags[-1] 9494 tags.append("$") 9495 else: 9496 self.raise_error("No closing $ found") 9497 9498 heredoc_start = self._curr 9499 9500 while self._curr: 9501 if self._match_text_seq(*tags, advance=False): 9502 this = self._find_sql(heredoc_start, self._prev) 9503 self._advance(len(tags)) 9504 return self.expression(exp.Heredoc(this=this, tag=tag_text)) 9505 9506 self._advance() 9507 9508 self.raise_error(f"No closing {''.join(tags)} found") 9509 return None 9510 9511 def _find_parser(self, parsers: dict[str, t.Callable], trie: dict) -> t.Callable | None: 9512 if not self._curr: 9513 return None 9514 9515 index = self._index 9516 this = [] 9517 while True: 9518 # The current token might be multiple words 9519 curr = self._curr.text.upper() 9520 key = curr.split(" ") 9521 this.append(curr) 9522 9523 self._advance() 9524 result, trie = in_trie(trie, key) 9525 if result == TrieResult.FAILED: 9526 break 9527 9528 if result == TrieResult.EXISTS: 9529 subparser = parsers[" ".join(this)] 9530 return subparser 9531 9532 self._retreat(index) 9533 return None 9534 9535 def _match_l_paren(self, expression: exp.Expr | None = None) -> None: 9536 if not self._match(TokenType.L_PAREN, expression=expression): 9537 self.raise_error("Expecting (") 9538 9539 def _match_r_paren(self, expression: exp.Expr | None = None) -> None: 9540 if not self._match(TokenType.R_PAREN, expression=expression): 9541 self.raise_error("Expecting )") 9542 9543 def _replace_lambda( 9544 self, node: exp.Expr | None, expressions: list[exp.Expr] 9545 ) -> exp.Expr | None: 9546 if not node: 9547 return node 9548 9549 lambda_types = {e.name: e.args.get("to") or False for e in expressions} 9550 9551 for column in node.find_all(exp.Column): 9552 typ = lambda_types.get(column.parts[0].name) 9553 if typ is not None: 9554 dot_or_id = column.to_dot() if column.table else column.this 9555 9556 if typ: 9557 dot_or_id = self.expression(exp.Cast(this=dot_or_id, to=typ)) 9558 9559 parent = column.parent 9560 9561 while isinstance(parent, exp.Dot): 9562 if not isinstance(parent.parent, exp.Dot): 9563 parent.replace(dot_or_id) 9564 break 9565 parent = parent.parent 9566 else: 9567 if column is node: 9568 node = dot_or_id 9569 else: 9570 column.replace(dot_or_id) 9571 return node 9572 9573 def _parse_truncate_table(self) -> exp.TruncateTable | None | exp.Expr: 9574 start = self._prev 9575 9576 # Not to be confused with TRUNCATE(number, decimals) function call 9577 if self._match(TokenType.L_PAREN): 9578 self._retreat(self._index - 2) 9579 return self._parse_function() 9580 9581 # Clickhouse supports TRUNCATE DATABASE as well 9582 is_database = self._match(TokenType.DATABASE) 9583 9584 self._match(TokenType.TABLE) 9585 9586 exists = self._parse_exists(not_=False) 9587 9588 expressions = self._parse_csv( 9589 lambda: self._parse_table(schema=True, is_db_reference=is_database) 9590 ) 9591 9592 cluster = self._parse_on_property() if self._match(TokenType.ON) else None 9593 9594 if self._match_text_seq("RESTART", "IDENTITY"): 9595 identity = "RESTART" 9596 elif self._match_text_seq("CONTINUE", "IDENTITY"): 9597 identity = "CONTINUE" 9598 else: 9599 identity = None 9600 9601 if self._match_text_seq("CASCADE") or self._match_text_seq("RESTRICT"): 9602 option = self._prev.text 9603 else: 9604 option = None 9605 9606 partition = self._parse_partition() 9607 9608 # Fallback case 9609 if self._curr: 9610 return self._parse_as_command(start) 9611 9612 return self.expression( 9613 exp.TruncateTable( 9614 expressions=expressions, 9615 is_database=is_database, 9616 exists=exists, 9617 cluster=cluster, 9618 identity=identity, 9619 option=option, 9620 partition=partition, 9621 ) 9622 ) 9623 9624 def _parse_indexed_column(self) -> exp.Expr | None: 9625 return self._parse_ordered(self._parse_opclass) 9626 9627 def _parse_with_operator(self) -> exp.Expr | None: 9628 this = self._parse_indexed_column() 9629 9630 if not self._match(TokenType.WITH): 9631 return this 9632 9633 op = self._parse_var(any_token=True, tokens=self.RESERVED_TOKENS) 9634 9635 return self.expression(exp.WithOperator(this=this, op=op)) 9636 9637 def _parse_wrapped_options(self) -> list[exp.Expr]: 9638 self._match(TokenType.EQ) 9639 self._match(TokenType.L_PAREN) 9640 9641 opts: list[exp.Expr] = [] 9642 option: exp.Expr | list[exp.Expr] | None 9643 while self._curr and not self._match(TokenType.R_PAREN): 9644 if self._match_text_seq("FORMAT_NAME", "="): 9645 # The FORMAT_NAME can be set to an identifier for Snowflake and T-SQL 9646 option = self._parse_format_name() 9647 else: 9648 option = self._parse_property() 9649 9650 if option is None: 9651 self.raise_error("Unable to parse option") 9652 break 9653 9654 opts.extend(ensure_list(option)) 9655 9656 return opts 9657 9658 def _parse_copy_parameters(self) -> list[exp.CopyParameter]: 9659 sep = TokenType.COMMA if self.dialect.COPY_PARAMS_ARE_CSV else None 9660 9661 options = [] 9662 while self._curr and not self._match(TokenType.R_PAREN, advance=False): 9663 option = self._parse_var(any_token=True) 9664 prev = self._prev.text.upper() 9665 9666 # Different dialects might separate options and values by white space, "=" and "AS" 9667 self._match(TokenType.EQ) 9668 self._match(TokenType.ALIAS) 9669 9670 param = self.expression(exp.CopyParameter(this=option)) 9671 9672 if prev in self.COPY_INTO_VARLEN_OPTIONS and self._match( 9673 TokenType.L_PAREN, advance=False 9674 ): 9675 # Snowflake FILE_FORMAT case, Databricks COPY & FORMAT options 9676 param.set("expressions", self._parse_wrapped_options()) 9677 elif prev == "FILE_FORMAT": 9678 # T-SQL's external file format case 9679 param.set("expression", self._parse_field()) 9680 elif ( 9681 prev == "FORMAT" 9682 and self._prev.token_type == TokenType.ALIAS 9683 and self._match_texts(("AVRO", "JSON")) 9684 ): 9685 param.set("this", exp.var(f"FORMAT AS {self._prev.text.upper()}")) 9686 param.set("expression", self._parse_field()) 9687 else: 9688 param.set("expression", self._parse_unquoted_field() or self._parse_bracket()) 9689 9690 options.append(param) 9691 9692 if sep: 9693 self._match(sep) 9694 9695 return options 9696 9697 def _parse_credentials(self) -> exp.Credentials | None: 9698 expr = self.expression(exp.Credentials()) 9699 9700 if self._match_text_seq("STORAGE_INTEGRATION", "="): 9701 expr.set("storage", self._parse_field()) 9702 if self._match_text_seq("CREDENTIALS"): 9703 # Snowflake case: CREDENTIALS = (...), Redshift case: CREDENTIALS <string> 9704 creds = ( 9705 self._parse_wrapped_options() if self._match(TokenType.EQ) else self._parse_field() 9706 ) 9707 expr.set("credentials", creds) 9708 if self._match_text_seq("ENCRYPTION"): 9709 expr.set("encryption", self._parse_wrapped_options()) 9710 if self._match_text_seq("IAM_ROLE"): 9711 expr.set( 9712 "iam_role", 9713 exp.var(self._prev.text) if self._match(TokenType.DEFAULT) else self._parse_field(), 9714 ) 9715 if self._match_text_seq("REGION"): 9716 expr.set("region", self._parse_field()) 9717 9718 return expr 9719 9720 def _parse_file_location(self) -> exp.Expr | None: 9721 return self._parse_field() 9722 9723 def _parse_copy(self) -> exp.Copy | exp.Command: 9724 start = self._prev 9725 9726 self._match(TokenType.INTO) 9727 9728 this = ( 9729 self._parse_select(nested=True, parse_subquery_alias=False) 9730 if self._match(TokenType.L_PAREN, advance=False) 9731 else self._parse_table(schema=True) 9732 ) 9733 9734 kind = self._match(TokenType.FROM) or not self._match_text_seq("TO") 9735 9736 files = self._parse_csv(self._parse_file_location) 9737 if self._match(TokenType.EQ, advance=False): 9738 # Backtrack one token since we've consumed the lhs of a parameter assignment here. 9739 # This can happen for Snowflake dialect. Instead, we'd like to parse the parameter 9740 # list via `_parse_wrapped(..)` below. 9741 self._advance(-1) 9742 files = [] 9743 9744 credentials = self._parse_credentials() 9745 9746 self._match_text_seq("WITH") 9747 9748 params = self._parse_wrapped(self._parse_copy_parameters, optional=True) 9749 9750 # Fallback case 9751 if self._curr: 9752 return self._parse_as_command(start) 9753 9754 return self.expression( 9755 exp.Copy(this=this, kind=kind, credentials=credentials, files=files, params=params) 9756 ) 9757 9758 def _parse_normalize(self) -> exp.Normalize: 9759 return self.expression( 9760 exp.Normalize( 9761 this=self._parse_bitwise(), form=self._match(TokenType.COMMA) and self._parse_var() 9762 ) 9763 ) 9764 9765 def _parse_ceil_floor(self, expr_type: type[TCeilFloor]) -> TCeilFloor: 9766 args = self._parse_csv(lambda: self._parse_lambda()) 9767 9768 this = seq_get(args, 0) 9769 decimals = seq_get(args, 1) 9770 9771 return expr_type( 9772 this=this, 9773 decimals=decimals, 9774 to=self._parse_var() if self._match_text_seq("TO") else None, 9775 ) 9776 9777 def _parse_star_ops(self) -> exp.Expr | None: 9778 star_token = self._prev 9779 9780 if self._match_text_seq("COLUMNS", "(", advance=False): 9781 this = self._parse_function() 9782 if isinstance(this, exp.Columns): 9783 this.set("unpack", True) 9784 return this 9785 9786 index = self._index 9787 ilike = self._parse_string() if self._match(TokenType.ILIKE) else None 9788 if not ilike: 9789 # ILIKE without a string pattern is not a star filter, e.g. `* ILIKE (foo)` 9790 self._retreat(index) 9791 9792 return self.expression( 9793 exp.Star( 9794 ilike=ilike, 9795 except_=self._parse_star_op("EXCEPT", "EXCLUDE"), 9796 replace=self._parse_star_op("REPLACE"), 9797 rename=self._parse_star_op("RENAME"), 9798 ) 9799 ).update_positions(star_token) 9800 9801 def _parse_grant_privilege(self) -> exp.GrantPrivilege | None: 9802 privilege_parts = [] 9803 9804 # Keep consuming consecutive keywords until comma (end of this privilege) or ON 9805 # (end of privilege list) or L_PAREN (start of column list) are met 9806 while self._curr and not self._match_set(self.PRIVILEGE_FOLLOW_TOKENS, advance=False): 9807 privilege_parts.append(self._curr.text.upper()) 9808 self._advance() 9809 9810 this = exp.var(" ".join(privilege_parts)) 9811 expressions = ( 9812 self._parse_wrapped_csv(self._parse_column) 9813 if self._match(TokenType.L_PAREN, advance=False) 9814 else None 9815 ) 9816 9817 return self.expression(exp.GrantPrivilege(this=this, expressions=expressions)) 9818 9819 def _parse_grant_principal(self) -> exp.GrantPrincipal | None: 9820 kind = self._match_texts(("ROLE", "GROUP")) and self._prev.text.upper() 9821 principal = self._parse_id_var() 9822 9823 if not principal: 9824 return None 9825 9826 return self.expression(exp.GrantPrincipal(this=principal, kind=kind)) 9827 9828 def _parse_grant_revoke_common( 9829 self, 9830 ) -> tuple[list | None, str | None, exp.Expr | None]: 9831 privileges = self._parse_csv(self._parse_grant_privilege) 9832 9833 self._match(TokenType.ON) 9834 kind = self._prev.text.upper() if self._match_set(self.CREATABLES) else None 9835 9836 # Attempt to parse the securable e.g. MySQL allows names 9837 # such as "foo.*", "*.*" which are not easily parseable yet 9838 securable = self._try_parse(self._parse_table_parts) 9839 9840 return privileges, kind, securable 9841 9842 def _parse_grant(self) -> exp.Grant | exp.Command: 9843 start = self._prev 9844 9845 privileges, kind, securable = self._parse_grant_revoke_common() 9846 9847 if not securable or not self._match_text_seq("TO"): 9848 return self._parse_as_command(start) 9849 9850 principals = self._parse_csv(self._parse_grant_principal) 9851 9852 grant_option = self._match_text_seq("WITH", "GRANT", "OPTION") 9853 9854 if self._curr: 9855 return self._parse_as_command(start) 9856 9857 return self.expression( 9858 exp.Grant( 9859 privileges=privileges, 9860 kind=kind, 9861 securable=securable, 9862 principals=principals, 9863 grant_option=grant_option, 9864 ) 9865 ) 9866 9867 def _parse_revoke(self) -> exp.Revoke | exp.Command: 9868 start = self._prev 9869 9870 grant_option = self._match_text_seq("GRANT", "OPTION", "FOR") 9871 9872 privileges, kind, securable = self._parse_grant_revoke_common() 9873 9874 if not securable or not self._match_text_seq("FROM"): 9875 return self._parse_as_command(start) 9876 9877 principals = self._parse_csv(self._parse_grant_principal) 9878 9879 cascade = None 9880 if self._match_texts(("CASCADE", "RESTRICT")): 9881 cascade = self._prev.text.upper() 9882 9883 if self._curr: 9884 return self._parse_as_command(start) 9885 9886 return self.expression( 9887 exp.Revoke( 9888 privileges=privileges, 9889 kind=kind, 9890 securable=securable, 9891 principals=principals, 9892 grant_option=grant_option, 9893 cascade=cascade, 9894 ) 9895 ) 9896 9897 def _parse_overlay(self) -> exp.Overlay: 9898 def _parse_overlay_arg(text: str) -> exp.Expr | None: 9899 return ( 9900 self._parse_bitwise() 9901 if self._match(TokenType.COMMA) or self._match_text_seq(text) 9902 else None 9903 ) 9904 9905 return self.expression( 9906 exp.Overlay( 9907 this=self._parse_bitwise(), 9908 expression=_parse_overlay_arg("PLACING"), 9909 from_=_parse_overlay_arg("FROM"), 9910 for_=_parse_overlay_arg("FOR"), 9911 ) 9912 ) 9913 9914 def _parse_format_name(self) -> exp.Property: 9915 # Note: Although not specified in the docs, Snowflake does accept a string/identifier 9916 # for FILE_FORMAT = <format_name> 9917 return self.expression( 9918 exp.Property( 9919 this=exp.var("FORMAT_NAME"), value=self._parse_string() or self._parse_table_parts() 9920 ) 9921 ) 9922 9923 def _parse_distinct_arg_function(self, func: type[F], distinct_index: int = 0) -> F: 9924 is_distinct = self._match(TokenType.DISTINCT) 9925 if not is_distinct: 9926 self._match(TokenType.ALL) 9927 9928 args = [self._parse_lambda()] 9929 if self._match(TokenType.COMMA): 9930 args.extend(self._parse_function_args()) 9931 9932 target = seq_get(args, distinct_index) 9933 if is_distinct and target: 9934 args[distinct_index] = self.expression(exp.Distinct(expressions=[target])) 9935 9936 return func.from_arg_list(args) 9937 9938 def _identifier_expression( 9939 self, token: Token | None = None, quoted: bool | None = None 9940 ) -> exp.Identifier: 9941 token = token or self._prev 9942 return self.expression(exp.Identifier(this=token.text, quoted=quoted), token) 9943 9944 def _build_pipe_cte( 9945 self, 9946 query: exp.Query, 9947 expressions: list[exp.Expr], 9948 alias_cte: exp.TableAlias | None = None, 9949 ) -> exp.Select: 9950 new_cte: str | exp.TableAlias | None 9951 if alias_cte: 9952 new_cte = alias_cte 9953 else: 9954 self._pipe_cte_counter += 1 9955 new_cte = f"__tmp{self._pipe_cte_counter}" 9956 9957 with_ = query.args.get("with_") 9958 ctes = with_.pop() if with_ else None 9959 9960 new_select = exp.select(*expressions, copy=False).from_(new_cte, copy=False) 9961 if ctes: 9962 new_select.set("with_", ctes) 9963 9964 return new_select.with_(new_cte, as_=query, copy=False) 9965 9966 def _parse_pipe_syntax_select(self, query: exp.Select) -> exp.Select: 9967 select = self._parse_select(consume_pipe=False) 9968 if not select: 9969 return query 9970 9971 return self._build_pipe_cte( 9972 query=query.select(*select.expressions, append=False), expressions=[exp.Star()] 9973 ) 9974 9975 def _parse_pipe_syntax_limit(self, query: exp.Select) -> exp.Select: 9976 limit = self._parse_limit() 9977 offset = self._parse_offset() 9978 if limit: 9979 curr_limit = query.args.get("limit", limit) 9980 if curr_limit.expression.to_py() >= limit.expression.to_py(): 9981 query.limit(limit, copy=False) 9982 if offset: 9983 curr_offset = query.args.get("offset") 9984 curr_offset = curr_offset.expression.to_py() if curr_offset else 0 9985 query.offset(exp.Literal.number(curr_offset + offset.expression.to_py()), copy=False) 9986 9987 return query 9988 9989 def _parse_pipe_syntax_aggregate_fields(self) -> exp.Expr | None: 9990 this = self._parse_disjunction() 9991 if self._match_text_seq("GROUP", "AND", advance=False): 9992 return this 9993 9994 this = self._parse_alias(this) 9995 9996 if self._match_set((TokenType.ASC, TokenType.DESC), advance=False): 9997 return self._parse_ordered(lambda: this) 9998 9999 return this 10000 10001 def _parse_pipe_syntax_aggregate_group_order_by( 10002 self, query: exp.Select, group_by_exists: bool = True 10003 ) -> exp.Select: 10004 expr = self._parse_csv(self._parse_pipe_syntax_aggregate_fields) 10005 aggregates_or_groups, orders = [], [] 10006 for element in expr: 10007 if isinstance(element, exp.Ordered): 10008 this = element.this 10009 if isinstance(this, exp.Alias): 10010 element.set("this", this.args["alias"]) 10011 orders.append(element) 10012 else: 10013 this = element 10014 aggregates_or_groups.append(this) 10015 10016 if group_by_exists: 10017 query.select( 10018 *aggregates_or_groups, *query.expressions, append=False, copy=False 10019 ).group_by( 10020 *[projection.args.get("alias", projection) for projection in aggregates_or_groups], 10021 copy=False, 10022 ) 10023 else: 10024 query.select(*aggregates_or_groups, append=False, copy=False) 10025 10026 if orders: 10027 return query.order_by(*orders, append=False, copy=False) 10028 10029 return query 10030 10031 def _parse_pipe_syntax_aggregate(self, query: exp.Select) -> exp.Select: 10032 self._match_text_seq("AGGREGATE") 10033 query = self._parse_pipe_syntax_aggregate_group_order_by(query, group_by_exists=False) 10034 10035 if self._match(TokenType.GROUP_BY) or ( 10036 self._match_text_seq("GROUP", "AND") and self._match(TokenType.ORDER_BY) 10037 ): 10038 query = self._parse_pipe_syntax_aggregate_group_order_by(query) 10039 10040 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 10041 10042 def _parse_pipe_syntax_set_operator(self, query: exp.Query) -> exp.Query | None: 10043 first_setop = self.parse_set_operation(this=query) 10044 if not first_setop: 10045 return None 10046 10047 def _parse_and_unwrap_query() -> exp.Expr | None: 10048 expr = self._parse_paren() 10049 return expr.assert_is(exp.Subquery).unnest() if expr else None 10050 10051 first_setop.this.pop() 10052 10053 setops = [ 10054 first_setop.expression.pop().assert_is(exp.Subquery).unnest(), 10055 *self._parse_csv(_parse_and_unwrap_query), 10056 ] 10057 10058 query = self._build_pipe_cte(query=query, expressions=[exp.Star()]) 10059 with_ = query.args.get("with_") 10060 ctes = with_.pop() if with_ else None 10061 10062 if isinstance(first_setop, exp.Union): 10063 query = query.union(*setops, copy=False, **first_setop.args) 10064 elif isinstance(first_setop, exp.Except): 10065 query = query.except_(*setops, copy=False, **first_setop.args) 10066 else: 10067 query = query.intersect(*setops, copy=False, **first_setop.args) 10068 10069 query.set("with_", ctes) 10070 10071 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 10072 10073 def _parse_pipe_syntax_join(self, query: exp.Query) -> exp.Query | None: 10074 join = self._parse_join() 10075 if not join: 10076 return None 10077 10078 if isinstance(query, exp.Select): 10079 return query.join(join, copy=False) 10080 10081 return query 10082 10083 def _parse_pipe_syntax_pivot(self, query: exp.Select) -> exp.Select: 10084 pivots = self._parse_pivots() 10085 if not pivots: 10086 return query 10087 10088 from_ = query.args.get("from_") 10089 if from_: 10090 from_.this.set("pivots", pivots) 10091 10092 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 10093 10094 def _parse_pipe_syntax_extend(self, query: exp.Select) -> exp.Select: 10095 self._match_text_seq("EXTEND") 10096 query.select(*[exp.Star(), *self._parse_expressions()], append=False, copy=False) 10097 return self._build_pipe_cte(query=query, expressions=[exp.Star()]) 10098 10099 def _parse_pipe_syntax_tablesample(self, query: exp.Select) -> exp.Select: 10100 sample = self._parse_table_sample() 10101 10102 with_ = query.args.get("with_") 10103 if with_: 10104 with_.expressions[-1].this.set("sample", sample) 10105 else: 10106 query.set("sample", sample) 10107 10108 return query 10109 10110 def _parse_pipe_syntax_query(self, query: exp.Query) -> exp.Query | None: 10111 if isinstance(query, exp.Subquery): 10112 query = exp.select("*").from_(query, copy=False) 10113 10114 if not query.args.get("from_"): 10115 query = exp.select("*").from_(query.subquery(copy=False), copy=False) 10116 10117 while self._match(TokenType.PIPE_GT): 10118 start_index = self._index 10119 start_text = self._curr.text.upper() 10120 parser = self.PIPE_SYNTAX_TRANSFORM_PARSERS.get(start_text) 10121 if not parser: 10122 # The set operators (UNION, etc) and the JOIN operator have a few common starting 10123 # keywords, making it tricky to disambiguate them without lookahead. The approach 10124 # here is to try and parse a set operation and if that fails, then try to parse a 10125 # join operator. If that fails as well, then the operator is not supported. 10126 parsed_query = self._parse_pipe_syntax_set_operator(query) 10127 parsed_query = parsed_query or self._parse_pipe_syntax_join(query) 10128 if not parsed_query: 10129 self._retreat(start_index) 10130 self.raise_error(f"Unsupported pipe syntax operator: '{start_text}'.") 10131 break 10132 query = parsed_query 10133 else: 10134 query = parser(self, query) 10135 10136 return query 10137 10138 def _parse_declareitem(self) -> exp.DeclareItem | None: 10139 self._match_texts(("VAR", "VARIABLE")) 10140 10141 vars = self._parse_csv(self._parse_id_var) 10142 if not vars: 10143 return None 10144 10145 self._match(TokenType.ALIAS) 10146 kind = self._parse_schema() if self._match(TokenType.TABLE) else self._parse_types() 10147 default = ( 10148 self._match(TokenType.DEFAULT) or self._match(TokenType.EQ) 10149 ) and self._parse_bitwise() 10150 10151 return self.expression(exp.DeclareItem(this=vars, kind=kind, default=default)) 10152 10153 def _parse_declare(self) -> exp.Declare | exp.Command: 10154 start = self._prev 10155 replace = self._match_text_seq("OR", "REPLACE") 10156 expressions = self._try_parse(lambda: self._parse_csv(self._parse_declareitem)) 10157 10158 if not expressions or self._curr: 10159 return self._parse_as_command(start) 10160 10161 return self.expression(exp.Declare(expressions=expressions, replace=replace)) 10162 10163 def build_cast(self, strict: bool, **kwargs) -> exp.Expr: 10164 exp_class = exp.Cast if strict else exp.TryCast 10165 10166 if exp_class == exp.TryCast: 10167 kwargs["requires_string"] = self.dialect.TRY_CAST_REQUIRES_STRING 10168 10169 return self.expression(exp_class(**kwargs)) 10170 10171 def _parse_json_value(self) -> exp.JSONValue: 10172 this = self._parse_bitwise() 10173 self._match(TokenType.COMMA) 10174 path = self._parse_bitwise() 10175 10176 returning = self._match(TokenType.RETURNING) and self._parse_type() 10177 10178 return self.expression( 10179 exp.JSONValue( 10180 this=this, 10181 path=self.dialect.to_json_path(path), 10182 returning=returning, 10183 on_condition=self._parse_on_condition(), 10184 ) 10185 ) 10186 10187 def _parse_group_concat(self) -> exp.Expr | None: 10188 def concat_exprs(node: exp.Expr | None, exprs: list[exp.Expr]) -> exp.Expr: 10189 if isinstance(node, exp.Distinct) and len(node.expressions) > 1: 10190 concat_exprs = [ 10191 self.expression( 10192 exp.Concat( 10193 expressions=node.expressions, 10194 safe=True, 10195 coalesce=self.dialect.CONCAT_COALESCE, 10196 ) 10197 ) 10198 ] 10199 node.set("expressions", concat_exprs) 10200 return node 10201 if len(exprs) == 1: 10202 return exprs[0] 10203 return self.expression( 10204 exp.Concat(expressions=args, safe=True, coalesce=self.dialect.CONCAT_COALESCE) 10205 ) 10206 10207 args = self._parse_csv(self._parse_lambda) 10208 10209 if args: 10210 order = args[-1] if isinstance(args[-1], exp.Order) else None 10211 10212 if order: 10213 # Order By is the last (or only) expression in the list and has consumed the 'expr' before it, 10214 # remove 'expr' from exp.Order and add it back to args 10215 args[-1] = order.this 10216 order.set("this", concat_exprs(order.this, args)) 10217 10218 this = order or concat_exprs(args[0], args) 10219 else: 10220 this = None 10221 10222 separator = self._parse_field() if self._match(TokenType.SEPARATOR) else None 10223 10224 return self.expression(exp.GroupConcat(this=this, separator=separator)) 10225 10226 def _parse_initcap(self) -> exp.Initcap: 10227 expr = exp.Initcap.from_arg_list(self._parse_function_args()) 10228 10229 # attach dialect's default delimiters 10230 if expr.args.get("expression") is None: 10231 expr.set("expression", exp.Literal.string(self.dialect.INITCAP_DEFAULT_DELIMITER_CHARS)) 10232 10233 return expr 10234 10235 def _parse_operator(self, this: exp.Expr | None) -> exp.Expr | None: 10236 while True: 10237 if not self._match(TokenType.L_PAREN): 10238 break 10239 10240 op = "" 10241 while self._curr and not self._match(TokenType.R_PAREN): 10242 op += self._curr.text 10243 self._advance() 10244 10245 comments = self._prev_comments 10246 this = self.expression( 10247 exp.Operator(this=this, operator=op, expression=self._parse_bitwise()), 10248 comments=comments, 10249 ) 10250 10251 if not self._match(TokenType.OPERATOR): 10252 break 10253 10254 return this
Parser consumes a list of tokens produced by the Tokenizer and produces a parsed syntax tree.
Arguments:
- error_level: The desired error level. Default: ErrorLevel.IMMEDIATE
- error_message_context: The amount of context to capture from a query string when displaying the error message (in number of characters). Default: 100
- max_errors: Maximum number of error messages to include in a raised ParseError. This is only relevant if error_level is ErrorLevel.RAISE. Default: 3
- max_nodes: Maximum number of AST nodes to prevent memory exhaustion. Set to -1 (default) to disable the check.
1886 def __init__( 1887 self, 1888 error_level: ErrorLevel | None = None, 1889 error_message_context: int = 100, 1890 max_errors: int = 3, 1891 max_nodes: int = -1, 1892 dialect: DialectType = None, 1893 ): 1894 self.error_level: ErrorLevel = error_level or ErrorLevel.IMMEDIATE 1895 self.error_message_context: int = error_message_context 1896 self.max_errors: int = max_errors 1897 self.max_nodes: int = max_nodes 1898 self.dialect: t.Any = _resolve_dialect(dialect) 1899 self.sql: str = "" 1900 self.errors: list[ParseError] = [] 1901 self._tokens: list[Token] = [] 1902 self._tokens_size: i64 = 0 1903 self._index: i64 = 0 1904 self._curr: Token = SENTINEL_NONE 1905 self._next: Token = SENTINEL_NONE 1906 self._prev: Token = SENTINEL_NONE 1907 self._prev_comments: list[str] = [] 1908 self._pipe_cte_counter: int = 0 1909 self._chunks: list[list[Token]] = [] 1910 self._chunk_index: i64 = 0 1911 self._node_count: int = 0
1913 def reset(self) -> None: 1914 self.sql = "" 1915 self.errors = [] 1916 self._tokens = [] 1917 self._tokens_size = 0 1918 self._index = 0 1919 self._curr = SENTINEL_NONE 1920 self._next = SENTINEL_NONE 1921 self._prev = SENTINEL_NONE 1922 self._prev_comments = [] 1923 self._pipe_cte_counter = 0 1924 self._chunks = [] 1925 self._chunk_index = 0 1926 self._node_count = 0
2019 def raise_error(self, message: str, token: Token = SENTINEL_NONE) -> None: 2020 token = token or self._curr or self._prev or Token.string("") 2021 formatted_sql, start_context, highlight, end_context = highlight_sql( 2022 sql=self.sql, 2023 positions=[(token.start, token.end)], 2024 context_length=self.error_message_context, 2025 ) 2026 formatted_message = f"{message}. Line {token.line}, Col: {token.col}.\n {formatted_sql}" 2027 2028 error = ParseError.new( 2029 formatted_message, 2030 description=message, 2031 line=token.line, 2032 col=token.col, 2033 start_context=start_context, 2034 highlight=highlight, 2035 end_context=end_context, 2036 ) 2037 2038 if self.error_level == ErrorLevel.IMMEDIATE: 2039 raise error 2040 2041 self.errors.append(error)
2043 def validate_expression(self, expression: E, args: list | None = None) -> E: 2044 if self.max_nodes > -1: 2045 self._node_count += 1 2046 if self._node_count > self.max_nodes: 2047 self.raise_error(f"Maximum number of AST nodes ({self.max_nodes}) exceeded") 2048 if self.error_level != ErrorLevel.IGNORE: 2049 for error_message in expression.error_messages(args): 2050 self.raise_error(error_message) 2051 return expression
2070 def parse(self, raw_tokens: list[Token], sql: str) -> list[exp.Expr | None]: 2071 """ 2072 Parses a list of tokens and returns a list of syntax trees, one tree 2073 per parsed SQL statement. 2074 2075 Args: 2076 raw_tokens: The list of tokens. 2077 sql: The original SQL string. 2078 2079 Returns: 2080 The list of the produced syntax trees. 2081 """ 2082 return self._parse( 2083 parse_method=self.__class__._parse_statement, raw_tokens=raw_tokens, sql=sql 2084 )
Parses a list of tokens and returns a list of syntax trees, one tree per parsed SQL statement.
Arguments:
- raw_tokens: The list of tokens.
- sql: The original SQL string.
Returns:
The list of the produced syntax trees.
2086 def parse_into( 2087 self, 2088 expression_types: exp.IntoType, 2089 raw_tokens: list[Token], 2090 sql: str | None = None, 2091 ) -> list[exp.Expr | None]: 2092 """ 2093 Parses a list of tokens into a given Expr type. If a collection of Expr 2094 types is given instead, this method will try to parse the token list into each one 2095 of them, stopping at the first for which the parsing succeeds. 2096 2097 Args: 2098 expression_types: The expression type(s) to try and parse the token list into. 2099 raw_tokens: The list of tokens. 2100 sql: The original SQL string, used to produce helpful debug messages. 2101 2102 Returns: 2103 The target Expr. 2104 """ 2105 errors = [] 2106 for expression_type in ensure_list(expression_types): 2107 parser = self.EXPRESSION_PARSERS.get(t.cast(type[exp.Expr], expression_type)) 2108 if not parser: 2109 raise TypeError(f"No parser registered for {expression_type}") 2110 2111 try: 2112 return self._parse(parser, raw_tokens, sql) 2113 except ParseError as e: 2114 e.errors[0]["into_expression"] = expression_type 2115 errors.append(e) 2116 2117 raise ParseError( 2118 f"Failed to parse '{sql or raw_tokens}' into {expression_types}", 2119 errors=merge_errors(errors), 2120 ) from errors[-1]
Parses a list of tokens into a given Expr type. If a collection of Expr types is given instead, this method will try to parse the token list into each one of them, stopping at the first for which the parsing succeeds.
Arguments:
- expression_types: The expression type(s) to try and parse the token list into.
- raw_tokens: The list of tokens.
- sql: The original SQL string, used to produce helpful debug messages.
Returns:
The target Expr.
2122 def check_errors(self) -> None: 2123 """Logs or raises any found errors, depending on the chosen error level setting.""" 2124 if self.error_level == ErrorLevel.WARN: 2125 for error in self.errors: 2126 logger.error(str(error)) 2127 elif self.error_level == ErrorLevel.RAISE and self.errors: 2128 raise ParseError( 2129 concat_messages(self.errors, self.max_errors), 2130 errors=merge_errors(self.errors), 2131 )
Logs or raises any found errors, depending on the chosen error level setting.
2133 def expression( 2134 self, 2135 instance: E, 2136 token: Token | None = None, 2137 comments: list[str] | None = None, 2138 ) -> E: 2139 if token: 2140 instance.update_positions(token) 2141 instance.add_comments(comments) if comments else self._add_comments(instance) 2142 if not instance.is_primitive: 2143 instance = self.validate_expression(instance) 2144 return instance
5778 def parse_set_operation( 5779 self, this: exp.Expr | None, consume_pipe: bool = False 5780 ) -> exp.Expr | None: 5781 start = self._index 5782 _, side_token, kind_token = self._parse_join_parts() 5783 5784 side = side_token.text if side_token else None 5785 kind = kind_token.text if kind_token else None 5786 5787 if not self._match_set(self.SET_OPERATIONS): 5788 self._retreat(start) 5789 return None 5790 5791 token_type = self._prev.token_type 5792 5793 if token_type == TokenType.UNION: 5794 operation: type[exp.SetOperation] = exp.Union 5795 elif token_type == TokenType.EXCEPT: 5796 operation = exp.Except 5797 else: 5798 operation = exp.Intersect 5799 5800 comments = self._prev.comments 5801 5802 if self._match(TokenType.DISTINCT): 5803 distinct: bool | None = True 5804 elif self._match(TokenType.ALL): 5805 distinct = False 5806 else: 5807 distinct = self.dialect.SET_OP_DISTINCT_BY_DEFAULT[operation] 5808 if distinct is None: 5809 self.raise_error(f"Expected DISTINCT or ALL for {operation.__name__}") 5810 5811 by_name = ( 5812 self._match_text_seq("BY", "NAME") 5813 or self._match_text_seq("STRICT", "CORRESPONDING") 5814 or None 5815 ) 5816 if self._match_text_seq("CORRESPONDING"): 5817 by_name = True 5818 if not side and not kind: 5819 kind = "INNER" 5820 5821 on_column_list = None 5822 if by_name and self._match_texts(("ON", "BY")): 5823 on_column_list = self._parse_wrapped_csv(self._parse_column) 5824 5825 expression = self._parse_select( 5826 nested=True, parse_set_operation=False, consume_pipe=consume_pipe 5827 ) 5828 5829 # Wrap VALUES operands in selects, both for consistency with the CTE canonicalization 5830 # in _parse_cte and so that alias pushdown can reach into set operation branches 5831 if isinstance(this, exp.Values): 5832 this = self._values_to_select(this) 5833 if isinstance(expression, exp.Values): 5834 expression = self._values_to_select(expression) 5835 5836 return self.expression( 5837 operation( 5838 this=this, 5839 distinct=distinct, 5840 by_name=by_name, 5841 expression=expression, 5842 side=side, 5843 kind=kind, 5844 on=on_column_list, 5845 ), 5846 comments=comments, 5847 )