sqlglot.tokens
1from __future__ import annotations 2 3import typing as t 4 5from sqlglot import tokenizer_core 6from sqlglot.trie import new_trie 7 8from sqlglot.tokenizer_core import Token, TokenizerCore, TokenType 9 10 11# The sqlglotc distribution ships no importable `sqlglotc` module; it overlays 12# compiled .so files onto sqlglot's modules, so detect it via the compiled core. 13SQLGLOTC_INSTALLED = not getattr(tokenizer_core, "__file__", "py").endswith(".py") 14 15try: 16 import sqlglotrs # type: ignore # noqa: F401 17 18 if not SQLGLOTC_INSTALLED: 19 import warnings 20 21 warnings.warn( 22 "sqlglot[rs] is deprecated and no longer compatible with sqlglot. " 23 "Please use sqlglotc instead for faster parsing: pip install sqlglot[c]", 24 ) 25except ImportError: 26 pass 27 28if t.TYPE_CHECKING: 29 from sqlglot.dialects.dialect import DialectType 30 31 32def _convert_quotes(arr: list[str | tuple[str, str]]) -> dict[str, str]: 33 return dict((item, item) if isinstance(item, str) else (item[0], item[1]) for item in arr) 34 35 36def _quotes_to_format( 37 token_type: TokenType, arr: list[str | tuple[str, str]] 38) -> dict[str, tuple[str, TokenType]]: 39 return {k: (v, token_type) for k, v in _convert_quotes(arr).items()} 40 41 42class _TokenizerBase: 43 QUOTES: t.ClassVar[list[tuple[str, str] | str]] 44 IDENTIFIERS: t.ClassVar[list[str | tuple[str, str]]] 45 BIT_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 46 BYTE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 47 HEX_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 48 NATIONAL_RAW_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 49 RAW_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 50 HEREDOC_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 51 UNICODE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 52 STRING_ESCAPES: t.ClassVar[list[str]] 53 BYTE_STRING_ESCAPES: t.ClassVar[list[str]] 54 IDENTIFIER_ESCAPES: t.ClassVar[list[str]] 55 HINT_START: t.ClassVar[str] 56 KEYWORDS: t.ClassVar[dict[str, TokenType]] 57 SINGLE_TOKENS: t.ClassVar[dict[str, TokenType]] 58 NUMERIC_LITERALS: t.ClassVar[dict[str, str]] 59 VAR_SINGLE_TOKENS: t.ClassVar[set[str]] 60 COMMANDS: t.ClassVar[set[TokenType]] 61 COMMAND_PREFIX_TOKENS: t.ClassVar[set[TokenType]] 62 HEREDOC_TAG_IS_IDENTIFIER: t.ClassVar[bool] 63 STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS: t.ClassVar[bool] 64 NUMERIC_ESCAPES: t.ClassVar[dict[str, tuple[int, int, int, int]]] 65 DROP_UNKNOWN_ESCAPES: t.ClassVar[bool] 66 NUMERIC_ESCAPES_ARE_BYTES: t.ClassVar[bool] 67 LONE_SURROGATE_REPLACEMENT: t.ClassVar[str] 68 NESTED_COMMENTS: t.ClassVar[bool] 69 DASH_COMMENT_REQUIRES_BOUNDARY: t.ClassVar[bool] 70 COMMENTS_TERMINATE_AT_NEWLINE_ONLY: t.ClassVar[bool] 71 TOKENS_PRECEDING_HINT: t.ClassVar[set[TokenType]] 72 HEREDOC_STRING_ALTERNATIVE: t.ClassVar[TokenType] 73 COMMENTS: t.ClassVar[list[str | tuple[str, str]]] 74 _QUOTES: t.ClassVar[dict[str, str]] 75 _IDENTIFIERS: t.ClassVar[dict[str, str]] 76 _FORMAT_STRINGS: t.ClassVar[dict[str, tuple[str, TokenType]]] 77 _STRING_ESCAPES: t.ClassVar[set[str]] 78 _BYTE_STRING_ESCAPES: t.ClassVar[set[str]] 79 _IDENTIFIER_ESCAPES: t.ClassVar[set[str]] 80 _COMMENTS: t.ClassVar[dict[str, str | None]] 81 _KEYWORD_TRIE: t.ClassVar[dict[str | int, object]] 82 83 @classmethod 84 def __init_subclass__(cls, **kwargs: t.Any) -> None: 85 super().__init_subclass__(**kwargs) 86 cls._QUOTES = _convert_quotes(cls.QUOTES) 87 cls._IDENTIFIERS = _convert_quotes(cls.IDENTIFIERS) 88 cls._FORMAT_STRINGS = { 89 **{ 90 p + s: (e, TokenType.NATIONAL_STRING) 91 for s, e in cls._QUOTES.items() 92 for p in ("n", "N") 93 }, 94 **_quotes_to_format(TokenType.BIT_STRING, cls.BIT_STRINGS), 95 **_quotes_to_format(TokenType.BYTE_STRING, cls.BYTE_STRINGS), 96 **_quotes_to_format(TokenType.HEX_STRING, cls.HEX_STRINGS), 97 **_quotes_to_format(TokenType.NATIONAL_RAW_STRING, cls.NATIONAL_RAW_STRINGS), 98 **_quotes_to_format(TokenType.RAW_STRING, cls.RAW_STRINGS), 99 **_quotes_to_format(TokenType.HEREDOC_STRING, cls.HEREDOC_STRINGS), 100 **_quotes_to_format(TokenType.UNICODE_STRING, cls.UNICODE_STRINGS), 101 } 102 if not any( 103 "BYTE_STRING_ESCAPES" in k.__dict__ for k in cls.__mro__ if k.__module__ != __name__ 104 ): 105 cls.BYTE_STRING_ESCAPES = cls.STRING_ESCAPES.copy() 106 cls._STRING_ESCAPES = set(cls.STRING_ESCAPES) 107 cls._BYTE_STRING_ESCAPES = set(cls.BYTE_STRING_ESCAPES) 108 cls._IDENTIFIER_ESCAPES = set(cls.IDENTIFIER_ESCAPES) 109 cls._COMMENTS = { 110 **{c: None for c in cls.COMMENTS if isinstance(c, str)}, 111 **{c[0]: c[1] for c in cls.COMMENTS if not isinstance(c, str)}, 112 "{#": "#}", # Ensure Jinja comments are tokenized correctly in all dialects 113 } 114 if cls.HINT_START in cls.KEYWORDS: 115 cls._COMMENTS[cls.HINT_START] = "*/" 116 cls._KEYWORD_TRIE = new_trie( 117 key.upper() 118 for key in ( 119 *cls.KEYWORDS, 120 *cls._COMMENTS, 121 *cls._QUOTES, 122 *cls._FORMAT_STRINGS, 123 ) 124 if " " in key or any(single in key for single in cls.SINGLE_TOKENS) 125 ) 126 127 128class Tokenizer(_TokenizerBase): 129 SINGLE_TOKENS = { 130 "(": TokenType.L_PAREN, 131 ")": TokenType.R_PAREN, 132 "[": TokenType.L_BRACKET, 133 "]": TokenType.R_BRACKET, 134 "{": TokenType.L_BRACE, 135 "}": TokenType.R_BRACE, 136 "&": TokenType.AMP, 137 "^": TokenType.CARET, 138 ":": TokenType.COLON, 139 ",": TokenType.COMMA, 140 ".": TokenType.DOT, 141 "-": TokenType.DASH, 142 "=": TokenType.EQ, 143 ">": TokenType.GT, 144 "<": TokenType.LT, 145 "%": TokenType.MOD, 146 "!": TokenType.NOT, 147 "|": TokenType.PIPE, 148 "+": TokenType.PLUS, 149 ";": TokenType.SEMICOLON, 150 "/": TokenType.SLASH, 151 "\\": TokenType.BACKSLASH, 152 "*": TokenType.STAR, 153 "~": TokenType.TILDE, 154 "?": TokenType.PLACEHOLDER, 155 "@": TokenType.PARAMETER, 156 "#": TokenType.HASH, 157 # Used for breaking a var like x'y' but nothing else the token type doesn't matter 158 "'": TokenType.UNKNOWN, 159 "`": TokenType.UNKNOWN, 160 '"': TokenType.UNKNOWN, 161 } 162 163 BIT_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 164 BYTE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 165 HEX_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 166 NATIONAL_RAW_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 167 RAW_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 168 HEREDOC_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 169 UNICODE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 170 IDENTIFIERS: t.ClassVar[list[str | tuple[str, str]]] = ['"'] 171 QUOTES: t.ClassVar[list[tuple[str, str] | str]] = ["'"] 172 STRING_ESCAPES: t.ClassVar[list[str]] = ["'"] 173 BYTE_STRING_ESCAPES: t.ClassVar[list[str]] = [] 174 VAR_SINGLE_TOKENS: t.ClassVar[set[str]] = set() 175 176 # The strings in this list can always be used as escapes, regardless of the surrounding 177 # identifier delimiters. By default, the closing delimiter is assumed to also act as an 178 # identifier escape, e.g. if we use double-quotes, then they also act as escapes: "x""" 179 IDENTIFIER_ESCAPES: t.ClassVar[list[str]] = [] 180 181 # Whether the heredoc tags follow the same lexical rules as unquoted identifiers 182 HEREDOC_TAG_IS_IDENTIFIER = False 183 184 # Token that we'll generate as a fallback if the heredoc prefix doesn't correspond to a heredoc 185 HEREDOC_STRING_ALTERNATIVE = TokenType.VAR 186 187 # Whether string escape characters function as such when placed within raw strings 188 STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS = True 189 190 # Maps the char after a backslash to (base, min digits, max digits, max value) for escape 191 # sequences that encode a code point, e.g. {"x": (16, 2, 2, 0xFF)} decodes '\x41' as 'A'. 192 # Digits are read until the next one would exceed the max value; if fewer than the min digits 193 # are read, the sequence isn't decoded. Octal escapes, e.g. '\101', are keyed by "0". 194 NUMERIC_ESCAPES: t.ClassVar[dict[str, tuple[int, int, int, int]]] = {} 195 196 # Whether the backslash is dropped from escape sequences that aren't otherwise decoded, 197 # e.g. '\z' is 'z' 198 DROP_UNKNOWN_ESCAPES = False 199 200 # Whether byte-valued numeric escapes (e.g. \xhh) are raw bytes, so that consecutive ones are 201 # decoded together as UTF-8, e.g. '\xC3\xA9' is 'é' instead of 'é' 202 NUMERIC_ESCAPES_ARE_BYTES = False 203 204 # What an unpaired surrogate escape (e.g. \uD800) decodes to. If empty, it isn't decoded 205 LONE_SURROGATE_REPLACEMENT = "" 206 207 NESTED_COMMENTS = True 208 209 # Whether "--" only starts a comment when followed by whitespace or a control character 210 DASH_COMMENT_REQUIRES_BOUNDARY = False 211 212 # Whether "--"/"#" line comments are terminated only by "\n", and not also by "\r" 213 COMMENTS_TERMINATE_AT_NEWLINE_ONLY = False 214 215 HINT_START = "/*+" 216 217 TOKENS_PRECEDING_HINT = {TokenType.SELECT, TokenType.INSERT, TokenType.UPDATE, TokenType.DELETE} 218 219 # Autofilled 220 _COMMENTS: t.ClassVar[dict[str, str | None]] = {} 221 _FORMAT_STRINGS: t.ClassVar[dict[str, tuple[str, TokenType]]] = {} 222 _IDENTIFIERS: t.ClassVar[dict[str, str]] = {} 223 _IDENTIFIER_ESCAPES: t.ClassVar[set[str]] = set() 224 _QUOTES: t.ClassVar[dict[str, str]] = {} 225 _STRING_ESCAPES: t.ClassVar[set[str]] = set() 226 _BYTE_STRING_ESCAPES: t.ClassVar[set[str]] = set() 227 _KEYWORD_TRIE: t.ClassVar[dict[str | int, object]] = {} 228 229 KEYWORDS: t.ClassVar[dict[str, TokenType]] = { 230 **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")}, 231 **{f"{prefix}%}}": TokenType.BLOCK_END for prefix in ("", "+", "-")}, 232 **{f"{{{{{postfix}": TokenType.BLOCK_START for postfix in ("+", "-")}, 233 **{f"{prefix}}}}}": TokenType.BLOCK_END for prefix in ("+", "-")}, 234 HINT_START: TokenType.HINT, 235 "&<": TokenType.AMP_LT, 236 "&>": TokenType.AMP_GT, 237 "==": TokenType.EQ, 238 "::": TokenType.DCOLON, 239 "?::": TokenType.QDCOLON, 240 "||": TokenType.DPIPE, 241 "|>": TokenType.PIPE_GT, 242 ">=": TokenType.GTE, 243 "<=": TokenType.LTE, 244 "<>": TokenType.NEQ, 245 "!=": TokenType.NEQ, 246 ":=": TokenType.COLON_EQ, 247 "<=>": TokenType.NULLSAFE_EQ, 248 "->": TokenType.ARROW, 249 "->>": TokenType.DARROW, 250 "=>": TokenType.FARROW, 251 "#>": TokenType.HASH_ARROW, 252 "#>>": TokenType.DHASH_ARROW, 253 "<->": TokenType.LR_ARROW, 254 "<<->>": TokenType.LLRR_ARROW, 255 "&&": TokenType.DAMP, 256 "??": TokenType.DQMARK, 257 "~~~": TokenType.GLOB, 258 "~~": TokenType.LIKE, 259 "~~*": TokenType.ILIKE, 260 "~*": TokenType.IRLIKE, 261 "-|-": TokenType.ADJACENT, 262 "ALL": TokenType.ALL, 263 "AND": TokenType.AND, 264 "ANTI": TokenType.ANTI, 265 "ANY": TokenType.ANY, 266 "ASC": TokenType.ASC, 267 "AS": TokenType.ALIAS, 268 "ASOF": TokenType.ASOF, 269 "AUTOINCREMENT": TokenType.AUTO_INCREMENT, 270 "AUTO_INCREMENT": TokenType.AUTO_INCREMENT, 271 "BEGIN": TokenType.BEGIN, 272 "BETWEEN": TokenType.BETWEEN, 273 "CACHE": TokenType.CACHE, 274 "UNCACHE": TokenType.UNCACHE, 275 "CASE": TokenType.CASE, 276 "CLUSTER BY": TokenType.CLUSTER_BY, 277 "COLLATE": TokenType.COLLATE, 278 "COLUMN": TokenType.COLUMN, 279 "COMMIT": TokenType.COMMIT, 280 "CONNECT BY": TokenType.CONNECT_BY, 281 "CONSTRAINT": TokenType.CONSTRAINT, 282 "COPY": TokenType.COPY, 283 "CREATE": TokenType.CREATE, 284 "CROSS": TokenType.CROSS, 285 "CUBE": TokenType.CUBE, 286 "CURRENT_DATE": TokenType.CURRENT_DATE, 287 "CURRENT_SCHEMA": TokenType.CURRENT_SCHEMA, 288 "CURRENT_TIME": TokenType.CURRENT_TIME, 289 "CURRENT_TIMESTAMP": TokenType.CURRENT_TIMESTAMP, 290 "CURRENT_USER": TokenType.CURRENT_USER, 291 "CURRENT_CATALOG": TokenType.CURRENT_CATALOG, 292 "DATABASE": TokenType.DATABASE, 293 "DEFAULT": TokenType.DEFAULT, 294 "DELETE": TokenType.DELETE, 295 "DESC": TokenType.DESC, 296 "DESCRIBE": TokenType.DESCRIBE, 297 "DISTINCT": TokenType.DISTINCT, 298 "DISTRIBUTE BY": TokenType.DISTRIBUTE_BY, 299 "DIV": TokenType.DIV, 300 "DROP": TokenType.DROP, 301 "ELSE": TokenType.ELSE, 302 "END": TokenType.END, 303 "ENUM": TokenType.ENUM, 304 "ESCAPE": TokenType.ESCAPE, 305 "EXCEPT": TokenType.EXCEPT, 306 "EXECUTE": TokenType.EXECUTE, 307 "EXISTS": TokenType.EXISTS, 308 "FALSE": TokenType.FALSE, 309 "FETCH": TokenType.FETCH, 310 "FILTER": TokenType.FILTER, 311 "FILE": TokenType.FILE, 312 "FIRST": TokenType.FIRST, 313 "FULL": TokenType.FULL, 314 "FUNCTION": TokenType.FUNCTION, 315 "FOR": TokenType.FOR, 316 "FOREIGN KEY": TokenType.FOREIGN_KEY, 317 "FORMAT": TokenType.FORMAT, 318 "FROM": TokenType.FROM, 319 "GEOGRAPHY": TokenType.GEOGRAPHY, 320 "GEOMETRY": TokenType.GEOMETRY, 321 "GLOB": TokenType.GLOB, 322 "GROUP BY": TokenType.GROUP_BY, 323 "GROUPING SETS": TokenType.GROUPING_SETS, 324 "HAVING": TokenType.HAVING, 325 "ILIKE": TokenType.ILIKE, 326 "IN": TokenType.IN, 327 "INDEX": TokenType.INDEX, 328 "INET": TokenType.INET, 329 "INNER": TokenType.INNER, 330 "INSERT": TokenType.INSERT, 331 "INTERVAL": TokenType.INTERVAL, 332 "INTERSECT": TokenType.INTERSECT, 333 "INTO": TokenType.INTO, 334 "IS": TokenType.IS, 335 "ISNULL": TokenType.ISNULL, 336 "JOIN": TokenType.JOIN, 337 "KEEP": TokenType.KEEP, 338 "KILL": TokenType.KILL, 339 "LATERAL": TokenType.LATERAL, 340 "LEFT": TokenType.LEFT, 341 "LIKE": TokenType.LIKE, 342 "LIMIT": TokenType.LIMIT, 343 "LOAD": TokenType.LOAD, 344 "LOCALTIME": TokenType.LOCALTIME, 345 "LOCALTIMESTAMP": TokenType.LOCALTIMESTAMP, 346 "LOCK": TokenType.LOCK, 347 "MERGE": TokenType.MERGE, 348 "NAMESPACE": TokenType.NAMESPACE, 349 "NATURAL": TokenType.NATURAL, 350 "NEXT": TokenType.NEXT, 351 "NOT": TokenType.NOT, 352 "NOTNULL": TokenType.NOTNULL, 353 "NULL": TokenType.NULL, 354 "OBJECT": TokenType.OBJECT, 355 "REF": TokenType.REF, 356 "OFFSET": TokenType.OFFSET, 357 "ON": TokenType.ON, 358 "OR": TokenType.OR, 359 "XOR": TokenType.XOR, 360 "ORDER BY": TokenType.ORDER_BY, 361 "ORDINALITY": TokenType.ORDINALITY, 362 "OUT": TokenType.OUT, 363 "OUTER": TokenType.OUTER, 364 "OVER": TokenType.OVER, 365 "OVERLAPS": TokenType.OVERLAPS, 366 "OVERWRITE": TokenType.OVERWRITE, 367 "PARTITION": TokenType.PARTITION, 368 "PARTITION BY": TokenType.PARTITION_BY, 369 "PARTITIONED BY": TokenType.PARTITION_BY, 370 "PARTITIONED_BY": TokenType.PARTITION_BY, 371 "PERCENT": TokenType.PERCENT, 372 "PIVOT": TokenType.PIVOT, 373 "PRAGMA": TokenType.PRAGMA, 374 "PRIMARY KEY": TokenType.PRIMARY_KEY, 375 "PROCEDURE": TokenType.PROCEDURE, 376 "OPERATOR": TokenType.OPERATOR, 377 "QUALIFY": TokenType.QUALIFY, 378 "RANGE": TokenType.RANGE, 379 "RECURSIVE": TokenType.RECURSIVE, 380 "REGEXP": TokenType.RLIKE, 381 "RENAME": TokenType.RENAME, 382 "REPLACE": TokenType.REPLACE, 383 "RETURNING": TokenType.RETURNING, 384 "REFERENCES": TokenType.REFERENCES, 385 "RIGHT": TokenType.RIGHT, 386 "RLIKE": TokenType.RLIKE, 387 "ROLLBACK": TokenType.ROLLBACK, 388 "ROLLUP": TokenType.ROLLUP, 389 "ROW": TokenType.ROW, 390 "ROWS": TokenType.ROWS, 391 "SCHEMA": TokenType.SCHEMA, 392 "SELECT": TokenType.SELECT, 393 "SEMI": TokenType.SEMI, 394 "SESSION": TokenType.SESSION, 395 "SESSION_USER": TokenType.SESSION_USER, 396 "SET": TokenType.SET, 397 "SETTINGS": TokenType.SETTINGS, 398 "SHOW": TokenType.SHOW, 399 "SIMILAR TO": TokenType.SIMILAR_TO, 400 "SOME": TokenType.SOME, 401 "SORT BY": TokenType.SORT_BY, 402 "SQL SECURITY": TokenType.SQL_SECURITY, 403 "STRAIGHT_JOIN": TokenType.STRAIGHT_JOIN, 404 "TABLE": TokenType.TABLE, 405 "TABLESAMPLE": TokenType.TABLE_SAMPLE, 406 "TEMP": TokenType.TEMPORARY, 407 "TEMPORARY": TokenType.TEMPORARY, 408 "THEN": TokenType.THEN, 409 "TRUE": TokenType.TRUE, 410 "TRUNCATE": TokenType.TRUNCATE, 411 "TRIGGER": TokenType.TRIGGER, 412 "UNION": TokenType.UNION, 413 "UNKNOWN": TokenType.UNKNOWN, 414 "UNNEST": TokenType.UNNEST, 415 "UNPIVOT": TokenType.UNPIVOT, 416 "UPDATE": TokenType.UPDATE, 417 "USE": TokenType.USE, 418 "USING": TokenType.USING, 419 "UUID": TokenType.UUID, 420 "VALUES": TokenType.VALUES, 421 "VIEW": TokenType.VIEW, 422 "VOLATILE": TokenType.VOLATILE, 423 "WHEN": TokenType.WHEN, 424 "WHERE": TokenType.WHERE, 425 "WINDOW": TokenType.WINDOW, 426 "WITH": TokenType.WITH, 427 "APPLY": TokenType.APPLY, 428 "ARRAY": TokenType.ARRAY, 429 "BIT": TokenType.BIT, 430 "BOOL": TokenType.BOOLEAN, 431 "BOOLEAN": TokenType.BOOLEAN, 432 "BYTE": TokenType.TINYINT, 433 "MEDIUMINT": TokenType.MEDIUMINT, 434 "INT1": TokenType.TINYINT, 435 "TINYINT": TokenType.TINYINT, 436 "INT16": TokenType.SMALLINT, 437 "SHORT": TokenType.SMALLINT, 438 "SMALLINT": TokenType.SMALLINT, 439 "HUGEINT": TokenType.INT128, 440 "UHUGEINT": TokenType.UINT128, 441 "INT2": TokenType.SMALLINT, 442 "INTEGER": TokenType.INT, 443 "INT": TokenType.INT, 444 "INT4": TokenType.INT, 445 "INT32": TokenType.INT, 446 "INT64": TokenType.BIGINT, 447 "INT128": TokenType.INT128, 448 "INT256": TokenType.INT256, 449 "LONG": TokenType.BIGINT, 450 "BIGINT": TokenType.BIGINT, 451 "INT8": TokenType.TINYINT, 452 "UINT": TokenType.UINT, 453 "UINT128": TokenType.UINT128, 454 "UINT256": TokenType.UINT256, 455 "DEC": TokenType.DECIMAL, 456 "DECIMAL": TokenType.DECIMAL, 457 "DECIMAL32": TokenType.DECIMAL32, 458 "DECIMAL64": TokenType.DECIMAL64, 459 "DECIMAL128": TokenType.DECIMAL128, 460 "DECIMAL256": TokenType.DECIMAL256, 461 "DECFLOAT": TokenType.DECFLOAT, 462 "BIGDECIMAL": TokenType.BIGDECIMAL, 463 "BIGNUMERIC": TokenType.BIGDECIMAL, 464 "BIGNUM": TokenType.BIGNUM, 465 "LIST": TokenType.LIST, 466 "MAP": TokenType.MAP, 467 "NULLABLE": TokenType.NULLABLE, 468 "NUMBER": TokenType.DECIMAL, 469 "NUMERIC": TokenType.DECIMAL, 470 "FIXED": TokenType.DECIMAL, 471 "REAL": TokenType.FLOAT, 472 "FLOAT": TokenType.FLOAT, 473 "FLOAT4": TokenType.FLOAT, 474 "FLOAT8": TokenType.DOUBLE, 475 "DOUBLE": TokenType.DOUBLE, 476 "DOUBLE PRECISION": TokenType.DOUBLE, 477 "JSON": TokenType.JSON, 478 "JSONB": TokenType.JSONB, 479 "CHAR": TokenType.CHAR, 480 "CHARACTER": TokenType.CHAR, 481 "CHAR VARYING": TokenType.VARCHAR, 482 "CHARACTER VARYING": TokenType.VARCHAR, 483 "NCHAR": TokenType.NCHAR, 484 "VARCHAR": TokenType.VARCHAR, 485 "VARCHAR2": TokenType.VARCHAR, 486 "NVARCHAR": TokenType.NVARCHAR, 487 "NVARCHAR2": TokenType.NVARCHAR, 488 "BPCHAR": TokenType.BPCHAR, 489 "STR": TokenType.TEXT, 490 "STRING": TokenType.TEXT, 491 "TEXT": TokenType.TEXT, 492 "LONGTEXT": TokenType.LONGTEXT, 493 "MEDIUMTEXT": TokenType.MEDIUMTEXT, 494 "TINYTEXT": TokenType.TINYTEXT, 495 "CLOB": TokenType.TEXT, 496 "LONGVARCHAR": TokenType.TEXT, 497 "BINARY": TokenType.BINARY, 498 "BLOB": TokenType.VARBINARY, 499 "LONGBLOB": TokenType.LONGBLOB, 500 "MEDIUMBLOB": TokenType.MEDIUMBLOB, 501 "TINYBLOB": TokenType.TINYBLOB, 502 "BYTEA": TokenType.VARBINARY, 503 "VARBINARY": TokenType.VARBINARY, 504 "TIME": TokenType.TIME, 505 "TIMETZ": TokenType.TIMETZ, 506 "TIME_NS": TokenType.TIME_NS, 507 "TIMESTAMP": TokenType.TIMESTAMP, 508 "TIMESTAMPTZ": TokenType.TIMESTAMPTZ, 509 "TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ, 510 "TIMESTAMP_LTZ": TokenType.TIMESTAMPLTZ, 511 "TIMESTAMPNTZ": TokenType.TIMESTAMPNTZ, 512 "TIMESTAMP_NTZ": TokenType.TIMESTAMPNTZ, 513 "DATE": TokenType.DATE, 514 "DATETIME": TokenType.DATETIME, 515 "INT4RANGE": TokenType.INT4RANGE, 516 "INT4MULTIRANGE": TokenType.INT4MULTIRANGE, 517 "INT8RANGE": TokenType.INT8RANGE, 518 "INT8MULTIRANGE": TokenType.INT8MULTIRANGE, 519 "NUMRANGE": TokenType.NUMRANGE, 520 "NUMMULTIRANGE": TokenType.NUMMULTIRANGE, 521 "TSRANGE": TokenType.TSRANGE, 522 "TSMULTIRANGE": TokenType.TSMULTIRANGE, 523 "TSTZRANGE": TokenType.TSTZRANGE, 524 "TSTZMULTIRANGE": TokenType.TSTZMULTIRANGE, 525 "DATERANGE": TokenType.DATERANGE, 526 "DATEMULTIRANGE": TokenType.DATEMULTIRANGE, 527 "UNIQUE": TokenType.UNIQUE, 528 "VECTOR": TokenType.VECTOR, 529 "STRUCT": TokenType.STRUCT, 530 "SEQUENCE": TokenType.SEQUENCE, 531 "VARIANT": TokenType.VARIANT, 532 "ALTER": TokenType.ALTER, 533 "ANALYZE": TokenType.ANALYZE, 534 "CALL": TokenType.COMMAND, 535 "COMMENT": TokenType.COMMENT, 536 "EXPLAIN": TokenType.COMMAND, 537 "GRANT": TokenType.GRANT, 538 "REVOKE": TokenType.REVOKE, 539 "OPTIMIZE": TokenType.COMMAND, 540 "PREPARE": TokenType.COMMAND, 541 "VACUUM": TokenType.COMMAND, 542 "USER-DEFINED": TokenType.USERDEFINED, 543 } 544 545 COMMANDS = { 546 TokenType.COMMAND, 547 TokenType.EXECUTE, 548 TokenType.FETCH, 549 TokenType.SHOW, 550 TokenType.RENAME, 551 } 552 553 COMMAND_PREFIX_TOKENS = {TokenType.SEMICOLON, TokenType.BEGIN} 554 555 # Handle numeric literals like in hive (3L = BIGINT) 556 NUMERIC_LITERALS: t.ClassVar[dict[str, str]] = {} 557 558 # In tokenizers like JSONPath, dots are always key separators, never decimal points 559 NUMBERS_CAN_HAVE_DECIMALS: t.ClassVar[bool] = True 560 561 COMMENTS = ["--", ("/*", "*/")] 562 563 __slots__ = ( 564 "dialect", 565 "_core", 566 ) 567 568 def __init__(self, dialect: DialectType = None) -> None: 569 from sqlglot.dialects.dialect import Dialect 570 571 self.dialect = Dialect.get_or_raise(dialect) 572 self._core = self._init_core() 573 574 def _init_core(self) -> TokenizerCore: 575 return TokenizerCore( 576 single_tokens=self.SINGLE_TOKENS, 577 keywords=self.KEYWORDS, 578 quotes=self._QUOTES, 579 format_strings=self._FORMAT_STRINGS, 580 identifiers=self._IDENTIFIERS, 581 comments=self._COMMENTS, 582 string_escapes=self._STRING_ESCAPES, 583 byte_string_escapes=self._BYTE_STRING_ESCAPES, 584 identifier_escapes=self._IDENTIFIER_ESCAPES, 585 commands=self.COMMANDS, 586 command_prefix_tokens=self.COMMAND_PREFIX_TOKENS, 587 nested_comments=self.NESTED_COMMENTS, 588 dash_comment_requires_boundary=self.DASH_COMMENT_REQUIRES_BOUNDARY, 589 comments_terminate_at_newline_only=self.COMMENTS_TERMINATE_AT_NEWLINE_ONLY, 590 hint_start=self.HINT_START, 591 tokens_preceding_hint=self.TOKENS_PRECEDING_HINT, 592 has_bit_strings=bool(self.BIT_STRINGS), 593 has_hex_strings=bool(self.HEX_STRINGS), 594 numeric_literals=self.NUMERIC_LITERALS, 595 var_single_tokens=self.VAR_SINGLE_TOKENS, 596 string_escapes_allowed_in_raw_strings=self.STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS, 597 heredoc_tag_is_identifier=self.HEREDOC_TAG_IS_IDENTIFIER, 598 heredoc_string_alternative=self.HEREDOC_STRING_ALTERNATIVE, 599 keyword_trie=self._KEYWORD_TRIE, 600 numbers_can_be_underscore_separated=self.dialect.NUMBERS_CAN_BE_UNDERSCORE_SEPARATED, 601 numbers_can_have_decimals=self.NUMBERS_CAN_HAVE_DECIMALS, 602 identifiers_can_start_with_digit=self.dialect.IDENTIFIERS_CAN_START_WITH_DIGIT, 603 unescaped_sequences=self.dialect.UNESCAPED_SEQUENCES, 604 numeric_escapes=self.NUMERIC_ESCAPES, 605 drop_unknown_escapes=self.DROP_UNKNOWN_ESCAPES, 606 numeric_escapes_are_bytes=self.NUMERIC_ESCAPES_ARE_BYTES, 607 byte_strings_are_bytes=self.dialect.BYTE_STRING_IS_BYTES_TYPE, 608 lone_surrogate_replacement=self.LONE_SURROGATE_REPLACEMENT, 609 ) 610 611 def tokenize(self, sql: str) -> list[Token]: 612 """Returns a list of tokens corresponding to the SQL string `sql`.""" 613 return self._core.tokenize(sql) # type: ignore 614 615 @property 616 def sql(self) -> str: 617 """The SQL string being tokenized.""" 618 return self._core.sql 619 620 @property 621 def size(self) -> int: 622 """Length of the SQL string.""" 623 return self._core.size 624 625 @property 626 def tokens(self) -> list[Token]: 627 """The list of tokens produced by tokenization.""" 628 return self._core.tokens
SQLGLOTC_INSTALLED =
False
129class Tokenizer(_TokenizerBase): 130 SINGLE_TOKENS = { 131 "(": TokenType.L_PAREN, 132 ")": TokenType.R_PAREN, 133 "[": TokenType.L_BRACKET, 134 "]": TokenType.R_BRACKET, 135 "{": TokenType.L_BRACE, 136 "}": TokenType.R_BRACE, 137 "&": TokenType.AMP, 138 "^": TokenType.CARET, 139 ":": TokenType.COLON, 140 ",": TokenType.COMMA, 141 ".": TokenType.DOT, 142 "-": TokenType.DASH, 143 "=": TokenType.EQ, 144 ">": TokenType.GT, 145 "<": TokenType.LT, 146 "%": TokenType.MOD, 147 "!": TokenType.NOT, 148 "|": TokenType.PIPE, 149 "+": TokenType.PLUS, 150 ";": TokenType.SEMICOLON, 151 "/": TokenType.SLASH, 152 "\\": TokenType.BACKSLASH, 153 "*": TokenType.STAR, 154 "~": TokenType.TILDE, 155 "?": TokenType.PLACEHOLDER, 156 "@": TokenType.PARAMETER, 157 "#": TokenType.HASH, 158 # Used for breaking a var like x'y' but nothing else the token type doesn't matter 159 "'": TokenType.UNKNOWN, 160 "`": TokenType.UNKNOWN, 161 '"': TokenType.UNKNOWN, 162 } 163 164 BIT_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 165 BYTE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 166 HEX_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 167 NATIONAL_RAW_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 168 RAW_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 169 HEREDOC_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 170 UNICODE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 171 IDENTIFIERS: t.ClassVar[list[str | tuple[str, str]]] = ['"'] 172 QUOTES: t.ClassVar[list[tuple[str, str] | str]] = ["'"] 173 STRING_ESCAPES: t.ClassVar[list[str]] = ["'"] 174 BYTE_STRING_ESCAPES: t.ClassVar[list[str]] = [] 175 VAR_SINGLE_TOKENS: t.ClassVar[set[str]] = set() 176 177 # The strings in this list can always be used as escapes, regardless of the surrounding 178 # identifier delimiters. By default, the closing delimiter is assumed to also act as an 179 # identifier escape, e.g. if we use double-quotes, then they also act as escapes: "x""" 180 IDENTIFIER_ESCAPES: t.ClassVar[list[str]] = [] 181 182 # Whether the heredoc tags follow the same lexical rules as unquoted identifiers 183 HEREDOC_TAG_IS_IDENTIFIER = False 184 185 # Token that we'll generate as a fallback if the heredoc prefix doesn't correspond to a heredoc 186 HEREDOC_STRING_ALTERNATIVE = TokenType.VAR 187 188 # Whether string escape characters function as such when placed within raw strings 189 STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS = True 190 191 # Maps the char after a backslash to (base, min digits, max digits, max value) for escape 192 # sequences that encode a code point, e.g. {"x": (16, 2, 2, 0xFF)} decodes '\x41' as 'A'. 193 # Digits are read until the next one would exceed the max value; if fewer than the min digits 194 # are read, the sequence isn't decoded. Octal escapes, e.g. '\101', are keyed by "0". 195 NUMERIC_ESCAPES: t.ClassVar[dict[str, tuple[int, int, int, int]]] = {} 196 197 # Whether the backslash is dropped from escape sequences that aren't otherwise decoded, 198 # e.g. '\z' is 'z' 199 DROP_UNKNOWN_ESCAPES = False 200 201 # Whether byte-valued numeric escapes (e.g. \xhh) are raw bytes, so that consecutive ones are 202 # decoded together as UTF-8, e.g. '\xC3\xA9' is 'é' instead of 'é' 203 NUMERIC_ESCAPES_ARE_BYTES = False 204 205 # What an unpaired surrogate escape (e.g. \uD800) decodes to. If empty, it isn't decoded 206 LONE_SURROGATE_REPLACEMENT = "" 207 208 NESTED_COMMENTS = True 209 210 # Whether "--" only starts a comment when followed by whitespace or a control character 211 DASH_COMMENT_REQUIRES_BOUNDARY = False 212 213 # Whether "--"/"#" line comments are terminated only by "\n", and not also by "\r" 214 COMMENTS_TERMINATE_AT_NEWLINE_ONLY = False 215 216 HINT_START = "/*+" 217 218 TOKENS_PRECEDING_HINT = {TokenType.SELECT, TokenType.INSERT, TokenType.UPDATE, TokenType.DELETE} 219 220 # Autofilled 221 _COMMENTS: t.ClassVar[dict[str, str | None]] = {} 222 _FORMAT_STRINGS: t.ClassVar[dict[str, tuple[str, TokenType]]] = {} 223 _IDENTIFIERS: t.ClassVar[dict[str, str]] = {} 224 _IDENTIFIER_ESCAPES: t.ClassVar[set[str]] = set() 225 _QUOTES: t.ClassVar[dict[str, str]] = {} 226 _STRING_ESCAPES: t.ClassVar[set[str]] = set() 227 _BYTE_STRING_ESCAPES: t.ClassVar[set[str]] = set() 228 _KEYWORD_TRIE: t.ClassVar[dict[str | int, object]] = {} 229 230 KEYWORDS: t.ClassVar[dict[str, TokenType]] = { 231 **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")}, 232 **{f"{prefix}%}}": TokenType.BLOCK_END for prefix in ("", "+", "-")}, 233 **{f"{{{{{postfix}": TokenType.BLOCK_START for postfix in ("+", "-")}, 234 **{f"{prefix}}}}}": TokenType.BLOCK_END for prefix in ("+", "-")}, 235 HINT_START: TokenType.HINT, 236 "&<": TokenType.AMP_LT, 237 "&>": TokenType.AMP_GT, 238 "==": TokenType.EQ, 239 "::": TokenType.DCOLON, 240 "?::": TokenType.QDCOLON, 241 "||": TokenType.DPIPE, 242 "|>": TokenType.PIPE_GT, 243 ">=": TokenType.GTE, 244 "<=": TokenType.LTE, 245 "<>": TokenType.NEQ, 246 "!=": TokenType.NEQ, 247 ":=": TokenType.COLON_EQ, 248 "<=>": TokenType.NULLSAFE_EQ, 249 "->": TokenType.ARROW, 250 "->>": TokenType.DARROW, 251 "=>": TokenType.FARROW, 252 "#>": TokenType.HASH_ARROW, 253 "#>>": TokenType.DHASH_ARROW, 254 "<->": TokenType.LR_ARROW, 255 "<<->>": TokenType.LLRR_ARROW, 256 "&&": TokenType.DAMP, 257 "??": TokenType.DQMARK, 258 "~~~": TokenType.GLOB, 259 "~~": TokenType.LIKE, 260 "~~*": TokenType.ILIKE, 261 "~*": TokenType.IRLIKE, 262 "-|-": TokenType.ADJACENT, 263 "ALL": TokenType.ALL, 264 "AND": TokenType.AND, 265 "ANTI": TokenType.ANTI, 266 "ANY": TokenType.ANY, 267 "ASC": TokenType.ASC, 268 "AS": TokenType.ALIAS, 269 "ASOF": TokenType.ASOF, 270 "AUTOINCREMENT": TokenType.AUTO_INCREMENT, 271 "AUTO_INCREMENT": TokenType.AUTO_INCREMENT, 272 "BEGIN": TokenType.BEGIN, 273 "BETWEEN": TokenType.BETWEEN, 274 "CACHE": TokenType.CACHE, 275 "UNCACHE": TokenType.UNCACHE, 276 "CASE": TokenType.CASE, 277 "CLUSTER BY": TokenType.CLUSTER_BY, 278 "COLLATE": TokenType.COLLATE, 279 "COLUMN": TokenType.COLUMN, 280 "COMMIT": TokenType.COMMIT, 281 "CONNECT BY": TokenType.CONNECT_BY, 282 "CONSTRAINT": TokenType.CONSTRAINT, 283 "COPY": TokenType.COPY, 284 "CREATE": TokenType.CREATE, 285 "CROSS": TokenType.CROSS, 286 "CUBE": TokenType.CUBE, 287 "CURRENT_DATE": TokenType.CURRENT_DATE, 288 "CURRENT_SCHEMA": TokenType.CURRENT_SCHEMA, 289 "CURRENT_TIME": TokenType.CURRENT_TIME, 290 "CURRENT_TIMESTAMP": TokenType.CURRENT_TIMESTAMP, 291 "CURRENT_USER": TokenType.CURRENT_USER, 292 "CURRENT_CATALOG": TokenType.CURRENT_CATALOG, 293 "DATABASE": TokenType.DATABASE, 294 "DEFAULT": TokenType.DEFAULT, 295 "DELETE": TokenType.DELETE, 296 "DESC": TokenType.DESC, 297 "DESCRIBE": TokenType.DESCRIBE, 298 "DISTINCT": TokenType.DISTINCT, 299 "DISTRIBUTE BY": TokenType.DISTRIBUTE_BY, 300 "DIV": TokenType.DIV, 301 "DROP": TokenType.DROP, 302 "ELSE": TokenType.ELSE, 303 "END": TokenType.END, 304 "ENUM": TokenType.ENUM, 305 "ESCAPE": TokenType.ESCAPE, 306 "EXCEPT": TokenType.EXCEPT, 307 "EXECUTE": TokenType.EXECUTE, 308 "EXISTS": TokenType.EXISTS, 309 "FALSE": TokenType.FALSE, 310 "FETCH": TokenType.FETCH, 311 "FILTER": TokenType.FILTER, 312 "FILE": TokenType.FILE, 313 "FIRST": TokenType.FIRST, 314 "FULL": TokenType.FULL, 315 "FUNCTION": TokenType.FUNCTION, 316 "FOR": TokenType.FOR, 317 "FOREIGN KEY": TokenType.FOREIGN_KEY, 318 "FORMAT": TokenType.FORMAT, 319 "FROM": TokenType.FROM, 320 "GEOGRAPHY": TokenType.GEOGRAPHY, 321 "GEOMETRY": TokenType.GEOMETRY, 322 "GLOB": TokenType.GLOB, 323 "GROUP BY": TokenType.GROUP_BY, 324 "GROUPING SETS": TokenType.GROUPING_SETS, 325 "HAVING": TokenType.HAVING, 326 "ILIKE": TokenType.ILIKE, 327 "IN": TokenType.IN, 328 "INDEX": TokenType.INDEX, 329 "INET": TokenType.INET, 330 "INNER": TokenType.INNER, 331 "INSERT": TokenType.INSERT, 332 "INTERVAL": TokenType.INTERVAL, 333 "INTERSECT": TokenType.INTERSECT, 334 "INTO": TokenType.INTO, 335 "IS": TokenType.IS, 336 "ISNULL": TokenType.ISNULL, 337 "JOIN": TokenType.JOIN, 338 "KEEP": TokenType.KEEP, 339 "KILL": TokenType.KILL, 340 "LATERAL": TokenType.LATERAL, 341 "LEFT": TokenType.LEFT, 342 "LIKE": TokenType.LIKE, 343 "LIMIT": TokenType.LIMIT, 344 "LOAD": TokenType.LOAD, 345 "LOCALTIME": TokenType.LOCALTIME, 346 "LOCALTIMESTAMP": TokenType.LOCALTIMESTAMP, 347 "LOCK": TokenType.LOCK, 348 "MERGE": TokenType.MERGE, 349 "NAMESPACE": TokenType.NAMESPACE, 350 "NATURAL": TokenType.NATURAL, 351 "NEXT": TokenType.NEXT, 352 "NOT": TokenType.NOT, 353 "NOTNULL": TokenType.NOTNULL, 354 "NULL": TokenType.NULL, 355 "OBJECT": TokenType.OBJECT, 356 "REF": TokenType.REF, 357 "OFFSET": TokenType.OFFSET, 358 "ON": TokenType.ON, 359 "OR": TokenType.OR, 360 "XOR": TokenType.XOR, 361 "ORDER BY": TokenType.ORDER_BY, 362 "ORDINALITY": TokenType.ORDINALITY, 363 "OUT": TokenType.OUT, 364 "OUTER": TokenType.OUTER, 365 "OVER": TokenType.OVER, 366 "OVERLAPS": TokenType.OVERLAPS, 367 "OVERWRITE": TokenType.OVERWRITE, 368 "PARTITION": TokenType.PARTITION, 369 "PARTITION BY": TokenType.PARTITION_BY, 370 "PARTITIONED BY": TokenType.PARTITION_BY, 371 "PARTITIONED_BY": TokenType.PARTITION_BY, 372 "PERCENT": TokenType.PERCENT, 373 "PIVOT": TokenType.PIVOT, 374 "PRAGMA": TokenType.PRAGMA, 375 "PRIMARY KEY": TokenType.PRIMARY_KEY, 376 "PROCEDURE": TokenType.PROCEDURE, 377 "OPERATOR": TokenType.OPERATOR, 378 "QUALIFY": TokenType.QUALIFY, 379 "RANGE": TokenType.RANGE, 380 "RECURSIVE": TokenType.RECURSIVE, 381 "REGEXP": TokenType.RLIKE, 382 "RENAME": TokenType.RENAME, 383 "REPLACE": TokenType.REPLACE, 384 "RETURNING": TokenType.RETURNING, 385 "REFERENCES": TokenType.REFERENCES, 386 "RIGHT": TokenType.RIGHT, 387 "RLIKE": TokenType.RLIKE, 388 "ROLLBACK": TokenType.ROLLBACK, 389 "ROLLUP": TokenType.ROLLUP, 390 "ROW": TokenType.ROW, 391 "ROWS": TokenType.ROWS, 392 "SCHEMA": TokenType.SCHEMA, 393 "SELECT": TokenType.SELECT, 394 "SEMI": TokenType.SEMI, 395 "SESSION": TokenType.SESSION, 396 "SESSION_USER": TokenType.SESSION_USER, 397 "SET": TokenType.SET, 398 "SETTINGS": TokenType.SETTINGS, 399 "SHOW": TokenType.SHOW, 400 "SIMILAR TO": TokenType.SIMILAR_TO, 401 "SOME": TokenType.SOME, 402 "SORT BY": TokenType.SORT_BY, 403 "SQL SECURITY": TokenType.SQL_SECURITY, 404 "STRAIGHT_JOIN": TokenType.STRAIGHT_JOIN, 405 "TABLE": TokenType.TABLE, 406 "TABLESAMPLE": TokenType.TABLE_SAMPLE, 407 "TEMP": TokenType.TEMPORARY, 408 "TEMPORARY": TokenType.TEMPORARY, 409 "THEN": TokenType.THEN, 410 "TRUE": TokenType.TRUE, 411 "TRUNCATE": TokenType.TRUNCATE, 412 "TRIGGER": TokenType.TRIGGER, 413 "UNION": TokenType.UNION, 414 "UNKNOWN": TokenType.UNKNOWN, 415 "UNNEST": TokenType.UNNEST, 416 "UNPIVOT": TokenType.UNPIVOT, 417 "UPDATE": TokenType.UPDATE, 418 "USE": TokenType.USE, 419 "USING": TokenType.USING, 420 "UUID": TokenType.UUID, 421 "VALUES": TokenType.VALUES, 422 "VIEW": TokenType.VIEW, 423 "VOLATILE": TokenType.VOLATILE, 424 "WHEN": TokenType.WHEN, 425 "WHERE": TokenType.WHERE, 426 "WINDOW": TokenType.WINDOW, 427 "WITH": TokenType.WITH, 428 "APPLY": TokenType.APPLY, 429 "ARRAY": TokenType.ARRAY, 430 "BIT": TokenType.BIT, 431 "BOOL": TokenType.BOOLEAN, 432 "BOOLEAN": TokenType.BOOLEAN, 433 "BYTE": TokenType.TINYINT, 434 "MEDIUMINT": TokenType.MEDIUMINT, 435 "INT1": TokenType.TINYINT, 436 "TINYINT": TokenType.TINYINT, 437 "INT16": TokenType.SMALLINT, 438 "SHORT": TokenType.SMALLINT, 439 "SMALLINT": TokenType.SMALLINT, 440 "HUGEINT": TokenType.INT128, 441 "UHUGEINT": TokenType.UINT128, 442 "INT2": TokenType.SMALLINT, 443 "INTEGER": TokenType.INT, 444 "INT": TokenType.INT, 445 "INT4": TokenType.INT, 446 "INT32": TokenType.INT, 447 "INT64": TokenType.BIGINT, 448 "INT128": TokenType.INT128, 449 "INT256": TokenType.INT256, 450 "LONG": TokenType.BIGINT, 451 "BIGINT": TokenType.BIGINT, 452 "INT8": TokenType.TINYINT, 453 "UINT": TokenType.UINT, 454 "UINT128": TokenType.UINT128, 455 "UINT256": TokenType.UINT256, 456 "DEC": TokenType.DECIMAL, 457 "DECIMAL": TokenType.DECIMAL, 458 "DECIMAL32": TokenType.DECIMAL32, 459 "DECIMAL64": TokenType.DECIMAL64, 460 "DECIMAL128": TokenType.DECIMAL128, 461 "DECIMAL256": TokenType.DECIMAL256, 462 "DECFLOAT": TokenType.DECFLOAT, 463 "BIGDECIMAL": TokenType.BIGDECIMAL, 464 "BIGNUMERIC": TokenType.BIGDECIMAL, 465 "BIGNUM": TokenType.BIGNUM, 466 "LIST": TokenType.LIST, 467 "MAP": TokenType.MAP, 468 "NULLABLE": TokenType.NULLABLE, 469 "NUMBER": TokenType.DECIMAL, 470 "NUMERIC": TokenType.DECIMAL, 471 "FIXED": TokenType.DECIMAL, 472 "REAL": TokenType.FLOAT, 473 "FLOAT": TokenType.FLOAT, 474 "FLOAT4": TokenType.FLOAT, 475 "FLOAT8": TokenType.DOUBLE, 476 "DOUBLE": TokenType.DOUBLE, 477 "DOUBLE PRECISION": TokenType.DOUBLE, 478 "JSON": TokenType.JSON, 479 "JSONB": TokenType.JSONB, 480 "CHAR": TokenType.CHAR, 481 "CHARACTER": TokenType.CHAR, 482 "CHAR VARYING": TokenType.VARCHAR, 483 "CHARACTER VARYING": TokenType.VARCHAR, 484 "NCHAR": TokenType.NCHAR, 485 "VARCHAR": TokenType.VARCHAR, 486 "VARCHAR2": TokenType.VARCHAR, 487 "NVARCHAR": TokenType.NVARCHAR, 488 "NVARCHAR2": TokenType.NVARCHAR, 489 "BPCHAR": TokenType.BPCHAR, 490 "STR": TokenType.TEXT, 491 "STRING": TokenType.TEXT, 492 "TEXT": TokenType.TEXT, 493 "LONGTEXT": TokenType.LONGTEXT, 494 "MEDIUMTEXT": TokenType.MEDIUMTEXT, 495 "TINYTEXT": TokenType.TINYTEXT, 496 "CLOB": TokenType.TEXT, 497 "LONGVARCHAR": TokenType.TEXT, 498 "BINARY": TokenType.BINARY, 499 "BLOB": TokenType.VARBINARY, 500 "LONGBLOB": TokenType.LONGBLOB, 501 "MEDIUMBLOB": TokenType.MEDIUMBLOB, 502 "TINYBLOB": TokenType.TINYBLOB, 503 "BYTEA": TokenType.VARBINARY, 504 "VARBINARY": TokenType.VARBINARY, 505 "TIME": TokenType.TIME, 506 "TIMETZ": TokenType.TIMETZ, 507 "TIME_NS": TokenType.TIME_NS, 508 "TIMESTAMP": TokenType.TIMESTAMP, 509 "TIMESTAMPTZ": TokenType.TIMESTAMPTZ, 510 "TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ, 511 "TIMESTAMP_LTZ": TokenType.TIMESTAMPLTZ, 512 "TIMESTAMPNTZ": TokenType.TIMESTAMPNTZ, 513 "TIMESTAMP_NTZ": TokenType.TIMESTAMPNTZ, 514 "DATE": TokenType.DATE, 515 "DATETIME": TokenType.DATETIME, 516 "INT4RANGE": TokenType.INT4RANGE, 517 "INT4MULTIRANGE": TokenType.INT4MULTIRANGE, 518 "INT8RANGE": TokenType.INT8RANGE, 519 "INT8MULTIRANGE": TokenType.INT8MULTIRANGE, 520 "NUMRANGE": TokenType.NUMRANGE, 521 "NUMMULTIRANGE": TokenType.NUMMULTIRANGE, 522 "TSRANGE": TokenType.TSRANGE, 523 "TSMULTIRANGE": TokenType.TSMULTIRANGE, 524 "TSTZRANGE": TokenType.TSTZRANGE, 525 "TSTZMULTIRANGE": TokenType.TSTZMULTIRANGE, 526 "DATERANGE": TokenType.DATERANGE, 527 "DATEMULTIRANGE": TokenType.DATEMULTIRANGE, 528 "UNIQUE": TokenType.UNIQUE, 529 "VECTOR": TokenType.VECTOR, 530 "STRUCT": TokenType.STRUCT, 531 "SEQUENCE": TokenType.SEQUENCE, 532 "VARIANT": TokenType.VARIANT, 533 "ALTER": TokenType.ALTER, 534 "ANALYZE": TokenType.ANALYZE, 535 "CALL": TokenType.COMMAND, 536 "COMMENT": TokenType.COMMENT, 537 "EXPLAIN": TokenType.COMMAND, 538 "GRANT": TokenType.GRANT, 539 "REVOKE": TokenType.REVOKE, 540 "OPTIMIZE": TokenType.COMMAND, 541 "PREPARE": TokenType.COMMAND, 542 "VACUUM": TokenType.COMMAND, 543 "USER-DEFINED": TokenType.USERDEFINED, 544 } 545 546 COMMANDS = { 547 TokenType.COMMAND, 548 TokenType.EXECUTE, 549 TokenType.FETCH, 550 TokenType.SHOW, 551 TokenType.RENAME, 552 } 553 554 COMMAND_PREFIX_TOKENS = {TokenType.SEMICOLON, TokenType.BEGIN} 555 556 # Handle numeric literals like in hive (3L = BIGINT) 557 NUMERIC_LITERALS: t.ClassVar[dict[str, str]] = {} 558 559 # In tokenizers like JSONPath, dots are always key separators, never decimal points 560 NUMBERS_CAN_HAVE_DECIMALS: t.ClassVar[bool] = True 561 562 COMMENTS = ["--", ("/*", "*/")] 563 564 __slots__ = ( 565 "dialect", 566 "_core", 567 ) 568 569 def __init__(self, dialect: DialectType = None) -> None: 570 from sqlglot.dialects.dialect import Dialect 571 572 self.dialect = Dialect.get_or_raise(dialect) 573 self._core = self._init_core() 574 575 def _init_core(self) -> TokenizerCore: 576 return TokenizerCore( 577 single_tokens=self.SINGLE_TOKENS, 578 keywords=self.KEYWORDS, 579 quotes=self._QUOTES, 580 format_strings=self._FORMAT_STRINGS, 581 identifiers=self._IDENTIFIERS, 582 comments=self._COMMENTS, 583 string_escapes=self._STRING_ESCAPES, 584 byte_string_escapes=self._BYTE_STRING_ESCAPES, 585 identifier_escapes=self._IDENTIFIER_ESCAPES, 586 commands=self.COMMANDS, 587 command_prefix_tokens=self.COMMAND_PREFIX_TOKENS, 588 nested_comments=self.NESTED_COMMENTS, 589 dash_comment_requires_boundary=self.DASH_COMMENT_REQUIRES_BOUNDARY, 590 comments_terminate_at_newline_only=self.COMMENTS_TERMINATE_AT_NEWLINE_ONLY, 591 hint_start=self.HINT_START, 592 tokens_preceding_hint=self.TOKENS_PRECEDING_HINT, 593 has_bit_strings=bool(self.BIT_STRINGS), 594 has_hex_strings=bool(self.HEX_STRINGS), 595 numeric_literals=self.NUMERIC_LITERALS, 596 var_single_tokens=self.VAR_SINGLE_TOKENS, 597 string_escapes_allowed_in_raw_strings=self.STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS, 598 heredoc_tag_is_identifier=self.HEREDOC_TAG_IS_IDENTIFIER, 599 heredoc_string_alternative=self.HEREDOC_STRING_ALTERNATIVE, 600 keyword_trie=self._KEYWORD_TRIE, 601 numbers_can_be_underscore_separated=self.dialect.NUMBERS_CAN_BE_UNDERSCORE_SEPARATED, 602 numbers_can_have_decimals=self.NUMBERS_CAN_HAVE_DECIMALS, 603 identifiers_can_start_with_digit=self.dialect.IDENTIFIERS_CAN_START_WITH_DIGIT, 604 unescaped_sequences=self.dialect.UNESCAPED_SEQUENCES, 605 numeric_escapes=self.NUMERIC_ESCAPES, 606 drop_unknown_escapes=self.DROP_UNKNOWN_ESCAPES, 607 numeric_escapes_are_bytes=self.NUMERIC_ESCAPES_ARE_BYTES, 608 byte_strings_are_bytes=self.dialect.BYTE_STRING_IS_BYTES_TYPE, 609 lone_surrogate_replacement=self.LONE_SURROGATE_REPLACEMENT, 610 ) 611 612 def tokenize(self, sql: str) -> list[Token]: 613 """Returns a list of tokens corresponding to the SQL string `sql`.""" 614 return self._core.tokenize(sql) # type: ignore 615 616 @property 617 def sql(self) -> str: 618 """The SQL string being tokenized.""" 619 return self._core.sql 620 621 @property 622 def size(self) -> int: 623 """Length of the SQL string.""" 624 return self._core.size 625 626 @property 627 def tokens(self) -> list[Token]: 628 """The list of tokens produced by tokenization.""" 629 return self._core.tokens
Tokenizer( dialect: Union[str, sqlglot.dialects.Dialect, type[sqlglot.dialects.Dialect], NoneType] = None)
SINGLE_TOKENS =
{'(': <TokenType.L_PAREN: 1>, ')': <TokenType.R_PAREN: 2>, '[': <TokenType.L_BRACKET: 3>, ']': <TokenType.R_BRACKET: 4>, '{': <TokenType.L_BRACE: 5>, '}': <TokenType.R_BRACE: 6>, '&': <TokenType.AMP: 36>, '^': <TokenType.CARET: 42>, ':': <TokenType.COLON: 11>, ',': <TokenType.COMMA: 7>, '.': <TokenType.DOT: 8>, '-': <TokenType.DASH: 9>, '=': <TokenType.EQ: 28>, '>': <TokenType.GT: 25>, '<': <TokenType.LT: 23>, '%': <TokenType.MOD: 333>, '!': <TokenType.NOT: 27>, '|': <TokenType.PIPE: 39>, '+': <TokenType.PLUS: 10>, ';': <TokenType.SEMICOLON: 19>, '/': <TokenType.SLASH: 22>, '\\': <TokenType.BACKSLASH: 21>, '*': <TokenType.STAR: 20>, '~': <TokenType.TILDE: 44>, '?': <TokenType.PLACEHOLDER: 361>, '@': <TokenType.PARAMETER: 58>, '#': <TokenType.HASH: 48>, "'": <TokenType.UNKNOWN: 216>, '`': <TokenType.UNKNOWN: 216>, '"': <TokenType.UNKNOWN: 216>}
TOKENS_PRECEDING_HINT =
{<TokenType.SELECT: 392>, <TokenType.INSERT: 304>, <TokenType.DELETE: 260>, <TokenType.UPDATE: 424>}
KEYWORDS: ClassVar[dict[str, sqlglot.tokenizer_core.TokenType]] =
{'{%': <TokenType.BLOCK_START: 73>, '{%+': <TokenType.BLOCK_START: 73>, '{%-': <TokenType.BLOCK_START: 73>, '%}': <TokenType.BLOCK_END: 74>, '+%}': <TokenType.BLOCK_END: 74>, '-%}': <TokenType.BLOCK_END: 74>, '{{+': <TokenType.BLOCK_START: 73>, '{{-': <TokenType.BLOCK_START: 73>, '+}}': <TokenType.BLOCK_END: 74>, '-}}': <TokenType.BLOCK_END: 74>, '/*+': <TokenType.HINT: 297>, '&<': <TokenType.AMP_LT: 63>, '&>': <TokenType.AMP_GT: 64>, '==': <TokenType.EQ: 28>, '::': <TokenType.DCOLON: 14>, '?::': <TokenType.QDCOLON: 375>, '||': <TokenType.DPIPE: 37>, '|>': <TokenType.PIPE_GT: 38>, '>=': <TokenType.GTE: 26>, '<=': <TokenType.LTE: 24>, '<>': <TokenType.NEQ: 29>, '!=': <TokenType.NEQ: 29>, ':=': <TokenType.COLON_EQ: 31>, '<=>': <TokenType.NULLSAFE_EQ: 30>, '->': <TokenType.ARROW: 45>, '->>': <TokenType.DARROW: 46>, '=>': <TokenType.FARROW: 47>, '#>': <TokenType.HASH_ARROW: 49>, '#>>': <TokenType.DHASH_ARROW: 50>, '<->': <TokenType.LR_ARROW: 51>, '<<->>': <TokenType.LLRR_ARROW: 52>, '&&': <TokenType.DAMP: 62>, '??': <TokenType.DQMARK: 18>, '~~~': <TokenType.GLOB: 291>, '~~': <TokenType.LIKE: 322>, '~~*': <TokenType.ILIKE: 299>, '~*': <TokenType.IRLIKE: 311>, '-|-': <TokenType.ADJACENT: 65>, 'ALL': <TokenType.ALL: 222>, 'AND': <TokenType.AND: 34>, 'ANTI': <TokenType.ANTI: 223>, 'ANY': <TokenType.ANY: 224>, 'ASC': <TokenType.ASC: 227>, 'AS': <TokenType.ALIAS: 220>, 'ASOF': <TokenType.ASOF: 228>, 'AUTOINCREMENT': <TokenType.AUTO_INCREMENT: 230>, 'AUTO_INCREMENT': <TokenType.AUTO_INCREMENT: 230>, 'BEGIN': <TokenType.BEGIN: 231>, 'BETWEEN': <TokenType.BETWEEN: 232>, 'CACHE': <TokenType.CACHE: 234>, 'UNCACHE': <TokenType.UNCACHE: 419>, 'CASE': <TokenType.CASE: 235>, 'CLUSTER BY': <TokenType.CLUSTER_BY: 237>, 'COLLATE': <TokenType.COLLATE: 238>, 'COLUMN': <TokenType.COLUMN: 81>, 'COMMIT': <TokenType.COMMIT: 241>, 'CONNECT BY': <TokenType.CONNECT_BY: 243>, 'CONSTRAINT': <TokenType.CONSTRAINT: 244>, 'COPY': <TokenType.COPY: 245>, 'CREATE': <TokenType.CREATE: 246>, 'CROSS': <TokenType.CROSS: 247>, 'CUBE': <TokenType.CUBE: 248>, 'CURRENT_DATE': <TokenType.CURRENT_DATE: 249>, 'CURRENT_SCHEMA': <TokenType.CURRENT_SCHEMA: 251>, 'CURRENT_TIME': <TokenType.CURRENT_TIME: 252>, 'CURRENT_TIMESTAMP': <TokenType.CURRENT_TIMESTAMP: 253>, 'CURRENT_USER': <TokenType.CURRENT_USER: 254>, 'CURRENT_CATALOG': <TokenType.CURRENT_CATALOG: 257>, 'DATABASE': <TokenType.DATABASE: 80>, 'DEFAULT': <TokenType.DEFAULT: 259>, 'DELETE': <TokenType.DELETE: 260>, 'DESC': <TokenType.DESC: 261>, 'DESCRIBE': <TokenType.DESCRIBE: 262>, 'DISTINCT': <TokenType.DISTINCT: 266>, 'DISTRIBUTE BY': <TokenType.DISTRIBUTE_BY: 267>, 'DIV': <TokenType.DIV: 268>, 'DROP': <TokenType.DROP: 269>, 'ELSE': <TokenType.ELSE: 270>, 'END': <TokenType.END: 271>, 'ENUM': <TokenType.ENUM: 207>, 'ESCAPE': <TokenType.ESCAPE: 272>, 'EXCEPT': <TokenType.EXCEPT: 273>, 'EXECUTE': <TokenType.EXECUTE: 274>, 'EXISTS': <TokenType.EXISTS: 275>, 'FALSE': <TokenType.FALSE: 276>, 'FETCH': <TokenType.FETCH: 277>, 'FILTER': <TokenType.FILTER: 280>, 'FILE': <TokenType.FILE: 278>, 'FIRST': <TokenType.FIRST: 282>, 'FULL': <TokenType.FULL: 288>, 'FUNCTION': <TokenType.FUNCTION: 289>, 'FOR': <TokenType.FOR: 283>, 'FOREIGN KEY': <TokenType.FOREIGN_KEY: 285>, 'FORMAT': <TokenType.FORMAT: 286>, 'FROM': <TokenType.FROM: 287>, 'GEOGRAPHY': <TokenType.GEOGRAPHY: 173>, 'GEOMETRY': <TokenType.GEOMETRY: 176>, 'GLOB': <TokenType.GLOB: 291>, 'GROUP BY': <TokenType.GROUP_BY: 294>, 'GROUPING SETS': <TokenType.GROUPING_SETS: 295>, 'HAVING': <TokenType.HAVING: 296>, 'ILIKE': <TokenType.ILIKE: 299>, 'IN': <TokenType.IN: 300>, 'INDEX': <TokenType.INDEX: 301>, 'INET': <TokenType.INET: 202>, 'INNER': <TokenType.INNER: 303>, 'INSERT': <TokenType.INSERT: 304>, 'INTERVAL': <TokenType.INTERVAL: 308>, 'INTERSECT': <TokenType.INTERSECT: 307>, 'INTO': <TokenType.INTO: 309>, 'IS': <TokenType.IS: 312>, 'ISNULL': <TokenType.ISNULL: 313>, 'JOIN': <TokenType.JOIN: 314>, 'KEEP': <TokenType.KEEP: 316>, 'KILL': <TokenType.KILL: 318>, 'LATERAL': <TokenType.LATERAL: 320>, 'LEFT': <TokenType.LEFT: 321>, 'LIKE': <TokenType.LIKE: 322>, 'LIMIT': <TokenType.LIMIT: 323>, 'LOAD': <TokenType.LOAD: 325>, 'LOCALTIME': <TokenType.LOCALTIME: 180>, 'LOCALTIMESTAMP': <TokenType.LOCALTIMESTAMP: 181>, 'LOCK': <TokenType.LOCK: 326>, 'MERGE': <TokenType.MERGE: 332>, 'NAMESPACE': <TokenType.NAMESPACE: 445>, 'NATURAL': <TokenType.NATURAL: 335>, 'NEXT': <TokenType.NEXT: 336>, 'NOT': <TokenType.NOT: 27>, 'NOTNULL': <TokenType.NOTNULL: 338>, 'NULL': <TokenType.NULL: 339>, 'OBJECT': <TokenType.OBJECT: 200>, 'REF': <TokenType.REF: 201>, 'OFFSET': <TokenType.OFFSET: 341>, 'ON': <TokenType.ON: 342>, 'OR': <TokenType.OR: 35>, 'XOR': <TokenType.XOR: 66>, 'ORDER BY': <TokenType.ORDER_BY: 345>, 'ORDINALITY': <TokenType.ORDINALITY: 348>, 'OUT': <TokenType.OUT: 349>, 'OUTER': <TokenType.OUTER: 351>, 'OVER': <TokenType.OVER: 352>, 'OVERLAPS': <TokenType.OVERLAPS: 353>, 'OVERWRITE': <TokenType.OVERWRITE: 354>, 'PARTITION': <TokenType.PARTITION: 356>, 'PARTITION BY': <TokenType.PARTITION_BY: 357>, 'PARTITIONED BY': <TokenType.PARTITION_BY: 357>, 'PARTITIONED_BY': <TokenType.PARTITION_BY: 357>, 'PERCENT': <TokenType.PERCENT: 359>, 'PIVOT': <TokenType.PIVOT: 360>, 'PRAGMA': <TokenType.PRAGMA: 365>, 'PRIMARY KEY': <TokenType.PRIMARY_KEY: 367>, 'PROCEDURE': <TokenType.PROCEDURE: 368>, 'OPERATOR': <TokenType.OPERATOR: 344>, 'QUALIFY': <TokenType.QUALIFY: 373>, 'RANGE': <TokenType.RANGE: 376>, 'RECURSIVE': <TokenType.RECURSIVE: 377>, 'REGEXP': <TokenType.RLIKE: 385>, 'RENAME': <TokenType.RENAME: 379>, 'REPLACE': <TokenType.REPLACE: 380>, 'RETURNING': <TokenType.RETURNING: 381>, 'REFERENCES': <TokenType.REFERENCES: 383>, 'RIGHT': <TokenType.RIGHT: 384>, 'RLIKE': <TokenType.RLIKE: 385>, 'ROLLBACK': <TokenType.ROLLBACK: 387>, 'ROLLUP': <TokenType.ROLLUP: 388>, 'ROW': <TokenType.ROW: 389>, 'ROWS': <TokenType.ROWS: 390>, 'SCHEMA': <TokenType.SCHEMA: 83>, 'SELECT': <TokenType.SELECT: 392>, 'SEMI': <TokenType.SEMI: 393>, 'SESSION': <TokenType.SESSION: 59>, 'SESSION_USER': <TokenType.SESSION_USER: 61>, 'SET': <TokenType.SET: 397>, 'SETTINGS': <TokenType.SETTINGS: 398>, 'SHOW': <TokenType.SHOW: 399>, 'SIMILAR TO': <TokenType.SIMILAR_TO: 400>, 'SOME': <TokenType.SOME: 401>, 'SORT BY': <TokenType.SORT_BY: 402>, 'SQL SECURITY': <TokenType.SQL_SECURITY: 404>, 'STRAIGHT_JOIN': <TokenType.STRAIGHT_JOIN: 407>, 'TABLE': <TokenType.TABLE: 84>, 'TABLESAMPLE': <TokenType.TABLE_SAMPLE: 410>, 'TEMP': <TokenType.TEMPORARY: 412>, 'TEMPORARY': <TokenType.TEMPORARY: 412>, 'THEN': <TokenType.THEN: 414>, 'TRUE': <TokenType.TRUE: 415>, 'TRUNCATE': <TokenType.TRUNCATE: 416>, 'TRIGGER': <TokenType.TRIGGER: 417>, 'UNION': <TokenType.UNION: 421>, 'UNKNOWN': <TokenType.UNKNOWN: 216>, 'UNNEST': <TokenType.UNNEST: 422>, 'UNPIVOT': <TokenType.UNPIVOT: 423>, 'UPDATE': <TokenType.UPDATE: 424>, 'USE': <TokenType.USE: 425>, 'USING': <TokenType.USING: 426>, 'UUID': <TokenType.UUID: 172>, 'VALUES': <TokenType.VALUES: 427>, 'VIEW': <TokenType.VIEW: 429>, 'VOLATILE': <TokenType.VOLATILE: 431>, 'WHEN': <TokenType.WHEN: 433>, 'WHERE': <TokenType.WHERE: 434>, 'WINDOW': <TokenType.WINDOW: 435>, 'WITH': <TokenType.WITH: 436>, 'APPLY': <TokenType.APPLY: 225>, 'ARRAY': <TokenType.ARRAY: 226>, 'BIT': <TokenType.BIT: 98>, 'BOOL': <TokenType.BOOLEAN: 99>, 'BOOLEAN': <TokenType.BOOLEAN: 99>, 'BYTE': <TokenType.TINYINT: 100>, 'MEDIUMINT': <TokenType.MEDIUMINT: 104>, 'INT1': <TokenType.TINYINT: 100>, 'TINYINT': <TokenType.TINYINT: 100>, 'INT16': <TokenType.SMALLINT: 102>, 'SHORT': <TokenType.SMALLINT: 102>, 'SMALLINT': <TokenType.SMALLINT: 102>, 'HUGEINT': <TokenType.INT128: 111>, 'UHUGEINT': <TokenType.UINT128: 112>, 'INT2': <TokenType.SMALLINT: 102>, 'INTEGER': <TokenType.INT: 106>, 'INT': <TokenType.INT: 106>, 'INT4': <TokenType.INT: 106>, 'INT32': <TokenType.INT: 106>, 'INT64': <TokenType.BIGINT: 108>, 'INT128': <TokenType.INT128: 111>, 'INT256': <TokenType.INT256: 113>, 'LONG': <TokenType.BIGINT: 108>, 'BIGINT': <TokenType.BIGINT: 108>, 'INT8': <TokenType.TINYINT: 100>, 'UINT': <TokenType.UINT: 107>, 'UINT128': <TokenType.UINT128: 112>, 'UINT256': <TokenType.UINT256: 114>, 'DEC': <TokenType.DECIMAL: 118>, 'DECIMAL': <TokenType.DECIMAL: 118>, 'DECIMAL32': <TokenType.DECIMAL32: 119>, 'DECIMAL64': <TokenType.DECIMAL64: 120>, 'DECIMAL128': <TokenType.DECIMAL128: 121>, 'DECIMAL256': <TokenType.DECIMAL256: 122>, 'DECFLOAT': <TokenType.DECFLOAT: 123>, 'BIGDECIMAL': <TokenType.BIGDECIMAL: 125>, 'BIGNUMERIC': <TokenType.BIGDECIMAL: 125>, 'BIGNUM': <TokenType.BIGNUM: 110>, 'LIST': <TokenType.LIST: 324>, 'MAP': <TokenType.MAP: 327>, 'NULLABLE': <TokenType.NULLABLE: 175>, 'NUMBER': <TokenType.DECIMAL: 118>, 'NUMERIC': <TokenType.DECIMAL: 118>, 'FIXED': <TokenType.DECIMAL: 118>, 'REAL': <TokenType.FLOAT: 115>, 'FLOAT': <TokenType.FLOAT: 115>, 'FLOAT4': <TokenType.FLOAT: 115>, 'FLOAT8': <TokenType.DOUBLE: 116>, 'DOUBLE': <TokenType.DOUBLE: 116>, 'DOUBLE PRECISION': <TokenType.DOUBLE: 116>, 'JSON': <TokenType.JSON: 142>, 'JSONB': <TokenType.JSONB: 143>, 'CHAR': <TokenType.CHAR: 126>, 'CHARACTER': <TokenType.CHAR: 126>, 'CHAR VARYING': <TokenType.VARCHAR: 128>, 'CHARACTER VARYING': <TokenType.VARCHAR: 128>, 'NCHAR': <TokenType.NCHAR: 127>, 'VARCHAR': <TokenType.VARCHAR: 128>, 'VARCHAR2': <TokenType.VARCHAR: 128>, 'NVARCHAR': <TokenType.NVARCHAR: 129>, 'NVARCHAR2': <TokenType.NVARCHAR: 129>, 'BPCHAR': <TokenType.BPCHAR: 130>, 'STR': <TokenType.TEXT: 131>, 'STRING': <TokenType.TEXT: 131>, 'TEXT': <TokenType.TEXT: 131>, 'LONGTEXT': <TokenType.LONGTEXT: 133>, 'MEDIUMTEXT': <TokenType.MEDIUMTEXT: 132>, 'TINYTEXT': <TokenType.TINYTEXT: 138>, 'CLOB': <TokenType.TEXT: 131>, 'LONGVARCHAR': <TokenType.TEXT: 131>, 'BINARY': <TokenType.BINARY: 140>, 'BLOB': <TokenType.VARBINARY: 141>, 'LONGBLOB': <TokenType.LONGBLOB: 136>, 'MEDIUMBLOB': <TokenType.MEDIUMBLOB: 135>, 'TINYBLOB': <TokenType.TINYBLOB: 137>, 'BYTEA': <TokenType.VARBINARY: 141>, 'VARBINARY': <TokenType.VARBINARY: 141>, 'TIME': <TokenType.TIME: 144>, 'TIMETZ': <TokenType.TIMETZ: 145>, 'TIME_NS': <TokenType.TIME_NS: 146>, 'TIMESTAMP': <TokenType.TIMESTAMP: 147>, 'TIMESTAMPTZ': <TokenType.TIMESTAMPTZ: 148>, 'TIMESTAMPLTZ': <TokenType.TIMESTAMPLTZ: 149>, 'TIMESTAMP_LTZ': <TokenType.TIMESTAMPLTZ: 149>, 'TIMESTAMPNTZ': <TokenType.TIMESTAMPNTZ: 150>, 'TIMESTAMP_NTZ': <TokenType.TIMESTAMPNTZ: 150>, 'DATE': <TokenType.DATE: 158>, 'DATETIME': <TokenType.DATETIME: 154>, 'INT4RANGE': <TokenType.INT4RANGE: 160>, 'INT4MULTIRANGE': <TokenType.INT4MULTIRANGE: 161>, 'INT8RANGE': <TokenType.INT8RANGE: 162>, 'INT8MULTIRANGE': <TokenType.INT8MULTIRANGE: 163>, 'NUMRANGE': <TokenType.NUMRANGE: 164>, 'NUMMULTIRANGE': <TokenType.NUMMULTIRANGE: 165>, 'TSRANGE': <TokenType.TSRANGE: 166>, 'TSMULTIRANGE': <TokenType.TSMULTIRANGE: 167>, 'TSTZRANGE': <TokenType.TSTZRANGE: 168>, 'TSTZMULTIRANGE': <TokenType.TSTZMULTIRANGE: 169>, 'DATERANGE': <TokenType.DATERANGE: 170>, 'DATEMULTIRANGE': <TokenType.DATEMULTIRANGE: 171>, 'UNIQUE': <TokenType.UNIQUE: 437>, 'VECTOR': <TokenType.VECTOR: 217>, 'STRUCT': <TokenType.STRUCT: 408>, 'SEQUENCE': <TokenType.SEQUENCE: 395>, 'VARIANT': <TokenType.VARIANT: 199>, 'ALTER': <TokenType.ALTER: 221>, 'ANALYZE': <TokenType.ANALYZE: 444>, 'CALL': <TokenType.COMMAND: 239>, 'COMMENT': <TokenType.COMMENT: 240>, 'EXPLAIN': <TokenType.COMMAND: 239>, 'GRANT': <TokenType.GRANT: 293>, 'REVOKE': <TokenType.REVOKE: 382>, 'OPTIMIZE': <TokenType.COMMAND: 239>, 'PREPARE': <TokenType.COMMAND: 239>, 'VACUUM': <TokenType.COMMAND: 239>, 'USER-DEFINED': <TokenType.USERDEFINED: 194>}
COMMANDS =
{<TokenType.SHOW: 399>, <TokenType.COMMAND: 239>, <TokenType.EXECUTE: 274>, <TokenType.FETCH: 277>, <TokenType.RENAME: 379>}
612 def tokenize(self, sql: str) -> list[Token]: 613 """Returns a list of tokens corresponding to the SQL string `sql`.""" 614 return self._core.tokenize(sql) # type: ignore
Returns a list of tokens corresponding to the SQL string sql.
sql: str
616 @property 617 def sql(self) -> str: 618 """The SQL string being tokenized.""" 619 return self._core.sql
The SQL string being tokenized.
size: int
621 @property 622 def size(self) -> int: 623 """Length of the SQL string.""" 624 return self._core.size
Length of the SQL string.
tokens: list[sqlglot.tokenizer_core.Token]
626 @property 627 def tokens(self) -> list[Token]: 628 """The list of tokens produced by tokenization.""" 629 return self._core.tokens
The list of tokens produced by tokenization.