sqlglot.tokenizer_core

   1from __future__ import annotations
   2
   3import typing as t
   4from enum import IntEnum, auto
   5
   6from sqlglot.errors import TokenError
   7
   8# dict lookup is faster than .upper() and .isdigit()
   9_CHAR_UPPER: dict[str, str] = {chr(i): chr(i).upper() for i in range(97, 123)}
  10_DIGIT_CHARS: frozenset[str] = frozenset("0123456789")
  11_DIGIT_VALUES: dict[str, int] = {c: int(c, 16) for c in "0123456789abcdefABCDEF"}
  12_OCTAL_CHARS: frozenset[str] = frozenset("01234567")
  13
  14
  15class TokenType(IntEnum):
  16    L_PAREN = auto()
  17    R_PAREN = auto()
  18    L_BRACKET = auto()
  19    R_BRACKET = auto()
  20    L_BRACE = auto()
  21    R_BRACE = auto()
  22    COMMA = auto()
  23    DOT = auto()
  24    DASH = auto()
  25    PLUS = auto()
  26    COLON = auto()
  27    DOTCOLON = auto()
  28    DOTCARET = auto()
  29    DCOLON = auto()
  30    DCOLONDOLLAR = auto()
  31    DCOLONPERCENT = auto()
  32    DCOLONQMARK = auto()
  33    DQMARK = auto()
  34    SEMICOLON = auto()
  35    STAR = auto()
  36    BACKSLASH = auto()
  37    SLASH = auto()
  38    LT = auto()
  39    LTE = auto()
  40    GT = auto()
  41    GTE = auto()
  42    NOT = auto()
  43    EQ = auto()
  44    NEQ = auto()
  45    NULLSAFE_EQ = auto()
  46    COLON_EQ = auto()
  47    COLON_GT = auto()
  48    NCOLON_GT = auto()
  49    AND = auto()
  50    OR = auto()
  51    AMP = auto()
  52    DPIPE = auto()
  53    PIPE_GT = auto()
  54    PIPE = auto()
  55    PIPE_SLASH = auto()
  56    DPIPE_SLASH = auto()
  57    CARET = auto()
  58    CARET_AT = auto()
  59    TILDE = auto()
  60    ARROW = auto()
  61    DARROW = auto()
  62    FARROW = auto()
  63    HASH = auto()
  64    HASH_ARROW = auto()
  65    DHASH_ARROW = auto()
  66    LR_ARROW = auto()
  67    LLRR_ARROW = auto()
  68    DAT = auto()
  69    AT_QMARK = auto()
  70    LT_AT = auto()
  71    AT_GT = auto()
  72    DOLLAR = auto()
  73    PARAMETER = auto()
  74    SESSION = auto()
  75    SESSION_PARAMETER = auto()
  76    SESSION_USER = auto()
  77    DAMP = auto()
  78    AMP_LT = auto()
  79    AMP_GT = auto()
  80    ADJACENT = auto()
  81    XOR = auto()
  82    DSTAR = auto()
  83    QMARK_AMP = auto()
  84    QMARK_PIPE = auto()
  85    HASH_DASH = auto()
  86    EXCLAMATION = auto()
  87
  88    URI_START = auto()
  89
  90    BLOCK_START = auto()
  91    BLOCK_END = auto()
  92
  93    SPACE = auto()
  94    BREAK = auto()
  95
  96    STRING = auto()
  97    NUMBER = auto()
  98    IDENTIFIER = auto()
  99    DATABASE = auto()
 100    COLUMN = auto()
 101    COLUMN_DEF = auto()
 102    SCHEMA = auto()
 103    TABLE = auto()
 104    WAREHOUSE = auto()
 105    STAGE = auto()
 106    STREAM = auto()
 107    STREAMLIT = auto()
 108    VAR = auto()
 109    BIT_STRING = auto()
 110    HEX_STRING = auto()
 111    BYTE_STRING = auto()
 112    NATIONAL_RAW_STRING = auto()
 113    NATIONAL_STRING = auto()
 114    RAW_STRING = auto()
 115    HEREDOC_STRING = auto()
 116    UNICODE_STRING = auto()
 117
 118    # types
 119    BIT = auto()
 120    BOOLEAN = auto()
 121    TINYINT = auto()
 122    UTINYINT = auto()
 123    SMALLINT = auto()
 124    USMALLINT = auto()
 125    MEDIUMINT = auto()
 126    UMEDIUMINT = auto()
 127    INT = auto()
 128    UINT = auto()
 129    BIGINT = auto()
 130    UBIGINT = auto()
 131    BIGNUM = auto()
 132    INT128 = auto()
 133    UINT128 = auto()
 134    INT256 = auto()
 135    UINT256 = auto()
 136    FLOAT = auto()
 137    DOUBLE = auto()
 138    UDOUBLE = auto()
 139    DECIMAL = auto()
 140    DECIMAL32 = auto()
 141    DECIMAL64 = auto()
 142    DECIMAL128 = auto()
 143    DECIMAL256 = auto()
 144    DECFLOAT = auto()
 145    UDECIMAL = auto()
 146    BIGDECIMAL = auto()
 147    CHAR = auto()
 148    NCHAR = auto()
 149    VARCHAR = auto()
 150    NVARCHAR = auto()
 151    BPCHAR = auto()
 152    TEXT = auto()
 153    MEDIUMTEXT = auto()
 154    LONGTEXT = auto()
 155    BLOB = auto()
 156    MEDIUMBLOB = auto()
 157    LONGBLOB = auto()
 158    TINYBLOB = auto()
 159    TINYTEXT = auto()
 160    NAME = auto()
 161    BINARY = auto()
 162    VARBINARY = auto()
 163    JSON = auto()
 164    JSONB = auto()
 165    TIME = auto()
 166    TIMETZ = auto()
 167    TIME_NS = auto()
 168    TIMESTAMP = auto()
 169    TIMESTAMPTZ = auto()
 170    TIMESTAMPLTZ = auto()
 171    TIMESTAMPNTZ = auto()
 172    TIMESTAMP_S = auto()
 173    TIMESTAMP_MS = auto()
 174    TIMESTAMP_NS = auto()
 175    DATETIME = auto()
 176    DATETIME2 = auto()
 177    DATETIME64 = auto()
 178    SMALLDATETIME = auto()
 179    DATE = auto()
 180    DATE32 = auto()
 181    INT4RANGE = auto()
 182    INT4MULTIRANGE = auto()
 183    INT8RANGE = auto()
 184    INT8MULTIRANGE = auto()
 185    NUMRANGE = auto()
 186    NUMMULTIRANGE = auto()
 187    TSRANGE = auto()
 188    TSMULTIRANGE = auto()
 189    TSTZRANGE = auto()
 190    TSTZMULTIRANGE = auto()
 191    DATERANGE = auto()
 192    DATEMULTIRANGE = auto()
 193    UUID = auto()
 194    GEOGRAPHY = auto()
 195    GEOGRAPHYPOINT = auto()
 196    NULLABLE = auto()
 197    GEOMETRY = auto()
 198    POINT = auto()
 199    RING = auto()
 200    LINESTRING = auto()
 201    LOCALTIME = auto()
 202    LOCALTIMESTAMP = auto()
 203    SYSTIMESTAMP = auto()
 204    MULTILINESTRING = auto()
 205    POLYGON = auto()
 206    MULTIPOLYGON = auto()
 207    HLLSKETCH = auto()
 208    HSTORE = auto()
 209    SUPER = auto()
 210    SERIAL = auto()
 211    SMALLSERIAL = auto()
 212    BIGSERIAL = auto()
 213    XML = auto()
 214    YEAR = auto()
 215    USERDEFINED = auto()
 216    MONEY = auto()
 217    SMALLMONEY = auto()
 218    ROWVERSION = auto()
 219    IMAGE = auto()
 220    VARIANT = auto()
 221    OBJECT = auto()
 222    REF = auto()
 223    INET = auto()
 224    IPADDRESS = auto()
 225    IPPREFIX = auto()
 226    IPV4 = auto()
 227    IPV6 = auto()
 228    ENUM = auto()
 229    ENUM8 = auto()
 230    ENUM16 = auto()
 231    FIXEDSTRING = auto()
 232    LOWCARDINALITY = auto()
 233    NESTED = auto()
 234    AGGREGATEFUNCTION = auto()
 235    SIMPLEAGGREGATEFUNCTION = auto()
 236    TDIGEST = auto()
 237    UNKNOWN = auto()
 238    VECTOR = auto()
 239    DYNAMIC = auto()
 240    VOID = auto()
 241
 242    # keywords
 243    ALIAS = auto()
 244    ALTER = auto()
 245    ALL = auto()
 246    ANTI = auto()
 247    ANY = auto()
 248    APPLY = auto()
 249    ARRAY = auto()
 250    ASC = auto()
 251    ASOF = auto()
 252    ATTACH = auto()
 253    AUTO_INCREMENT = auto()
 254    BEGIN = auto()
 255    BETWEEN = auto()
 256    BULK_COLLECT_INTO = auto()
 257    CACHE = auto()
 258    CASE = auto()
 259    CHARACTER_SET = auto()
 260    CLUSTER_BY = auto()
 261    COLLATE = auto()
 262    COMMAND = auto()
 263    COMMENT = auto()
 264    COMMIT = auto()
 265    CONNECT = auto()
 266    CONNECT_BY = auto()
 267    CONSTRAINT = auto()
 268    COPY = auto()
 269    CREATE = auto()
 270    CROSS = auto()
 271    CUBE = auto()
 272    CURRENT_DATE = auto()
 273    CURRENT_DATETIME = auto()
 274    CURRENT_SCHEMA = auto()
 275    CURRENT_TIME = auto()
 276    CURRENT_TIMESTAMP = auto()
 277    CURRENT_USER = auto()
 278    CURRENT_USER_ID = auto()
 279    CURRENT_ROLE = auto()
 280    CURRENT_CATALOG = auto()
 281    DECLARE = auto()
 282    DEFAULT = auto()
 283    DELETE = auto()
 284    DESC = auto()
 285    DESCRIBE = auto()
 286    DETACH = auto()
 287    DISCONNECT = auto()
 288    DICTIONARY = auto()
 289    DISTINCT = auto()
 290    DISTRIBUTE_BY = auto()
 291    DIV = auto()
 292    DROP = auto()
 293    ELSE = auto()
 294    END = auto()
 295    ESCAPE = auto()
 296    EXCEPT = auto()
 297    EXECUTE = auto()
 298    EXISTS = auto()
 299    FALSE = auto()
 300    FETCH = auto()
 301    FILE = auto()
 302    FILE_FORMAT = auto()
 303    FILTER = auto()
 304    FINAL = auto()
 305    FIRST = auto()
 306    FOR = auto()
 307    FORCE = auto()
 308    FOREIGN_KEY = auto()
 309    FORMAT = auto()
 310    FROM = auto()
 311    FULL = auto()
 312    FUNCTION = auto()
 313    GET = auto()
 314    GLOB = auto()
 315    GLOBAL = auto()
 316    GRANT = auto()
 317    GROUP_BY = auto()
 318    GROUPING_SETS = auto()
 319    HAVING = auto()
 320    HINT = auto()
 321    IGNORE = auto()
 322    ILIKE = auto()
 323    IN = auto()
 324    INDEX = auto()
 325    INDEXED_BY = auto()
 326    INNER = auto()
 327    INSERT = auto()
 328    INSTALL = auto()
 329    INTEGRATION = auto()
 330    INTERSECT = auto()
 331    INTERVAL = auto()
 332    INTO = auto()
 333    INTRODUCER = auto()
 334    IRLIKE = auto()
 335    IS = auto()
 336    ISNULL = auto()
 337    JOIN = auto()
 338    JOIN_MARKER = auto()
 339    KEEP = auto()
 340    KEY = auto()
 341    KILL = auto()
 342    LANGUAGE = auto()
 343    LATERAL = auto()
 344    LEFT = auto()
 345    LIKE = auto()
 346    LIMIT = auto()
 347    LIST = auto()
 348    LOAD = auto()
 349    LOCK = auto()
 350    MAP = auto()
 351    MATCH = auto()
 352    MATCH_CONDITION = auto()
 353    MATCH_RECOGNIZE = auto()
 354    MEMBER_OF = auto()
 355    MERGE = auto()
 356    MOD = auto()
 357    MODEL = auto()
 358    NATURAL = auto()
 359    NEXT = auto()
 360    NOTHING = auto()
 361    NOTNULL = auto()
 362    NULL = auto()
 363    OBJECT_IDENTIFIER = auto()
 364    OFFSET = auto()
 365    ON = auto()
 366    ONLY = auto()
 367    OPERATOR = auto()
 368    ORDER_BY = auto()
 369    ORDER_SIBLINGS_BY = auto()
 370    ORDERED = auto()
 371    ORDINALITY = auto()
 372    OUT = auto()
 373    INOUT = auto()
 374    OUTER = auto()
 375    OVER = auto()
 376    OVERLAPS = auto()
 377    OVERWRITE = auto()
 378    PACKAGE = auto()
 379    PARTITION = auto()
 380    PARTITION_BY = auto()
 381    PASTE = auto()
 382    PERCENT = auto()
 383    PIVOT = auto()
 384    PLACEHOLDER = auto()
 385    POLICY = auto()
 386    POOL = auto()
 387    POSITIONAL = auto()
 388    PRAGMA = auto()
 389    PREWHERE = auto()
 390    PRIMARY_KEY = auto()
 391    PROCEDURE = auto()
 392    PROPERTIES = auto()
 393    PROJECTION = auto()
 394    PSEUDO_TYPE = auto()
 395    PUT = auto()
 396    QUALIFY = auto()
 397    QUOTE = auto()
 398    QDCOLON = auto()
 399    RANGE = auto()
 400    RECURSIVE = auto()
 401    REFRESH = auto()
 402    RENAME = auto()
 403    REPLACE = auto()
 404    RETURNING = auto()
 405    REVOKE = auto()
 406    REFERENCES = auto()
 407    RIGHT = auto()
 408    RLIKE = auto()
 409    ROLE = auto()
 410    ROLLBACK = auto()
 411    ROLLUP = auto()
 412    ROW = auto()
 413    ROWS = auto()
 414    RULE = auto()
 415    SELECT = auto()
 416    SEMI = auto()
 417    SEPARATOR = auto()
 418    SEQUENCE = auto()
 419    SERDE_PROPERTIES = auto()
 420    SET = auto()
 421    SETTINGS = auto()
 422    SHOW = auto()
 423    SIMILAR_TO = auto()
 424    SOME = auto()
 425    SORT_BY = auto()
 426    SOUNDS_LIKE = auto()
 427    SQL_SECURITY = auto()
 428    START_WITH = auto()
 429    STORAGE_INTEGRATION = auto()
 430    STRAIGHT_JOIN = auto()
 431    STRUCT = auto()
 432    SUMMARIZE = auto()
 433    TABLE_SAMPLE = auto()
 434    TAG = auto()
 435    TEMPORARY = auto()
 436    TOP = auto()
 437    THEN = auto()
 438    TRUE = auto()
 439    TRUNCATE = auto()
 440    TRIGGER = auto()
 441    TYPE = auto()
 442    UNCACHE = auto()
 443    UNDROP = auto()
 444    UNION = auto()
 445    UNNEST = auto()
 446    UNPIVOT = auto()
 447    UPDATE = auto()
 448    USE = auto()
 449    USING = auto()
 450    VALUES = auto()
 451    VARIADIC = auto()
 452    VIEW = auto()
 453    SEMANTIC_VIEW = auto()
 454    VOLATILE = auto()
 455    VOLUME = auto()
 456    WHEN = auto()
 457    WHERE = auto()
 458    WINDOW = auto()
 459    WITH = auto()
 460    UNIQUE = auto()
 461    UTC_DATE = auto()
 462    UTC_TIME = auto()
 463    UTC_TIMESTAMP = auto()
 464    OPTION = auto()
 465    SINK = auto()
 466    SOURCE = auto()
 467    ANALYZE = auto()
 468    NAMESPACE = auto()
 469    EXPORT = auto()
 470
 471    # sentinels
 472    HIVE_TOKEN_STREAM = auto()
 473    SENTINEL = auto()
 474
 475    def __str__(self) -> str:
 476        return f"TokenType.{self.name}"
 477
 478
 479class Token:
 480    # mypyc doesn't expose slots
 481    _attrs: t.ClassVar[tuple[str, ...]] = (
 482        "token_type",
 483        "text",
 484        "line",
 485        "col",
 486        "start",
 487        "end",
 488        "comments",
 489    )
 490    __slots__ = _attrs
 491
 492    @classmethod
 493    def number(cls, number: int) -> Token:
 494        """Returns a NUMBER token with `number` as its text."""
 495        return cls(TokenType.NUMBER, str(number))
 496
 497    @classmethod
 498    def string(cls, string: str) -> Token:
 499        """Returns a STRING token with `string` as its text."""
 500        return cls(TokenType.STRING, string)
 501
 502    @classmethod
 503    def identifier(cls, identifier: str) -> Token:
 504        """Returns an IDENTIFIER token with `identifier` as its text."""
 505        return cls(TokenType.IDENTIFIER, identifier)
 506
 507    @classmethod
 508    def var(cls, var: str) -> Token:
 509        """Returns an VAR token with `var` as its text."""
 510        return cls(TokenType.VAR, var)
 511
 512    def __init__(
 513        self,
 514        token_type: TokenType,
 515        text: str,
 516        line: int = 1,
 517        col: int = 1,
 518        start: int = 0,
 519        end: int = 0,
 520        comments: list[str] | None = None,
 521    ) -> None:
 522        self.token_type = token_type
 523        self.text = text
 524        self.line = line
 525        self.col = col
 526        self.start = start
 527        self.end = end
 528        self.comments = [] if comments is None else comments
 529
 530    def __bool__(self) -> bool:
 531        return self.token_type != TokenType.SENTINEL
 532
 533    def __repr__(self) -> str:
 534        attributes = ", ".join(
 535            f"{k}: TokenType.{self.token_type.name}"
 536            if k == "token_type"
 537            else f"{k}: {getattr(self, k)}"
 538            for k in self._attrs
 539        )
 540        return f"<Token {attributes}>"
 541
 542
 543class TokenizerCore:
 544    __slots__ = (
 545        "sql",
 546        "size",
 547        "tokens",
 548        "_start",
 549        "_current",
 550        "_line",
 551        "_col",
 552        "_comments",
 553        "_char",
 554        "_end",
 555        "_peek",
 556        "_prev_token_line",
 557        "single_tokens",
 558        "keywords",
 559        "quotes",
 560        "format_strings",
 561        "identifiers",
 562        "comments",
 563        "string_escapes",
 564        "byte_string_escapes",
 565        "identifier_escapes",
 566        "commands",
 567        "command_prefix_tokens",
 568        "nested_comments",
 569        "dash_comment_requires_boundary",
 570        "comments_terminate_at_newline_only",
 571        "hint_start",
 572        "tokens_preceding_hint",
 573        "has_bit_strings",
 574        "has_hex_strings",
 575        "numeric_literals",
 576        "var_single_tokens",
 577        "string_escapes_allowed_in_raw_strings",
 578        "heredoc_tag_is_identifier",
 579        "heredoc_string_alternative",
 580        "keyword_trie",
 581        "numbers_can_be_underscore_separated",
 582        "numbers_can_have_decimals",
 583        "identifiers_can_start_with_digit",
 584        "unescaped_sequences",
 585        "numeric_escapes",
 586        "drop_unknown_escapes",
 587        "numeric_escapes_are_bytes",
 588        "byte_strings_are_bytes",
 589        "lone_surrogate_replacement",
 590    )
 591
 592    def __init__(
 593        self,
 594        single_tokens: dict[str, TokenType],
 595        keywords: dict[str, TokenType],
 596        quotes: dict[str, str],
 597        format_strings: dict[str, tuple[str, TokenType]],
 598        identifiers: dict[str, str],
 599        comments: dict[str, str | None],
 600        string_escapes: set[str],
 601        byte_string_escapes: set[str],
 602        identifier_escapes: set[str],
 603        commands: set[TokenType],
 604        command_prefix_tokens: set[TokenType],
 605        nested_comments: bool,
 606        dash_comment_requires_boundary: bool,
 607        comments_terminate_at_newline_only: bool,
 608        hint_start: str,
 609        tokens_preceding_hint: set[TokenType],
 610        has_bit_strings: bool,
 611        has_hex_strings: bool,
 612        numeric_literals: dict[str, str],
 613        var_single_tokens: set[str],
 614        string_escapes_allowed_in_raw_strings: bool,
 615        heredoc_tag_is_identifier: bool,
 616        heredoc_string_alternative: TokenType,
 617        keyword_trie: dict,
 618        numbers_can_be_underscore_separated: bool,
 619        numbers_can_have_decimals: bool,
 620        identifiers_can_start_with_digit: bool,
 621        unescaped_sequences: dict[str, str],
 622        numeric_escapes: dict[str, tuple[int, int, int, int]],
 623        drop_unknown_escapes: bool,
 624        numeric_escapes_are_bytes: bool,
 625        byte_strings_are_bytes: bool,
 626        lone_surrogate_replacement: str,
 627    ) -> None:
 628        self.single_tokens = single_tokens
 629        self.keywords = keywords
 630        self.quotes = quotes
 631        self.format_strings = format_strings
 632        self.identifiers = identifiers
 633        self.comments = comments
 634        self.string_escapes = string_escapes
 635        self.byte_string_escapes = byte_string_escapes
 636        self.identifier_escapes = identifier_escapes
 637        self.commands = commands
 638        self.command_prefix_tokens = command_prefix_tokens
 639        self.nested_comments = nested_comments
 640        self.dash_comment_requires_boundary = dash_comment_requires_boundary
 641        self.comments_terminate_at_newline_only = comments_terminate_at_newline_only
 642        self.hint_start = hint_start
 643        self.tokens_preceding_hint = tokens_preceding_hint
 644        self.has_bit_strings = has_bit_strings
 645        self.has_hex_strings = has_hex_strings
 646        self.numeric_literals = numeric_literals
 647        self.var_single_tokens = var_single_tokens
 648        self.string_escapes_allowed_in_raw_strings = string_escapes_allowed_in_raw_strings
 649        self.heredoc_tag_is_identifier = heredoc_tag_is_identifier
 650        self.heredoc_string_alternative = heredoc_string_alternative
 651        self.keyword_trie = keyword_trie
 652        self.numbers_can_be_underscore_separated = numbers_can_be_underscore_separated
 653        self.numbers_can_have_decimals = numbers_can_have_decimals
 654        self.identifiers_can_start_with_digit = identifiers_can_start_with_digit
 655        self.unescaped_sequences = unescaped_sequences
 656        self.numeric_escapes = numeric_escapes
 657        self.drop_unknown_escapes = drop_unknown_escapes
 658        self.numeric_escapes_are_bytes = numeric_escapes_are_bytes
 659        self.byte_strings_are_bytes = byte_strings_are_bytes
 660        self.lone_surrogate_replacement = lone_surrogate_replacement
 661        self.sql = ""
 662        self.size = 0
 663        self.tokens: list[Token] = []
 664        self._start = 0
 665        self._current = 0
 666        self._line = 1
 667        self._col = 0
 668        self._comments: list[str] = []
 669        self._char = ""
 670        self._end = False
 671        self._peek = ""
 672        self._prev_token_line = -1
 673
 674    def reset(self) -> None:
 675        self.sql = ""
 676        self.size = 0
 677        self.tokens = []
 678        self._start = 0
 679        self._current = 0
 680        self._line = 1
 681        self._col = 0
 682        self._comments = []
 683        self._char = ""
 684        self._end = False
 685        self._peek = ""
 686        self._prev_token_line = -1
 687
 688    def tokenize(self, sql: str) -> list[Token]:
 689        """Returns a list of tokens corresponding to the SQL string `sql`."""
 690        self.reset()
 691        self.sql = sql
 692        self.size = len(sql)
 693
 694        try:
 695            self._scan()
 696        except Exception as e:
 697            start = max(self._current - 50, 0)
 698            end = min(self._current + 50, self.size - 1)
 699            context = self.sql[start:end]
 700            raise TokenError(f"Error tokenizing '{context}'", start=start, end=end) from e
 701
 702        return self.tokens
 703
 704    def _scan(self, check_semicolon: bool = False) -> None:
 705        identifiers = self.identifiers
 706        digit_chars = _DIGIT_CHARS
 707
 708        while self.size and not self._end:
 709            current = self._current
 710
 711            # Skip spaces here rather than iteratively calling advance() for performance reasons
 712            while current < self.size:
 713                char = self.sql[current]
 714
 715                if char == " " or char == "\t":
 716                    current += 1
 717                else:
 718                    break
 719
 720            offset = current - self._current if current > self._current else 1
 721
 722            self._start = current
 723            self._advance(offset)
 724
 725            if not self._char.isspace():
 726                if self._char in digit_chars:
 727                    self._scan_number()
 728                elif self._char in identifiers:
 729                    self._scan_identifier(identifiers[self._char])
 730                else:
 731                    self._scan_keywords()
 732
 733            if check_semicolon and self._peek == ";":
 734                break
 735
 736        if self.tokens and self._comments:
 737            self.tokens[-1].comments.extend(self._comments)
 738
 739    def _chars(self, size: int) -> str:
 740        if size == 1:
 741            return self._char
 742
 743        start = self._current - 1
 744        end = start + size
 745
 746        return self.sql[start:end] if end <= self.size else ""
 747
 748    def _advance(self, i: int = 1, alnum: bool = False) -> None:
 749        char = self._char
 750
 751        if char == "\n" or char == "\r":
 752            # Ensures we don't count an extra line if we get a \r\n line break sequence
 753            if not (char == "\r" and self._peek == "\n"):
 754                self._col = i
 755                self._line += 1
 756        else:
 757            self._col += i
 758
 759        self._current += i
 760        sql = self.sql
 761        size = self.size
 762        self._end = self._current >= size
 763        self._char = sql[self._current - 1]
 764        self._peek = "" if self._end else sql[self._current]
 765
 766        if alnum and self._char.isalnum():
 767            # Cache to local variables instead of attributes for better performance
 768            _col = self._col
 769            _current = self._current
 770            _end = self._end
 771            _peek = self._peek
 772
 773            while _peek.isalnum():
 774                _col += 1
 775                _current += 1
 776                _end = _current >= size
 777                _peek = "" if _end else sql[_current]
 778
 779            self._col = _col
 780            self._current = _current
 781            self._end = _end
 782            self._peek = _peek
 783            self._char = sql[_current - 1]
 784
 785    @property
 786    def _text(self) -> str:
 787        return self.sql[self._start : self._current]
 788
 789    def _add(self, token_type: TokenType, text: str | None = None) -> None:
 790        self._prev_token_line = self._line
 791
 792        if self._comments and token_type == TokenType.SEMICOLON and self.tokens:
 793            self.tokens[-1].comments.extend(self._comments)
 794            self._comments = []
 795
 796        if text is None:
 797            text = self.sql[self._start : self._current]
 798
 799        self.tokens.append(
 800            Token(
 801                token_type,
 802                text=text,
 803                line=self._line,
 804                col=self._col,
 805                start=self._start,
 806                end=self._current - 1,
 807                comments=self._comments,
 808            )
 809        )
 810        self._comments = []
 811
 812        # If we have either a semicolon or a begin token before the command's token, we'll parse
 813        # whatever follows the command's token as a string
 814        if (
 815            token_type in self.commands
 816            and self._peek != ";"
 817            and (len(self.tokens) == 1 or self.tokens[-2].token_type in self.command_prefix_tokens)
 818        ):
 819            start = self._current
 820            tokens = len(self.tokens)
 821            self._scan(check_semicolon=True)
 822            self.tokens = self.tokens[:tokens]
 823            text = self.sql[start : self._current].strip()
 824            if text:
 825                self._add(TokenType.STRING, text)
 826
 827    def _scan_keywords(self) -> None:
 828        sql = self.sql
 829        sql_size = self.size
 830        single_tokens = self.single_tokens
 831        char_upper = _CHAR_UPPER
 832        size = 0
 833        word = None
 834        chars = self._char
 835        char = chars
 836        prev_space = False
 837        skip = False
 838        trie = self.keyword_trie
 839        single_token = char in single_tokens
 840
 841        while chars:
 842            if not skip:
 843                sub = trie.get(char_upper.get(char, char))
 844                if sub is None:
 845                    break
 846                trie = sub
 847                if 0 in trie:
 848                    word = chars
 849
 850            end = self._current + size
 851            size += 1
 852
 853            if end < sql_size:
 854                char = sql[end]
 855                single_token = single_token or char in single_tokens
 856                is_space = char.isspace()
 857
 858                if not is_space or not prev_space:
 859                    if is_space:
 860                        char = " "
 861                    chars += char
 862                    prev_space = is_space
 863                    skip = False
 864                else:
 865                    skip = True
 866            else:
 867                char = ""
 868                break
 869
 870        if word:
 871            if self._scan_string(word):
 872                return
 873            if self._scan_comment(word):
 874                return
 875            if prev_space or single_token or not char:
 876                word = word.upper()
 877                if word in self.keywords:
 878                    self._advance(size - 1)
 879                    self._add(self.keywords[word], text=word)
 880                    return
 881
 882        if self._char in single_tokens:
 883            self._add(single_tokens[self._char], text=self._char)
 884            return
 885
 886        self._scan_var()
 887
 888    def _scan_comment(self, comment_start: str) -> bool:
 889        if comment_start not in self.comments:
 890            return False
 891
 892        if comment_start == "--" and self.dash_comment_requires_boundary:
 893            boundary_index = self._current + 1
 894            if boundary_index < self.size:
 895                boundary_char = self.sql[boundary_index]
 896                if not (
 897                    boundary_char.isspace() or ord(boundary_char) < 32 or ord(boundary_char) == 127
 898                ):
 899                    return False
 900
 901        comment_start_line = self._line
 902        comment_start_size = len(comment_start)
 903        comment_end = self.comments[comment_start]
 904
 905        if comment_end:
 906            # Skip the comment's start delimiter
 907            self._advance(comment_start_size)
 908
 909            comment_count = 1
 910            comment_end_size = len(comment_end)
 911            nested_comments = self.nested_comments
 912
 913            while not self._end:
 914                if self._chars(comment_end_size) == comment_end:
 915                    comment_count -= 1
 916                    if not comment_count:
 917                        break
 918
 919                self._advance(alnum=True)
 920
 921                # Nested comments are allowed by some dialects, e.g. databricks, duckdb, postgres
 922                if (
 923                    nested_comments
 924                    and not self._end
 925                    and self._chars(comment_end_size) == comment_start
 926                ):
 927                    self._advance(comment_start_size)
 928                    comment_count += 1
 929
 930            self._comments.append(self._text[comment_start_size : -comment_end_size + 1])
 931            self._advance(comment_end_size - 1)
 932        else:
 933            _peek = self._peek
 934            terminates_at_carriage_return = not self.comments_terminate_at_newline_only
 935            while (
 936                not self._end
 937                and _peek != "\n"
 938                and (not terminates_at_carriage_return or _peek != "\r")
 939            ):
 940                self._advance(alnum=True)
 941                _peek = self._peek
 942            self._comments.append(self._text[comment_start_size:])
 943
 944        if (
 945            comment_start == self.hint_start
 946            and self.tokens
 947            and self.tokens[-1].token_type in self.tokens_preceding_hint
 948        ):
 949            self._add(TokenType.HINT)
 950
 951        # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.
 952        # Multiple consecutive comments are preserved by appending them to the current comments list.
 953        if comment_start_line == self._prev_token_line:
 954            self.tokens[-1].comments.extend(self._comments)
 955            self._comments = []
 956            self._prev_token_line = self._line
 957
 958        return True
 959
 960    def _scan_number(self) -> None:
 961        if self._char == "0":
 962            peek = _CHAR_UPPER.get(self._peek, self._peek)
 963            if peek == "B" and self.has_bit_strings:
 964                return self._scan_bits()
 965            elif peek == "X":
 966                return self._scan_hex() if self.has_hex_strings else self._add(TokenType.NUMBER)
 967
 968        decimal = False
 969        scientific = 0
 970        numbers_can_be_underscore_separated = self.numbers_can_be_underscore_separated
 971        single_tokens = self.single_tokens
 972        keywords = self.keywords
 973        numeric_literals = self.numeric_literals
 974        identifiers_can_start_with_digit = self.identifiers_can_start_with_digit
 975
 976        is_underscore_separated: bool = False
 977        number_text: str = ""
 978        numeric_literal: str = ""
 979        numeric_type: TokenType | None = None
 980
 981        while True:
 982            if self._peek in _DIGIT_CHARS:
 983                # Batch consecutive digits: scan ahead to find how many
 984                sql = self.sql
 985                end = self._current + 1
 986                size = self.size
 987                while end < size and sql[end] in _DIGIT_CHARS:
 988                    end += 1
 989                self._advance(end - self._current)
 990            elif self._peek == "." and not decimal:
 991                if (
 992                    self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
 993                ) or not self.numbers_can_have_decimals:
 994                    break
 995                decimal = True
 996                self._advance()
 997            elif self._peek in ("-", "+") and scientific == 1:
 998                # Only consume +/- if followed by a digit
 999                if self._current + 1 < self.size and self.sql[self._current + 1] in _DIGIT_CHARS:
1000                    scientific += 1
1001                    self._advance()
1002                else:
1003                    break
1004            elif (
1005                _CHAR_UPPER.get(self._peek, self._peek) == "E"
1006                and not scientific
1007                and not (
1008                    numbers_can_be_underscore_separated
1009                    and self._current + 1 < self.size
1010                    and self.sql[self._current + 1] == "_"
1011                )
1012            ):
1013                scientific += 1
1014                self._advance()
1015            elif (
1016                self._peek == "_"
1017                and numbers_can_be_underscore_separated
1018                and self._char in _DIGIT_CHARS
1019                and self._current + 1 < self.size
1020                and self.sql[self._current + 1] in _DIGIT_CHARS
1021            ):
1022                is_underscore_separated = True
1023                self._advance()
1024            elif self._peek.isidentifier():
1025                number_text = self._text
1026
1027                while self._peek and not self._peek.isspace() and self._peek not in single_tokens:
1028                    numeric_literal += self._peek
1029                    self._advance()
1030
1031                numeric_type = keywords.get(numeric_literals.get(numeric_literal.upper(), ""))
1032
1033                if numeric_type:
1034                    break
1035                elif identifiers_can_start_with_digit:
1036                    return self._add(TokenType.VAR)
1037
1038                self._advance(-len(numeric_literal))
1039                break
1040            else:
1041                break
1042
1043        number_text = number_text or self.sql[self._start : self._current]
1044
1045        # Normalize inputs such as 100_000 to 100000
1046        if is_underscore_separated:
1047            number_text = number_text.replace("_", "")
1048
1049        self._add(TokenType.NUMBER, number_text)
1050
1051        # Normalize inputs such as 123L to 123::BIGINT so that they're parsed as casts
1052        if numeric_type:
1053            self._add(TokenType.DCOLON, "::")
1054            self._add(numeric_type, numeric_literal)
1055
1056    def _scan_bits(self) -> None:
1057        self._advance()
1058        value = self._extract_value()
1059        try:
1060            # If `value` can't be converted to a binary, fallback to tokenizing it as an identifier
1061            int(value, 2)
1062            self._add(TokenType.BIT_STRING, value[2:])  # Drop the 0b
1063        except ValueError:
1064            self._add(TokenType.IDENTIFIER)
1065
1066    def _scan_hex(self) -> None:
1067        self._advance()
1068        value = self._extract_value()
1069        try:
1070            # If `value` can't be converted to a hex, fallback to tokenizing it as an identifier
1071            int(value, 16)
1072            self._add(TokenType.HEX_STRING, value[2:])  # Drop the 0x
1073        except ValueError:
1074            self._add(TokenType.IDENTIFIER)
1075
1076    def _extract_value(self) -> str:
1077        single_tokens = self.single_tokens
1078
1079        while True:
1080            char = self._peek.strip()
1081            if char and char not in single_tokens:
1082                self._advance(alnum=True)
1083            else:
1084                break
1085
1086        return self._text
1087
1088    def _scan_string(self, start: str) -> bool:
1089        base = None
1090        token_type = TokenType.STRING
1091
1092        if start in self.quotes:
1093            end = self.quotes[start]
1094        elif start in self.format_strings:
1095            end, token_type = self.format_strings[start]
1096
1097            if token_type == TokenType.HEX_STRING:
1098                base = 16
1099            elif token_type == TokenType.BIT_STRING:
1100                base = 2
1101            elif token_type == TokenType.HEREDOC_STRING:
1102                self._advance()
1103
1104                if self._char == end:
1105                    tag = ""
1106                else:
1107                    tag = self._extract_string(
1108                        end,
1109                        raw_string=True,
1110                        raise_unmatched=not self.heredoc_tag_is_identifier,
1111                    )
1112
1113                if (
1114                    tag
1115                    and self.heredoc_tag_is_identifier
1116                    and (self._end or tag.isdigit() or any(c.isspace() for c in tag))
1117                ):
1118                    if not self._end:
1119                        self._advance(-1)
1120
1121                    self._advance(-len(tag))
1122                    self._add(self.heredoc_string_alternative)
1123                    return True
1124
1125                end = f"{start}{tag}{end}"
1126        else:
1127            return False
1128
1129        self._advance(len(start))
1130        text = self._extract_string(
1131            end,
1132            escapes=(
1133                self.byte_string_escapes
1134                if token_type == TokenType.BYTE_STRING
1135                else self.string_escapes
1136            ),
1137            raw_string=token_type
1138            in (TokenType.RAW_STRING, TokenType.NATIONAL_RAW_STRING, TokenType.HEREDOC_STRING),
1139            bytes_literal=token_type == TokenType.BYTE_STRING and self.byte_strings_are_bytes,
1140        )
1141
1142        if base and text:
1143            try:
1144                int(text, base)
1145            except Exception:
1146                raise TokenError(
1147                    f"Numeric string contains invalid characters from {self._line}:{self._start}"
1148                )
1149
1150        self._add(token_type, text)
1151        return True
1152
1153    def _scan_identifier(self, identifier_end: str) -> None:
1154        self._advance()
1155        text = self._extract_string(
1156            identifier_end, escapes=self.identifier_escapes | {identifier_end}
1157        )
1158        self._add(TokenType.IDENTIFIER, text)
1159
1160    def _scan_var(self) -> None:
1161        var_single_tokens = self.var_single_tokens
1162        single_tokens = self.single_tokens
1163
1164        while True:
1165            peek = self._peek
1166            if not peek or peek.isspace():
1167                break
1168            if peek not in var_single_tokens and peek in single_tokens:
1169                break
1170            self._advance(alnum=True)
1171
1172        self._add(
1173            TokenType.VAR
1174            if self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
1175            else self.keywords.get(self.sql[self._start : self._current].upper(), TokenType.VAR)
1176        )
1177
1178    def _read_numeric_escape(
1179        self, start: int, base: int, min_digits: int, max_digits: int, max_value: int
1180    ) -> tuple[int, int]:
1181        """
1182        Reads up to `max_digits` digits from `start`, stopping before the value exceeds `max_value`.
1183        Returns the value and the end index, or -1 as the value if fewer than `min_digits` were read.
1184        """
1185        sql = self.sql
1186        value = 0
1187        end = start
1188        limit = min(start + max_digits, self.size)
1189
1190        while end < limit:
1191            digit = _DIGIT_VALUES.get(sql[end], 16)
1192            if digit >= base or value * base + digit > max_value:
1193                break
1194            value = value * base + digit
1195            end += 1
1196
1197        return (value, end) if end - start >= min_digits else (-1, end)
1198
1199    def _read_byte_escape(self, pos: int) -> tuple[int, int]:
1200        """Reads the numeric escape at `pos` (a backslash) if it encodes a byte, else returns -1."""
1201        peek = self.sql[pos + 1 : pos + 2]
1202        is_octal = peek in _OCTAL_CHARS
1203        spec = self.numeric_escapes.get("0" if is_octal else peek)
1204        if not spec or (spec[0] != 8 and spec[3] > 0xFF):
1205            return -1, pos
1206
1207        base, min_digits, max_digits, max_value = spec
1208        value, end = self._read_numeric_escape(
1209            pos + 1 if is_octal else pos + 2, base, min_digits, max_digits, max_value
1210        )
1211        return (value & 0xFF if value >= 0 else value), end
1212
1213    def _scan_numeric_escape(self) -> str:
1214        """
1215        Decodes the numeric escape sequence starting at the current char, which is a backslash,
1216        according to `numeric_escapes`. Returns an empty string if nothing is decoded.
1217        """
1218        sql = self.sql
1219        start = self._current
1220        peek = self._peek
1221
1222        # Octal escapes start with a digit (e.g. \101), the others with a letter (e.g. \x41)
1223        is_octal = peek in _OCTAL_CHARS
1224        spec = self.numeric_escapes.get("0" if is_octal else peek)
1225
1226        if spec:
1227            base, min_digits, max_digits, max_value = spec
1228            value, end = self._read_numeric_escape(
1229                start if is_octal else start + 1, base, min_digits, max_digits, max_value
1230            )
1231
1232            # With byte semantics, octal and hex escapes encode a byte, so octal values wrap around,
1233            # e.g. \541 is \x61
1234            is_byte = self.numeric_escapes_are_bytes and (base == 8 or max_value <= 0xFF)
1235            if is_byte and value >= 0:
1236                value &= 0xFF
1237
1238            # A UTF-16 surrogate is only valid as a high half (D800-DBFF) followed by an escaped
1239            # low half (DC00-DFFF), e.g. \uD83D\uDE00 is U+1F600
1240            if 0xD800 <= value <= 0xDFFF:
1241                low, low_end = -1, end
1242                if value <= 0xDBFF and sql[end : end + 2] == "\\" + peek:
1243                    low, low_end = self._read_numeric_escape(
1244                        end + 2, base, min_digits, max_digits, max_value
1245                    )
1246
1247                if 0xDC00 <= low <= 0xDFFF:
1248                    # Combine the two halves into a single code point (standard formula)
1249                    value = 0x10000 + ((value - 0xD800) << 10) + (low - 0xDC00)
1250                    end = low_end
1251                elif self.lone_surrogate_replacement and end < self.size:
1252                    # A lone surrogate isn't a character, so some engines replace it (e.g. with ?)
1253                    self._advance(end - start + 1)
1254                    return self.lone_surrogate_replacement
1255                else:
1256                    # A lone surrogate isn't a character, so the escape isn't decoded
1257                    value = -1
1258
1259            # Decode one UTF-8 character at a time to avoid rescanning long invalid byte runs.
1260            # Invalid UTF-8 falls back to decoding each byte as a code point. This first check
1261            # detects whether the byte can start a multi-byte UTF-8 character.
1262            if is_byte and 0xC2 <= value <= 0xF4:
1263                run = bytearray([value])
1264                run_end = end
1265                byte_count = 2 if value < 0xE0 else 3 if value < 0xF0 else 4
1266                while len(run) < byte_count and sql[run_end : run_end + 1] == "\\":
1267                    byte, byte_end = self._read_byte_escape(run_end)
1268                    if not 0x80 <= byte <= 0xBF:
1269                        break
1270                    run.append(byte)
1271                    run_end = byte_end
1272
1273                try:
1274                    decoded = run.decode("utf-8")
1275                except UnicodeDecodeError:
1276                    decoded = ""
1277
1278                if decoded and run_end < self.size:
1279                    self._advance(run_end - start + 1)
1280                    return decoded
1281
1282            # -1 means the escape isn't decoded, e.g. because it has too few digits. An escape at the
1283            # end of the input is left to the caller, which reports the unterminated string
1284            if value >= 0 and end < self.size:
1285                # Move past the whole escape, from the backslash through its last digit
1286                self._advance(end - start + 1)
1287                return chr(value)
1288
1289        return ""
1290
1291    def _extract_string(
1292        self,
1293        delimiter: str,
1294        escapes: set[str] | None = None,
1295        raw_string: bool = False,
1296        raise_unmatched: bool = True,
1297        bytes_literal: bool = False,
1298    ) -> str:
1299        """
1300        Extracts a string up to the delimiter, decoding its escape sequences. For byte literals,
1301        each char of the result is a byte, so non-ASCII chars are stored as their UTF-8 bytes.
1302        """
1303        text = ""
1304        delim_size = len(delimiter)
1305        escapes = self.string_escapes if escapes is None else escapes
1306        unescaped_sequences = self.unescaped_sequences
1307        numeric_escapes = self.numeric_escapes
1308        drop_unknown_escapes = self.drop_unknown_escapes
1309        string_escapes_allowed_in_raw_strings = self.string_escapes_allowed_in_raw_strings
1310        quotes = self.quotes
1311        sql = self.sql
1312
1313        # use str.find() when the string is simple... no \ or other escapes
1314        if delim_size == 1:
1315            pos = self._current - 1
1316            end = sql.find(delimiter, pos)
1317
1318            if (
1319                # the closing delimiter was found
1320                end != -1
1321                # there's no doubled delimiter (e.g. '' escape), or the delimiter isn't an escape char
1322                and (end + 1 >= self.size or sql[end + 1] != delimiter or delimiter not in escapes)
1323                # no backslash in the string that would need escape processing
1324                and (not (unescaped_sequences or "\\" in escapes) or sql.find("\\", pos, end) == -1)
1325            ):
1326                newlines = sql.count("\n", pos, end)
1327                if newlines:
1328                    self._line += newlines
1329                    self._col = end - sql.rfind("\n", pos, end)
1330                else:
1331                    self._col += end - pos
1332
1333                self._current = end + 1
1334                self._end = self._current >= self.size
1335                self._char = sql[end]
1336                self._peek = "" if self._end else sql[self._current]
1337                text = sql[pos:end]
1338                if bytes_literal and not text.isascii():
1339                    text = text.encode("utf-8").decode("latin-1")
1340                return text
1341
1342        while True:
1343            backslash_escape = not raw_string and self._char == "\\" and "\\" in escapes
1344
1345            # Numeric escapes come first, since they may overlap fixed sequences (e.g. \000 vs \0)
1346            if backslash_escape and numeric_escapes:
1347                decoded = self._scan_numeric_escape()
1348                if decoded:
1349                    text += decoded
1350                    continue
1351
1352            # An escape at the end of the input is left to the checks below, which report the
1353            # unterminated string
1354            if (
1355                not raw_string
1356                and unescaped_sequences
1357                and self._char in escapes
1358                and self._current + 1 < self.size
1359            ):
1360                unescaped_sequence = unescaped_sequences.get(self._char + self._peek)
1361                if unescaped_sequence is not None:
1362                    # Advance one char at a time so that line numbers are updated for a newline
1363                    if self._peek in "\n\r":
1364                        self._advance()
1365                        self._advance()
1366                    else:
1367                        self._advance(2)
1368                    text += unescaped_sequence
1369                    continue
1370
1371            # Not decoded: drop the backslash if configured, unless the string is unterminated
1372            if backslash_escape and drop_unknown_escapes and self._current + 1 < self.size:
1373                # Advance one char at a time so that line numbers are updated if `peek` is a newline
1374                peek = self._peek
1375                self._advance()
1376                self._advance()
1377                text += peek.encode("utf-8").decode("latin-1") if bytes_literal else peek
1378                continue
1379
1380            # An escaped quote before the closing delimiter (e.g. \" in """a\"""") must be
1381            # consumed here, otherwise it'd be picked up by the delimiter check below and
1382            # terminate the string early. This is only relevant for multi-char delimiters
1383            # made up of quote chars, e.g. it shouldn't apply to Snowflake's $$ strings
1384            escaped_delimiter = self._peek == delimiter or (
1385                delim_size > 1 and self._peek == delimiter[0] and self._peek in quotes
1386            )
1387
1388            if (
1389                (string_escapes_allowed_in_raw_strings or not raw_string)
1390                and self._char in escapes
1391                and (escaped_delimiter or self._peek in escapes)
1392                and (self._char not in quotes or self._char == self._peek)
1393            ):
1394                if escaped_delimiter:
1395                    text += self._peek if not raw_string else self._char + self._peek
1396                else:
1397                    text += self._char + self._peek
1398
1399                if self._current + 1 < self.size:
1400                    self._advance(2)
1401                else:
1402                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._current}")
1403            else:
1404                if self._chars(delim_size) == delimiter:
1405                    if delim_size > 1:
1406                        self._advance(delim_size - 1)
1407                    break
1408
1409                if self._end:
1410                    if not raise_unmatched:
1411                        return text + self._char
1412
1413                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._start}")
1414
1415                current = self._current - 1
1416                self._advance(alnum=True)
1417                chunk = sql[current : self._current - 1]
1418                if bytes_literal and not chunk.isascii():
1419                    chunk = chunk.encode("utf-8").decode("latin-1")
1420                text += chunk
1421
1422        return text