sqlglot.tokenizer_core

   1from __future__ import annotations
   2
   3import typing as t
   4from enum import IntEnum, auto
   5
   6from sqlglot.errors import TokenError
   7
   8# dict lookup is faster than .upper() and .isdigit()
   9_CHAR_UPPER: dict[str, str] = {chr(i): chr(i).upper() for i in range(97, 123)}
  10_DIGIT_CHARS: frozenset[str] = frozenset("0123456789")
  11
  12
  13class TokenType(IntEnum):
  14    L_PAREN = auto()
  15    R_PAREN = auto()
  16    L_BRACKET = auto()
  17    R_BRACKET = auto()
  18    L_BRACE = auto()
  19    R_BRACE = auto()
  20    COMMA = auto()
  21    DOT = auto()
  22    DASH = auto()
  23    PLUS = auto()
  24    COLON = auto()
  25    DOTCOLON = auto()
  26    DOTCARET = auto()
  27    DCOLON = auto()
  28    DCOLONDOLLAR = auto()
  29    DCOLONPERCENT = auto()
  30    DCOLONQMARK = auto()
  31    DQMARK = auto()
  32    SEMICOLON = auto()
  33    STAR = auto()
  34    BACKSLASH = auto()
  35    SLASH = auto()
  36    LT = auto()
  37    LTE = auto()
  38    GT = auto()
  39    GTE = auto()
  40    NOT = auto()
  41    EQ = auto()
  42    NEQ = auto()
  43    NULLSAFE_EQ = auto()
  44    COLON_EQ = auto()
  45    COLON_GT = auto()
  46    NCOLON_GT = auto()
  47    AND = auto()
  48    OR = auto()
  49    AMP = auto()
  50    DPIPE = auto()
  51    PIPE_GT = auto()
  52    PIPE = auto()
  53    PIPE_SLASH = auto()
  54    DPIPE_SLASH = auto()
  55    CARET = auto()
  56    CARET_AT = auto()
  57    TILDE = auto()
  58    ARROW = auto()
  59    DARROW = auto()
  60    FARROW = auto()
  61    HASH = auto()
  62    HASH_ARROW = auto()
  63    DHASH_ARROW = auto()
  64    LR_ARROW = auto()
  65    LLRR_ARROW = auto()
  66    DAT = auto()
  67    AT_QMARK = auto()
  68    LT_AT = auto()
  69    AT_GT = auto()
  70    DOLLAR = auto()
  71    PARAMETER = auto()
  72    SESSION = auto()
  73    SESSION_PARAMETER = auto()
  74    SESSION_USER = auto()
  75    DAMP = auto()
  76    AMP_LT = auto()
  77    AMP_GT = auto()
  78    ADJACENT = auto()
  79    XOR = auto()
  80    DSTAR = auto()
  81    QMARK_AMP = auto()
  82    QMARK_PIPE = auto()
  83    HASH_DASH = auto()
  84    EXCLAMATION = auto()
  85
  86    URI_START = auto()
  87
  88    BLOCK_START = auto()
  89    BLOCK_END = auto()
  90
  91    SPACE = auto()
  92    BREAK = auto()
  93
  94    STRING = auto()
  95    NUMBER = auto()
  96    IDENTIFIER = auto()
  97    DATABASE = auto()
  98    COLUMN = auto()
  99    COLUMN_DEF = auto()
 100    SCHEMA = auto()
 101    TABLE = auto()
 102    WAREHOUSE = auto()
 103    STAGE = auto()
 104    STREAM = auto()
 105    STREAMLIT = auto()
 106    VAR = auto()
 107    BIT_STRING = auto()
 108    HEX_STRING = auto()
 109    BYTE_STRING = auto()
 110    NATIONAL_STRING = auto()
 111    RAW_STRING = auto()
 112    HEREDOC_STRING = auto()
 113    UNICODE_STRING = auto()
 114
 115    # types
 116    BIT = auto()
 117    BOOLEAN = auto()
 118    TINYINT = auto()
 119    UTINYINT = auto()
 120    SMALLINT = auto()
 121    USMALLINT = auto()
 122    MEDIUMINT = auto()
 123    UMEDIUMINT = auto()
 124    INT = auto()
 125    UINT = auto()
 126    BIGINT = auto()
 127    UBIGINT = auto()
 128    BIGNUM = auto()
 129    INT128 = auto()
 130    UINT128 = auto()
 131    INT256 = auto()
 132    UINT256 = auto()
 133    FLOAT = auto()
 134    DOUBLE = auto()
 135    UDOUBLE = auto()
 136    DECIMAL = auto()
 137    DECIMAL32 = auto()
 138    DECIMAL64 = auto()
 139    DECIMAL128 = auto()
 140    DECIMAL256 = auto()
 141    DECFLOAT = auto()
 142    UDECIMAL = auto()
 143    BIGDECIMAL = auto()
 144    CHAR = auto()
 145    NCHAR = auto()
 146    VARCHAR = auto()
 147    NVARCHAR = auto()
 148    BPCHAR = auto()
 149    TEXT = auto()
 150    MEDIUMTEXT = auto()
 151    LONGTEXT = auto()
 152    BLOB = auto()
 153    MEDIUMBLOB = auto()
 154    LONGBLOB = auto()
 155    TINYBLOB = auto()
 156    TINYTEXT = auto()
 157    NAME = auto()
 158    BINARY = auto()
 159    VARBINARY = auto()
 160    JSON = auto()
 161    JSONB = auto()
 162    TIME = auto()
 163    TIMETZ = auto()
 164    TIME_NS = auto()
 165    TIMESTAMP = auto()
 166    TIMESTAMPTZ = auto()
 167    TIMESTAMPLTZ = auto()
 168    TIMESTAMPNTZ = auto()
 169    TIMESTAMP_S = auto()
 170    TIMESTAMP_MS = auto()
 171    TIMESTAMP_NS = auto()
 172    DATETIME = auto()
 173    DATETIME2 = auto()
 174    DATETIME64 = auto()
 175    SMALLDATETIME = auto()
 176    DATE = auto()
 177    DATE32 = auto()
 178    INT4RANGE = auto()
 179    INT4MULTIRANGE = auto()
 180    INT8RANGE = auto()
 181    INT8MULTIRANGE = auto()
 182    NUMRANGE = auto()
 183    NUMMULTIRANGE = auto()
 184    TSRANGE = auto()
 185    TSMULTIRANGE = auto()
 186    TSTZRANGE = auto()
 187    TSTZMULTIRANGE = auto()
 188    DATERANGE = auto()
 189    DATEMULTIRANGE = auto()
 190    UUID = auto()
 191    GEOGRAPHY = auto()
 192    GEOGRAPHYPOINT = auto()
 193    NULLABLE = auto()
 194    GEOMETRY = auto()
 195    POINT = auto()
 196    RING = auto()
 197    LINESTRING = auto()
 198    LOCALTIME = auto()
 199    LOCALTIMESTAMP = auto()
 200    SYSTIMESTAMP = auto()
 201    MULTILINESTRING = auto()
 202    POLYGON = auto()
 203    MULTIPOLYGON = auto()
 204    HLLSKETCH = auto()
 205    HSTORE = auto()
 206    SUPER = auto()
 207    SERIAL = auto()
 208    SMALLSERIAL = auto()
 209    BIGSERIAL = auto()
 210    XML = auto()
 211    YEAR = auto()
 212    USERDEFINED = auto()
 213    MONEY = auto()
 214    SMALLMONEY = auto()
 215    ROWVERSION = auto()
 216    IMAGE = auto()
 217    VARIANT = auto()
 218    OBJECT = auto()
 219    INET = auto()
 220    IPADDRESS = auto()
 221    IPPREFIX = auto()
 222    IPV4 = auto()
 223    IPV6 = auto()
 224    ENUM = auto()
 225    ENUM8 = auto()
 226    ENUM16 = auto()
 227    FIXEDSTRING = auto()
 228    LOWCARDINALITY = auto()
 229    NESTED = auto()
 230    AGGREGATEFUNCTION = auto()
 231    SIMPLEAGGREGATEFUNCTION = auto()
 232    TDIGEST = auto()
 233    UNKNOWN = auto()
 234    VECTOR = auto()
 235    DYNAMIC = auto()
 236    VOID = auto()
 237
 238    # keywords
 239    ALIAS = auto()
 240    ALTER = auto()
 241    ALL = auto()
 242    ANTI = auto()
 243    ANY = auto()
 244    APPLY = auto()
 245    ARRAY = auto()
 246    ASC = auto()
 247    ASOF = auto()
 248    ATTACH = auto()
 249    AUTO_INCREMENT = auto()
 250    BEGIN = auto()
 251    BETWEEN = auto()
 252    BULK_COLLECT_INTO = auto()
 253    CACHE = auto()
 254    CASE = auto()
 255    CHARACTER_SET = auto()
 256    CLUSTER_BY = auto()
 257    COLLATE = auto()
 258    COMMAND = auto()
 259    COMMENT = auto()
 260    COMMIT = auto()
 261    CONNECT_BY = auto()
 262    CONSTRAINT = auto()
 263    COPY = auto()
 264    CREATE = auto()
 265    CROSS = auto()
 266    CUBE = auto()
 267    CURRENT_DATE = auto()
 268    CURRENT_DATETIME = auto()
 269    CURRENT_SCHEMA = auto()
 270    CURRENT_TIME = auto()
 271    CURRENT_TIMESTAMP = auto()
 272    CURRENT_USER = auto()
 273    CURRENT_USER_ID = auto()
 274    CURRENT_ROLE = auto()
 275    CURRENT_CATALOG = auto()
 276    DECLARE = auto()
 277    DEFAULT = auto()
 278    DELETE = auto()
 279    DESC = auto()
 280    DESCRIBE = auto()
 281    DETACH = auto()
 282    DICTIONARY = auto()
 283    DISTINCT = auto()
 284    DISTRIBUTE_BY = auto()
 285    DIV = auto()
 286    DROP = auto()
 287    ELSE = auto()
 288    END = auto()
 289    ESCAPE = auto()
 290    EXCEPT = auto()
 291    EXECUTE = auto()
 292    EXISTS = auto()
 293    FALSE = auto()
 294    FETCH = auto()
 295    FILE = auto()
 296    FILE_FORMAT = auto()
 297    FILTER = auto()
 298    FINAL = auto()
 299    FIRST = auto()
 300    FOR = auto()
 301    FORCE = auto()
 302    FOREIGN_KEY = auto()
 303    FORMAT = auto()
 304    FROM = auto()
 305    FULL = auto()
 306    FUNCTION = auto()
 307    GET = auto()
 308    GLOB = auto()
 309    GLOBAL = auto()
 310    GRANT = auto()
 311    GROUP_BY = auto()
 312    GROUPING_SETS = auto()
 313    HAVING = auto()
 314    HINT = auto()
 315    IGNORE = auto()
 316    ILIKE = auto()
 317    IN = auto()
 318    INDEX = auto()
 319    INDEXED_BY = auto()
 320    INNER = auto()
 321    INSERT = auto()
 322    INSTALL = auto()
 323    INTEGRATION = auto()
 324    INTERSECT = auto()
 325    INTERVAL = auto()
 326    INTO = auto()
 327    INTRODUCER = auto()
 328    IRLIKE = auto()
 329    IS = auto()
 330    ISNULL = auto()
 331    JOIN = auto()
 332    JOIN_MARKER = auto()
 333    KEEP = auto()
 334    KEY = auto()
 335    KILL = auto()
 336    LANGUAGE = auto()
 337    LATERAL = auto()
 338    LEFT = auto()
 339    LIKE = auto()
 340    LIMIT = auto()
 341    LIST = auto()
 342    LOAD = auto()
 343    LOCK = auto()
 344    MAP = auto()
 345    MATCH = auto()
 346    MATCH_CONDITION = auto()
 347    MATCH_RECOGNIZE = auto()
 348    MEMBER_OF = auto()
 349    MERGE = auto()
 350    MOD = auto()
 351    MODEL = auto()
 352    NATURAL = auto()
 353    NEXT = auto()
 354    NOTHING = auto()
 355    NOTNULL = auto()
 356    NULL = auto()
 357    OBJECT_IDENTIFIER = auto()
 358    OFFSET = auto()
 359    ON = auto()
 360    ONLY = auto()
 361    OPERATOR = auto()
 362    ORDER_BY = auto()
 363    ORDER_SIBLINGS_BY = auto()
 364    ORDERED = auto()
 365    ORDINALITY = auto()
 366    OUT = auto()
 367    INOUT = auto()
 368    OUTER = auto()
 369    OVER = auto()
 370    OVERLAPS = auto()
 371    OVERWRITE = auto()
 372    PACKAGE = auto()
 373    PARTITION = auto()
 374    PARTITION_BY = auto()
 375    PERCENT = auto()
 376    PIVOT = auto()
 377    PLACEHOLDER = auto()
 378    POLICY = auto()
 379    POOL = auto()
 380    POSITIONAL = auto()
 381    PRAGMA = auto()
 382    PREWHERE = auto()
 383    PRIMARY_KEY = auto()
 384    PROCEDURE = auto()
 385    PROPERTIES = auto()
 386    PSEUDO_TYPE = auto()
 387    PUT = auto()
 388    QUALIFY = auto()
 389    QUOTE = auto()
 390    QDCOLON = auto()
 391    RANGE = auto()
 392    RECURSIVE = auto()
 393    REFRESH = auto()
 394    RENAME = auto()
 395    REPLACE = auto()
 396    RETURNING = auto()
 397    REVOKE = auto()
 398    REFERENCES = auto()
 399    RIGHT = auto()
 400    RLIKE = auto()
 401    ROLE = auto()
 402    ROLLBACK = auto()
 403    ROLLUP = auto()
 404    ROW = auto()
 405    ROWS = auto()
 406    RULE = auto()
 407    SELECT = auto()
 408    SEMI = auto()
 409    SEPARATOR = auto()
 410    SEQUENCE = auto()
 411    SERDE_PROPERTIES = auto()
 412    SET = auto()
 413    SETTINGS = auto()
 414    SHOW = auto()
 415    SIMILAR_TO = auto()
 416    SOME = auto()
 417    SORT_BY = auto()
 418    SOUNDS_LIKE = auto()
 419    SQL_SECURITY = auto()
 420    START_WITH = auto()
 421    STORAGE_INTEGRATION = auto()
 422    STRAIGHT_JOIN = auto()
 423    STRUCT = auto()
 424    SUMMARIZE = auto()
 425    TABLE_SAMPLE = auto()
 426    TAG = auto()
 427    TEMPORARY = auto()
 428    TOP = auto()
 429    THEN = auto()
 430    TRUE = auto()
 431    TRUNCATE = auto()
 432    TRIGGER = auto()
 433    TYPE = auto()
 434    UNCACHE = auto()
 435    UNDROP = auto()
 436    UNION = auto()
 437    UNNEST = auto()
 438    UNPIVOT = auto()
 439    UPDATE = auto()
 440    USE = auto()
 441    USING = auto()
 442    VALUES = auto()
 443    VARIADIC = auto()
 444    VIEW = auto()
 445    SEMANTIC_VIEW = auto()
 446    VOLATILE = auto()
 447    VOLUME = auto()
 448    WHEN = auto()
 449    WHERE = auto()
 450    WINDOW = auto()
 451    WITH = auto()
 452    UNIQUE = auto()
 453    UTC_DATE = auto()
 454    UTC_TIME = auto()
 455    UTC_TIMESTAMP = auto()
 456    VERSION_SNAPSHOT = auto()
 457    TIMESTAMP_SNAPSHOT = auto()
 458    OPTION = auto()
 459    SINK = auto()
 460    SOURCE = auto()
 461    ANALYZE = auto()
 462    NAMESPACE = auto()
 463    EXPORT = auto()
 464
 465    # sentinels
 466    HIVE_TOKEN_STREAM = auto()
 467    SENTINEL = auto()
 468
 469    def __str__(self) -> str:
 470        return f"TokenType.{self.name}"
 471
 472
 473class Token:
 474    # mypyc doesn't expose slots
 475    _attrs: t.ClassVar[tuple[str, ...]] = (
 476        "token_type",
 477        "text",
 478        "line",
 479        "col",
 480        "start",
 481        "end",
 482        "comments",
 483    )
 484    __slots__ = _attrs
 485
 486    @classmethod
 487    def number(cls, number: int) -> Token:
 488        """Returns a NUMBER token with `number` as its text."""
 489        return cls(TokenType.NUMBER, str(number))
 490
 491    @classmethod
 492    def string(cls, string: str) -> Token:
 493        """Returns a STRING token with `string` as its text."""
 494        return cls(TokenType.STRING, string)
 495
 496    @classmethod
 497    def identifier(cls, identifier: str) -> Token:
 498        """Returns an IDENTIFIER token with `identifier` as its text."""
 499        return cls(TokenType.IDENTIFIER, identifier)
 500
 501    @classmethod
 502    def var(cls, var: str) -> Token:
 503        """Returns an VAR token with `var` as its text."""
 504        return cls(TokenType.VAR, var)
 505
 506    def __init__(
 507        self,
 508        token_type: TokenType,
 509        text: str,
 510        line: int = 1,
 511        col: int = 1,
 512        start: int = 0,
 513        end: int = 0,
 514        comments: list[str] | None = None,
 515    ) -> None:
 516        self.token_type = token_type
 517        self.text = text
 518        self.line = line
 519        self.col = col
 520        self.start = start
 521        self.end = end
 522        self.comments = [] if comments is None else comments
 523
 524    def __bool__(self) -> bool:
 525        return self.token_type != TokenType.SENTINEL
 526
 527    def __repr__(self) -> str:
 528        attributes = ", ".join(
 529            f"{k}: TokenType.{self.token_type.name}"
 530            if k == "token_type"
 531            else f"{k}: {getattr(self, k)}"
 532            for k in self._attrs
 533        )
 534        return f"<Token {attributes}>"
 535
 536
 537class TokenizerCore:
 538    __slots__ = (
 539        "sql",
 540        "size",
 541        "tokens",
 542        "_start",
 543        "_current",
 544        "_line",
 545        "_col",
 546        "_comments",
 547        "_char",
 548        "_end",
 549        "_peek",
 550        "_prev_token_line",
 551        "single_tokens",
 552        "keywords",
 553        "quotes",
 554        "format_strings",
 555        "identifiers",
 556        "comments",
 557        "string_escapes",
 558        "byte_string_escapes",
 559        "identifier_escapes",
 560        "escape_follow_chars",
 561        "commands",
 562        "command_prefix_tokens",
 563        "nested_comments",
 564        "hint_start",
 565        "tokens_preceding_hint",
 566        "has_bit_strings",
 567        "has_hex_strings",
 568        "numeric_literals",
 569        "var_single_tokens",
 570        "string_escapes_allowed_in_raw_strings",
 571        "heredoc_tag_is_identifier",
 572        "heredoc_string_alternative",
 573        "keyword_trie",
 574        "numbers_can_be_underscore_separated",
 575        "numbers_can_have_decimals",
 576        "identifiers_can_start_with_digit",
 577        "unescaped_sequences",
 578    )
 579
 580    def __init__(
 581        self,
 582        single_tokens: dict[str, TokenType],
 583        keywords: dict[str, TokenType],
 584        quotes: dict[str, str],
 585        format_strings: dict[str, tuple[str, TokenType]],
 586        identifiers: dict[str, str],
 587        comments: dict[str, str | None],
 588        string_escapes: set[str],
 589        byte_string_escapes: set[str],
 590        identifier_escapes: set[str],
 591        escape_follow_chars: set[str],
 592        commands: set[TokenType],
 593        command_prefix_tokens: set[TokenType],
 594        nested_comments: bool,
 595        hint_start: str,
 596        tokens_preceding_hint: set[TokenType],
 597        has_bit_strings: bool,
 598        has_hex_strings: bool,
 599        numeric_literals: dict[str, str],
 600        var_single_tokens: set[str],
 601        string_escapes_allowed_in_raw_strings: bool,
 602        heredoc_tag_is_identifier: bool,
 603        heredoc_string_alternative: TokenType,
 604        keyword_trie: dict,
 605        numbers_can_be_underscore_separated: bool,
 606        numbers_can_have_decimals: bool,
 607        identifiers_can_start_with_digit: bool,
 608        unescaped_sequences: dict[str, str],
 609    ) -> None:
 610        self.single_tokens = single_tokens
 611        self.keywords = keywords
 612        self.quotes = quotes
 613        self.format_strings = format_strings
 614        self.identifiers = identifiers
 615        self.comments = comments
 616        self.string_escapes = string_escapes
 617        self.byte_string_escapes = byte_string_escapes
 618        self.identifier_escapes = identifier_escapes
 619        self.escape_follow_chars = escape_follow_chars
 620        self.commands = commands
 621        self.command_prefix_tokens = command_prefix_tokens
 622        self.nested_comments = nested_comments
 623        self.hint_start = hint_start
 624        self.tokens_preceding_hint = tokens_preceding_hint
 625        self.has_bit_strings = has_bit_strings
 626        self.has_hex_strings = has_hex_strings
 627        self.numeric_literals = numeric_literals
 628        self.var_single_tokens = var_single_tokens
 629        self.string_escapes_allowed_in_raw_strings = string_escapes_allowed_in_raw_strings
 630        self.heredoc_tag_is_identifier = heredoc_tag_is_identifier
 631        self.heredoc_string_alternative = heredoc_string_alternative
 632        self.keyword_trie = keyword_trie
 633        self.numbers_can_be_underscore_separated = numbers_can_be_underscore_separated
 634        self.numbers_can_have_decimals = numbers_can_have_decimals
 635        self.identifiers_can_start_with_digit = identifiers_can_start_with_digit
 636        self.unescaped_sequences = unescaped_sequences
 637        self.sql = ""
 638        self.size = 0
 639        self.tokens: list[Token] = []
 640        self._start = 0
 641        self._current = 0
 642        self._line = 1
 643        self._col = 0
 644        self._comments: list[str] = []
 645        self._char = ""
 646        self._end = False
 647        self._peek = ""
 648        self._prev_token_line = -1
 649
 650    def reset(self) -> None:
 651        self.sql = ""
 652        self.size = 0
 653        self.tokens = []
 654        self._start = 0
 655        self._current = 0
 656        self._line = 1
 657        self._col = 0
 658        self._comments = []
 659        self._char = ""
 660        self._end = False
 661        self._peek = ""
 662        self._prev_token_line = -1
 663
 664    def tokenize(self, sql: str) -> list[Token]:
 665        """Returns a list of tokens corresponding to the SQL string `sql`."""
 666        self.reset()
 667        self.sql = sql
 668        self.size = len(sql)
 669
 670        try:
 671            self._scan()
 672        except Exception as e:
 673            start = max(self._current - 50, 0)
 674            end = min(self._current + 50, self.size - 1)
 675            context = self.sql[start:end]
 676            raise TokenError(f"Error tokenizing '{context}'") from e
 677
 678        return self.tokens
 679
 680    def _scan(self, check_semicolon: bool = False) -> None:
 681        identifiers = self.identifiers
 682        digit_chars = _DIGIT_CHARS
 683
 684        while self.size and not self._end:
 685            current = self._current
 686
 687            # Skip spaces here rather than iteratively calling advance() for performance reasons
 688            while current < self.size:
 689                char = self.sql[current]
 690
 691                if char == " " or char == "\t":
 692                    current += 1
 693                else:
 694                    break
 695
 696            offset = current - self._current if current > self._current else 1
 697
 698            self._start = current
 699            self._advance(offset)
 700
 701            if not self._char.isspace():
 702                if self._char in digit_chars:
 703                    self._scan_number()
 704                elif self._char in identifiers:
 705                    self._scan_identifier(identifiers[self._char])
 706                else:
 707                    self._scan_keywords()
 708
 709            if check_semicolon and self._peek == ";":
 710                break
 711
 712        if self.tokens and self._comments:
 713            self.tokens[-1].comments.extend(self._comments)
 714
 715    def _chars(self, size: int) -> str:
 716        if size == 1:
 717            return self._char
 718
 719        start = self._current - 1
 720        end = start + size
 721
 722        return self.sql[start:end] if end <= self.size else ""
 723
 724    def _advance(self, i: int = 1, alnum: bool = False) -> None:
 725        char = self._char
 726
 727        if char == "\n" or char == "\r":
 728            # Ensures we don't count an extra line if we get a \r\n line break sequence
 729            if not (char == "\r" and self._peek == "\n"):
 730                self._col = i
 731                self._line += 1
 732        else:
 733            self._col += i
 734
 735        self._current += i
 736        sql = self.sql
 737        size = self.size
 738        self._end = self._current >= size
 739        self._char = sql[self._current - 1]
 740        self._peek = "" if self._end else sql[self._current]
 741
 742        if alnum and self._char.isalnum():
 743            # Cache to local variables instead of attributes for better performance
 744            _col = self._col
 745            _current = self._current
 746            _end = self._end
 747            _peek = self._peek
 748
 749            while _peek.isalnum():
 750                _col += 1
 751                _current += 1
 752                _end = _current >= size
 753                _peek = "" if _end else sql[_current]
 754
 755            self._col = _col
 756            self._current = _current
 757            self._end = _end
 758            self._peek = _peek
 759            self._char = sql[_current - 1]
 760
 761    @property
 762    def _text(self) -> str:
 763        return self.sql[self._start : self._current]
 764
 765    def _add(self, token_type: TokenType, text: str | None = None) -> None:
 766        self._prev_token_line = self._line
 767
 768        if self._comments and token_type == TokenType.SEMICOLON and self.tokens:
 769            self.tokens[-1].comments.extend(self._comments)
 770            self._comments = []
 771
 772        if text is None:
 773            text = self.sql[self._start : self._current]
 774
 775        self.tokens.append(
 776            Token(
 777                token_type,
 778                text=text,
 779                line=self._line,
 780                col=self._col,
 781                start=self._start,
 782                end=self._current - 1,
 783                comments=self._comments,
 784            )
 785        )
 786        self._comments = []
 787
 788        # If we have either a semicolon or a begin token before the command's token, we'll parse
 789        # whatever follows the command's token as a string
 790        if (
 791            token_type in self.commands
 792            and self._peek != ";"
 793            and (len(self.tokens) == 1 or self.tokens[-2].token_type in self.command_prefix_tokens)
 794        ):
 795            start = self._current
 796            tokens = len(self.tokens)
 797            self._scan(check_semicolon=True)
 798            self.tokens = self.tokens[:tokens]
 799            text = self.sql[start : self._current].strip()
 800            if text:
 801                self._add(TokenType.STRING, text)
 802
 803    def _scan_keywords(self) -> None:
 804        sql = self.sql
 805        sql_size = self.size
 806        single_tokens = self.single_tokens
 807        char_upper = _CHAR_UPPER
 808        size = 0
 809        word = None
 810        chars = self._char
 811        char = chars
 812        prev_space = False
 813        skip = False
 814        trie = self.keyword_trie
 815        single_token = char in single_tokens
 816
 817        while chars:
 818            if not skip:
 819                sub = trie.get(char_upper.get(char, char))
 820                if sub is None:
 821                    break
 822                trie = sub
 823                if 0 in trie:
 824                    word = chars
 825
 826            end = self._current + size
 827            size += 1
 828
 829            if end < sql_size:
 830                char = sql[end]
 831                single_token = single_token or char in single_tokens
 832                is_space = char.isspace()
 833
 834                if not is_space or not prev_space:
 835                    if is_space:
 836                        char = " "
 837                    chars += char
 838                    prev_space = is_space
 839                    skip = False
 840                else:
 841                    skip = True
 842            else:
 843                char = ""
 844                break
 845
 846        if word:
 847            if self._scan_string(word):
 848                return
 849            if self._scan_comment(word):
 850                return
 851            if prev_space or single_token or not char:
 852                self._advance(size - 1)
 853                word = word.upper()
 854                self._add(self.keywords[word], text=word)
 855                return
 856
 857        if self._char in single_tokens:
 858            self._add(single_tokens[self._char], text=self._char)
 859            return
 860
 861        self._scan_var()
 862
 863    def _scan_comment(self, comment_start: str) -> bool:
 864        if comment_start not in self.comments:
 865            return False
 866
 867        comment_start_line = self._line
 868        comment_start_size = len(comment_start)
 869        comment_end = self.comments[comment_start]
 870
 871        if comment_end:
 872            # Skip the comment's start delimiter
 873            self._advance(comment_start_size)
 874
 875            comment_count = 1
 876            comment_end_size = len(comment_end)
 877            nested_comments = self.nested_comments
 878
 879            while not self._end:
 880                if self._chars(comment_end_size) == comment_end:
 881                    comment_count -= 1
 882                    if not comment_count:
 883                        break
 884
 885                self._advance(alnum=True)
 886
 887                # Nested comments are allowed by some dialects, e.g. databricks, duckdb, postgres
 888                if (
 889                    nested_comments
 890                    and not self._end
 891                    and self._chars(comment_end_size) == comment_start
 892                ):
 893                    self._advance(comment_start_size)
 894                    comment_count += 1
 895
 896            self._comments.append(self._text[comment_start_size : -comment_end_size + 1])
 897            self._advance(comment_end_size - 1)
 898        else:
 899            _peek = self._peek
 900            while not self._end and _peek != "\n" and _peek != "\r":
 901                self._advance(alnum=True)
 902                _peek = self._peek
 903            self._comments.append(self._text[comment_start_size:])
 904
 905        if (
 906            comment_start == self.hint_start
 907            and self.tokens
 908            and self.tokens[-1].token_type in self.tokens_preceding_hint
 909        ):
 910            self._add(TokenType.HINT)
 911
 912        # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.
 913        # Multiple consecutive comments are preserved by appending them to the current comments list.
 914        if comment_start_line == self._prev_token_line:
 915            self.tokens[-1].comments.extend(self._comments)
 916            self._comments = []
 917            self._prev_token_line = self._line
 918
 919        return True
 920
 921    def _scan_number(self) -> None:
 922        if self._char == "0":
 923            peek = _CHAR_UPPER.get(self._peek, self._peek)
 924            if peek == "B":
 925                return self._scan_bits() if self.has_bit_strings else self._add(TokenType.NUMBER)
 926            elif peek == "X":
 927                return self._scan_hex() if self.has_hex_strings else self._add(TokenType.NUMBER)
 928
 929        decimal = False
 930        scientific = 0
 931        numbers_can_be_underscore_separated = self.numbers_can_be_underscore_separated
 932        single_tokens = self.single_tokens
 933        keywords = self.keywords
 934        numeric_literals = self.numeric_literals
 935        identifiers_can_start_with_digit = self.identifiers_can_start_with_digit
 936
 937        is_underscore_separated: bool = False
 938        number_text: str = ""
 939        numeric_literal: str = ""
 940        numeric_type: TokenType | None = None
 941
 942        while True:
 943            if self._peek in _DIGIT_CHARS:
 944                # Batch consecutive digits: scan ahead to find how many
 945                sql = self.sql
 946                end = self._current + 1
 947                size = self.size
 948                while end < size and sql[end] in _DIGIT_CHARS:
 949                    end += 1
 950                self._advance(end - self._current)
 951            elif self._peek == "." and not decimal:
 952                if (
 953                    self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
 954                ) or not self.numbers_can_have_decimals:
 955                    break
 956                decimal = True
 957                self._advance()
 958            elif self._peek in ("-", "+") and scientific == 1:
 959                # Only consume +/- if followed by a digit
 960                if self._current + 1 < self.size and self.sql[self._current + 1] in _DIGIT_CHARS:
 961                    scientific += 1
 962                    self._advance()
 963                else:
 964                    break
 965            elif _CHAR_UPPER.get(self._peek, self._peek) == "E" and not scientific:
 966                scientific += 1
 967                self._advance()
 968            elif self._peek == "_" and numbers_can_be_underscore_separated:
 969                is_underscore_separated = True
 970                self._advance()
 971            elif self._peek.isidentifier():
 972                number_text = self._text
 973
 974                while self._peek and not self._peek.isspace() and self._peek not in single_tokens:
 975                    numeric_literal += self._peek
 976                    self._advance()
 977
 978                numeric_type = keywords.get(numeric_literals.get(numeric_literal.upper(), ""))
 979
 980                if numeric_type:
 981                    break
 982                elif identifiers_can_start_with_digit:
 983                    return self._add(TokenType.VAR)
 984
 985                self._advance(-len(numeric_literal))
 986                break
 987            else:
 988                break
 989
 990        number_text = number_text or self.sql[self._start : self._current]
 991
 992        # Normalize inputs such as 100_000 to 100000
 993        if is_underscore_separated:
 994            number_text = number_text.replace("_", "")
 995
 996        self._add(TokenType.NUMBER, number_text)
 997
 998        # Normalize inputs such as 123L to 123::BIGINT so that they're parsed as casts
 999        if numeric_type:
1000            self._add(TokenType.DCOLON, "::")
1001            self._add(numeric_type, numeric_literal)
1002
1003    def _scan_bits(self) -> None:
1004        self._advance()
1005        value = self._extract_value()
1006        try:
1007            # If `value` can't be converted to a binary, fallback to tokenizing it as an identifier
1008            int(value, 2)
1009            self._add(TokenType.BIT_STRING, value[2:])  # Drop the 0b
1010        except ValueError:
1011            self._add(TokenType.IDENTIFIER)
1012
1013    def _scan_hex(self) -> None:
1014        self._advance()
1015        value = self._extract_value()
1016        try:
1017            # If `value` can't be converted to a hex, fallback to tokenizing it as an identifier
1018            int(value, 16)
1019            self._add(TokenType.HEX_STRING, value[2:])  # Drop the 0x
1020        except ValueError:
1021            self._add(TokenType.IDENTIFIER)
1022
1023    def _extract_value(self) -> str:
1024        single_tokens = self.single_tokens
1025
1026        while True:
1027            char = self._peek.strip()
1028            if char and char not in single_tokens:
1029                self._advance(alnum=True)
1030            else:
1031                break
1032
1033        return self._text
1034
1035    def _scan_string(self, start: str) -> bool:
1036        base = None
1037        token_type = TokenType.STRING
1038
1039        if start in self.quotes:
1040            end = self.quotes[start]
1041        elif start in self.format_strings:
1042            end, token_type = self.format_strings[start]
1043
1044            if token_type == TokenType.HEX_STRING:
1045                base = 16
1046            elif token_type == TokenType.BIT_STRING:
1047                base = 2
1048            elif token_type == TokenType.HEREDOC_STRING:
1049                self._advance()
1050
1051                if self._char == end:
1052                    tag = ""
1053                else:
1054                    tag = self._extract_string(
1055                        end,
1056                        raw_string=True,
1057                        raise_unmatched=not self.heredoc_tag_is_identifier,
1058                    )
1059
1060                if (
1061                    tag
1062                    and self.heredoc_tag_is_identifier
1063                    and (self._end or tag.isdigit() or any(c.isspace() for c in tag))
1064                ):
1065                    if not self._end:
1066                        self._advance(-1)
1067
1068                    self._advance(-len(tag))
1069                    self._add(self.heredoc_string_alternative)
1070                    return True
1071
1072                end = f"{start}{tag}{end}"
1073        else:
1074            return False
1075
1076        self._advance(len(start))
1077        text = self._extract_string(
1078            end,
1079            escapes=(
1080                self.byte_string_escapes
1081                if token_type == TokenType.BYTE_STRING
1082                else self.string_escapes
1083            ),
1084            raw_string=token_type == TokenType.RAW_STRING,
1085        )
1086
1087        if base and text:
1088            try:
1089                int(text, base)
1090            except Exception:
1091                raise TokenError(
1092                    f"Numeric string contains invalid characters from {self._line}:{self._start}"
1093                )
1094
1095        self._add(token_type, text)
1096        return True
1097
1098    def _scan_identifier(self, identifier_end: str) -> None:
1099        self._advance()
1100        text = self._extract_string(
1101            identifier_end, escapes=self.identifier_escapes | {identifier_end}
1102        )
1103        self._add(TokenType.IDENTIFIER, text)
1104
1105    def _scan_var(self) -> None:
1106        var_single_tokens = self.var_single_tokens
1107        single_tokens = self.single_tokens
1108
1109        while True:
1110            peek = self._peek
1111            if not peek or peek.isspace():
1112                break
1113            if peek not in var_single_tokens and peek in single_tokens:
1114                break
1115            self._advance(alnum=True)
1116
1117        self._add(
1118            TokenType.VAR
1119            if self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
1120            else self.keywords.get(self.sql[self._start : self._current].upper(), TokenType.VAR)
1121        )
1122
1123    def _extract_string(
1124        self,
1125        delimiter: str,
1126        escapes: set[str] | None = None,
1127        raw_string: bool = False,
1128        raise_unmatched: bool = True,
1129    ) -> str:
1130        text = ""
1131        delim_size = len(delimiter)
1132        escapes = self.string_escapes if escapes is None else escapes
1133        unescaped_sequences = self.unescaped_sequences
1134        escape_follow_chars = self.escape_follow_chars
1135        string_escapes_allowed_in_raw_strings = self.string_escapes_allowed_in_raw_strings
1136        quotes = self.quotes
1137        sql = self.sql
1138
1139        # use str.find() when the string is simple... no \ or other escapes
1140        if delim_size == 1:
1141            pos = self._current - 1
1142            end = sql.find(delimiter, pos)
1143
1144            if (
1145                # the closing delimiter was found
1146                end != -1
1147                # there's no doubled delimiter (e.g. '' escape), or the delimiter isn't an escape char
1148                and (end + 1 >= self.size or sql[end + 1] != delimiter or delimiter not in escapes)
1149                # no backslash in the string that would need escape processing
1150                and (not (unescaped_sequences or "\\" in escapes) or sql.find("\\", pos, end) == -1)
1151            ):
1152                newlines = sql.count("\n", pos, end)
1153                if newlines:
1154                    self._line += newlines
1155                    self._col = end - sql.rfind("\n", pos, end)
1156                else:
1157                    self._col += end - pos
1158
1159                self._current = end + 1
1160                self._end = self._current >= self.size
1161                self._char = sql[end]
1162                self._peek = "" if self._end else sql[self._current]
1163                return sql[pos:end]
1164
1165        while True:
1166            if not raw_string and unescaped_sequences and self._peek and self._char in escapes:
1167                unescaped_sequence = unescaped_sequences.get(self._char + self._peek)
1168                if unescaped_sequence:
1169                    self._advance(2)
1170                    text += unescaped_sequence
1171                    continue
1172
1173            is_valid_custom_escape = (
1174                escape_follow_chars and self._char == "\\" and self._peek not in escape_follow_chars
1175            )
1176
1177            if (
1178                (string_escapes_allowed_in_raw_strings or not raw_string)
1179                and self._char in escapes
1180                and (self._peek == delimiter or self._peek in escapes or is_valid_custom_escape)
1181                and (self._char not in quotes or self._char == self._peek)
1182            ):
1183                if self._peek == delimiter:
1184                    text += self._peek
1185                elif is_valid_custom_escape and self._char != self._peek:
1186                    text += self._peek
1187                else:
1188                    text += self._char + self._peek
1189
1190                if self._current + 1 < self.size:
1191                    self._advance(2)
1192                else:
1193                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._current}")
1194            else:
1195                if self._chars(delim_size) == delimiter:
1196                    if delim_size > 1:
1197                        self._advance(delim_size - 1)
1198                    break
1199
1200                if self._end:
1201                    if not raise_unmatched:
1202                        return text + self._char
1203
1204                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._start}")
1205
1206                current = self._current - 1
1207                self._advance(alnum=True)
1208                text += sql[current : self._current - 1]
1209
1210        return text