sqlglot.tokenizer_core

   1from __future__ import annotations
   2
   3import typing as t
   4from enum import IntEnum, auto
   5
   6from sqlglot.errors import TokenError
   7
   8# dict lookup is faster than .upper() and .isdigit()
   9_CHAR_UPPER: dict[str, str] = {chr(i): chr(i).upper() for i in range(97, 123)}
  10_DIGIT_CHARS: frozenset[str] = frozenset("0123456789")
  11_DIGIT_VALUES: dict[str, int] = {c: int(c, 16) for c in "0123456789abcdefABCDEF"}
  12_OCTAL_CHARS: frozenset[str] = frozenset("01234567")
  13
  14
  15class TokenType(IntEnum):
  16    L_PAREN = auto()
  17    R_PAREN = auto()
  18    L_BRACKET = auto()
  19    R_BRACKET = auto()
  20    L_BRACE = auto()
  21    R_BRACE = auto()
  22    COMMA = auto()
  23    DOT = auto()
  24    DASH = auto()
  25    PLUS = auto()
  26    COLON = auto()
  27    DOTCOLON = auto()
  28    DOTCARET = auto()
  29    DCOLON = auto()
  30    DCOLONDOLLAR = auto()
  31    DCOLONPERCENT = auto()
  32    DCOLONQMARK = auto()
  33    DQMARK = auto()
  34    SEMICOLON = auto()
  35    STAR = auto()
  36    BACKSLASH = auto()
  37    SLASH = auto()
  38    LT = auto()
  39    LTE = auto()
  40    GT = auto()
  41    GTE = auto()
  42    NOT = auto()
  43    EQ = auto()
  44    NEQ = auto()
  45    NULLSAFE_EQ = auto()
  46    COLON_EQ = auto()
  47    COLON_GT = auto()
  48    NCOLON_GT = auto()
  49    AND = auto()
  50    OR = auto()
  51    AMP = auto()
  52    DPIPE = auto()
  53    PIPE_GT = auto()
  54    PIPE = auto()
  55    PIPE_SLASH = auto()
  56    DPIPE_SLASH = auto()
  57    CARET = auto()
  58    CARET_AT = auto()
  59    TILDE = auto()
  60    ARROW = auto()
  61    DARROW = auto()
  62    FARROW = auto()
  63    HASH = auto()
  64    HASH_ARROW = auto()
  65    DHASH_ARROW = auto()
  66    LR_ARROW = auto()
  67    LLRR_ARROW = auto()
  68    DAT = auto()
  69    AT_QMARK = auto()
  70    LT_AT = auto()
  71    AT_GT = auto()
  72    DOLLAR = auto()
  73    PARAMETER = auto()
  74    SESSION = auto()
  75    SESSION_PARAMETER = auto()
  76    SESSION_USER = auto()
  77    DAMP = auto()
  78    AMP_LT = auto()
  79    AMP_GT = auto()
  80    ADJACENT = auto()
  81    XOR = auto()
  82    DSTAR = auto()
  83    QMARK_AMP = auto()
  84    QMARK_PIPE = auto()
  85    HASH_DASH = auto()
  86    EXCLAMATION = auto()
  87
  88    URI_START = auto()
  89
  90    BLOCK_START = auto()
  91    BLOCK_END = auto()
  92
  93    SPACE = auto()
  94    BREAK = auto()
  95
  96    STRING = auto()
  97    NUMBER = auto()
  98    IDENTIFIER = auto()
  99    DATABASE = auto()
 100    COLUMN = auto()
 101    COLUMN_DEF = auto()
 102    SCHEMA = auto()
 103    TABLE = auto()
 104    WAREHOUSE = auto()
 105    STAGE = auto()
 106    STREAM = auto()
 107    STREAMLIT = auto()
 108    VAR = auto()
 109    BIT_STRING = auto()
 110    HEX_STRING = auto()
 111    BYTE_STRING = auto()
 112    NATIONAL_STRING = auto()
 113    RAW_STRING = auto()
 114    HEREDOC_STRING = auto()
 115    UNICODE_STRING = auto()
 116
 117    # types
 118    BIT = auto()
 119    BOOLEAN = auto()
 120    TINYINT = auto()
 121    UTINYINT = auto()
 122    SMALLINT = auto()
 123    USMALLINT = auto()
 124    MEDIUMINT = auto()
 125    UMEDIUMINT = auto()
 126    INT = auto()
 127    UINT = auto()
 128    BIGINT = auto()
 129    UBIGINT = auto()
 130    BIGNUM = auto()
 131    INT128 = auto()
 132    UINT128 = auto()
 133    INT256 = auto()
 134    UINT256 = auto()
 135    FLOAT = auto()
 136    DOUBLE = auto()
 137    UDOUBLE = auto()
 138    DECIMAL = auto()
 139    DECIMAL32 = auto()
 140    DECIMAL64 = auto()
 141    DECIMAL128 = auto()
 142    DECIMAL256 = auto()
 143    DECFLOAT = auto()
 144    UDECIMAL = auto()
 145    BIGDECIMAL = auto()
 146    CHAR = auto()
 147    NCHAR = auto()
 148    VARCHAR = auto()
 149    NVARCHAR = auto()
 150    BPCHAR = auto()
 151    TEXT = auto()
 152    MEDIUMTEXT = auto()
 153    LONGTEXT = auto()
 154    BLOB = auto()
 155    MEDIUMBLOB = auto()
 156    LONGBLOB = auto()
 157    TINYBLOB = auto()
 158    TINYTEXT = auto()
 159    NAME = auto()
 160    BINARY = auto()
 161    VARBINARY = auto()
 162    JSON = auto()
 163    JSONB = auto()
 164    TIME = auto()
 165    TIMETZ = auto()
 166    TIME_NS = auto()
 167    TIMESTAMP = auto()
 168    TIMESTAMPTZ = auto()
 169    TIMESTAMPLTZ = auto()
 170    TIMESTAMPNTZ = auto()
 171    TIMESTAMP_S = auto()
 172    TIMESTAMP_MS = auto()
 173    TIMESTAMP_NS = auto()
 174    DATETIME = auto()
 175    DATETIME2 = auto()
 176    DATETIME64 = auto()
 177    SMALLDATETIME = auto()
 178    DATE = auto()
 179    DATE32 = auto()
 180    INT4RANGE = auto()
 181    INT4MULTIRANGE = auto()
 182    INT8RANGE = auto()
 183    INT8MULTIRANGE = auto()
 184    NUMRANGE = auto()
 185    NUMMULTIRANGE = auto()
 186    TSRANGE = auto()
 187    TSMULTIRANGE = auto()
 188    TSTZRANGE = auto()
 189    TSTZMULTIRANGE = auto()
 190    DATERANGE = auto()
 191    DATEMULTIRANGE = auto()
 192    UUID = auto()
 193    GEOGRAPHY = auto()
 194    GEOGRAPHYPOINT = auto()
 195    NULLABLE = auto()
 196    GEOMETRY = auto()
 197    POINT = auto()
 198    RING = auto()
 199    LINESTRING = auto()
 200    LOCALTIME = auto()
 201    LOCALTIMESTAMP = auto()
 202    SYSTIMESTAMP = auto()
 203    MULTILINESTRING = auto()
 204    POLYGON = auto()
 205    MULTIPOLYGON = auto()
 206    HLLSKETCH = auto()
 207    HSTORE = auto()
 208    SUPER = auto()
 209    SERIAL = auto()
 210    SMALLSERIAL = auto()
 211    BIGSERIAL = auto()
 212    XML = auto()
 213    YEAR = auto()
 214    USERDEFINED = auto()
 215    MONEY = auto()
 216    SMALLMONEY = auto()
 217    ROWVERSION = auto()
 218    IMAGE = auto()
 219    VARIANT = auto()
 220    OBJECT = auto()
 221    INET = auto()
 222    IPADDRESS = auto()
 223    IPPREFIX = auto()
 224    IPV4 = auto()
 225    IPV6 = auto()
 226    ENUM = auto()
 227    ENUM8 = auto()
 228    ENUM16 = auto()
 229    FIXEDSTRING = auto()
 230    LOWCARDINALITY = auto()
 231    NESTED = auto()
 232    AGGREGATEFUNCTION = auto()
 233    SIMPLEAGGREGATEFUNCTION = auto()
 234    TDIGEST = auto()
 235    UNKNOWN = auto()
 236    VECTOR = auto()
 237    DYNAMIC = auto()
 238    VOID = auto()
 239
 240    # keywords
 241    ALIAS = auto()
 242    ALTER = auto()
 243    ALL = auto()
 244    ANTI = auto()
 245    ANY = auto()
 246    APPLY = auto()
 247    ARRAY = auto()
 248    ASC = auto()
 249    ASOF = auto()
 250    ATTACH = auto()
 251    AUTO_INCREMENT = auto()
 252    BEGIN = auto()
 253    BETWEEN = auto()
 254    BULK_COLLECT_INTO = auto()
 255    CACHE = auto()
 256    CASE = auto()
 257    CHARACTER_SET = auto()
 258    CLUSTER_BY = auto()
 259    COLLATE = auto()
 260    COMMAND = auto()
 261    COMMENT = auto()
 262    COMMIT = auto()
 263    CONNECT_BY = auto()
 264    CONSTRAINT = auto()
 265    COPY = auto()
 266    CREATE = auto()
 267    CROSS = auto()
 268    CUBE = auto()
 269    CURRENT_DATE = auto()
 270    CURRENT_DATETIME = auto()
 271    CURRENT_SCHEMA = auto()
 272    CURRENT_TIME = auto()
 273    CURRENT_TIMESTAMP = auto()
 274    CURRENT_USER = auto()
 275    CURRENT_USER_ID = auto()
 276    CURRENT_ROLE = auto()
 277    CURRENT_CATALOG = auto()
 278    DECLARE = auto()
 279    DEFAULT = auto()
 280    DELETE = auto()
 281    DESC = auto()
 282    DESCRIBE = auto()
 283    DETACH = auto()
 284    DICTIONARY = auto()
 285    DISTINCT = auto()
 286    DISTRIBUTE_BY = auto()
 287    DIV = auto()
 288    DROP = auto()
 289    ELSE = auto()
 290    END = auto()
 291    ESCAPE = auto()
 292    EXCEPT = auto()
 293    EXECUTE = auto()
 294    EXISTS = auto()
 295    FALSE = auto()
 296    FETCH = auto()
 297    FILE = auto()
 298    FILE_FORMAT = auto()
 299    FILTER = auto()
 300    FINAL = auto()
 301    FIRST = auto()
 302    FOR = auto()
 303    FORCE = auto()
 304    FOREIGN_KEY = auto()
 305    FORMAT = auto()
 306    FROM = auto()
 307    FULL = auto()
 308    FUNCTION = auto()
 309    GET = auto()
 310    GLOB = auto()
 311    GLOBAL = auto()
 312    GRANT = auto()
 313    GROUP_BY = auto()
 314    GROUPING_SETS = auto()
 315    HAVING = auto()
 316    HINT = auto()
 317    IGNORE = auto()
 318    ILIKE = auto()
 319    IN = auto()
 320    INDEX = auto()
 321    INDEXED_BY = auto()
 322    INNER = auto()
 323    INSERT = auto()
 324    INSTALL = auto()
 325    INTEGRATION = auto()
 326    INTERSECT = auto()
 327    INTERVAL = auto()
 328    INTO = auto()
 329    INTRODUCER = auto()
 330    IRLIKE = auto()
 331    IS = auto()
 332    ISNULL = auto()
 333    JOIN = auto()
 334    JOIN_MARKER = auto()
 335    KEEP = auto()
 336    KEY = auto()
 337    KILL = auto()
 338    LANGUAGE = auto()
 339    LATERAL = auto()
 340    LEFT = auto()
 341    LIKE = auto()
 342    LIMIT = auto()
 343    LIST = auto()
 344    LOAD = auto()
 345    LOCK = auto()
 346    MAP = auto()
 347    MATCH = auto()
 348    MATCH_CONDITION = auto()
 349    MATCH_RECOGNIZE = auto()
 350    MEMBER_OF = auto()
 351    MERGE = auto()
 352    MOD = auto()
 353    MODEL = auto()
 354    NATURAL = auto()
 355    NEXT = auto()
 356    NOTHING = auto()
 357    NOTNULL = auto()
 358    NULL = auto()
 359    OBJECT_IDENTIFIER = auto()
 360    OFFSET = auto()
 361    ON = auto()
 362    ONLY = auto()
 363    OPERATOR = auto()
 364    ORDER_BY = auto()
 365    ORDER_SIBLINGS_BY = auto()
 366    ORDERED = auto()
 367    ORDINALITY = auto()
 368    OUT = auto()
 369    INOUT = auto()
 370    OUTER = auto()
 371    OVER = auto()
 372    OVERLAPS = auto()
 373    OVERWRITE = auto()
 374    PACKAGE = auto()
 375    PARTITION = auto()
 376    PARTITION_BY = auto()
 377    PERCENT = auto()
 378    PIVOT = auto()
 379    PLACEHOLDER = auto()
 380    POLICY = auto()
 381    POOL = auto()
 382    POSITIONAL = auto()
 383    PRAGMA = auto()
 384    PREWHERE = auto()
 385    PRIMARY_KEY = auto()
 386    PROCEDURE = auto()
 387    PROPERTIES = auto()
 388    PROJECTION = auto()
 389    PSEUDO_TYPE = auto()
 390    PUT = auto()
 391    QUALIFY = auto()
 392    QUOTE = auto()
 393    QDCOLON = auto()
 394    RANGE = auto()
 395    RECURSIVE = auto()
 396    REFRESH = auto()
 397    RENAME = auto()
 398    REPLACE = auto()
 399    RETURNING = auto()
 400    REVOKE = auto()
 401    REFERENCES = auto()
 402    RIGHT = auto()
 403    RLIKE = auto()
 404    ROLE = auto()
 405    ROLLBACK = auto()
 406    ROLLUP = auto()
 407    ROW = auto()
 408    ROWS = auto()
 409    RULE = auto()
 410    SELECT = auto()
 411    SEMI = auto()
 412    SEPARATOR = auto()
 413    SEQUENCE = auto()
 414    SERDE_PROPERTIES = auto()
 415    SET = auto()
 416    SETTINGS = auto()
 417    SHOW = auto()
 418    SIMILAR_TO = auto()
 419    SOME = auto()
 420    SORT_BY = auto()
 421    SOUNDS_LIKE = auto()
 422    SQL_SECURITY = auto()
 423    START_WITH = auto()
 424    STORAGE_INTEGRATION = auto()
 425    STRAIGHT_JOIN = auto()
 426    STRUCT = auto()
 427    SUMMARIZE = auto()
 428    TABLE_SAMPLE = auto()
 429    TAG = auto()
 430    TEMPORARY = auto()
 431    TOP = auto()
 432    THEN = auto()
 433    TRUE = auto()
 434    TRUNCATE = auto()
 435    TRIGGER = auto()
 436    TYPE = auto()
 437    UNCACHE = auto()
 438    UNDROP = auto()
 439    UNION = auto()
 440    UNNEST = auto()
 441    UNPIVOT = auto()
 442    UPDATE = auto()
 443    USE = auto()
 444    USING = auto()
 445    VALUES = auto()
 446    VARIADIC = auto()
 447    VIEW = auto()
 448    SEMANTIC_VIEW = auto()
 449    VOLATILE = auto()
 450    VOLUME = auto()
 451    WHEN = auto()
 452    WHERE = auto()
 453    WINDOW = auto()
 454    WITH = auto()
 455    UNIQUE = auto()
 456    UTC_DATE = auto()
 457    UTC_TIME = auto()
 458    UTC_TIMESTAMP = auto()
 459    OPTION = auto()
 460    SINK = auto()
 461    SOURCE = auto()
 462    ANALYZE = auto()
 463    NAMESPACE = auto()
 464    EXPORT = auto()
 465
 466    # sentinels
 467    HIVE_TOKEN_STREAM = auto()
 468    SENTINEL = auto()
 469
 470    def __str__(self) -> str:
 471        return f"TokenType.{self.name}"
 472
 473
 474class Token:
 475    # mypyc doesn't expose slots
 476    _attrs: t.ClassVar[tuple[str, ...]] = (
 477        "token_type",
 478        "text",
 479        "line",
 480        "col",
 481        "start",
 482        "end",
 483        "comments",
 484    )
 485    __slots__ = _attrs
 486
 487    @classmethod
 488    def number(cls, number: int) -> Token:
 489        """Returns a NUMBER token with `number` as its text."""
 490        return cls(TokenType.NUMBER, str(number))
 491
 492    @classmethod
 493    def string(cls, string: str) -> Token:
 494        """Returns a STRING token with `string` as its text."""
 495        return cls(TokenType.STRING, string)
 496
 497    @classmethod
 498    def identifier(cls, identifier: str) -> Token:
 499        """Returns an IDENTIFIER token with `identifier` as its text."""
 500        return cls(TokenType.IDENTIFIER, identifier)
 501
 502    @classmethod
 503    def var(cls, var: str) -> Token:
 504        """Returns an VAR token with `var` as its text."""
 505        return cls(TokenType.VAR, var)
 506
 507    def __init__(
 508        self,
 509        token_type: TokenType,
 510        text: str,
 511        line: int = 1,
 512        col: int = 1,
 513        start: int = 0,
 514        end: int = 0,
 515        comments: list[str] | None = None,
 516    ) -> None:
 517        self.token_type = token_type
 518        self.text = text
 519        self.line = line
 520        self.col = col
 521        self.start = start
 522        self.end = end
 523        self.comments = [] if comments is None else comments
 524
 525    def __bool__(self) -> bool:
 526        return self.token_type != TokenType.SENTINEL
 527
 528    def __repr__(self) -> str:
 529        attributes = ", ".join(
 530            f"{k}: TokenType.{self.token_type.name}"
 531            if k == "token_type"
 532            else f"{k}: {getattr(self, k)}"
 533            for k in self._attrs
 534        )
 535        return f"<Token {attributes}>"
 536
 537
 538class TokenizerCore:
 539    __slots__ = (
 540        "sql",
 541        "size",
 542        "tokens",
 543        "_start",
 544        "_current",
 545        "_line",
 546        "_col",
 547        "_comments",
 548        "_char",
 549        "_end",
 550        "_peek",
 551        "_prev_token_line",
 552        "single_tokens",
 553        "keywords",
 554        "quotes",
 555        "format_strings",
 556        "identifiers",
 557        "comments",
 558        "string_escapes",
 559        "byte_string_escapes",
 560        "identifier_escapes",
 561        "escape_follow_chars",
 562        "commands",
 563        "command_prefix_tokens",
 564        "nested_comments",
 565        "hint_start",
 566        "tokens_preceding_hint",
 567        "has_bit_strings",
 568        "has_hex_strings",
 569        "numeric_literals",
 570        "var_single_tokens",
 571        "string_escapes_allowed_in_raw_strings",
 572        "heredoc_tag_is_identifier",
 573        "heredoc_string_alternative",
 574        "keyword_trie",
 575        "numbers_can_be_underscore_separated",
 576        "numbers_can_have_decimals",
 577        "identifiers_can_start_with_digit",
 578        "unescaped_sequences",
 579        "numeric_escapes",
 580        "drop_unknown_escapes",
 581    )
 582
 583    def __init__(
 584        self,
 585        single_tokens: dict[str, TokenType],
 586        keywords: dict[str, TokenType],
 587        quotes: dict[str, str],
 588        format_strings: dict[str, tuple[str, TokenType]],
 589        identifiers: dict[str, str],
 590        comments: dict[str, str | None],
 591        string_escapes: set[str],
 592        byte_string_escapes: set[str],
 593        identifier_escapes: set[str],
 594        escape_follow_chars: set[str],
 595        commands: set[TokenType],
 596        command_prefix_tokens: set[TokenType],
 597        nested_comments: bool,
 598        hint_start: str,
 599        tokens_preceding_hint: set[TokenType],
 600        has_bit_strings: bool,
 601        has_hex_strings: bool,
 602        numeric_literals: dict[str, str],
 603        var_single_tokens: set[str],
 604        string_escapes_allowed_in_raw_strings: bool,
 605        heredoc_tag_is_identifier: bool,
 606        heredoc_string_alternative: TokenType,
 607        keyword_trie: dict,
 608        numbers_can_be_underscore_separated: bool,
 609        numbers_can_have_decimals: bool,
 610        identifiers_can_start_with_digit: bool,
 611        unescaped_sequences: dict[str, str],
 612        numeric_escapes: dict[str, tuple[int, int, int, int]],
 613        drop_unknown_escapes: bool,
 614    ) -> None:
 615        self.single_tokens = single_tokens
 616        self.keywords = keywords
 617        self.quotes = quotes
 618        self.format_strings = format_strings
 619        self.identifiers = identifiers
 620        self.comments = comments
 621        self.string_escapes = string_escapes
 622        self.byte_string_escapes = byte_string_escapes
 623        self.identifier_escapes = identifier_escapes
 624        self.escape_follow_chars = escape_follow_chars
 625        self.commands = commands
 626        self.command_prefix_tokens = command_prefix_tokens
 627        self.nested_comments = nested_comments
 628        self.hint_start = hint_start
 629        self.tokens_preceding_hint = tokens_preceding_hint
 630        self.has_bit_strings = has_bit_strings
 631        self.has_hex_strings = has_hex_strings
 632        self.numeric_literals = numeric_literals
 633        self.var_single_tokens = var_single_tokens
 634        self.string_escapes_allowed_in_raw_strings = string_escapes_allowed_in_raw_strings
 635        self.heredoc_tag_is_identifier = heredoc_tag_is_identifier
 636        self.heredoc_string_alternative = heredoc_string_alternative
 637        self.keyword_trie = keyword_trie
 638        self.numbers_can_be_underscore_separated = numbers_can_be_underscore_separated
 639        self.numbers_can_have_decimals = numbers_can_have_decimals
 640        self.identifiers_can_start_with_digit = identifiers_can_start_with_digit
 641        self.unescaped_sequences = unescaped_sequences
 642        self.numeric_escapes = numeric_escapes
 643        self.drop_unknown_escapes = drop_unknown_escapes
 644        self.sql = ""
 645        self.size = 0
 646        self.tokens: list[Token] = []
 647        self._start = 0
 648        self._current = 0
 649        self._line = 1
 650        self._col = 0
 651        self._comments: list[str] = []
 652        self._char = ""
 653        self._end = False
 654        self._peek = ""
 655        self._prev_token_line = -1
 656
 657    def reset(self) -> None:
 658        self.sql = ""
 659        self.size = 0
 660        self.tokens = []
 661        self._start = 0
 662        self._current = 0
 663        self._line = 1
 664        self._col = 0
 665        self._comments = []
 666        self._char = ""
 667        self._end = False
 668        self._peek = ""
 669        self._prev_token_line = -1
 670
 671    def tokenize(self, sql: str) -> list[Token]:
 672        """Returns a list of tokens corresponding to the SQL string `sql`."""
 673        self.reset()
 674        self.sql = sql
 675        self.size = len(sql)
 676
 677        try:
 678            self._scan()
 679        except Exception as e:
 680            start = max(self._current - 50, 0)
 681            end = min(self._current + 50, self.size - 1)
 682            context = self.sql[start:end]
 683            raise TokenError(f"Error tokenizing '{context}'", start=start, end=end) from e
 684
 685        return self.tokens
 686
 687    def _scan(self, check_semicolon: bool = False) -> None:
 688        identifiers = self.identifiers
 689        digit_chars = _DIGIT_CHARS
 690
 691        while self.size and not self._end:
 692            current = self._current
 693
 694            # Skip spaces here rather than iteratively calling advance() for performance reasons
 695            while current < self.size:
 696                char = self.sql[current]
 697
 698                if char == " " or char == "\t":
 699                    current += 1
 700                else:
 701                    break
 702
 703            offset = current - self._current if current > self._current else 1
 704
 705            self._start = current
 706            self._advance(offset)
 707
 708            if not self._char.isspace():
 709                if self._char in digit_chars:
 710                    self._scan_number()
 711                elif self._char in identifiers:
 712                    self._scan_identifier(identifiers[self._char])
 713                else:
 714                    self._scan_keywords()
 715
 716            if check_semicolon and self._peek == ";":
 717                break
 718
 719        if self.tokens and self._comments:
 720            self.tokens[-1].comments.extend(self._comments)
 721
 722    def _chars(self, size: int) -> str:
 723        if size == 1:
 724            return self._char
 725
 726        start = self._current - 1
 727        end = start + size
 728
 729        return self.sql[start:end] if end <= self.size else ""
 730
 731    def _advance(self, i: int = 1, alnum: bool = False) -> None:
 732        char = self._char
 733
 734        if char == "\n" or char == "\r":
 735            # Ensures we don't count an extra line if we get a \r\n line break sequence
 736            if not (char == "\r" and self._peek == "\n"):
 737                self._col = i
 738                self._line += 1
 739        else:
 740            self._col += i
 741
 742        self._current += i
 743        sql = self.sql
 744        size = self.size
 745        self._end = self._current >= size
 746        self._char = sql[self._current - 1]
 747        self._peek = "" if self._end else sql[self._current]
 748
 749        if alnum and self._char.isalnum():
 750            # Cache to local variables instead of attributes for better performance
 751            _col = self._col
 752            _current = self._current
 753            _end = self._end
 754            _peek = self._peek
 755
 756            while _peek.isalnum():
 757                _col += 1
 758                _current += 1
 759                _end = _current >= size
 760                _peek = "" if _end else sql[_current]
 761
 762            self._col = _col
 763            self._current = _current
 764            self._end = _end
 765            self._peek = _peek
 766            self._char = sql[_current - 1]
 767
 768    @property
 769    def _text(self) -> str:
 770        return self.sql[self._start : self._current]
 771
 772    def _add(self, token_type: TokenType, text: str | None = None) -> None:
 773        self._prev_token_line = self._line
 774
 775        if self._comments and token_type == TokenType.SEMICOLON and self.tokens:
 776            self.tokens[-1].comments.extend(self._comments)
 777            self._comments = []
 778
 779        if text is None:
 780            text = self.sql[self._start : self._current]
 781
 782        self.tokens.append(
 783            Token(
 784                token_type,
 785                text=text,
 786                line=self._line,
 787                col=self._col,
 788                start=self._start,
 789                end=self._current - 1,
 790                comments=self._comments,
 791            )
 792        )
 793        self._comments = []
 794
 795        # If we have either a semicolon or a begin token before the command's token, we'll parse
 796        # whatever follows the command's token as a string
 797        if (
 798            token_type in self.commands
 799            and self._peek != ";"
 800            and (len(self.tokens) == 1 or self.tokens[-2].token_type in self.command_prefix_tokens)
 801        ):
 802            start = self._current
 803            tokens = len(self.tokens)
 804            self._scan(check_semicolon=True)
 805            self.tokens = self.tokens[:tokens]
 806            text = self.sql[start : self._current].strip()
 807            if text:
 808                self._add(TokenType.STRING, text)
 809
 810    def _scan_keywords(self) -> None:
 811        sql = self.sql
 812        sql_size = self.size
 813        single_tokens = self.single_tokens
 814        char_upper = _CHAR_UPPER
 815        size = 0
 816        word = None
 817        chars = self._char
 818        char = chars
 819        prev_space = False
 820        skip = False
 821        trie = self.keyword_trie
 822        single_token = char in single_tokens
 823
 824        while chars:
 825            if not skip:
 826                sub = trie.get(char_upper.get(char, char))
 827                if sub is None:
 828                    break
 829                trie = sub
 830                if 0 in trie:
 831                    word = chars
 832
 833            end = self._current + size
 834            size += 1
 835
 836            if end < sql_size:
 837                char = sql[end]
 838                single_token = single_token or char in single_tokens
 839                is_space = char.isspace()
 840
 841                if not is_space or not prev_space:
 842                    if is_space:
 843                        char = " "
 844                    chars += char
 845                    prev_space = is_space
 846                    skip = False
 847                else:
 848                    skip = True
 849            else:
 850                char = ""
 851                break
 852
 853        if word:
 854            if self._scan_string(word):
 855                return
 856            if self._scan_comment(word):
 857                return
 858            if prev_space or single_token or not char:
 859                self._advance(size - 1)
 860                word = word.upper()
 861                self._add(self.keywords[word], text=word)
 862                return
 863
 864        if self._char in single_tokens:
 865            self._add(single_tokens[self._char], text=self._char)
 866            return
 867
 868        self._scan_var()
 869
 870    def _scan_comment(self, comment_start: str) -> bool:
 871        if comment_start not in self.comments:
 872            return False
 873
 874        comment_start_line = self._line
 875        comment_start_size = len(comment_start)
 876        comment_end = self.comments[comment_start]
 877
 878        if comment_end:
 879            # Skip the comment's start delimiter
 880            self._advance(comment_start_size)
 881
 882            comment_count = 1
 883            comment_end_size = len(comment_end)
 884            nested_comments = self.nested_comments
 885
 886            while not self._end:
 887                if self._chars(comment_end_size) == comment_end:
 888                    comment_count -= 1
 889                    if not comment_count:
 890                        break
 891
 892                self._advance(alnum=True)
 893
 894                # Nested comments are allowed by some dialects, e.g. databricks, duckdb, postgres
 895                if (
 896                    nested_comments
 897                    and not self._end
 898                    and self._chars(comment_end_size) == comment_start
 899                ):
 900                    self._advance(comment_start_size)
 901                    comment_count += 1
 902
 903            self._comments.append(self._text[comment_start_size : -comment_end_size + 1])
 904            self._advance(comment_end_size - 1)
 905        else:
 906            _peek = self._peek
 907            while not self._end and _peek != "\n" and _peek != "\r":
 908                self._advance(alnum=True)
 909                _peek = self._peek
 910            self._comments.append(self._text[comment_start_size:])
 911
 912        if (
 913            comment_start == self.hint_start
 914            and self.tokens
 915            and self.tokens[-1].token_type in self.tokens_preceding_hint
 916        ):
 917            self._add(TokenType.HINT)
 918
 919        # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.
 920        # Multiple consecutive comments are preserved by appending them to the current comments list.
 921        if comment_start_line == self._prev_token_line:
 922            self.tokens[-1].comments.extend(self._comments)
 923            self._comments = []
 924            self._prev_token_line = self._line
 925
 926        return True
 927
 928    def _scan_number(self) -> None:
 929        if self._char == "0":
 930            peek = _CHAR_UPPER.get(self._peek, self._peek)
 931            if peek == "B" and self.has_bit_strings:
 932                return self._scan_bits()
 933            elif peek == "X":
 934                return self._scan_hex() if self.has_hex_strings else self._add(TokenType.NUMBER)
 935
 936        decimal = False
 937        scientific = 0
 938        numbers_can_be_underscore_separated = self.numbers_can_be_underscore_separated
 939        single_tokens = self.single_tokens
 940        keywords = self.keywords
 941        numeric_literals = self.numeric_literals
 942        identifiers_can_start_with_digit = self.identifiers_can_start_with_digit
 943
 944        is_underscore_separated: bool = False
 945        number_text: str = ""
 946        numeric_literal: str = ""
 947        numeric_type: TokenType | None = None
 948
 949        while True:
 950            if self._peek in _DIGIT_CHARS:
 951                # Batch consecutive digits: scan ahead to find how many
 952                sql = self.sql
 953                end = self._current + 1
 954                size = self.size
 955                while end < size and sql[end] in _DIGIT_CHARS:
 956                    end += 1
 957                self._advance(end - self._current)
 958            elif self._peek == "." and not decimal:
 959                if (
 960                    self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
 961                ) or not self.numbers_can_have_decimals:
 962                    break
 963                decimal = True
 964                self._advance()
 965            elif self._peek in ("-", "+") and scientific == 1:
 966                # Only consume +/- if followed by a digit
 967                if self._current + 1 < self.size and self.sql[self._current + 1] in _DIGIT_CHARS:
 968                    scientific += 1
 969                    self._advance()
 970                else:
 971                    break
 972            elif _CHAR_UPPER.get(self._peek, self._peek) == "E" and not scientific:
 973                scientific += 1
 974                self._advance()
 975            elif self._peek == "_" and numbers_can_be_underscore_separated:
 976                is_underscore_separated = True
 977                self._advance()
 978            elif self._peek.isidentifier():
 979                number_text = self._text
 980
 981                while self._peek and not self._peek.isspace() and self._peek not in single_tokens:
 982                    numeric_literal += self._peek
 983                    self._advance()
 984
 985                numeric_type = keywords.get(numeric_literals.get(numeric_literal.upper(), ""))
 986
 987                if numeric_type:
 988                    break
 989                elif identifiers_can_start_with_digit:
 990                    return self._add(TokenType.VAR)
 991
 992                self._advance(-len(numeric_literal))
 993                break
 994            else:
 995                break
 996
 997        number_text = number_text or self.sql[self._start : self._current]
 998
 999        # Normalize inputs such as 100_000 to 100000
1000        if is_underscore_separated:
1001            number_text = number_text.replace("_", "")
1002
1003        self._add(TokenType.NUMBER, number_text)
1004
1005        # Normalize inputs such as 123L to 123::BIGINT so that they're parsed as casts
1006        if numeric_type:
1007            self._add(TokenType.DCOLON, "::")
1008            self._add(numeric_type, numeric_literal)
1009
1010    def _scan_bits(self) -> None:
1011        self._advance()
1012        value = self._extract_value()
1013        try:
1014            # If `value` can't be converted to a binary, fallback to tokenizing it as an identifier
1015            int(value, 2)
1016            self._add(TokenType.BIT_STRING, value[2:])  # Drop the 0b
1017        except ValueError:
1018            self._add(TokenType.IDENTIFIER)
1019
1020    def _scan_hex(self) -> None:
1021        self._advance()
1022        value = self._extract_value()
1023        try:
1024            # If `value` can't be converted to a hex, fallback to tokenizing it as an identifier
1025            int(value, 16)
1026            self._add(TokenType.HEX_STRING, value[2:])  # Drop the 0x
1027        except ValueError:
1028            self._add(TokenType.IDENTIFIER)
1029
1030    def _extract_value(self) -> str:
1031        single_tokens = self.single_tokens
1032
1033        while True:
1034            char = self._peek.strip()
1035            if char and char not in single_tokens:
1036                self._advance(alnum=True)
1037            else:
1038                break
1039
1040        return self._text
1041
1042    def _scan_string(self, start: str) -> bool:
1043        base = None
1044        token_type = TokenType.STRING
1045
1046        if start in self.quotes:
1047            end = self.quotes[start]
1048        elif start in self.format_strings:
1049            end, token_type = self.format_strings[start]
1050
1051            if token_type == TokenType.HEX_STRING:
1052                base = 16
1053            elif token_type == TokenType.BIT_STRING:
1054                base = 2
1055            elif token_type == TokenType.HEREDOC_STRING:
1056                self._advance()
1057
1058                if self._char == end:
1059                    tag = ""
1060                else:
1061                    tag = self._extract_string(
1062                        end,
1063                        raw_string=True,
1064                        raise_unmatched=not self.heredoc_tag_is_identifier,
1065                    )
1066
1067                if (
1068                    tag
1069                    and self.heredoc_tag_is_identifier
1070                    and (self._end or tag.isdigit() or any(c.isspace() for c in tag))
1071                ):
1072                    if not self._end:
1073                        self._advance(-1)
1074
1075                    self._advance(-len(tag))
1076                    self._add(self.heredoc_string_alternative)
1077                    return True
1078
1079                end = f"{start}{tag}{end}"
1080        else:
1081            return False
1082
1083        self._advance(len(start))
1084        text = self._extract_string(
1085            end,
1086            escapes=(
1087                self.byte_string_escapes
1088                if token_type == TokenType.BYTE_STRING
1089                else self.string_escapes
1090            ),
1091            raw_string=token_type == TokenType.RAW_STRING,
1092        )
1093
1094        if base and text:
1095            try:
1096                int(text, base)
1097            except Exception:
1098                raise TokenError(
1099                    f"Numeric string contains invalid characters from {self._line}:{self._start}"
1100                )
1101
1102        self._add(token_type, text)
1103        return True
1104
1105    def _scan_identifier(self, identifier_end: str) -> None:
1106        self._advance()
1107        text = self._extract_string(
1108            identifier_end, escapes=self.identifier_escapes | {identifier_end}
1109        )
1110        self._add(TokenType.IDENTIFIER, text)
1111
1112    def _scan_var(self) -> None:
1113        var_single_tokens = self.var_single_tokens
1114        single_tokens = self.single_tokens
1115
1116        while True:
1117            peek = self._peek
1118            if not peek or peek.isspace():
1119                break
1120            if peek not in var_single_tokens and peek in single_tokens:
1121                break
1122            self._advance(alnum=True)
1123
1124        self._add(
1125            TokenType.VAR
1126            if self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
1127            else self.keywords.get(self.sql[self._start : self._current].upper(), TokenType.VAR)
1128        )
1129
1130    def _read_numeric_escape(
1131        self, start: int, base: int, min_digits: int, max_digits: int, max_value: int
1132    ) -> tuple[int, int]:
1133        """
1134        Reads up to `max_digits` digits from `start`, stopping before the value exceeds `max_value`.
1135        Returns the value and the end index, or -1 as the value if fewer than `min_digits` were read.
1136        """
1137        sql = self.sql
1138        value = 0
1139        end = start
1140        limit = min(start + max_digits, self.size)
1141
1142        while end < limit:
1143            digit = _DIGIT_VALUES.get(sql[end], 16)
1144            if digit >= base or value * base + digit > max_value:
1145                break
1146            value = value * base + digit
1147            end += 1
1148
1149        return (value, end) if end - start >= min_digits else (-1, end)
1150
1151    def _scan_numeric_escape(self) -> str:
1152        """
1153        Decodes the numeric escape sequence starting at the current char, which is a backslash,
1154        according to `numeric_escapes`. Returns an empty string if nothing is decoded.
1155        """
1156        sql = self.sql
1157        start = self._current
1158        peek = self._peek
1159
1160        # Octal escapes start with a digit (e.g. \101), the others with a letter (e.g. \x41)
1161        is_octal = peek in _OCTAL_CHARS
1162        spec = self.numeric_escapes.get("0" if is_octal else peek)
1163
1164        if spec:
1165            base, min_digits, max_digits, max_value = spec
1166            value, end = self._read_numeric_escape(
1167                start if is_octal else start + 1, base, min_digits, max_digits, max_value
1168            )
1169
1170            # A UTF-16 surrogate is only valid as a high half (D800-DBFF) followed by an escaped
1171            # low half (DC00-DFFF), e.g. \uD83D\uDE00 is U+1F600
1172            if 0xD800 <= value <= 0xDFFF:
1173                low, low_end = -1, end
1174                if value <= 0xDBFF and sql[end : end + 2] == "\\" + peek:
1175                    low, low_end = self._read_numeric_escape(
1176                        end + 2, base, min_digits, max_digits, max_value
1177                    )
1178
1179                if 0xDC00 <= low <= 0xDFFF:
1180                    # Combine the two halves into a single code point (standard formula)
1181                    value = 0x10000 + ((value - 0xD800) << 10) + (low - 0xDC00)
1182                    end = low_end
1183                else:
1184                    # A lone surrogate isn't a character, so the escape isn't decoded
1185                    value = -1
1186
1187            # -1 means the escape isn't decoded, e.g. because it has too few digits. An escape at the
1188            # end of the input is left to the caller, which reports the unterminated string
1189            if value >= 0 and end < self.size:
1190                # Move past the whole escape, from the backslash through its last digit
1191                self._advance(end - start + 1)
1192                return chr(value)
1193
1194        return ""
1195
1196    def _extract_string(
1197        self,
1198        delimiter: str,
1199        escapes: set[str] | None = None,
1200        raw_string: bool = False,
1201        raise_unmatched: bool = True,
1202    ) -> str:
1203        text = ""
1204        delim_size = len(delimiter)
1205        escapes = self.string_escapes if escapes is None else escapes
1206        unescaped_sequences = self.unescaped_sequences
1207        escape_follow_chars = self.escape_follow_chars
1208        numeric_escapes = self.numeric_escapes
1209        drop_unknown_escapes = self.drop_unknown_escapes
1210        string_escapes_allowed_in_raw_strings = self.string_escapes_allowed_in_raw_strings
1211        quotes = self.quotes
1212        sql = self.sql
1213
1214        # use str.find() when the string is simple... no \ or other escapes
1215        if delim_size == 1:
1216            pos = self._current - 1
1217            end = sql.find(delimiter, pos)
1218
1219            if (
1220                # the closing delimiter was found
1221                end != -1
1222                # there's no doubled delimiter (e.g. '' escape), or the delimiter isn't an escape char
1223                and (end + 1 >= self.size or sql[end + 1] != delimiter or delimiter not in escapes)
1224                # no backslash in the string that would need escape processing
1225                and (not (unescaped_sequences or "\\" in escapes) or sql.find("\\", pos, end) == -1)
1226            ):
1227                newlines = sql.count("\n", pos, end)
1228                if newlines:
1229                    self._line += newlines
1230                    self._col = end - sql.rfind("\n", pos, end)
1231                else:
1232                    self._col += end - pos
1233
1234                self._current = end + 1
1235                self._end = self._current >= self.size
1236                self._char = sql[end]
1237                self._peek = "" if self._end else sql[self._current]
1238                return sql[pos:end]
1239
1240        while True:
1241            backslash_escape = not raw_string and self._char == "\\" and "\\" in escapes
1242
1243            # Numeric escapes come first, since they may overlap fixed sequences (e.g. \000 vs \0)
1244            if backslash_escape and numeric_escapes:
1245                decoded = self._scan_numeric_escape()
1246                if decoded:
1247                    text += decoded
1248                    continue
1249
1250            # An escape at the end of the input is left to the checks below, which report the
1251            # unterminated string
1252            if (
1253                not raw_string
1254                and unescaped_sequences
1255                and self._char in escapes
1256                and self._current + 1 < self.size
1257            ):
1258                unescaped_sequence = unescaped_sequences.get(self._char + self._peek)
1259                if unescaped_sequence:
1260                    # Advance one char at a time so that line numbers are updated for a newline
1261                    if self._peek in "\n\r":
1262                        self._advance()
1263                        self._advance()
1264                    else:
1265                        self._advance(2)
1266                    text += unescaped_sequence
1267                    continue
1268
1269            # Not decoded: drop the backslash if configured, unless the string is unterminated
1270            if backslash_escape and drop_unknown_escapes and self._current + 1 < self.size:
1271                # Advance one char at a time so that line numbers are updated if `peek` is a newline
1272                peek = self._peek
1273                self._advance()
1274                self._advance()
1275                text += peek
1276                continue
1277
1278            is_valid_custom_escape = (
1279                escape_follow_chars and self._char == "\\" and self._peek not in escape_follow_chars
1280            )
1281
1282            # An escaped quote before the closing delimiter (e.g. \" in """a\"""") must be
1283            # consumed here, otherwise it'd be picked up by the delimiter check below and
1284            # terminate the string early. This is only relevant for multi-char delimiters
1285            # made up of quote chars, e.g. it shouldn't apply to Snowflake's $$ strings
1286            escaped_delimiter = self._peek == delimiter or (
1287                delim_size > 1 and self._peek == delimiter[0] and self._peek in quotes
1288            )
1289
1290            if (
1291                (string_escapes_allowed_in_raw_strings or not raw_string)
1292                and self._char in escapes
1293                and (escaped_delimiter or self._peek in escapes or is_valid_custom_escape)
1294                and (self._char not in quotes or self._char == self._peek)
1295            ):
1296                if escaped_delimiter:
1297                    text += self._peek if not raw_string else self._char + self._peek
1298                elif is_valid_custom_escape and self._char != self._peek:
1299                    text += self._peek
1300                else:
1301                    text += self._char + self._peek
1302
1303                if self._current + 1 < self.size:
1304                    self._advance(2)
1305                else:
1306                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._current}")
1307            else:
1308                if self._chars(delim_size) == delimiter:
1309                    if delim_size > 1:
1310                        self._advance(delim_size - 1)
1311                    break
1312
1313                if self._end:
1314                    if not raise_unmatched:
1315                        return text + self._char
1316
1317                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._start}")
1318
1319                current = self._current - 1
1320                self._advance(alnum=True)
1321                text += sql[current : self._current - 1]
1322
1323        return text