sqlglot.tokenizer_core

   1from __future__ import annotations
   2
   3import typing as t
   4from enum import IntEnum, auto
   5
   6from sqlglot.errors import TokenError
   7
   8# dict lookup is faster than .upper() and .isdigit()
   9_CHAR_UPPER: dict[str, str] = {chr(i): chr(i).upper() for i in range(97, 123)}
  10_DIGIT_CHARS: frozenset[str] = frozenset("0123456789")
  11
  12
  13class TokenType(IntEnum):
  14    L_PAREN = auto()
  15    R_PAREN = auto()
  16    L_BRACKET = auto()
  17    R_BRACKET = auto()
  18    L_BRACE = auto()
  19    R_BRACE = auto()
  20    COMMA = auto()
  21    DOT = auto()
  22    DASH = auto()
  23    PLUS = auto()
  24    COLON = auto()
  25    DOTCOLON = auto()
  26    DOTCARET = auto()
  27    DCOLON = auto()
  28    DCOLONDOLLAR = auto()
  29    DCOLONPERCENT = auto()
  30    DCOLONQMARK = auto()
  31    DQMARK = auto()
  32    SEMICOLON = auto()
  33    STAR = auto()
  34    BACKSLASH = auto()
  35    SLASH = auto()
  36    LT = auto()
  37    LTE = auto()
  38    GT = auto()
  39    GTE = auto()
  40    NOT = auto()
  41    EQ = auto()
  42    NEQ = auto()
  43    NULLSAFE_EQ = auto()
  44    COLON_EQ = auto()
  45    COLON_GT = auto()
  46    NCOLON_GT = auto()
  47    AND = auto()
  48    OR = auto()
  49    AMP = auto()
  50    DPIPE = auto()
  51    PIPE_GT = auto()
  52    PIPE = auto()
  53    PIPE_SLASH = auto()
  54    DPIPE_SLASH = auto()
  55    CARET = auto()
  56    CARET_AT = auto()
  57    TILDE = auto()
  58    ARROW = auto()
  59    DARROW = auto()
  60    FARROW = auto()
  61    HASH = auto()
  62    HASH_ARROW = auto()
  63    DHASH_ARROW = auto()
  64    LR_ARROW = auto()
  65    LLRR_ARROW = auto()
  66    DAT = auto()
  67    AT_QMARK = auto()
  68    LT_AT = auto()
  69    AT_GT = auto()
  70    DOLLAR = auto()
  71    PARAMETER = auto()
  72    SESSION = auto()
  73    SESSION_PARAMETER = auto()
  74    SESSION_USER = auto()
  75    DAMP = auto()
  76    AMP_LT = auto()
  77    AMP_GT = auto()
  78    ADJACENT = auto()
  79    XOR = auto()
  80    DSTAR = auto()
  81    QMARK_AMP = auto()
  82    QMARK_PIPE = auto()
  83    HASH_DASH = auto()
  84    EXCLAMATION = auto()
  85
  86    URI_START = auto()
  87
  88    BLOCK_START = auto()
  89    BLOCK_END = auto()
  90
  91    SPACE = auto()
  92    BREAK = auto()
  93
  94    STRING = auto()
  95    NUMBER = auto()
  96    IDENTIFIER = auto()
  97    DATABASE = auto()
  98    COLUMN = auto()
  99    COLUMN_DEF = auto()
 100    SCHEMA = auto()
 101    TABLE = auto()
 102    WAREHOUSE = auto()
 103    STAGE = auto()
 104    STREAM = auto()
 105    STREAMLIT = auto()
 106    VAR = auto()
 107    BIT_STRING = auto()
 108    HEX_STRING = auto()
 109    BYTE_STRING = auto()
 110    NATIONAL_STRING = auto()
 111    RAW_STRING = auto()
 112    HEREDOC_STRING = auto()
 113    UNICODE_STRING = auto()
 114
 115    # types
 116    BIT = auto()
 117    BOOLEAN = auto()
 118    TINYINT = auto()
 119    UTINYINT = auto()
 120    SMALLINT = auto()
 121    USMALLINT = auto()
 122    MEDIUMINT = auto()
 123    UMEDIUMINT = auto()
 124    INT = auto()
 125    UINT = auto()
 126    BIGINT = auto()
 127    UBIGINT = auto()
 128    BIGNUM = auto()
 129    INT128 = auto()
 130    UINT128 = auto()
 131    INT256 = auto()
 132    UINT256 = auto()
 133    FLOAT = auto()
 134    DOUBLE = auto()
 135    UDOUBLE = auto()
 136    DECIMAL = auto()
 137    DECIMAL32 = auto()
 138    DECIMAL64 = auto()
 139    DECIMAL128 = auto()
 140    DECIMAL256 = auto()
 141    DECFLOAT = auto()
 142    UDECIMAL = auto()
 143    BIGDECIMAL = auto()
 144    CHAR = auto()
 145    NCHAR = auto()
 146    VARCHAR = auto()
 147    NVARCHAR = auto()
 148    BPCHAR = auto()
 149    TEXT = auto()
 150    MEDIUMTEXT = auto()
 151    LONGTEXT = auto()
 152    BLOB = auto()
 153    MEDIUMBLOB = auto()
 154    LONGBLOB = auto()
 155    TINYBLOB = auto()
 156    TINYTEXT = auto()
 157    NAME = auto()
 158    BINARY = auto()
 159    VARBINARY = auto()
 160    JSON = auto()
 161    JSONB = auto()
 162    TIME = auto()
 163    TIMETZ = auto()
 164    TIME_NS = auto()
 165    TIMESTAMP = auto()
 166    TIMESTAMPTZ = auto()
 167    TIMESTAMPLTZ = auto()
 168    TIMESTAMPNTZ = auto()
 169    TIMESTAMP_S = auto()
 170    TIMESTAMP_MS = auto()
 171    TIMESTAMP_NS = auto()
 172    DATETIME = auto()
 173    DATETIME2 = auto()
 174    DATETIME64 = auto()
 175    SMALLDATETIME = auto()
 176    DATE = auto()
 177    DATE32 = auto()
 178    INT4RANGE = auto()
 179    INT4MULTIRANGE = auto()
 180    INT8RANGE = auto()
 181    INT8MULTIRANGE = auto()
 182    NUMRANGE = auto()
 183    NUMMULTIRANGE = auto()
 184    TSRANGE = auto()
 185    TSMULTIRANGE = auto()
 186    TSTZRANGE = auto()
 187    TSTZMULTIRANGE = auto()
 188    DATERANGE = auto()
 189    DATEMULTIRANGE = auto()
 190    UUID = auto()
 191    GEOGRAPHY = auto()
 192    GEOGRAPHYPOINT = auto()
 193    NULLABLE = auto()
 194    GEOMETRY = auto()
 195    POINT = auto()
 196    RING = auto()
 197    LINESTRING = auto()
 198    LOCALTIME = auto()
 199    LOCALTIMESTAMP = auto()
 200    SYSTIMESTAMP = auto()
 201    MULTILINESTRING = auto()
 202    POLYGON = auto()
 203    MULTIPOLYGON = auto()
 204    HLLSKETCH = auto()
 205    HSTORE = auto()
 206    SUPER = auto()
 207    SERIAL = auto()
 208    SMALLSERIAL = auto()
 209    BIGSERIAL = auto()
 210    XML = auto()
 211    YEAR = auto()
 212    USERDEFINED = auto()
 213    MONEY = auto()
 214    SMALLMONEY = auto()
 215    ROWVERSION = auto()
 216    IMAGE = auto()
 217    VARIANT = auto()
 218    OBJECT = auto()
 219    INET = auto()
 220    IPADDRESS = auto()
 221    IPPREFIX = auto()
 222    IPV4 = auto()
 223    IPV6 = auto()
 224    ENUM = auto()
 225    ENUM8 = auto()
 226    ENUM16 = auto()
 227    FIXEDSTRING = auto()
 228    LOWCARDINALITY = auto()
 229    NESTED = auto()
 230    AGGREGATEFUNCTION = auto()
 231    SIMPLEAGGREGATEFUNCTION = auto()
 232    TDIGEST = auto()
 233    UNKNOWN = auto()
 234    VECTOR = auto()
 235    DYNAMIC = auto()
 236    VOID = auto()
 237
 238    # keywords
 239    ALIAS = auto()
 240    ALTER = auto()
 241    ALL = auto()
 242    ANTI = auto()
 243    ANY = auto()
 244    APPLY = auto()
 245    ARRAY = auto()
 246    ASC = auto()
 247    ASOF = auto()
 248    ATTACH = auto()
 249    AUTO_INCREMENT = auto()
 250    BEGIN = auto()
 251    BETWEEN = auto()
 252    BULK_COLLECT_INTO = auto()
 253    CACHE = auto()
 254    CASE = auto()
 255    CHARACTER_SET = auto()
 256    CLUSTER_BY = auto()
 257    COLLATE = auto()
 258    COMMAND = auto()
 259    COMMENT = auto()
 260    COMMIT = auto()
 261    CONNECT_BY = auto()
 262    CONSTRAINT = auto()
 263    COPY = auto()
 264    CREATE = auto()
 265    CROSS = auto()
 266    CUBE = auto()
 267    CURRENT_DATE = auto()
 268    CURRENT_DATETIME = auto()
 269    CURRENT_SCHEMA = auto()
 270    CURRENT_TIME = auto()
 271    CURRENT_TIMESTAMP = auto()
 272    CURRENT_USER = auto()
 273    CURRENT_USER_ID = auto()
 274    CURRENT_ROLE = auto()
 275    CURRENT_CATALOG = auto()
 276    DECLARE = auto()
 277    DEFAULT = auto()
 278    DELETE = auto()
 279    DESC = auto()
 280    DESCRIBE = auto()
 281    DETACH = auto()
 282    DICTIONARY = auto()
 283    DISTINCT = auto()
 284    DISTRIBUTE_BY = auto()
 285    DIV = auto()
 286    DROP = auto()
 287    ELSE = auto()
 288    END = auto()
 289    ESCAPE = auto()
 290    EXCEPT = auto()
 291    EXECUTE = auto()
 292    EXISTS = auto()
 293    FALSE = auto()
 294    FETCH = auto()
 295    FILE = auto()
 296    FILE_FORMAT = auto()
 297    FILTER = auto()
 298    FINAL = auto()
 299    FIRST = auto()
 300    FOR = auto()
 301    FORCE = auto()
 302    FOREIGN_KEY = auto()
 303    FORMAT = auto()
 304    FROM = auto()
 305    FULL = auto()
 306    FUNCTION = auto()
 307    GET = auto()
 308    GLOB = auto()
 309    GLOBAL = auto()
 310    GRANT = auto()
 311    GROUP_BY = auto()
 312    GROUPING_SETS = auto()
 313    HAVING = auto()
 314    HINT = auto()
 315    IGNORE = auto()
 316    ILIKE = auto()
 317    IN = auto()
 318    INDEX = auto()
 319    INDEXED_BY = auto()
 320    INNER = auto()
 321    INSERT = auto()
 322    INSTALL = auto()
 323    INTEGRATION = auto()
 324    INTERSECT = auto()
 325    INTERVAL = auto()
 326    INTO = auto()
 327    INTRODUCER = auto()
 328    IRLIKE = auto()
 329    IS = auto()
 330    ISNULL = auto()
 331    JOIN = auto()
 332    JOIN_MARKER = auto()
 333    KEEP = auto()
 334    KEY = auto()
 335    KILL = auto()
 336    LANGUAGE = auto()
 337    LATERAL = auto()
 338    LEFT = auto()
 339    LIKE = auto()
 340    LIMIT = auto()
 341    LIST = auto()
 342    LOAD = auto()
 343    LOCK = auto()
 344    MAP = auto()
 345    MATCH = auto()
 346    MATCH_CONDITION = auto()
 347    MATCH_RECOGNIZE = auto()
 348    MEMBER_OF = auto()
 349    MERGE = auto()
 350    MOD = auto()
 351    MODEL = auto()
 352    NATURAL = auto()
 353    NEXT = auto()
 354    NOTHING = auto()
 355    NOTNULL = auto()
 356    NULL = auto()
 357    OBJECT_IDENTIFIER = auto()
 358    OFFSET = auto()
 359    ON = auto()
 360    ONLY = auto()
 361    OPERATOR = auto()
 362    ORDER_BY = auto()
 363    ORDER_SIBLINGS_BY = auto()
 364    ORDERED = auto()
 365    ORDINALITY = auto()
 366    OUT = auto()
 367    INOUT = auto()
 368    OUTER = auto()
 369    OVER = auto()
 370    OVERLAPS = auto()
 371    OVERWRITE = auto()
 372    PACKAGE = auto()
 373    PARTITION = auto()
 374    PARTITION_BY = auto()
 375    PERCENT = auto()
 376    PIVOT = auto()
 377    PLACEHOLDER = auto()
 378    POLICY = auto()
 379    POOL = auto()
 380    POSITIONAL = auto()
 381    PRAGMA = auto()
 382    PREWHERE = auto()
 383    PRIMARY_KEY = auto()
 384    PROCEDURE = auto()
 385    PROPERTIES = auto()
 386    PROJECTION = auto()
 387    PSEUDO_TYPE = auto()
 388    PUT = auto()
 389    QUALIFY = auto()
 390    QUOTE = auto()
 391    QDCOLON = auto()
 392    RANGE = auto()
 393    RECURSIVE = auto()
 394    REFRESH = auto()
 395    RENAME = auto()
 396    REPLACE = auto()
 397    RETURNING = auto()
 398    REVOKE = auto()
 399    REFERENCES = auto()
 400    RIGHT = auto()
 401    RLIKE = auto()
 402    ROLE = auto()
 403    ROLLBACK = auto()
 404    ROLLUP = auto()
 405    ROW = auto()
 406    ROWS = auto()
 407    RULE = auto()
 408    SELECT = auto()
 409    SEMI = auto()
 410    SEPARATOR = auto()
 411    SEQUENCE = auto()
 412    SERDE_PROPERTIES = auto()
 413    SET = auto()
 414    SETTINGS = auto()
 415    SHOW = auto()
 416    SIMILAR_TO = auto()
 417    SOME = auto()
 418    SORT_BY = auto()
 419    SOUNDS_LIKE = auto()
 420    SQL_SECURITY = auto()
 421    START_WITH = auto()
 422    STORAGE_INTEGRATION = auto()
 423    STRAIGHT_JOIN = auto()
 424    STRUCT = auto()
 425    SUMMARIZE = auto()
 426    TABLE_SAMPLE = auto()
 427    TAG = auto()
 428    TEMPORARY = auto()
 429    TOP = auto()
 430    THEN = auto()
 431    TRUE = auto()
 432    TRUNCATE = auto()
 433    TRIGGER = auto()
 434    TYPE = auto()
 435    UNCACHE = auto()
 436    UNDROP = auto()
 437    UNION = auto()
 438    UNNEST = auto()
 439    UNPIVOT = auto()
 440    UPDATE = auto()
 441    USE = auto()
 442    USING = auto()
 443    VALUES = auto()
 444    VARIADIC = auto()
 445    VIEW = auto()
 446    SEMANTIC_VIEW = auto()
 447    VOLATILE = auto()
 448    VOLUME = auto()
 449    WHEN = auto()
 450    WHERE = auto()
 451    WINDOW = auto()
 452    WITH = auto()
 453    UNIQUE = auto()
 454    UTC_DATE = auto()
 455    UTC_TIME = auto()
 456    UTC_TIMESTAMP = auto()
 457    OPTION = auto()
 458    SINK = auto()
 459    SOURCE = auto()
 460    ANALYZE = auto()
 461    NAMESPACE = auto()
 462    EXPORT = auto()
 463
 464    # sentinels
 465    HIVE_TOKEN_STREAM = auto()
 466    SENTINEL = auto()
 467
 468    def __str__(self) -> str:
 469        return f"TokenType.{self.name}"
 470
 471
 472class Token:
 473    # mypyc doesn't expose slots
 474    _attrs: t.ClassVar[tuple[str, ...]] = (
 475        "token_type",
 476        "text",
 477        "line",
 478        "col",
 479        "start",
 480        "end",
 481        "comments",
 482    )
 483    __slots__ = _attrs
 484
 485    @classmethod
 486    def number(cls, number: int) -> Token:
 487        """Returns a NUMBER token with `number` as its text."""
 488        return cls(TokenType.NUMBER, str(number))
 489
 490    @classmethod
 491    def string(cls, string: str) -> Token:
 492        """Returns a STRING token with `string` as its text."""
 493        return cls(TokenType.STRING, string)
 494
 495    @classmethod
 496    def identifier(cls, identifier: str) -> Token:
 497        """Returns an IDENTIFIER token with `identifier` as its text."""
 498        return cls(TokenType.IDENTIFIER, identifier)
 499
 500    @classmethod
 501    def var(cls, var: str) -> Token:
 502        """Returns an VAR token with `var` as its text."""
 503        return cls(TokenType.VAR, var)
 504
 505    def __init__(
 506        self,
 507        token_type: TokenType,
 508        text: str,
 509        line: int = 1,
 510        col: int = 1,
 511        start: int = 0,
 512        end: int = 0,
 513        comments: list[str] | None = None,
 514    ) -> None:
 515        self.token_type = token_type
 516        self.text = text
 517        self.line = line
 518        self.col = col
 519        self.start = start
 520        self.end = end
 521        self.comments = [] if comments is None else comments
 522
 523    def __bool__(self) -> bool:
 524        return self.token_type != TokenType.SENTINEL
 525
 526    def __repr__(self) -> str:
 527        attributes = ", ".join(
 528            f"{k}: TokenType.{self.token_type.name}"
 529            if k == "token_type"
 530            else f"{k}: {getattr(self, k)}"
 531            for k in self._attrs
 532        )
 533        return f"<Token {attributes}>"
 534
 535
 536class TokenizerCore:
 537    __slots__ = (
 538        "sql",
 539        "size",
 540        "tokens",
 541        "_start",
 542        "_current",
 543        "_line",
 544        "_col",
 545        "_comments",
 546        "_char",
 547        "_end",
 548        "_peek",
 549        "_prev_token_line",
 550        "single_tokens",
 551        "keywords",
 552        "quotes",
 553        "format_strings",
 554        "identifiers",
 555        "comments",
 556        "string_escapes",
 557        "byte_string_escapes",
 558        "identifier_escapes",
 559        "escape_follow_chars",
 560        "commands",
 561        "command_prefix_tokens",
 562        "nested_comments",
 563        "hint_start",
 564        "tokens_preceding_hint",
 565        "has_bit_strings",
 566        "has_hex_strings",
 567        "numeric_literals",
 568        "var_single_tokens",
 569        "string_escapes_allowed_in_raw_strings",
 570        "heredoc_tag_is_identifier",
 571        "heredoc_string_alternative",
 572        "keyword_trie",
 573        "numbers_can_be_underscore_separated",
 574        "numbers_can_have_decimals",
 575        "identifiers_can_start_with_digit",
 576        "unescaped_sequences",
 577    )
 578
 579    def __init__(
 580        self,
 581        single_tokens: dict[str, TokenType],
 582        keywords: dict[str, TokenType],
 583        quotes: dict[str, str],
 584        format_strings: dict[str, tuple[str, TokenType]],
 585        identifiers: dict[str, str],
 586        comments: dict[str, str | None],
 587        string_escapes: set[str],
 588        byte_string_escapes: set[str],
 589        identifier_escapes: set[str],
 590        escape_follow_chars: set[str],
 591        commands: set[TokenType],
 592        command_prefix_tokens: set[TokenType],
 593        nested_comments: bool,
 594        hint_start: str,
 595        tokens_preceding_hint: set[TokenType],
 596        has_bit_strings: bool,
 597        has_hex_strings: bool,
 598        numeric_literals: dict[str, str],
 599        var_single_tokens: set[str],
 600        string_escapes_allowed_in_raw_strings: bool,
 601        heredoc_tag_is_identifier: bool,
 602        heredoc_string_alternative: TokenType,
 603        keyword_trie: dict,
 604        numbers_can_be_underscore_separated: bool,
 605        numbers_can_have_decimals: bool,
 606        identifiers_can_start_with_digit: bool,
 607        unescaped_sequences: dict[str, str],
 608    ) -> None:
 609        self.single_tokens = single_tokens
 610        self.keywords = keywords
 611        self.quotes = quotes
 612        self.format_strings = format_strings
 613        self.identifiers = identifiers
 614        self.comments = comments
 615        self.string_escapes = string_escapes
 616        self.byte_string_escapes = byte_string_escapes
 617        self.identifier_escapes = identifier_escapes
 618        self.escape_follow_chars = escape_follow_chars
 619        self.commands = commands
 620        self.command_prefix_tokens = command_prefix_tokens
 621        self.nested_comments = nested_comments
 622        self.hint_start = hint_start
 623        self.tokens_preceding_hint = tokens_preceding_hint
 624        self.has_bit_strings = has_bit_strings
 625        self.has_hex_strings = has_hex_strings
 626        self.numeric_literals = numeric_literals
 627        self.var_single_tokens = var_single_tokens
 628        self.string_escapes_allowed_in_raw_strings = string_escapes_allowed_in_raw_strings
 629        self.heredoc_tag_is_identifier = heredoc_tag_is_identifier
 630        self.heredoc_string_alternative = heredoc_string_alternative
 631        self.keyword_trie = keyword_trie
 632        self.numbers_can_be_underscore_separated = numbers_can_be_underscore_separated
 633        self.numbers_can_have_decimals = numbers_can_have_decimals
 634        self.identifiers_can_start_with_digit = identifiers_can_start_with_digit
 635        self.unescaped_sequences = unescaped_sequences
 636        self.sql = ""
 637        self.size = 0
 638        self.tokens: list[Token] = []
 639        self._start = 0
 640        self._current = 0
 641        self._line = 1
 642        self._col = 0
 643        self._comments: list[str] = []
 644        self._char = ""
 645        self._end = False
 646        self._peek = ""
 647        self._prev_token_line = -1
 648
 649    def reset(self) -> None:
 650        self.sql = ""
 651        self.size = 0
 652        self.tokens = []
 653        self._start = 0
 654        self._current = 0
 655        self._line = 1
 656        self._col = 0
 657        self._comments = []
 658        self._char = ""
 659        self._end = False
 660        self._peek = ""
 661        self._prev_token_line = -1
 662
 663    def tokenize(self, sql: str) -> list[Token]:
 664        """Returns a list of tokens corresponding to the SQL string `sql`."""
 665        self.reset()
 666        self.sql = sql
 667        self.size = len(sql)
 668
 669        try:
 670            self._scan()
 671        except Exception as e:
 672            start = max(self._current - 50, 0)
 673            end = min(self._current + 50, self.size - 1)
 674            context = self.sql[start:end]
 675            raise TokenError(f"Error tokenizing '{context}'") from e
 676
 677        return self.tokens
 678
 679    def _scan(self, check_semicolon: bool = False) -> None:
 680        identifiers = self.identifiers
 681        digit_chars = _DIGIT_CHARS
 682
 683        while self.size and not self._end:
 684            current = self._current
 685
 686            # Skip spaces here rather than iteratively calling advance() for performance reasons
 687            while current < self.size:
 688                char = self.sql[current]
 689
 690                if char == " " or char == "\t":
 691                    current += 1
 692                else:
 693                    break
 694
 695            offset = current - self._current if current > self._current else 1
 696
 697            self._start = current
 698            self._advance(offset)
 699
 700            if not self._char.isspace():
 701                if self._char in digit_chars:
 702                    self._scan_number()
 703                elif self._char in identifiers:
 704                    self._scan_identifier(identifiers[self._char])
 705                else:
 706                    self._scan_keywords()
 707
 708            if check_semicolon and self._peek == ";":
 709                break
 710
 711        if self.tokens and self._comments:
 712            self.tokens[-1].comments.extend(self._comments)
 713
 714    def _chars(self, size: int) -> str:
 715        if size == 1:
 716            return self._char
 717
 718        start = self._current - 1
 719        end = start + size
 720
 721        return self.sql[start:end] if end <= self.size else ""
 722
 723    def _advance(self, i: int = 1, alnum: bool = False) -> None:
 724        char = self._char
 725
 726        if char == "\n" or char == "\r":
 727            # Ensures we don't count an extra line if we get a \r\n line break sequence
 728            if not (char == "\r" and self._peek == "\n"):
 729                self._col = i
 730                self._line += 1
 731        else:
 732            self._col += i
 733
 734        self._current += i
 735        sql = self.sql
 736        size = self.size
 737        self._end = self._current >= size
 738        self._char = sql[self._current - 1]
 739        self._peek = "" if self._end else sql[self._current]
 740
 741        if alnum and self._char.isalnum():
 742            # Cache to local variables instead of attributes for better performance
 743            _col = self._col
 744            _current = self._current
 745            _end = self._end
 746            _peek = self._peek
 747
 748            while _peek.isalnum():
 749                _col += 1
 750                _current += 1
 751                _end = _current >= size
 752                _peek = "" if _end else sql[_current]
 753
 754            self._col = _col
 755            self._current = _current
 756            self._end = _end
 757            self._peek = _peek
 758            self._char = sql[_current - 1]
 759
 760    @property
 761    def _text(self) -> str:
 762        return self.sql[self._start : self._current]
 763
 764    def _add(self, token_type: TokenType, text: str | None = None) -> None:
 765        self._prev_token_line = self._line
 766
 767        if self._comments and token_type == TokenType.SEMICOLON and self.tokens:
 768            self.tokens[-1].comments.extend(self._comments)
 769            self._comments = []
 770
 771        if text is None:
 772            text = self.sql[self._start : self._current]
 773
 774        self.tokens.append(
 775            Token(
 776                token_type,
 777                text=text,
 778                line=self._line,
 779                col=self._col,
 780                start=self._start,
 781                end=self._current - 1,
 782                comments=self._comments,
 783            )
 784        )
 785        self._comments = []
 786
 787        # If we have either a semicolon or a begin token before the command's token, we'll parse
 788        # whatever follows the command's token as a string
 789        if (
 790            token_type in self.commands
 791            and self._peek != ";"
 792            and (len(self.tokens) == 1 or self.tokens[-2].token_type in self.command_prefix_tokens)
 793        ):
 794            start = self._current
 795            tokens = len(self.tokens)
 796            self._scan(check_semicolon=True)
 797            self.tokens = self.tokens[:tokens]
 798            text = self.sql[start : self._current].strip()
 799            if text:
 800                self._add(TokenType.STRING, text)
 801
 802    def _scan_keywords(self) -> None:
 803        sql = self.sql
 804        sql_size = self.size
 805        single_tokens = self.single_tokens
 806        char_upper = _CHAR_UPPER
 807        size = 0
 808        word = None
 809        chars = self._char
 810        char = chars
 811        prev_space = False
 812        skip = False
 813        trie = self.keyword_trie
 814        single_token = char in single_tokens
 815
 816        while chars:
 817            if not skip:
 818                sub = trie.get(char_upper.get(char, char))
 819                if sub is None:
 820                    break
 821                trie = sub
 822                if 0 in trie:
 823                    word = chars
 824
 825            end = self._current + size
 826            size += 1
 827
 828            if end < sql_size:
 829                char = sql[end]
 830                single_token = single_token or char in single_tokens
 831                is_space = char.isspace()
 832
 833                if not is_space or not prev_space:
 834                    if is_space:
 835                        char = " "
 836                    chars += char
 837                    prev_space = is_space
 838                    skip = False
 839                else:
 840                    skip = True
 841            else:
 842                char = ""
 843                break
 844
 845        if word:
 846            if self._scan_string(word):
 847                return
 848            if self._scan_comment(word):
 849                return
 850            if prev_space or single_token or not char:
 851                self._advance(size - 1)
 852                word = word.upper()
 853                self._add(self.keywords[word], text=word)
 854                return
 855
 856        if self._char in single_tokens:
 857            self._add(single_tokens[self._char], text=self._char)
 858            return
 859
 860        self._scan_var()
 861
 862    def _scan_comment(self, comment_start: str) -> bool:
 863        if comment_start not in self.comments:
 864            return False
 865
 866        comment_start_line = self._line
 867        comment_start_size = len(comment_start)
 868        comment_end = self.comments[comment_start]
 869
 870        if comment_end:
 871            # Skip the comment's start delimiter
 872            self._advance(comment_start_size)
 873
 874            comment_count = 1
 875            comment_end_size = len(comment_end)
 876            nested_comments = self.nested_comments
 877
 878            while not self._end:
 879                if self._chars(comment_end_size) == comment_end:
 880                    comment_count -= 1
 881                    if not comment_count:
 882                        break
 883
 884                self._advance(alnum=True)
 885
 886                # Nested comments are allowed by some dialects, e.g. databricks, duckdb, postgres
 887                if (
 888                    nested_comments
 889                    and not self._end
 890                    and self._chars(comment_end_size) == comment_start
 891                ):
 892                    self._advance(comment_start_size)
 893                    comment_count += 1
 894
 895            self._comments.append(self._text[comment_start_size : -comment_end_size + 1])
 896            self._advance(comment_end_size - 1)
 897        else:
 898            _peek = self._peek
 899            while not self._end and _peek != "\n" and _peek != "\r":
 900                self._advance(alnum=True)
 901                _peek = self._peek
 902            self._comments.append(self._text[comment_start_size:])
 903
 904        if (
 905            comment_start == self.hint_start
 906            and self.tokens
 907            and self.tokens[-1].token_type in self.tokens_preceding_hint
 908        ):
 909            self._add(TokenType.HINT)
 910
 911        # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.
 912        # Multiple consecutive comments are preserved by appending them to the current comments list.
 913        if comment_start_line == self._prev_token_line:
 914            self.tokens[-1].comments.extend(self._comments)
 915            self._comments = []
 916            self._prev_token_line = self._line
 917
 918        return True
 919
 920    def _scan_number(self) -> None:
 921        if self._char == "0":
 922            peek = _CHAR_UPPER.get(self._peek, self._peek)
 923            if peek == "B":
 924                return self._scan_bits() if self.has_bit_strings else self._add(TokenType.NUMBER)
 925            elif peek == "X":
 926                return self._scan_hex() if self.has_hex_strings else self._add(TokenType.NUMBER)
 927
 928        decimal = False
 929        scientific = 0
 930        numbers_can_be_underscore_separated = self.numbers_can_be_underscore_separated
 931        single_tokens = self.single_tokens
 932        keywords = self.keywords
 933        numeric_literals = self.numeric_literals
 934        identifiers_can_start_with_digit = self.identifiers_can_start_with_digit
 935
 936        is_underscore_separated: bool = False
 937        number_text: str = ""
 938        numeric_literal: str = ""
 939        numeric_type: TokenType | None = None
 940
 941        while True:
 942            if self._peek in _DIGIT_CHARS:
 943                # Batch consecutive digits: scan ahead to find how many
 944                sql = self.sql
 945                end = self._current + 1
 946                size = self.size
 947                while end < size and sql[end] in _DIGIT_CHARS:
 948                    end += 1
 949                self._advance(end - self._current)
 950            elif self._peek == "." and not decimal:
 951                if (
 952                    self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
 953                ) or not self.numbers_can_have_decimals:
 954                    break
 955                decimal = True
 956                self._advance()
 957            elif self._peek in ("-", "+") and scientific == 1:
 958                # Only consume +/- if followed by a digit
 959                if self._current + 1 < self.size and self.sql[self._current + 1] in _DIGIT_CHARS:
 960                    scientific += 1
 961                    self._advance()
 962                else:
 963                    break
 964            elif _CHAR_UPPER.get(self._peek, self._peek) == "E" and not scientific:
 965                scientific += 1
 966                self._advance()
 967            elif self._peek == "_" and numbers_can_be_underscore_separated:
 968                is_underscore_separated = True
 969                self._advance()
 970            elif self._peek.isidentifier():
 971                number_text = self._text
 972
 973                while self._peek and not self._peek.isspace() and self._peek not in single_tokens:
 974                    numeric_literal += self._peek
 975                    self._advance()
 976
 977                numeric_type = keywords.get(numeric_literals.get(numeric_literal.upper(), ""))
 978
 979                if numeric_type:
 980                    break
 981                elif identifiers_can_start_with_digit:
 982                    return self._add(TokenType.VAR)
 983
 984                self._advance(-len(numeric_literal))
 985                break
 986            else:
 987                break
 988
 989        number_text = number_text or self.sql[self._start : self._current]
 990
 991        # Normalize inputs such as 100_000 to 100000
 992        if is_underscore_separated:
 993            number_text = number_text.replace("_", "")
 994
 995        self._add(TokenType.NUMBER, number_text)
 996
 997        # Normalize inputs such as 123L to 123::BIGINT so that they're parsed as casts
 998        if numeric_type:
 999            self._add(TokenType.DCOLON, "::")
1000            self._add(numeric_type, numeric_literal)
1001
1002    def _scan_bits(self) -> None:
1003        self._advance()
1004        value = self._extract_value()
1005        try:
1006            # If `value` can't be converted to a binary, fallback to tokenizing it as an identifier
1007            int(value, 2)
1008            self._add(TokenType.BIT_STRING, value[2:])  # Drop the 0b
1009        except ValueError:
1010            self._add(TokenType.IDENTIFIER)
1011
1012    def _scan_hex(self) -> None:
1013        self._advance()
1014        value = self._extract_value()
1015        try:
1016            # If `value` can't be converted to a hex, fallback to tokenizing it as an identifier
1017            int(value, 16)
1018            self._add(TokenType.HEX_STRING, value[2:])  # Drop the 0x
1019        except ValueError:
1020            self._add(TokenType.IDENTIFIER)
1021
1022    def _extract_value(self) -> str:
1023        single_tokens = self.single_tokens
1024
1025        while True:
1026            char = self._peek.strip()
1027            if char and char not in single_tokens:
1028                self._advance(alnum=True)
1029            else:
1030                break
1031
1032        return self._text
1033
1034    def _scan_string(self, start: str) -> bool:
1035        base = None
1036        token_type = TokenType.STRING
1037
1038        if start in self.quotes:
1039            end = self.quotes[start]
1040        elif start in self.format_strings:
1041            end, token_type = self.format_strings[start]
1042
1043            if token_type == TokenType.HEX_STRING:
1044                base = 16
1045            elif token_type == TokenType.BIT_STRING:
1046                base = 2
1047            elif token_type == TokenType.HEREDOC_STRING:
1048                self._advance()
1049
1050                if self._char == end:
1051                    tag = ""
1052                else:
1053                    tag = self._extract_string(
1054                        end,
1055                        raw_string=True,
1056                        raise_unmatched=not self.heredoc_tag_is_identifier,
1057                    )
1058
1059                if (
1060                    tag
1061                    and self.heredoc_tag_is_identifier
1062                    and (self._end or tag.isdigit() or any(c.isspace() for c in tag))
1063                ):
1064                    if not self._end:
1065                        self._advance(-1)
1066
1067                    self._advance(-len(tag))
1068                    self._add(self.heredoc_string_alternative)
1069                    return True
1070
1071                end = f"{start}{tag}{end}"
1072        else:
1073            return False
1074
1075        self._advance(len(start))
1076        text = self._extract_string(
1077            end,
1078            escapes=(
1079                self.byte_string_escapes
1080                if token_type == TokenType.BYTE_STRING
1081                else self.string_escapes
1082            ),
1083            raw_string=token_type == TokenType.RAW_STRING,
1084        )
1085
1086        if base and text:
1087            try:
1088                int(text, base)
1089            except Exception:
1090                raise TokenError(
1091                    f"Numeric string contains invalid characters from {self._line}:{self._start}"
1092                )
1093
1094        self._add(token_type, text)
1095        return True
1096
1097    def _scan_identifier(self, identifier_end: str) -> None:
1098        self._advance()
1099        text = self._extract_string(
1100            identifier_end, escapes=self.identifier_escapes | {identifier_end}
1101        )
1102        self._add(TokenType.IDENTIFIER, text)
1103
1104    def _scan_var(self) -> None:
1105        var_single_tokens = self.var_single_tokens
1106        single_tokens = self.single_tokens
1107
1108        while True:
1109            peek = self._peek
1110            if not peek or peek.isspace():
1111                break
1112            if peek not in var_single_tokens and peek in single_tokens:
1113                break
1114            self._advance(alnum=True)
1115
1116        self._add(
1117            TokenType.VAR
1118            if self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
1119            else self.keywords.get(self.sql[self._start : self._current].upper(), TokenType.VAR)
1120        )
1121
1122    def _extract_string(
1123        self,
1124        delimiter: str,
1125        escapes: set[str] | None = None,
1126        raw_string: bool = False,
1127        raise_unmatched: bool = True,
1128    ) -> str:
1129        text = ""
1130        delim_size = len(delimiter)
1131        escapes = self.string_escapes if escapes is None else escapes
1132        unescaped_sequences = self.unescaped_sequences
1133        escape_follow_chars = self.escape_follow_chars
1134        string_escapes_allowed_in_raw_strings = self.string_escapes_allowed_in_raw_strings
1135        quotes = self.quotes
1136        sql = self.sql
1137
1138        # use str.find() when the string is simple... no \ or other escapes
1139        if delim_size == 1:
1140            pos = self._current - 1
1141            end = sql.find(delimiter, pos)
1142
1143            if (
1144                # the closing delimiter was found
1145                end != -1
1146                # there's no doubled delimiter (e.g. '' escape), or the delimiter isn't an escape char
1147                and (end + 1 >= self.size or sql[end + 1] != delimiter or delimiter not in escapes)
1148                # no backslash in the string that would need escape processing
1149                and (not (unescaped_sequences or "\\" in escapes) or sql.find("\\", pos, end) == -1)
1150            ):
1151                newlines = sql.count("\n", pos, end)
1152                if newlines:
1153                    self._line += newlines
1154                    self._col = end - sql.rfind("\n", pos, end)
1155                else:
1156                    self._col += end - pos
1157
1158                self._current = end + 1
1159                self._end = self._current >= self.size
1160                self._char = sql[end]
1161                self._peek = "" if self._end else sql[self._current]
1162                return sql[pos:end]
1163
1164        while True:
1165            if not raw_string and unescaped_sequences and self._peek and self._char in escapes:
1166                unescaped_sequence = unescaped_sequences.get(self._char + self._peek)
1167                if unescaped_sequence:
1168                    self._advance(2)
1169                    text += unescaped_sequence
1170                    continue
1171
1172            is_valid_custom_escape = (
1173                escape_follow_chars and self._char == "\\" and self._peek not in escape_follow_chars
1174            )
1175
1176            # An escaped quote before the closing delimiter (e.g. \" in """a\"""") must be
1177            # consumed here, otherwise it'd be picked up by the delimiter check below and
1178            # terminate the string early. This is only relevant for multi-char delimiters
1179            # made up of quote chars, e.g. it shouldn't apply to Snowflake's $$ strings
1180            escaped_delimiter = self._peek == delimiter or (
1181                delim_size > 1 and self._peek == delimiter[0] and self._peek in quotes
1182            )
1183
1184            if (
1185                (string_escapes_allowed_in_raw_strings or not raw_string)
1186                and self._char in escapes
1187                and (escaped_delimiter or self._peek in escapes or is_valid_custom_escape)
1188                and (self._char not in quotes or self._char == self._peek)
1189            ):
1190                if escaped_delimiter:
1191                    text += self._peek if not raw_string else self._char + self._peek
1192                elif is_valid_custom_escape and self._char != self._peek:
1193                    text += self._peek
1194                else:
1195                    text += self._char + self._peek
1196
1197                if self._current + 1 < self.size:
1198                    self._advance(2)
1199                else:
1200                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._current}")
1201            else:
1202                if self._chars(delim_size) == delimiter:
1203                    if delim_size > 1:
1204                        self._advance(delim_size - 1)
1205                    break
1206
1207                if self._end:
1208                    if not raise_unmatched:
1209                        return text + self._char
1210
1211                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._start}")
1212
1213                current = self._current - 1
1214                self._advance(alnum=True)
1215                text += sql[current : self._current - 1]
1216
1217        return text