sqlglot.tokenizer_core

   1from __future__ import annotations
   2
   3import typing as t
   4from enum import IntEnum, auto
   5
   6from sqlglot.errors import TokenError
   7
   8# dict lookup is faster than .upper() and .isdigit()
   9_CHAR_UPPER: dict[str, str] = {chr(i): chr(i).upper() for i in range(97, 123)}
  10_DIGIT_CHARS: frozenset[str] = frozenset("0123456789")
  11
  12
  13class TokenType(IntEnum):
  14    L_PAREN = auto()
  15    R_PAREN = auto()
  16    L_BRACKET = auto()
  17    R_BRACKET = auto()
  18    L_BRACE = auto()
  19    R_BRACE = auto()
  20    COMMA = auto()
  21    DOT = auto()
  22    DASH = auto()
  23    PLUS = auto()
  24    COLON = auto()
  25    DOTCOLON = auto()
  26    DOTCARET = auto()
  27    DCOLON = auto()
  28    DCOLONDOLLAR = auto()
  29    DCOLONPERCENT = auto()
  30    DCOLONQMARK = auto()
  31    DQMARK = auto()
  32    SEMICOLON = auto()
  33    STAR = auto()
  34    BACKSLASH = auto()
  35    SLASH = auto()
  36    LT = auto()
  37    LTE = auto()
  38    GT = auto()
  39    GTE = auto()
  40    NOT = auto()
  41    EQ = auto()
  42    NEQ = auto()
  43    NULLSAFE_EQ = auto()
  44    COLON_EQ = auto()
  45    COLON_GT = auto()
  46    NCOLON_GT = auto()
  47    AND = auto()
  48    OR = auto()
  49    AMP = auto()
  50    DPIPE = auto()
  51    PIPE_GT = auto()
  52    PIPE = auto()
  53    PIPE_SLASH = auto()
  54    DPIPE_SLASH = auto()
  55    CARET = auto()
  56    CARET_AT = auto()
  57    TILDE = auto()
  58    ARROW = auto()
  59    DARROW = auto()
  60    FARROW = auto()
  61    HASH = auto()
  62    HASH_ARROW = auto()
  63    DHASH_ARROW = auto()
  64    LR_ARROW = auto()
  65    DAT = auto()
  66    LT_AT = auto()
  67    AT_GT = auto()
  68    DOLLAR = auto()
  69    PARAMETER = auto()
  70    SESSION = auto()
  71    SESSION_PARAMETER = auto()
  72    SESSION_USER = auto()
  73    DAMP = auto()
  74    AMP_LT = auto()
  75    AMP_GT = auto()
  76    ADJACENT = auto()
  77    XOR = auto()
  78    DSTAR = auto()
  79    QMARK_AMP = auto()
  80    QMARK_PIPE = auto()
  81    HASH_DASH = auto()
  82    EXCLAMATION = auto()
  83
  84    URI_START = auto()
  85
  86    BLOCK_START = auto()
  87    BLOCK_END = auto()
  88
  89    SPACE = auto()
  90    BREAK = auto()
  91
  92    STRING = auto()
  93    NUMBER = auto()
  94    IDENTIFIER = auto()
  95    DATABASE = auto()
  96    COLUMN = auto()
  97    COLUMN_DEF = auto()
  98    SCHEMA = auto()
  99    TABLE = auto()
 100    WAREHOUSE = auto()
 101    STAGE = auto()
 102    STREAM = auto()
 103    STREAMLIT = auto()
 104    VAR = auto()
 105    BIT_STRING = auto()
 106    HEX_STRING = auto()
 107    BYTE_STRING = auto()
 108    NATIONAL_STRING = auto()
 109    RAW_STRING = auto()
 110    HEREDOC_STRING = auto()
 111    UNICODE_STRING = auto()
 112
 113    # types
 114    BIT = auto()
 115    BOOLEAN = auto()
 116    TINYINT = auto()
 117    UTINYINT = auto()
 118    SMALLINT = auto()
 119    USMALLINT = auto()
 120    MEDIUMINT = auto()
 121    UMEDIUMINT = auto()
 122    INT = auto()
 123    UINT = auto()
 124    BIGINT = auto()
 125    UBIGINT = auto()
 126    BIGNUM = auto()
 127    INT128 = auto()
 128    UINT128 = auto()
 129    INT256 = auto()
 130    UINT256 = auto()
 131    FLOAT = auto()
 132    DOUBLE = auto()
 133    UDOUBLE = auto()
 134    DECIMAL = auto()
 135    DECIMAL32 = auto()
 136    DECIMAL64 = auto()
 137    DECIMAL128 = auto()
 138    DECIMAL256 = auto()
 139    DECFLOAT = auto()
 140    UDECIMAL = auto()
 141    BIGDECIMAL = auto()
 142    CHAR = auto()
 143    NCHAR = auto()
 144    VARCHAR = auto()
 145    NVARCHAR = auto()
 146    BPCHAR = auto()
 147    TEXT = auto()
 148    MEDIUMTEXT = auto()
 149    LONGTEXT = auto()
 150    BLOB = auto()
 151    MEDIUMBLOB = auto()
 152    LONGBLOB = auto()
 153    TINYBLOB = auto()
 154    TINYTEXT = auto()
 155    NAME = auto()
 156    BINARY = auto()
 157    VARBINARY = auto()
 158    JSON = auto()
 159    JSONB = auto()
 160    TIME = auto()
 161    TIMETZ = auto()
 162    TIME_NS = auto()
 163    TIMESTAMP = auto()
 164    TIMESTAMPTZ = auto()
 165    TIMESTAMPLTZ = auto()
 166    TIMESTAMPNTZ = auto()
 167    TIMESTAMP_S = auto()
 168    TIMESTAMP_MS = auto()
 169    TIMESTAMP_NS = auto()
 170    DATETIME = auto()
 171    DATETIME2 = auto()
 172    DATETIME64 = auto()
 173    SMALLDATETIME = auto()
 174    DATE = auto()
 175    DATE32 = auto()
 176    INT4RANGE = auto()
 177    INT4MULTIRANGE = auto()
 178    INT8RANGE = auto()
 179    INT8MULTIRANGE = auto()
 180    NUMRANGE = auto()
 181    NUMMULTIRANGE = auto()
 182    TSRANGE = auto()
 183    TSMULTIRANGE = auto()
 184    TSTZRANGE = auto()
 185    TSTZMULTIRANGE = auto()
 186    DATERANGE = auto()
 187    DATEMULTIRANGE = auto()
 188    UUID = auto()
 189    GEOGRAPHY = auto()
 190    GEOGRAPHYPOINT = auto()
 191    NULLABLE = auto()
 192    GEOMETRY = auto()
 193    POINT = auto()
 194    RING = auto()
 195    LINESTRING = auto()
 196    LOCALTIME = auto()
 197    LOCALTIMESTAMP = auto()
 198    SYSTIMESTAMP = auto()
 199    MULTILINESTRING = auto()
 200    POLYGON = auto()
 201    MULTIPOLYGON = auto()
 202    HLLSKETCH = auto()
 203    HSTORE = auto()
 204    SUPER = auto()
 205    SERIAL = auto()
 206    SMALLSERIAL = auto()
 207    BIGSERIAL = auto()
 208    XML = auto()
 209    YEAR = auto()
 210    USERDEFINED = auto()
 211    MONEY = auto()
 212    SMALLMONEY = auto()
 213    ROWVERSION = auto()
 214    IMAGE = auto()
 215    VARIANT = auto()
 216    OBJECT = auto()
 217    INET = auto()
 218    IPADDRESS = auto()
 219    IPPREFIX = auto()
 220    IPV4 = auto()
 221    IPV6 = auto()
 222    ENUM = auto()
 223    ENUM8 = auto()
 224    ENUM16 = auto()
 225    FIXEDSTRING = auto()
 226    LOWCARDINALITY = auto()
 227    NESTED = auto()
 228    AGGREGATEFUNCTION = auto()
 229    SIMPLEAGGREGATEFUNCTION = auto()
 230    TDIGEST = auto()
 231    UNKNOWN = auto()
 232    VECTOR = auto()
 233    DYNAMIC = auto()
 234    VOID = auto()
 235
 236    # keywords
 237    ALIAS = auto()
 238    ALTER = auto()
 239    ALL = auto()
 240    ANTI = auto()
 241    ANY = auto()
 242    APPLY = auto()
 243    ARRAY = auto()
 244    ASC = auto()
 245    ASOF = auto()
 246    ATTACH = auto()
 247    AUTO_INCREMENT = auto()
 248    BEGIN = auto()
 249    BETWEEN = auto()
 250    BULK_COLLECT_INTO = auto()
 251    CACHE = auto()
 252    CASE = auto()
 253    CHARACTER_SET = auto()
 254    CLUSTER_BY = auto()
 255    COLLATE = auto()
 256    COMMAND = auto()
 257    COMMENT = auto()
 258    COMMIT = auto()
 259    CONNECT_BY = auto()
 260    CONSTRAINT = auto()
 261    COPY = auto()
 262    CREATE = auto()
 263    CROSS = auto()
 264    CUBE = auto()
 265    CURRENT_DATE = auto()
 266    CURRENT_DATETIME = auto()
 267    CURRENT_SCHEMA = auto()
 268    CURRENT_TIME = auto()
 269    CURRENT_TIMESTAMP = auto()
 270    CURRENT_USER = auto()
 271    CURRENT_ROLE = auto()
 272    CURRENT_CATALOG = auto()
 273    DECLARE = auto()
 274    DEFAULT = auto()
 275    DELETE = auto()
 276    DESC = auto()
 277    DESCRIBE = auto()
 278    DETACH = auto()
 279    DICTIONARY = auto()
 280    DISTINCT = auto()
 281    DISTRIBUTE_BY = auto()
 282    DIV = auto()
 283    DROP = auto()
 284    ELSE = auto()
 285    END = auto()
 286    ESCAPE = auto()
 287    EXCEPT = auto()
 288    EXECUTE = auto()
 289    EXISTS = auto()
 290    FALSE = auto()
 291    FETCH = auto()
 292    FILE = auto()
 293    FILE_FORMAT = auto()
 294    FILTER = auto()
 295    FINAL = auto()
 296    FIRST = auto()
 297    FOR = auto()
 298    FORCE = auto()
 299    FOREIGN_KEY = auto()
 300    FORMAT = auto()
 301    FROM = auto()
 302    FULL = auto()
 303    FUNCTION = auto()
 304    GET = auto()
 305    GLOB = auto()
 306    GLOBAL = auto()
 307    GRANT = auto()
 308    GROUP_BY = auto()
 309    GROUPING_SETS = auto()
 310    HAVING = auto()
 311    HINT = auto()
 312    IGNORE = auto()
 313    ILIKE = auto()
 314    IN = auto()
 315    INDEX = auto()
 316    INDEXED_BY = auto()
 317    INNER = auto()
 318    INSERT = auto()
 319    INSTALL = auto()
 320    INTEGRATION = auto()
 321    INTERSECT = auto()
 322    INTERVAL = auto()
 323    INTO = auto()
 324    INTRODUCER = auto()
 325    IRLIKE = auto()
 326    IS = auto()
 327    ISNULL = auto()
 328    JOIN = auto()
 329    JOIN_MARKER = auto()
 330    KEEP = auto()
 331    KEY = auto()
 332    KILL = auto()
 333    LANGUAGE = auto()
 334    LATERAL = auto()
 335    LEFT = auto()
 336    LIKE = auto()
 337    LIMIT = auto()
 338    LIST = auto()
 339    LOAD = auto()
 340    LOCK = auto()
 341    MAP = auto()
 342    MATCH = auto()
 343    MATCH_CONDITION = auto()
 344    MATCH_RECOGNIZE = auto()
 345    MEMBER_OF = auto()
 346    MERGE = auto()
 347    MOD = auto()
 348    MODEL = auto()
 349    NATURAL = auto()
 350    NEXT = auto()
 351    NOTHING = auto()
 352    NOTNULL = auto()
 353    NULL = auto()
 354    OBJECT_IDENTIFIER = auto()
 355    OFFSET = auto()
 356    ON = auto()
 357    ONLY = auto()
 358    OPERATOR = auto()
 359    ORDER_BY = auto()
 360    ORDER_SIBLINGS_BY = auto()
 361    ORDERED = auto()
 362    ORDINALITY = auto()
 363    OUT = auto()
 364    INOUT = auto()
 365    OUTER = auto()
 366    OVER = auto()
 367    OVERLAPS = auto()
 368    OVERWRITE = auto()
 369    PACKAGE = auto()
 370    PARTITION = auto()
 371    PARTITION_BY = auto()
 372    PERCENT = auto()
 373    PIVOT = auto()
 374    PLACEHOLDER = auto()
 375    POLICY = auto()
 376    POOL = auto()
 377    POSITIONAL = auto()
 378    PRAGMA = auto()
 379    PREWHERE = auto()
 380    PRIMARY_KEY = auto()
 381    PROCEDURE = auto()
 382    PROPERTIES = auto()
 383    PSEUDO_TYPE = auto()
 384    PUT = auto()
 385    QUALIFY = auto()
 386    QUOTE = auto()
 387    QDCOLON = auto()
 388    RANGE = auto()
 389    RECURSIVE = auto()
 390    REFRESH = auto()
 391    RENAME = auto()
 392    REPLACE = auto()
 393    RETURNING = auto()
 394    REVOKE = auto()
 395    REFERENCES = auto()
 396    RIGHT = auto()
 397    RLIKE = auto()
 398    ROLE = auto()
 399    ROLLBACK = auto()
 400    ROLLUP = auto()
 401    ROW = auto()
 402    ROWS = auto()
 403    RULE = auto()
 404    SELECT = auto()
 405    SEMI = auto()
 406    SEPARATOR = auto()
 407    SEQUENCE = auto()
 408    SERDE_PROPERTIES = auto()
 409    SET = auto()
 410    SETTINGS = auto()
 411    SHOW = auto()
 412    SIMILAR_TO = auto()
 413    SOME = auto()
 414    SORT_BY = auto()
 415    SOUNDS_LIKE = auto()
 416    SQL_SECURITY = auto()
 417    START_WITH = auto()
 418    STORAGE_INTEGRATION = auto()
 419    STRAIGHT_JOIN = auto()
 420    STRUCT = auto()
 421    SUMMARIZE = auto()
 422    TABLE_SAMPLE = auto()
 423    TAG = auto()
 424    TEMPORARY = auto()
 425    TOP = auto()
 426    THEN = auto()
 427    TRUE = auto()
 428    TRUNCATE = auto()
 429    TRIGGER = auto()
 430    UNCACHE = auto()
 431    UNION = auto()
 432    UNNEST = auto()
 433    UNPIVOT = auto()
 434    UPDATE = auto()
 435    USE = auto()
 436    USING = auto()
 437    VALUES = auto()
 438    VARIADIC = auto()
 439    VIEW = auto()
 440    SEMANTIC_VIEW = auto()
 441    VOLATILE = auto()
 442    VOLUME = auto()
 443    WHEN = auto()
 444    WHERE = auto()
 445    WINDOW = auto()
 446    WITH = auto()
 447    UNIQUE = auto()
 448    UTC_DATE = auto()
 449    UTC_TIME = auto()
 450    UTC_TIMESTAMP = auto()
 451    VERSION_SNAPSHOT = auto()
 452    TIMESTAMP_SNAPSHOT = auto()
 453    OPTION = auto()
 454    SINK = auto()
 455    SOURCE = auto()
 456    ANALYZE = auto()
 457    NAMESPACE = auto()
 458    EXPORT = auto()
 459
 460    # sentinels
 461    HIVE_TOKEN_STREAM = auto()
 462    SENTINEL = auto()
 463
 464    def __str__(self) -> str:
 465        return f"TokenType.{self.name}"
 466
 467
 468class Token:
 469    # mypyc doesn't expose slots
 470    _attrs: t.ClassVar[tuple[str, ...]] = (
 471        "token_type",
 472        "text",
 473        "line",
 474        "col",
 475        "start",
 476        "end",
 477        "comments",
 478    )
 479    __slots__ = _attrs
 480
 481    @classmethod
 482    def number(cls, number: int) -> Token:
 483        """Returns a NUMBER token with `number` as its text."""
 484        return cls(TokenType.NUMBER, str(number))
 485
 486    @classmethod
 487    def string(cls, string: str) -> Token:
 488        """Returns a STRING token with `string` as its text."""
 489        return cls(TokenType.STRING, string)
 490
 491    @classmethod
 492    def identifier(cls, identifier: str) -> Token:
 493        """Returns an IDENTIFIER token with `identifier` as its text."""
 494        return cls(TokenType.IDENTIFIER, identifier)
 495
 496    @classmethod
 497    def var(cls, var: str) -> Token:
 498        """Returns an VAR token with `var` as its text."""
 499        return cls(TokenType.VAR, var)
 500
 501    def __init__(
 502        self,
 503        token_type: TokenType,
 504        text: str,
 505        line: int = 1,
 506        col: int = 1,
 507        start: int = 0,
 508        end: int = 0,
 509        comments: list[str] | None = None,
 510    ) -> None:
 511        self.token_type = token_type
 512        self.text = text
 513        self.line = line
 514        self.col = col
 515        self.start = start
 516        self.end = end
 517        self.comments = [] if comments is None else comments
 518
 519    def __bool__(self) -> bool:
 520        return self.token_type != TokenType.SENTINEL
 521
 522    def __repr__(self) -> str:
 523        attributes = ", ".join(
 524            f"{k}: TokenType.{self.token_type.name}"
 525            if k == "token_type"
 526            else f"{k}: {getattr(self, k)}"
 527            for k in self._attrs
 528        )
 529        return f"<Token {attributes}>"
 530
 531
 532class TokenizerCore:
 533    __slots__ = (
 534        "sql",
 535        "size",
 536        "tokens",
 537        "_start",
 538        "_current",
 539        "_line",
 540        "_col",
 541        "_comments",
 542        "_char",
 543        "_end",
 544        "_peek",
 545        "_prev_token_line",
 546        "single_tokens",
 547        "keywords",
 548        "quotes",
 549        "format_strings",
 550        "identifiers",
 551        "comments",
 552        "string_escapes",
 553        "byte_string_escapes",
 554        "identifier_escapes",
 555        "escape_follow_chars",
 556        "commands",
 557        "command_prefix_tokens",
 558        "nested_comments",
 559        "hint_start",
 560        "tokens_preceding_hint",
 561        "bit_strings",
 562        "hex_strings",
 563        "numeric_literals",
 564        "var_single_tokens",
 565        "string_escapes_allowed_in_raw_strings",
 566        "heredoc_tag_is_identifier",
 567        "heredoc_string_alternative",
 568        "keyword_trie",
 569        "numbers_can_be_underscore_separated",
 570        "numbers_can_have_decimals",
 571        "identifiers_can_start_with_digit",
 572        "unescaped_sequences",
 573    )
 574
 575    def __init__(
 576        self,
 577        single_tokens: dict[str, TokenType],
 578        keywords: dict[str, TokenType],
 579        quotes: dict[str, str],
 580        format_strings: dict[str, tuple[str, TokenType]],
 581        identifiers: dict[str, str],
 582        comments: dict[str, str | None],
 583        string_escapes: set[str],
 584        byte_string_escapes: set[str],
 585        identifier_escapes: set[str],
 586        escape_follow_chars: set[str],
 587        commands: set[TokenType],
 588        command_prefix_tokens: set[TokenType],
 589        nested_comments: bool,
 590        hint_start: str,
 591        tokens_preceding_hint: set[TokenType],
 592        bit_strings: list[str | tuple[str, str]],
 593        hex_strings: list[str | tuple[str, str]],
 594        numeric_literals: dict[str, str],
 595        var_single_tokens: set[str],
 596        string_escapes_allowed_in_raw_strings: bool,
 597        heredoc_tag_is_identifier: bool,
 598        heredoc_string_alternative: TokenType,
 599        keyword_trie: dict,
 600        numbers_can_be_underscore_separated: bool,
 601        numbers_can_have_decimals: bool,
 602        identifiers_can_start_with_digit: bool,
 603        unescaped_sequences: dict[str, str],
 604    ) -> None:
 605        self.single_tokens = single_tokens
 606        self.keywords = keywords
 607        self.quotes = quotes
 608        self.format_strings = format_strings
 609        self.identifiers = identifiers
 610        self.comments = comments
 611        self.string_escapes = string_escapes
 612        self.byte_string_escapes = byte_string_escapes
 613        self.identifier_escapes = identifier_escapes
 614        self.escape_follow_chars = escape_follow_chars
 615        self.commands = commands
 616        self.command_prefix_tokens = command_prefix_tokens
 617        self.nested_comments = nested_comments
 618        self.hint_start = hint_start
 619        self.tokens_preceding_hint = tokens_preceding_hint
 620        self.bit_strings = bit_strings
 621        self.hex_strings = hex_strings
 622        self.numeric_literals = numeric_literals
 623        self.var_single_tokens = var_single_tokens
 624        self.string_escapes_allowed_in_raw_strings = string_escapes_allowed_in_raw_strings
 625        self.heredoc_tag_is_identifier = heredoc_tag_is_identifier
 626        self.heredoc_string_alternative = heredoc_string_alternative
 627        self.keyword_trie = keyword_trie
 628        self.numbers_can_be_underscore_separated = numbers_can_be_underscore_separated
 629        self.numbers_can_have_decimals = numbers_can_have_decimals
 630        self.identifiers_can_start_with_digit = identifiers_can_start_with_digit
 631        self.unescaped_sequences = unescaped_sequences
 632        self.sql = ""
 633        self.size = 0
 634        self.tokens: list[Token] = []
 635        self._start = 0
 636        self._current = 0
 637        self._line = 1
 638        self._col = 0
 639        self._comments: list[str] = []
 640        self._char = ""
 641        self._end = False
 642        self._peek = ""
 643        self._prev_token_line = -1
 644
 645    def reset(self) -> None:
 646        self.sql = ""
 647        self.size = 0
 648        self.tokens = []
 649        self._start = 0
 650        self._current = 0
 651        self._line = 1
 652        self._col = 0
 653        self._comments = []
 654        self._char = ""
 655        self._end = False
 656        self._peek = ""
 657        self._prev_token_line = -1
 658
 659    def tokenize(self, sql: str) -> list[Token]:
 660        """Returns a list of tokens corresponding to the SQL string `sql`."""
 661        self.reset()
 662        self.sql = sql
 663        self.size = len(sql)
 664
 665        try:
 666            self._scan()
 667        except Exception as e:
 668            start = max(self._current - 50, 0)
 669            end = min(self._current + 50, self.size - 1)
 670            context = self.sql[start:end]
 671            raise TokenError(f"Error tokenizing '{context}'") from e
 672
 673        return self.tokens
 674
 675    def _scan(self, check_semicolon: bool = False) -> None:
 676        identifiers = self.identifiers
 677        digit_chars = _DIGIT_CHARS
 678
 679        while self.size and not self._end:
 680            current = self._current
 681
 682            # Skip spaces here rather than iteratively calling advance() for performance reasons
 683            while current < self.size:
 684                char = self.sql[current]
 685
 686                if char == " " or char == "\t":
 687                    current += 1
 688                else:
 689                    break
 690
 691            offset = current - self._current if current > self._current else 1
 692
 693            self._start = current
 694            self._advance(offset)
 695
 696            if not self._char.isspace():
 697                if self._char in digit_chars:
 698                    self._scan_number()
 699                elif self._char in identifiers:
 700                    self._scan_identifier(identifiers[self._char])
 701                else:
 702                    self._scan_keywords()
 703
 704            if check_semicolon and self._peek == ";":
 705                break
 706
 707        if self.tokens and self._comments:
 708            self.tokens[-1].comments.extend(self._comments)
 709
 710    def _chars(self, size: int) -> str:
 711        if size == 1:
 712            return self._char
 713
 714        start = self._current - 1
 715        end = start + size
 716
 717        return self.sql[start:end] if end <= self.size else ""
 718
 719    def _advance(self, i: int = 1, alnum: bool = False) -> None:
 720        char = self._char
 721
 722        if char == "\n" or char == "\r":
 723            # Ensures we don't count an extra line if we get a \r\n line break sequence
 724            if not (char == "\r" and self._peek == "\n"):
 725                self._col = i
 726                self._line += 1
 727        else:
 728            self._col += i
 729
 730        self._current += i
 731        sql = self.sql
 732        size = self.size
 733        self._end = self._current >= size
 734        self._char = sql[self._current - 1]
 735        self._peek = "" if self._end else sql[self._current]
 736
 737        if alnum and self._char.isalnum():
 738            # Cache to local variables instead of attributes for better performance
 739            _col = self._col
 740            _current = self._current
 741            _end = self._end
 742            _peek = self._peek
 743
 744            while _peek.isalnum():
 745                _col += 1
 746                _current += 1
 747                _end = _current >= size
 748                _peek = "" if _end else sql[_current]
 749
 750            self._col = _col
 751            self._current = _current
 752            self._end = _end
 753            self._peek = _peek
 754            self._char = sql[_current - 1]
 755
 756    @property
 757    def _text(self) -> str:
 758        return self.sql[self._start : self._current]
 759
 760    def _add(self, token_type: TokenType, text: str | None = None) -> None:
 761        self._prev_token_line = self._line
 762
 763        if self._comments and token_type == TokenType.SEMICOLON and self.tokens:
 764            self.tokens[-1].comments.extend(self._comments)
 765            self._comments = []
 766
 767        if text is None:
 768            text = self.sql[self._start : self._current]
 769
 770        self.tokens.append(
 771            Token(
 772                token_type,
 773                text=text,
 774                line=self._line,
 775                col=self._col,
 776                start=self._start,
 777                end=self._current - 1,
 778                comments=self._comments,
 779            )
 780        )
 781        self._comments = []
 782
 783        # If we have either a semicolon or a begin token before the command's token, we'll parse
 784        # whatever follows the command's token as a string
 785        if (
 786            token_type in self.commands
 787            and self._peek != ";"
 788            and (len(self.tokens) == 1 or self.tokens[-2].token_type in self.command_prefix_tokens)
 789        ):
 790            start = self._current
 791            tokens = len(self.tokens)
 792            self._scan(check_semicolon=True)
 793            self.tokens = self.tokens[:tokens]
 794            text = self.sql[start : self._current].strip()
 795            if text:
 796                self._add(TokenType.STRING, text)
 797
 798    def _scan_keywords(self) -> None:
 799        sql = self.sql
 800        sql_size = self.size
 801        single_tokens = self.single_tokens
 802        char_upper = _CHAR_UPPER
 803        size = 0
 804        word = None
 805        chars = self._char
 806        char = chars
 807        prev_space = False
 808        skip = False
 809        trie = self.keyword_trie
 810        single_token = char in single_tokens
 811
 812        while chars:
 813            if not skip:
 814                sub = trie.get(char_upper.get(char, char))
 815                if sub is None:
 816                    break
 817                trie = sub
 818                if 0 in trie:
 819                    word = chars
 820
 821            end = self._current + size
 822            size += 1
 823
 824            if end < sql_size:
 825                char = sql[end]
 826                single_token = single_token or char in single_tokens
 827                is_space = char.isspace()
 828
 829                if not is_space or not prev_space:
 830                    if is_space:
 831                        char = " "
 832                    chars += char
 833                    prev_space = is_space
 834                    skip = False
 835                else:
 836                    skip = True
 837            else:
 838                char = ""
 839                break
 840
 841        if word:
 842            if self._scan_string(word):
 843                return
 844            if self._scan_comment(word):
 845                return
 846            if prev_space or single_token or not char:
 847                self._advance(size - 1)
 848                word = word.upper()
 849                self._add(self.keywords[word], text=word)
 850                return
 851
 852        if self._char in single_tokens:
 853            self._add(single_tokens[self._char], text=self._char)
 854            return
 855
 856        self._scan_var()
 857
 858    def _scan_comment(self, comment_start: str) -> bool:
 859        if comment_start not in self.comments:
 860            return False
 861
 862        comment_start_line = self._line
 863        comment_start_size = len(comment_start)
 864        comment_end = self.comments[comment_start]
 865
 866        if comment_end:
 867            # Skip the comment's start delimiter
 868            self._advance(comment_start_size)
 869
 870            comment_count = 1
 871            comment_end_size = len(comment_end)
 872            nested_comments = self.nested_comments
 873
 874            while not self._end:
 875                if self._chars(comment_end_size) == comment_end:
 876                    comment_count -= 1
 877                    if not comment_count:
 878                        break
 879
 880                self._advance(alnum=True)
 881
 882                # Nested comments are allowed by some dialects, e.g. databricks, duckdb, postgres
 883                if (
 884                    nested_comments
 885                    and not self._end
 886                    and self._chars(comment_end_size) == comment_start
 887                ):
 888                    self._advance(comment_start_size)
 889                    comment_count += 1
 890
 891            self._comments.append(self._text[comment_start_size : -comment_end_size + 1])
 892            self._advance(comment_end_size - 1)
 893        else:
 894            _peek = self._peek
 895            while not self._end and _peek != "\n" and _peek != "\r":
 896                self._advance(alnum=True)
 897                _peek = self._peek
 898            self._comments.append(self._text[comment_start_size:])
 899
 900        if (
 901            comment_start == self.hint_start
 902            and self.tokens
 903            and self.tokens[-1].token_type in self.tokens_preceding_hint
 904        ):
 905            self._add(TokenType.HINT)
 906
 907        # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.
 908        # Multiple consecutive comments are preserved by appending them to the current comments list.
 909        if comment_start_line == self._prev_token_line:
 910            self.tokens[-1].comments.extend(self._comments)
 911            self._comments = []
 912            self._prev_token_line = self._line
 913
 914        return True
 915
 916    def _scan_number(self) -> None:
 917        if self._char == "0":
 918            peek = _CHAR_UPPER.get(self._peek, self._peek)
 919            if peek == "B":
 920                return self._scan_bits() if self.bit_strings else self._add(TokenType.NUMBER)
 921            elif peek == "X":
 922                return self._scan_hex() if self.hex_strings else self._add(TokenType.NUMBER)
 923
 924        decimal = False
 925        scientific = 0
 926        numbers_can_be_underscore_separated = self.numbers_can_be_underscore_separated
 927        single_tokens = self.single_tokens
 928        keywords = self.keywords
 929        numeric_literals = self.numeric_literals
 930        identifiers_can_start_with_digit = self.identifiers_can_start_with_digit
 931
 932        is_underscore_separated: bool = False
 933        number_text: str = ""
 934        numeric_literal: str = ""
 935        numeric_type: TokenType | None = None
 936
 937        while True:
 938            if self._peek in _DIGIT_CHARS:
 939                # Batch consecutive digits: scan ahead to find how many
 940                sql = self.sql
 941                end = self._current + 1
 942                size = self.size
 943                while end < size and sql[end] in _DIGIT_CHARS:
 944                    end += 1
 945                self._advance(end - self._current)
 946            elif self._peek == "." and not decimal:
 947                if (
 948                    self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
 949                ) or not self.numbers_can_have_decimals:
 950                    break
 951                decimal = True
 952                self._advance()
 953            elif self._peek in ("-", "+") and scientific == 1:
 954                # Only consume +/- if followed by a digit
 955                if self._current + 1 < self.size and self.sql[self._current + 1] in _DIGIT_CHARS:
 956                    scientific += 1
 957                    self._advance()
 958                else:
 959                    break
 960            elif _CHAR_UPPER.get(self._peek, self._peek) == "E" and not scientific:
 961                scientific += 1
 962                self._advance()
 963            elif self._peek == "_" and numbers_can_be_underscore_separated:
 964                is_underscore_separated = True
 965                self._advance()
 966            elif self._peek.isidentifier():
 967                number_text = self._text
 968
 969                while self._peek and not self._peek.isspace() and self._peek not in single_tokens:
 970                    numeric_literal += self._peek
 971                    self._advance()
 972
 973                numeric_type = keywords.get(numeric_literals.get(numeric_literal.upper(), ""))
 974
 975                if numeric_type:
 976                    break
 977                elif identifiers_can_start_with_digit:
 978                    return self._add(TokenType.VAR)
 979
 980                self._advance(-len(numeric_literal))
 981                break
 982            else:
 983                break
 984
 985        number_text = number_text or self.sql[self._start : self._current]
 986
 987        # Normalize inputs such as 100_000 to 100000
 988        if is_underscore_separated:
 989            number_text = number_text.replace("_", "")
 990
 991        self._add(TokenType.NUMBER, number_text)
 992
 993        # Normalize inputs such as 123L to 123::BIGINT so that they're parsed as casts
 994        if numeric_type:
 995            self._add(TokenType.DCOLON, "::")
 996            self._add(numeric_type, numeric_literal)
 997
 998    def _scan_bits(self) -> None:
 999        self._advance()
1000        value = self._extract_value()
1001        try:
1002            # If `value` can't be converted to a binary, fallback to tokenizing it as an identifier
1003            int(value, 2)
1004            self._add(TokenType.BIT_STRING, value[2:])  # Drop the 0b
1005        except ValueError:
1006            self._add(TokenType.IDENTIFIER)
1007
1008    def _scan_hex(self) -> None:
1009        self._advance()
1010        value = self._extract_value()
1011        try:
1012            # If `value` can't be converted to a hex, fallback to tokenizing it as an identifier
1013            int(value, 16)
1014            self._add(TokenType.HEX_STRING, value[2:])  # Drop the 0x
1015        except ValueError:
1016            self._add(TokenType.IDENTIFIER)
1017
1018    def _extract_value(self) -> str:
1019        single_tokens = self.single_tokens
1020
1021        while True:
1022            char = self._peek.strip()
1023            if char and char not in single_tokens:
1024                self._advance(alnum=True)
1025            else:
1026                break
1027
1028        return self._text
1029
1030    def _scan_string(self, start: str) -> bool:
1031        base = None
1032        token_type = TokenType.STRING
1033
1034        if start in self.quotes:
1035            end = self.quotes[start]
1036        elif start in self.format_strings:
1037            end, token_type = self.format_strings[start]
1038
1039            if token_type == TokenType.HEX_STRING:
1040                base = 16
1041            elif token_type == TokenType.BIT_STRING:
1042                base = 2
1043            elif token_type == TokenType.HEREDOC_STRING:
1044                self._advance()
1045
1046                if self._char == end:
1047                    tag = ""
1048                else:
1049                    tag = self._extract_string(
1050                        end,
1051                        raw_string=True,
1052                        raise_unmatched=not self.heredoc_tag_is_identifier,
1053                    )
1054
1055                if (
1056                    tag
1057                    and self.heredoc_tag_is_identifier
1058                    and (self._end or tag.isdigit() or any(c.isspace() for c in tag))
1059                ):
1060                    if not self._end:
1061                        self._advance(-1)
1062
1063                    self._advance(-len(tag))
1064                    self._add(self.heredoc_string_alternative)
1065                    return True
1066
1067                end = f"{start}{tag}{end}"
1068        else:
1069            return False
1070
1071        self._advance(len(start))
1072        text = self._extract_string(
1073            end,
1074            escapes=(
1075                self.byte_string_escapes
1076                if token_type == TokenType.BYTE_STRING
1077                else self.string_escapes
1078            ),
1079            raw_string=token_type == TokenType.RAW_STRING,
1080        )
1081
1082        if base and text:
1083            try:
1084                int(text, base)
1085            except Exception:
1086                raise TokenError(
1087                    f"Numeric string contains invalid characters from {self._line}:{self._start}"
1088                )
1089
1090        self._add(token_type, text)
1091        return True
1092
1093    def _scan_identifier(self, identifier_end: str) -> None:
1094        self._advance()
1095        text = self._extract_string(
1096            identifier_end, escapes=self.identifier_escapes | {identifier_end}
1097        )
1098        self._add(TokenType.IDENTIFIER, text)
1099
1100    def _scan_var(self) -> None:
1101        var_single_tokens = self.var_single_tokens
1102        single_tokens = self.single_tokens
1103
1104        while True:
1105            peek = self._peek
1106            if not peek or peek.isspace():
1107                break
1108            if peek not in var_single_tokens and peek in single_tokens:
1109                break
1110            self._advance(alnum=True)
1111
1112        self._add(
1113            TokenType.VAR
1114            if self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
1115            else self.keywords.get(self.sql[self._start : self._current].upper(), TokenType.VAR)
1116        )
1117
1118    def _extract_string(
1119        self,
1120        delimiter: str,
1121        escapes: set[str] | None = None,
1122        raw_string: bool = False,
1123        raise_unmatched: bool = True,
1124    ) -> str:
1125        text = ""
1126        delim_size = len(delimiter)
1127        escapes = self.string_escapes if escapes is None else escapes
1128        unescaped_sequences = self.unescaped_sequences
1129        escape_follow_chars = self.escape_follow_chars
1130        string_escapes_allowed_in_raw_strings = self.string_escapes_allowed_in_raw_strings
1131        quotes = self.quotes
1132        sql = self.sql
1133
1134        # use str.find() when the string is simple... no \ or other escapes
1135        if delim_size == 1:
1136            pos = self._current - 1
1137            end = sql.find(delimiter, pos)
1138
1139            if (
1140                # the closing delimiter was found
1141                end != -1
1142                # there's no doubled delimiter (e.g. '' escape), or the delimiter isn't an escape char
1143                and (end + 1 >= self.size or sql[end + 1] != delimiter or delimiter not in escapes)
1144                # no backslash in the string that would need escape processing
1145                and (not (unescaped_sequences or "\\" in escapes) or sql.find("\\", pos, end) == -1)
1146            ):
1147                newlines = sql.count("\n", pos, end)
1148                if newlines:
1149                    self._line += newlines
1150                    self._col = end - sql.rfind("\n", pos, end)
1151                else:
1152                    self._col += end - pos
1153
1154                self._current = end + 1
1155                self._end = self._current >= self.size
1156                self._char = sql[end]
1157                self._peek = "" if self._end else sql[self._current]
1158                return sql[pos:end]
1159
1160        while True:
1161            if not raw_string and unescaped_sequences and self._peek and self._char in escapes:
1162                unescaped_sequence = unescaped_sequences.get(self._char + self._peek)
1163                if unescaped_sequence:
1164                    self._advance(2)
1165                    text += unescaped_sequence
1166                    continue
1167
1168            is_valid_custom_escape = (
1169                escape_follow_chars and self._char == "\\" and self._peek not in escape_follow_chars
1170            )
1171
1172            if (
1173                (string_escapes_allowed_in_raw_strings or not raw_string)
1174                and self._char in escapes
1175                and (self._peek == delimiter or self._peek in escapes or is_valid_custom_escape)
1176                and (self._char not in quotes or self._char == self._peek)
1177            ):
1178                if self._peek == delimiter:
1179                    text += self._peek
1180                elif is_valid_custom_escape and self._char != self._peek:
1181                    text += self._peek
1182                else:
1183                    text += self._char + self._peek
1184
1185                if self._current + 1 < self.size:
1186                    self._advance(2)
1187                else:
1188                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._current}")
1189            else:
1190                if self._chars(delim_size) == delimiter:
1191                    if delim_size > 1:
1192                        self._advance(delim_size - 1)
1193                    break
1194
1195                if self._end:
1196                    if not raise_unmatched:
1197                        return text + self._char
1198
1199                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._start}")
1200
1201                current = self._current - 1
1202                self._advance(alnum=True)
1203                text += sql[current : self._current - 1]
1204
1205        return text