sqlglot.tokenizer_core

   1from __future__ import annotations
   2
   3import typing as t
   4from enum import IntEnum, auto
   5
   6from sqlglot.errors import TokenError
   7
   8# dict lookup is faster than .upper() and .isdigit()
   9_CHAR_UPPER: dict[str, str] = {chr(i): chr(i).upper() for i in range(97, 123)}
  10_DIGIT_CHARS: frozenset[str] = frozenset("0123456789")
  11
  12
  13class TokenType(IntEnum):
  14    L_PAREN = auto()
  15    R_PAREN = auto()
  16    L_BRACKET = auto()
  17    R_BRACKET = auto()
  18    L_BRACE = auto()
  19    R_BRACE = auto()
  20    COMMA = auto()
  21    DOT = auto()
  22    DASH = auto()
  23    PLUS = auto()
  24    COLON = auto()
  25    DOTCOLON = auto()
  26    DOTCARET = auto()
  27    DCOLON = auto()
  28    DCOLONDOLLAR = auto()
  29    DCOLONPERCENT = auto()
  30    DCOLONQMARK = auto()
  31    DQMARK = auto()
  32    SEMICOLON = auto()
  33    STAR = auto()
  34    BACKSLASH = auto()
  35    SLASH = auto()
  36    LT = auto()
  37    LTE = auto()
  38    GT = auto()
  39    GTE = auto()
  40    NOT = auto()
  41    EQ = auto()
  42    NEQ = auto()
  43    NULLSAFE_EQ = auto()
  44    COLON_EQ = auto()
  45    COLON_GT = auto()
  46    NCOLON_GT = auto()
  47    AND = auto()
  48    OR = auto()
  49    AMP = auto()
  50    DPIPE = auto()
  51    PIPE_GT = auto()
  52    PIPE = auto()
  53    PIPE_SLASH = auto()
  54    DPIPE_SLASH = auto()
  55    CARET = auto()
  56    CARET_AT = auto()
  57    TILDE = auto()
  58    ARROW = auto()
  59    DARROW = auto()
  60    FARROW = auto()
  61    HASH = auto()
  62    HASH_ARROW = auto()
  63    DHASH_ARROW = auto()
  64    LR_ARROW = auto()
  65    LLRR_ARROW = auto()
  66    DAT = auto()
  67    LT_AT = auto()
  68    AT_GT = auto()
  69    DOLLAR = auto()
  70    PARAMETER = auto()
  71    SESSION = auto()
  72    SESSION_PARAMETER = auto()
  73    SESSION_USER = auto()
  74    DAMP = auto()
  75    AMP_LT = auto()
  76    AMP_GT = auto()
  77    ADJACENT = auto()
  78    XOR = auto()
  79    DSTAR = auto()
  80    QMARK_AMP = auto()
  81    QMARK_PIPE = auto()
  82    HASH_DASH = auto()
  83    EXCLAMATION = auto()
  84
  85    URI_START = auto()
  86
  87    BLOCK_START = auto()
  88    BLOCK_END = auto()
  89
  90    SPACE = auto()
  91    BREAK = auto()
  92
  93    STRING = auto()
  94    NUMBER = auto()
  95    IDENTIFIER = auto()
  96    DATABASE = auto()
  97    COLUMN = auto()
  98    COLUMN_DEF = auto()
  99    SCHEMA = auto()
 100    TABLE = auto()
 101    WAREHOUSE = auto()
 102    STAGE = auto()
 103    STREAM = auto()
 104    STREAMLIT = auto()
 105    VAR = auto()
 106    BIT_STRING = auto()
 107    HEX_STRING = auto()
 108    BYTE_STRING = auto()
 109    NATIONAL_STRING = auto()
 110    RAW_STRING = auto()
 111    HEREDOC_STRING = auto()
 112    UNICODE_STRING = auto()
 113
 114    # types
 115    BIT = auto()
 116    BOOLEAN = auto()
 117    TINYINT = auto()
 118    UTINYINT = auto()
 119    SMALLINT = auto()
 120    USMALLINT = auto()
 121    MEDIUMINT = auto()
 122    UMEDIUMINT = auto()
 123    INT = auto()
 124    UINT = auto()
 125    BIGINT = auto()
 126    UBIGINT = auto()
 127    BIGNUM = auto()
 128    INT128 = auto()
 129    UINT128 = auto()
 130    INT256 = auto()
 131    UINT256 = auto()
 132    FLOAT = auto()
 133    DOUBLE = auto()
 134    UDOUBLE = auto()
 135    DECIMAL = auto()
 136    DECIMAL32 = auto()
 137    DECIMAL64 = auto()
 138    DECIMAL128 = auto()
 139    DECIMAL256 = auto()
 140    DECFLOAT = auto()
 141    UDECIMAL = auto()
 142    BIGDECIMAL = auto()
 143    CHAR = auto()
 144    NCHAR = auto()
 145    VARCHAR = auto()
 146    NVARCHAR = auto()
 147    BPCHAR = auto()
 148    TEXT = auto()
 149    MEDIUMTEXT = auto()
 150    LONGTEXT = auto()
 151    BLOB = auto()
 152    MEDIUMBLOB = auto()
 153    LONGBLOB = auto()
 154    TINYBLOB = auto()
 155    TINYTEXT = auto()
 156    NAME = auto()
 157    BINARY = auto()
 158    VARBINARY = auto()
 159    JSON = auto()
 160    JSONB = auto()
 161    TIME = auto()
 162    TIMETZ = auto()
 163    TIME_NS = auto()
 164    TIMESTAMP = auto()
 165    TIMESTAMPTZ = auto()
 166    TIMESTAMPLTZ = auto()
 167    TIMESTAMPNTZ = auto()
 168    TIMESTAMP_S = auto()
 169    TIMESTAMP_MS = auto()
 170    TIMESTAMP_NS = auto()
 171    DATETIME = auto()
 172    DATETIME2 = auto()
 173    DATETIME64 = auto()
 174    SMALLDATETIME = auto()
 175    DATE = auto()
 176    DATE32 = auto()
 177    INT4RANGE = auto()
 178    INT4MULTIRANGE = auto()
 179    INT8RANGE = auto()
 180    INT8MULTIRANGE = auto()
 181    NUMRANGE = auto()
 182    NUMMULTIRANGE = auto()
 183    TSRANGE = auto()
 184    TSMULTIRANGE = auto()
 185    TSTZRANGE = auto()
 186    TSTZMULTIRANGE = auto()
 187    DATERANGE = auto()
 188    DATEMULTIRANGE = auto()
 189    UUID = auto()
 190    GEOGRAPHY = auto()
 191    GEOGRAPHYPOINT = auto()
 192    NULLABLE = auto()
 193    GEOMETRY = auto()
 194    POINT = auto()
 195    RING = auto()
 196    LINESTRING = auto()
 197    LOCALTIME = auto()
 198    LOCALTIMESTAMP = auto()
 199    SYSTIMESTAMP = auto()
 200    MULTILINESTRING = auto()
 201    POLYGON = auto()
 202    MULTIPOLYGON = auto()
 203    HLLSKETCH = auto()
 204    HSTORE = auto()
 205    SUPER = auto()
 206    SERIAL = auto()
 207    SMALLSERIAL = auto()
 208    BIGSERIAL = auto()
 209    XML = auto()
 210    YEAR = auto()
 211    USERDEFINED = auto()
 212    MONEY = auto()
 213    SMALLMONEY = auto()
 214    ROWVERSION = auto()
 215    IMAGE = auto()
 216    VARIANT = auto()
 217    OBJECT = auto()
 218    INET = auto()
 219    IPADDRESS = auto()
 220    IPPREFIX = auto()
 221    IPV4 = auto()
 222    IPV6 = auto()
 223    ENUM = auto()
 224    ENUM8 = auto()
 225    ENUM16 = auto()
 226    FIXEDSTRING = auto()
 227    LOWCARDINALITY = auto()
 228    NESTED = auto()
 229    AGGREGATEFUNCTION = auto()
 230    SIMPLEAGGREGATEFUNCTION = auto()
 231    TDIGEST = auto()
 232    UNKNOWN = auto()
 233    VECTOR = auto()
 234    DYNAMIC = auto()
 235    VOID = auto()
 236
 237    # keywords
 238    ALIAS = auto()
 239    ALTER = auto()
 240    ALL = auto()
 241    ANTI = auto()
 242    ANY = auto()
 243    APPLY = auto()
 244    ARRAY = auto()
 245    ASC = auto()
 246    ASOF = auto()
 247    ATTACH = auto()
 248    AUTO_INCREMENT = auto()
 249    BEGIN = auto()
 250    BETWEEN = auto()
 251    BULK_COLLECT_INTO = auto()
 252    CACHE = auto()
 253    CASE = auto()
 254    CHARACTER_SET = auto()
 255    CLUSTER_BY = auto()
 256    COLLATE = auto()
 257    COMMAND = auto()
 258    COMMENT = auto()
 259    COMMIT = auto()
 260    CONNECT_BY = auto()
 261    CONSTRAINT = auto()
 262    COPY = auto()
 263    CREATE = auto()
 264    CROSS = auto()
 265    CUBE = auto()
 266    CURRENT_DATE = auto()
 267    CURRENT_DATETIME = auto()
 268    CURRENT_SCHEMA = auto()
 269    CURRENT_TIME = auto()
 270    CURRENT_TIMESTAMP = auto()
 271    CURRENT_USER = auto()
 272    CURRENT_USER_ID = auto()
 273    CURRENT_ROLE = auto()
 274    CURRENT_CATALOG = auto()
 275    DECLARE = auto()
 276    DEFAULT = auto()
 277    DELETE = auto()
 278    DESC = auto()
 279    DESCRIBE = auto()
 280    DETACH = auto()
 281    DICTIONARY = auto()
 282    DISTINCT = auto()
 283    DISTRIBUTE_BY = auto()
 284    DIV = auto()
 285    DROP = auto()
 286    ELSE = auto()
 287    END = auto()
 288    ESCAPE = auto()
 289    EXCEPT = auto()
 290    EXECUTE = auto()
 291    EXISTS = auto()
 292    FALSE = auto()
 293    FETCH = auto()
 294    FILE = auto()
 295    FILE_FORMAT = auto()
 296    FILTER = auto()
 297    FINAL = auto()
 298    FIRST = auto()
 299    FOR = auto()
 300    FORCE = auto()
 301    FOREIGN_KEY = auto()
 302    FORMAT = auto()
 303    FROM = auto()
 304    FULL = auto()
 305    FUNCTION = auto()
 306    GET = auto()
 307    GLOB = auto()
 308    GLOBAL = auto()
 309    GRANT = auto()
 310    GROUP_BY = auto()
 311    GROUPING_SETS = auto()
 312    HAVING = auto()
 313    HINT = auto()
 314    IGNORE = auto()
 315    ILIKE = auto()
 316    IN = auto()
 317    INDEX = auto()
 318    INDEXED_BY = auto()
 319    INNER = auto()
 320    INSERT = auto()
 321    INSTALL = auto()
 322    INTEGRATION = auto()
 323    INTERSECT = auto()
 324    INTERVAL = auto()
 325    INTO = auto()
 326    INTRODUCER = auto()
 327    IRLIKE = auto()
 328    IS = auto()
 329    ISNULL = auto()
 330    JOIN = auto()
 331    JOIN_MARKER = auto()
 332    KEEP = auto()
 333    KEY = auto()
 334    KILL = auto()
 335    LANGUAGE = auto()
 336    LATERAL = auto()
 337    LEFT = auto()
 338    LIKE = auto()
 339    LIMIT = auto()
 340    LIST = auto()
 341    LOAD = auto()
 342    LOCK = auto()
 343    MAP = auto()
 344    MATCH = auto()
 345    MATCH_CONDITION = auto()
 346    MATCH_RECOGNIZE = auto()
 347    MEMBER_OF = auto()
 348    MERGE = auto()
 349    MOD = auto()
 350    MODEL = auto()
 351    NATURAL = auto()
 352    NEXT = auto()
 353    NOTHING = auto()
 354    NOTNULL = auto()
 355    NULL = auto()
 356    OBJECT_IDENTIFIER = auto()
 357    OFFSET = auto()
 358    ON = auto()
 359    ONLY = auto()
 360    OPERATOR = auto()
 361    ORDER_BY = auto()
 362    ORDER_SIBLINGS_BY = auto()
 363    ORDERED = auto()
 364    ORDINALITY = auto()
 365    OUT = auto()
 366    INOUT = auto()
 367    OUTER = auto()
 368    OVER = auto()
 369    OVERLAPS = auto()
 370    OVERWRITE = auto()
 371    PACKAGE = auto()
 372    PARTITION = auto()
 373    PARTITION_BY = auto()
 374    PERCENT = auto()
 375    PIVOT = auto()
 376    PLACEHOLDER = auto()
 377    POLICY = auto()
 378    POOL = auto()
 379    POSITIONAL = auto()
 380    PRAGMA = auto()
 381    PREWHERE = auto()
 382    PRIMARY_KEY = auto()
 383    PROCEDURE = auto()
 384    PROPERTIES = auto()
 385    PSEUDO_TYPE = auto()
 386    PUT = auto()
 387    QUALIFY = auto()
 388    QUOTE = auto()
 389    QDCOLON = auto()
 390    RANGE = auto()
 391    RECURSIVE = auto()
 392    REFRESH = auto()
 393    RENAME = auto()
 394    REPLACE = auto()
 395    RETURNING = auto()
 396    REVOKE = auto()
 397    REFERENCES = auto()
 398    RIGHT = auto()
 399    RLIKE = auto()
 400    ROLE = auto()
 401    ROLLBACK = auto()
 402    ROLLUP = auto()
 403    ROW = auto()
 404    ROWS = auto()
 405    RULE = auto()
 406    SELECT = auto()
 407    SEMI = auto()
 408    SEPARATOR = auto()
 409    SEQUENCE = auto()
 410    SERDE_PROPERTIES = auto()
 411    SET = auto()
 412    SETTINGS = auto()
 413    SHOW = auto()
 414    SIMILAR_TO = auto()
 415    SOME = auto()
 416    SORT_BY = auto()
 417    SOUNDS_LIKE = auto()
 418    SQL_SECURITY = auto()
 419    START_WITH = auto()
 420    STORAGE_INTEGRATION = auto()
 421    STRAIGHT_JOIN = auto()
 422    STRUCT = auto()
 423    SUMMARIZE = auto()
 424    TABLE_SAMPLE = auto()
 425    TAG = auto()
 426    TEMPORARY = auto()
 427    TOP = auto()
 428    THEN = auto()
 429    TRUE = auto()
 430    TRUNCATE = auto()
 431    TRIGGER = auto()
 432    TYPE = auto()
 433    UNCACHE = auto()
 434    UNION = auto()
 435    UNNEST = auto()
 436    UNPIVOT = auto()
 437    UPDATE = auto()
 438    USE = auto()
 439    USING = auto()
 440    VALUES = auto()
 441    VARIADIC = auto()
 442    VIEW = auto()
 443    SEMANTIC_VIEW = auto()
 444    VOLATILE = auto()
 445    VOLUME = auto()
 446    WHEN = auto()
 447    WHERE = auto()
 448    WINDOW = auto()
 449    WITH = auto()
 450    UNIQUE = auto()
 451    UTC_DATE = auto()
 452    UTC_TIME = auto()
 453    UTC_TIMESTAMP = auto()
 454    VERSION_SNAPSHOT = auto()
 455    TIMESTAMP_SNAPSHOT = auto()
 456    OPTION = auto()
 457    SINK = auto()
 458    SOURCE = auto()
 459    ANALYZE = auto()
 460    NAMESPACE = auto()
 461    EXPORT = auto()
 462
 463    # sentinels
 464    HIVE_TOKEN_STREAM = auto()
 465    SENTINEL = auto()
 466
 467    def __str__(self) -> str:
 468        return f"TokenType.{self.name}"
 469
 470
 471class Token:
 472    # mypyc doesn't expose slots
 473    _attrs: t.ClassVar[tuple[str, ...]] = (
 474        "token_type",
 475        "text",
 476        "line",
 477        "col",
 478        "start",
 479        "end",
 480        "comments",
 481    )
 482    __slots__ = _attrs
 483
 484    @classmethod
 485    def number(cls, number: int) -> Token:
 486        """Returns a NUMBER token with `number` as its text."""
 487        return cls(TokenType.NUMBER, str(number))
 488
 489    @classmethod
 490    def string(cls, string: str) -> Token:
 491        """Returns a STRING token with `string` as its text."""
 492        return cls(TokenType.STRING, string)
 493
 494    @classmethod
 495    def identifier(cls, identifier: str) -> Token:
 496        """Returns an IDENTIFIER token with `identifier` as its text."""
 497        return cls(TokenType.IDENTIFIER, identifier)
 498
 499    @classmethod
 500    def var(cls, var: str) -> Token:
 501        """Returns an VAR token with `var` as its text."""
 502        return cls(TokenType.VAR, var)
 503
 504    def __init__(
 505        self,
 506        token_type: TokenType,
 507        text: str,
 508        line: int = 1,
 509        col: int = 1,
 510        start: int = 0,
 511        end: int = 0,
 512        comments: list[str] | None = None,
 513    ) -> None:
 514        self.token_type = token_type
 515        self.text = text
 516        self.line = line
 517        self.col = col
 518        self.start = start
 519        self.end = end
 520        self.comments = [] if comments is None else comments
 521
 522    def __bool__(self) -> bool:
 523        return self.token_type != TokenType.SENTINEL
 524
 525    def __repr__(self) -> str:
 526        attributes = ", ".join(
 527            f"{k}: TokenType.{self.token_type.name}"
 528            if k == "token_type"
 529            else f"{k}: {getattr(self, k)}"
 530            for k in self._attrs
 531        )
 532        return f"<Token {attributes}>"
 533
 534
 535class TokenizerCore:
 536    __slots__ = (
 537        "sql",
 538        "size",
 539        "tokens",
 540        "_start",
 541        "_current",
 542        "_line",
 543        "_col",
 544        "_comments",
 545        "_char",
 546        "_end",
 547        "_peek",
 548        "_prev_token_line",
 549        "single_tokens",
 550        "keywords",
 551        "quotes",
 552        "format_strings",
 553        "identifiers",
 554        "comments",
 555        "string_escapes",
 556        "byte_string_escapes",
 557        "identifier_escapes",
 558        "escape_follow_chars",
 559        "commands",
 560        "command_prefix_tokens",
 561        "nested_comments",
 562        "hint_start",
 563        "tokens_preceding_hint",
 564        "has_bit_strings",
 565        "has_hex_strings",
 566        "numeric_literals",
 567        "var_single_tokens",
 568        "string_escapes_allowed_in_raw_strings",
 569        "heredoc_tag_is_identifier",
 570        "heredoc_string_alternative",
 571        "keyword_trie",
 572        "numbers_can_be_underscore_separated",
 573        "numbers_can_have_decimals",
 574        "identifiers_can_start_with_digit",
 575        "unescaped_sequences",
 576    )
 577
 578    def __init__(
 579        self,
 580        single_tokens: dict[str, TokenType],
 581        keywords: dict[str, TokenType],
 582        quotes: dict[str, str],
 583        format_strings: dict[str, tuple[str, TokenType]],
 584        identifiers: dict[str, str],
 585        comments: dict[str, str | None],
 586        string_escapes: set[str],
 587        byte_string_escapes: set[str],
 588        identifier_escapes: set[str],
 589        escape_follow_chars: set[str],
 590        commands: set[TokenType],
 591        command_prefix_tokens: set[TokenType],
 592        nested_comments: bool,
 593        hint_start: str,
 594        tokens_preceding_hint: set[TokenType],
 595        has_bit_strings: bool,
 596        has_hex_strings: bool,
 597        numeric_literals: dict[str, str],
 598        var_single_tokens: set[str],
 599        string_escapes_allowed_in_raw_strings: bool,
 600        heredoc_tag_is_identifier: bool,
 601        heredoc_string_alternative: TokenType,
 602        keyword_trie: dict,
 603        numbers_can_be_underscore_separated: bool,
 604        numbers_can_have_decimals: bool,
 605        identifiers_can_start_with_digit: bool,
 606        unescaped_sequences: dict[str, str],
 607    ) -> None:
 608        self.single_tokens = single_tokens
 609        self.keywords = keywords
 610        self.quotes = quotes
 611        self.format_strings = format_strings
 612        self.identifiers = identifiers
 613        self.comments = comments
 614        self.string_escapes = string_escapes
 615        self.byte_string_escapes = byte_string_escapes
 616        self.identifier_escapes = identifier_escapes
 617        self.escape_follow_chars = escape_follow_chars
 618        self.commands = commands
 619        self.command_prefix_tokens = command_prefix_tokens
 620        self.nested_comments = nested_comments
 621        self.hint_start = hint_start
 622        self.tokens_preceding_hint = tokens_preceding_hint
 623        self.has_bit_strings = has_bit_strings
 624        self.has_hex_strings = has_hex_strings
 625        self.numeric_literals = numeric_literals
 626        self.var_single_tokens = var_single_tokens
 627        self.string_escapes_allowed_in_raw_strings = string_escapes_allowed_in_raw_strings
 628        self.heredoc_tag_is_identifier = heredoc_tag_is_identifier
 629        self.heredoc_string_alternative = heredoc_string_alternative
 630        self.keyword_trie = keyword_trie
 631        self.numbers_can_be_underscore_separated = numbers_can_be_underscore_separated
 632        self.numbers_can_have_decimals = numbers_can_have_decimals
 633        self.identifiers_can_start_with_digit = identifiers_can_start_with_digit
 634        self.unescaped_sequences = unescaped_sequences
 635        self.sql = ""
 636        self.size = 0
 637        self.tokens: list[Token] = []
 638        self._start = 0
 639        self._current = 0
 640        self._line = 1
 641        self._col = 0
 642        self._comments: list[str] = []
 643        self._char = ""
 644        self._end = False
 645        self._peek = ""
 646        self._prev_token_line = -1
 647
 648    def reset(self) -> None:
 649        self.sql = ""
 650        self.size = 0
 651        self.tokens = []
 652        self._start = 0
 653        self._current = 0
 654        self._line = 1
 655        self._col = 0
 656        self._comments = []
 657        self._char = ""
 658        self._end = False
 659        self._peek = ""
 660        self._prev_token_line = -1
 661
 662    def tokenize(self, sql: str) -> list[Token]:
 663        """Returns a list of tokens corresponding to the SQL string `sql`."""
 664        self.reset()
 665        self.sql = sql
 666        self.size = len(sql)
 667
 668        try:
 669            self._scan()
 670        except Exception as e:
 671            start = max(self._current - 50, 0)
 672            end = min(self._current + 50, self.size - 1)
 673            context = self.sql[start:end]
 674            raise TokenError(f"Error tokenizing '{context}'") from e
 675
 676        return self.tokens
 677
 678    def _scan(self, check_semicolon: bool = False) -> None:
 679        identifiers = self.identifiers
 680        digit_chars = _DIGIT_CHARS
 681
 682        while self.size and not self._end:
 683            current = self._current
 684
 685            # Skip spaces here rather than iteratively calling advance() for performance reasons
 686            while current < self.size:
 687                char = self.sql[current]
 688
 689                if char == " " or char == "\t":
 690                    current += 1
 691                else:
 692                    break
 693
 694            offset = current - self._current if current > self._current else 1
 695
 696            self._start = current
 697            self._advance(offset)
 698
 699            if not self._char.isspace():
 700                if self._char in digit_chars:
 701                    self._scan_number()
 702                elif self._char in identifiers:
 703                    self._scan_identifier(identifiers[self._char])
 704                else:
 705                    self._scan_keywords()
 706
 707            if check_semicolon and self._peek == ";":
 708                break
 709
 710        if self.tokens and self._comments:
 711            self.tokens[-1].comments.extend(self._comments)
 712
 713    def _chars(self, size: int) -> str:
 714        if size == 1:
 715            return self._char
 716
 717        start = self._current - 1
 718        end = start + size
 719
 720        return self.sql[start:end] if end <= self.size else ""
 721
 722    def _advance(self, i: int = 1, alnum: bool = False) -> None:
 723        char = self._char
 724
 725        if char == "\n" or char == "\r":
 726            # Ensures we don't count an extra line if we get a \r\n line break sequence
 727            if not (char == "\r" and self._peek == "\n"):
 728                self._col = i
 729                self._line += 1
 730        else:
 731            self._col += i
 732
 733        self._current += i
 734        sql = self.sql
 735        size = self.size
 736        self._end = self._current >= size
 737        self._char = sql[self._current - 1]
 738        self._peek = "" if self._end else sql[self._current]
 739
 740        if alnum and self._char.isalnum():
 741            # Cache to local variables instead of attributes for better performance
 742            _col = self._col
 743            _current = self._current
 744            _end = self._end
 745            _peek = self._peek
 746
 747            while _peek.isalnum():
 748                _col += 1
 749                _current += 1
 750                _end = _current >= size
 751                _peek = "" if _end else sql[_current]
 752
 753            self._col = _col
 754            self._current = _current
 755            self._end = _end
 756            self._peek = _peek
 757            self._char = sql[_current - 1]
 758
 759    @property
 760    def _text(self) -> str:
 761        return self.sql[self._start : self._current]
 762
 763    def _add(self, token_type: TokenType, text: str | None = None) -> None:
 764        self._prev_token_line = self._line
 765
 766        if self._comments and token_type == TokenType.SEMICOLON and self.tokens:
 767            self.tokens[-1].comments.extend(self._comments)
 768            self._comments = []
 769
 770        if text is None:
 771            text = self.sql[self._start : self._current]
 772
 773        self.tokens.append(
 774            Token(
 775                token_type,
 776                text=text,
 777                line=self._line,
 778                col=self._col,
 779                start=self._start,
 780                end=self._current - 1,
 781                comments=self._comments,
 782            )
 783        )
 784        self._comments = []
 785
 786        # If we have either a semicolon or a begin token before the command's token, we'll parse
 787        # whatever follows the command's token as a string
 788        if (
 789            token_type in self.commands
 790            and self._peek != ";"
 791            and (len(self.tokens) == 1 or self.tokens[-2].token_type in self.command_prefix_tokens)
 792        ):
 793            start = self._current
 794            tokens = len(self.tokens)
 795            self._scan(check_semicolon=True)
 796            self.tokens = self.tokens[:tokens]
 797            text = self.sql[start : self._current].strip()
 798            if text:
 799                self._add(TokenType.STRING, text)
 800
 801    def _scan_keywords(self) -> None:
 802        sql = self.sql
 803        sql_size = self.size
 804        single_tokens = self.single_tokens
 805        char_upper = _CHAR_UPPER
 806        size = 0
 807        word = None
 808        chars = self._char
 809        char = chars
 810        prev_space = False
 811        skip = False
 812        trie = self.keyword_trie
 813        single_token = char in single_tokens
 814
 815        while chars:
 816            if not skip:
 817                sub = trie.get(char_upper.get(char, char))
 818                if sub is None:
 819                    break
 820                trie = sub
 821                if 0 in trie:
 822                    word = chars
 823
 824            end = self._current + size
 825            size += 1
 826
 827            if end < sql_size:
 828                char = sql[end]
 829                single_token = single_token or char in single_tokens
 830                is_space = char.isspace()
 831
 832                if not is_space or not prev_space:
 833                    if is_space:
 834                        char = " "
 835                    chars += char
 836                    prev_space = is_space
 837                    skip = False
 838                else:
 839                    skip = True
 840            else:
 841                char = ""
 842                break
 843
 844        if word:
 845            if self._scan_string(word):
 846                return
 847            if self._scan_comment(word):
 848                return
 849            if prev_space or single_token or not char:
 850                self._advance(size - 1)
 851                word = word.upper()
 852                self._add(self.keywords[word], text=word)
 853                return
 854
 855        if self._char in single_tokens:
 856            self._add(single_tokens[self._char], text=self._char)
 857            return
 858
 859        self._scan_var()
 860
 861    def _scan_comment(self, comment_start: str) -> bool:
 862        if comment_start not in self.comments:
 863            return False
 864
 865        comment_start_line = self._line
 866        comment_start_size = len(comment_start)
 867        comment_end = self.comments[comment_start]
 868
 869        if comment_end:
 870            # Skip the comment's start delimiter
 871            self._advance(comment_start_size)
 872
 873            comment_count = 1
 874            comment_end_size = len(comment_end)
 875            nested_comments = self.nested_comments
 876
 877            while not self._end:
 878                if self._chars(comment_end_size) == comment_end:
 879                    comment_count -= 1
 880                    if not comment_count:
 881                        break
 882
 883                self._advance(alnum=True)
 884
 885                # Nested comments are allowed by some dialects, e.g. databricks, duckdb, postgres
 886                if (
 887                    nested_comments
 888                    and not self._end
 889                    and self._chars(comment_end_size) == comment_start
 890                ):
 891                    self._advance(comment_start_size)
 892                    comment_count += 1
 893
 894            self._comments.append(self._text[comment_start_size : -comment_end_size + 1])
 895            self._advance(comment_end_size - 1)
 896        else:
 897            _peek = self._peek
 898            while not self._end and _peek != "\n" and _peek != "\r":
 899                self._advance(alnum=True)
 900                _peek = self._peek
 901            self._comments.append(self._text[comment_start_size:])
 902
 903        if (
 904            comment_start == self.hint_start
 905            and self.tokens
 906            and self.tokens[-1].token_type in self.tokens_preceding_hint
 907        ):
 908            self._add(TokenType.HINT)
 909
 910        # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.
 911        # Multiple consecutive comments are preserved by appending them to the current comments list.
 912        if comment_start_line == self._prev_token_line:
 913            self.tokens[-1].comments.extend(self._comments)
 914            self._comments = []
 915            self._prev_token_line = self._line
 916
 917        return True
 918
 919    def _scan_number(self) -> None:
 920        if self._char == "0":
 921            peek = _CHAR_UPPER.get(self._peek, self._peek)
 922            if peek == "B":
 923                return self._scan_bits() if self.has_bit_strings else self._add(TokenType.NUMBER)
 924            elif peek == "X":
 925                return self._scan_hex() if self.has_hex_strings else self._add(TokenType.NUMBER)
 926
 927        decimal = False
 928        scientific = 0
 929        numbers_can_be_underscore_separated = self.numbers_can_be_underscore_separated
 930        single_tokens = self.single_tokens
 931        keywords = self.keywords
 932        numeric_literals = self.numeric_literals
 933        identifiers_can_start_with_digit = self.identifiers_can_start_with_digit
 934
 935        is_underscore_separated: bool = False
 936        number_text: str = ""
 937        numeric_literal: str = ""
 938        numeric_type: TokenType | None = None
 939
 940        while True:
 941            if self._peek in _DIGIT_CHARS:
 942                # Batch consecutive digits: scan ahead to find how many
 943                sql = self.sql
 944                end = self._current + 1
 945                size = self.size
 946                while end < size and sql[end] in _DIGIT_CHARS:
 947                    end += 1
 948                self._advance(end - self._current)
 949            elif self._peek == "." and not decimal:
 950                if (
 951                    self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
 952                ) or not self.numbers_can_have_decimals:
 953                    break
 954                decimal = True
 955                self._advance()
 956            elif self._peek in ("-", "+") and scientific == 1:
 957                # Only consume +/- if followed by a digit
 958                if self._current + 1 < self.size and self.sql[self._current + 1] in _DIGIT_CHARS:
 959                    scientific += 1
 960                    self._advance()
 961                else:
 962                    break
 963            elif _CHAR_UPPER.get(self._peek, self._peek) == "E" and not scientific:
 964                scientific += 1
 965                self._advance()
 966            elif self._peek == "_" and numbers_can_be_underscore_separated:
 967                is_underscore_separated = True
 968                self._advance()
 969            elif self._peek.isidentifier():
 970                number_text = self._text
 971
 972                while self._peek and not self._peek.isspace() and self._peek not in single_tokens:
 973                    numeric_literal += self._peek
 974                    self._advance()
 975
 976                numeric_type = keywords.get(numeric_literals.get(numeric_literal.upper(), ""))
 977
 978                if numeric_type:
 979                    break
 980                elif identifiers_can_start_with_digit:
 981                    return self._add(TokenType.VAR)
 982
 983                self._advance(-len(numeric_literal))
 984                break
 985            else:
 986                break
 987
 988        number_text = number_text or self.sql[self._start : self._current]
 989
 990        # Normalize inputs such as 100_000 to 100000
 991        if is_underscore_separated:
 992            number_text = number_text.replace("_", "")
 993
 994        self._add(TokenType.NUMBER, number_text)
 995
 996        # Normalize inputs such as 123L to 123::BIGINT so that they're parsed as casts
 997        if numeric_type:
 998            self._add(TokenType.DCOLON, "::")
 999            self._add(numeric_type, numeric_literal)
1000
1001    def _scan_bits(self) -> None:
1002        self._advance()
1003        value = self._extract_value()
1004        try:
1005            # If `value` can't be converted to a binary, fallback to tokenizing it as an identifier
1006            int(value, 2)
1007            self._add(TokenType.BIT_STRING, value[2:])  # Drop the 0b
1008        except ValueError:
1009            self._add(TokenType.IDENTIFIER)
1010
1011    def _scan_hex(self) -> None:
1012        self._advance()
1013        value = self._extract_value()
1014        try:
1015            # If `value` can't be converted to a hex, fallback to tokenizing it as an identifier
1016            int(value, 16)
1017            self._add(TokenType.HEX_STRING, value[2:])  # Drop the 0x
1018        except ValueError:
1019            self._add(TokenType.IDENTIFIER)
1020
1021    def _extract_value(self) -> str:
1022        single_tokens = self.single_tokens
1023
1024        while True:
1025            char = self._peek.strip()
1026            if char and char not in single_tokens:
1027                self._advance(alnum=True)
1028            else:
1029                break
1030
1031        return self._text
1032
1033    def _scan_string(self, start: str) -> bool:
1034        base = None
1035        token_type = TokenType.STRING
1036
1037        if start in self.quotes:
1038            end = self.quotes[start]
1039        elif start in self.format_strings:
1040            end, token_type = self.format_strings[start]
1041
1042            if token_type == TokenType.HEX_STRING:
1043                base = 16
1044            elif token_type == TokenType.BIT_STRING:
1045                base = 2
1046            elif token_type == TokenType.HEREDOC_STRING:
1047                self._advance()
1048
1049                if self._char == end:
1050                    tag = ""
1051                else:
1052                    tag = self._extract_string(
1053                        end,
1054                        raw_string=True,
1055                        raise_unmatched=not self.heredoc_tag_is_identifier,
1056                    )
1057
1058                if (
1059                    tag
1060                    and self.heredoc_tag_is_identifier
1061                    and (self._end or tag.isdigit() or any(c.isspace() for c in tag))
1062                ):
1063                    if not self._end:
1064                        self._advance(-1)
1065
1066                    self._advance(-len(tag))
1067                    self._add(self.heredoc_string_alternative)
1068                    return True
1069
1070                end = f"{start}{tag}{end}"
1071        else:
1072            return False
1073
1074        self._advance(len(start))
1075        text = self._extract_string(
1076            end,
1077            escapes=(
1078                self.byte_string_escapes
1079                if token_type == TokenType.BYTE_STRING
1080                else self.string_escapes
1081            ),
1082            raw_string=token_type == TokenType.RAW_STRING,
1083        )
1084
1085        if base and text:
1086            try:
1087                int(text, base)
1088            except Exception:
1089                raise TokenError(
1090                    f"Numeric string contains invalid characters from {self._line}:{self._start}"
1091                )
1092
1093        self._add(token_type, text)
1094        return True
1095
1096    def _scan_identifier(self, identifier_end: str) -> None:
1097        self._advance()
1098        text = self._extract_string(
1099            identifier_end, escapes=self.identifier_escapes | {identifier_end}
1100        )
1101        self._add(TokenType.IDENTIFIER, text)
1102
1103    def _scan_var(self) -> None:
1104        var_single_tokens = self.var_single_tokens
1105        single_tokens = self.single_tokens
1106
1107        while True:
1108            peek = self._peek
1109            if not peek or peek.isspace():
1110                break
1111            if peek not in var_single_tokens and peek in single_tokens:
1112                break
1113            self._advance(alnum=True)
1114
1115        self._add(
1116            TokenType.VAR
1117            if self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
1118            else self.keywords.get(self.sql[self._start : self._current].upper(), TokenType.VAR)
1119        )
1120
1121    def _extract_string(
1122        self,
1123        delimiter: str,
1124        escapes: set[str] | None = None,
1125        raw_string: bool = False,
1126        raise_unmatched: bool = True,
1127    ) -> str:
1128        text = ""
1129        delim_size = len(delimiter)
1130        escapes = self.string_escapes if escapes is None else escapes
1131        unescaped_sequences = self.unescaped_sequences
1132        escape_follow_chars = self.escape_follow_chars
1133        string_escapes_allowed_in_raw_strings = self.string_escapes_allowed_in_raw_strings
1134        quotes = self.quotes
1135        sql = self.sql
1136
1137        # use str.find() when the string is simple... no \ or other escapes
1138        if delim_size == 1:
1139            pos = self._current - 1
1140            end = sql.find(delimiter, pos)
1141
1142            if (
1143                # the closing delimiter was found
1144                end != -1
1145                # there's no doubled delimiter (e.g. '' escape), or the delimiter isn't an escape char
1146                and (end + 1 >= self.size or sql[end + 1] != delimiter or delimiter not in escapes)
1147                # no backslash in the string that would need escape processing
1148                and (not (unescaped_sequences or "\\" in escapes) or sql.find("\\", pos, end) == -1)
1149            ):
1150                newlines = sql.count("\n", pos, end)
1151                if newlines:
1152                    self._line += newlines
1153                    self._col = end - sql.rfind("\n", pos, end)
1154                else:
1155                    self._col += end - pos
1156
1157                self._current = end + 1
1158                self._end = self._current >= self.size
1159                self._char = sql[end]
1160                self._peek = "" if self._end else sql[self._current]
1161                return sql[pos:end]
1162
1163        while True:
1164            if not raw_string and unescaped_sequences and self._peek and self._char in escapes:
1165                unescaped_sequence = unescaped_sequences.get(self._char + self._peek)
1166                if unescaped_sequence:
1167                    self._advance(2)
1168                    text += unescaped_sequence
1169                    continue
1170
1171            is_valid_custom_escape = (
1172                escape_follow_chars and self._char == "\\" and self._peek not in escape_follow_chars
1173            )
1174
1175            if (
1176                (string_escapes_allowed_in_raw_strings or not raw_string)
1177                and self._char in escapes
1178                and (self._peek == delimiter or self._peek in escapes or is_valid_custom_escape)
1179                and (self._char not in quotes or self._char == self._peek)
1180            ):
1181                if self._peek == delimiter:
1182                    text += self._peek
1183                elif is_valid_custom_escape and self._char != self._peek:
1184                    text += self._peek
1185                else:
1186                    text += self._char + self._peek
1187
1188                if self._current + 1 < self.size:
1189                    self._advance(2)
1190                else:
1191                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._current}")
1192            else:
1193                if self._chars(delim_size) == delimiter:
1194                    if delim_size > 1:
1195                        self._advance(delim_size - 1)
1196                    break
1197
1198                if self._end:
1199                    if not raise_unmatched:
1200                        return text + self._char
1201
1202                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._start}")
1203
1204                current = self._current - 1
1205                self._advance(alnum=True)
1206                text += sql[current : self._current - 1]
1207
1208        return text