sqlglot.tokenizer_core

   1from __future__ import annotations
   2
   3import typing as t
   4from enum import IntEnum, auto
   5
   6from sqlglot.errors import TokenError
   7
   8# dict lookup is faster than .upper() and .isdigit()
   9_CHAR_UPPER: dict[str, str] = {chr(i): chr(i).upper() for i in range(97, 123)}
  10_DIGIT_CHARS: frozenset[str] = frozenset("0123456789")
  11
  12
  13class TokenType(IntEnum):
  14    L_PAREN = auto()
  15    R_PAREN = auto()
  16    L_BRACKET = auto()
  17    R_BRACKET = auto()
  18    L_BRACE = auto()
  19    R_BRACE = auto()
  20    COMMA = auto()
  21    DOT = auto()
  22    DASH = auto()
  23    PLUS = auto()
  24    COLON = auto()
  25    DOTCOLON = auto()
  26    DOTCARET = auto()
  27    DCOLON = auto()
  28    DCOLONDOLLAR = auto()
  29    DCOLONPERCENT = auto()
  30    DCOLONQMARK = auto()
  31    DQMARK = auto()
  32    SEMICOLON = auto()
  33    STAR = auto()
  34    BACKSLASH = auto()
  35    SLASH = auto()
  36    LT = auto()
  37    LTE = auto()
  38    GT = auto()
  39    GTE = auto()
  40    NOT = auto()
  41    EQ = auto()
  42    NEQ = auto()
  43    NULLSAFE_EQ = auto()
  44    COLON_EQ = auto()
  45    COLON_GT = auto()
  46    NCOLON_GT = auto()
  47    AND = auto()
  48    OR = auto()
  49    AMP = auto()
  50    DPIPE = auto()
  51    PIPE_GT = auto()
  52    PIPE = auto()
  53    PIPE_SLASH = auto()
  54    DPIPE_SLASH = auto()
  55    CARET = auto()
  56    CARET_AT = auto()
  57    TILDE = auto()
  58    ARROW = auto()
  59    DARROW = auto()
  60    FARROW = auto()
  61    HASH = auto()
  62    HASH_ARROW = auto()
  63    DHASH_ARROW = auto()
  64    LR_ARROW = auto()
  65    DAT = auto()
  66    LT_AT = auto()
  67    AT_GT = auto()
  68    DOLLAR = auto()
  69    PARAMETER = auto()
  70    SESSION = auto()
  71    SESSION_PARAMETER = auto()
  72    SESSION_USER = auto()
  73    DAMP = auto()
  74    AMP_LT = auto()
  75    AMP_GT = auto()
  76    ADJACENT = auto()
  77    XOR = auto()
  78    DSTAR = auto()
  79    QMARK_AMP = auto()
  80    QMARK_PIPE = auto()
  81    HASH_DASH = auto()
  82    EXCLAMATION = auto()
  83
  84    URI_START = auto()
  85
  86    BLOCK_START = auto()
  87    BLOCK_END = auto()
  88
  89    SPACE = auto()
  90    BREAK = auto()
  91
  92    STRING = auto()
  93    NUMBER = auto()
  94    IDENTIFIER = auto()
  95    DATABASE = auto()
  96    COLUMN = auto()
  97    COLUMN_DEF = auto()
  98    SCHEMA = auto()
  99    TABLE = auto()
 100    WAREHOUSE = auto()
 101    STAGE = auto()
 102    STREAM = auto()
 103    STREAMLIT = auto()
 104    VAR = auto()
 105    BIT_STRING = auto()
 106    HEX_STRING = auto()
 107    BYTE_STRING = auto()
 108    NATIONAL_STRING = auto()
 109    RAW_STRING = auto()
 110    HEREDOC_STRING = auto()
 111    UNICODE_STRING = auto()
 112
 113    # types
 114    BIT = auto()
 115    BOOLEAN = auto()
 116    TINYINT = auto()
 117    UTINYINT = auto()
 118    SMALLINT = auto()
 119    USMALLINT = auto()
 120    MEDIUMINT = auto()
 121    UMEDIUMINT = auto()
 122    INT = auto()
 123    UINT = auto()
 124    BIGINT = auto()
 125    UBIGINT = auto()
 126    BIGNUM = auto()
 127    INT128 = auto()
 128    UINT128 = auto()
 129    INT256 = auto()
 130    UINT256 = auto()
 131    FLOAT = auto()
 132    DOUBLE = auto()
 133    UDOUBLE = auto()
 134    DECIMAL = auto()
 135    DECIMAL32 = auto()
 136    DECIMAL64 = auto()
 137    DECIMAL128 = auto()
 138    DECIMAL256 = auto()
 139    DECFLOAT = auto()
 140    UDECIMAL = auto()
 141    BIGDECIMAL = auto()
 142    CHAR = auto()
 143    NCHAR = auto()
 144    VARCHAR = auto()
 145    NVARCHAR = auto()
 146    BPCHAR = auto()
 147    TEXT = auto()
 148    MEDIUMTEXT = auto()
 149    LONGTEXT = auto()
 150    BLOB = auto()
 151    MEDIUMBLOB = auto()
 152    LONGBLOB = auto()
 153    TINYBLOB = auto()
 154    TINYTEXT = auto()
 155    NAME = auto()
 156    BINARY = auto()
 157    VARBINARY = auto()
 158    JSON = auto()
 159    JSONB = auto()
 160    TIME = auto()
 161    TIMETZ = auto()
 162    TIME_NS = auto()
 163    TIMESTAMP = auto()
 164    TIMESTAMPTZ = auto()
 165    TIMESTAMPLTZ = auto()
 166    TIMESTAMPNTZ = auto()
 167    TIMESTAMP_S = auto()
 168    TIMESTAMP_MS = auto()
 169    TIMESTAMP_NS = auto()
 170    DATETIME = auto()
 171    DATETIME2 = auto()
 172    DATETIME64 = auto()
 173    SMALLDATETIME = auto()
 174    DATE = auto()
 175    DATE32 = auto()
 176    INT4RANGE = auto()
 177    INT4MULTIRANGE = auto()
 178    INT8RANGE = auto()
 179    INT8MULTIRANGE = auto()
 180    NUMRANGE = auto()
 181    NUMMULTIRANGE = auto()
 182    TSRANGE = auto()
 183    TSMULTIRANGE = auto()
 184    TSTZRANGE = auto()
 185    TSTZMULTIRANGE = auto()
 186    DATERANGE = auto()
 187    DATEMULTIRANGE = auto()
 188    UUID = auto()
 189    GEOGRAPHY = auto()
 190    GEOGRAPHYPOINT = auto()
 191    NULLABLE = auto()
 192    GEOMETRY = auto()
 193    POINT = auto()
 194    RING = auto()
 195    LINESTRING = auto()
 196    LOCALTIME = auto()
 197    LOCALTIMESTAMP = auto()
 198    SYSTIMESTAMP = auto()
 199    MULTILINESTRING = auto()
 200    POLYGON = auto()
 201    MULTIPOLYGON = auto()
 202    HLLSKETCH = auto()
 203    HSTORE = auto()
 204    SUPER = auto()
 205    SERIAL = auto()
 206    SMALLSERIAL = auto()
 207    BIGSERIAL = auto()
 208    XML = auto()
 209    YEAR = auto()
 210    USERDEFINED = auto()
 211    MONEY = auto()
 212    SMALLMONEY = auto()
 213    ROWVERSION = auto()
 214    IMAGE = auto()
 215    VARIANT = auto()
 216    OBJECT = auto()
 217    INET = auto()
 218    IPADDRESS = auto()
 219    IPPREFIX = auto()
 220    IPV4 = auto()
 221    IPV6 = auto()
 222    ENUM = auto()
 223    ENUM8 = auto()
 224    ENUM16 = auto()
 225    FIXEDSTRING = auto()
 226    LOWCARDINALITY = auto()
 227    NESTED = auto()
 228    AGGREGATEFUNCTION = auto()
 229    SIMPLEAGGREGATEFUNCTION = auto()
 230    TDIGEST = auto()
 231    UNKNOWN = auto()
 232    VECTOR = auto()
 233    DYNAMIC = auto()
 234    VOID = auto()
 235
 236    # keywords
 237    ALIAS = auto()
 238    ALTER = auto()
 239    ALL = auto()
 240    ANTI = auto()
 241    ANY = auto()
 242    APPLY = auto()
 243    ARRAY = auto()
 244    ASC = auto()
 245    ASOF = auto()
 246    ATTACH = auto()
 247    AUTO_INCREMENT = auto()
 248    BEGIN = auto()
 249    BETWEEN = auto()
 250    BULK_COLLECT_INTO = auto()
 251    CACHE = auto()
 252    CASE = auto()
 253    CHARACTER_SET = auto()
 254    CLUSTER_BY = auto()
 255    COLLATE = auto()
 256    COMMAND = auto()
 257    COMMENT = auto()
 258    COMMIT = auto()
 259    CONNECT_BY = auto()
 260    CONSTRAINT = auto()
 261    COPY = auto()
 262    CREATE = auto()
 263    CROSS = auto()
 264    CUBE = auto()
 265    CURRENT_DATE = auto()
 266    CURRENT_DATETIME = auto()
 267    CURRENT_SCHEMA = auto()
 268    CURRENT_TIME = auto()
 269    CURRENT_TIMESTAMP = auto()
 270    CURRENT_USER = auto()
 271    CURRENT_USER_ID = auto()
 272    CURRENT_ROLE = auto()
 273    CURRENT_CATALOG = auto()
 274    DECLARE = auto()
 275    DEFAULT = auto()
 276    DELETE = auto()
 277    DESC = auto()
 278    DESCRIBE = auto()
 279    DETACH = auto()
 280    DICTIONARY = auto()
 281    DISTINCT = auto()
 282    DISTRIBUTE_BY = auto()
 283    DIV = auto()
 284    DROP = auto()
 285    ELSE = auto()
 286    END = auto()
 287    ESCAPE = auto()
 288    EXCEPT = auto()
 289    EXECUTE = auto()
 290    EXISTS = auto()
 291    FALSE = auto()
 292    FETCH = auto()
 293    FILE = auto()
 294    FILE_FORMAT = auto()
 295    FILTER = auto()
 296    FINAL = auto()
 297    FIRST = auto()
 298    FOR = auto()
 299    FORCE = auto()
 300    FOREIGN_KEY = auto()
 301    FORMAT = auto()
 302    FROM = auto()
 303    FULL = auto()
 304    FUNCTION = auto()
 305    GET = auto()
 306    GLOB = auto()
 307    GLOBAL = auto()
 308    GRANT = auto()
 309    GROUP_BY = auto()
 310    GROUPING_SETS = auto()
 311    HAVING = auto()
 312    HINT = auto()
 313    IGNORE = auto()
 314    ILIKE = auto()
 315    IN = auto()
 316    INDEX = auto()
 317    INDEXED_BY = auto()
 318    INNER = auto()
 319    INSERT = auto()
 320    INSTALL = auto()
 321    INTEGRATION = auto()
 322    INTERSECT = auto()
 323    INTERVAL = auto()
 324    INTO = auto()
 325    INTRODUCER = auto()
 326    IRLIKE = auto()
 327    IS = auto()
 328    ISNULL = auto()
 329    JOIN = auto()
 330    JOIN_MARKER = auto()
 331    KEEP = auto()
 332    KEY = auto()
 333    KILL = auto()
 334    LANGUAGE = auto()
 335    LATERAL = auto()
 336    LEFT = auto()
 337    LIKE = auto()
 338    LIMIT = auto()
 339    LIST = auto()
 340    LOAD = auto()
 341    LOCK = auto()
 342    MAP = auto()
 343    MATCH = auto()
 344    MATCH_CONDITION = auto()
 345    MATCH_RECOGNIZE = auto()
 346    MEMBER_OF = auto()
 347    MERGE = auto()
 348    MOD = auto()
 349    MODEL = auto()
 350    NATURAL = auto()
 351    NEXT = auto()
 352    NOTHING = auto()
 353    NOTNULL = auto()
 354    NULL = auto()
 355    OBJECT_IDENTIFIER = auto()
 356    OFFSET = auto()
 357    ON = auto()
 358    ONLY = auto()
 359    OPERATOR = auto()
 360    ORDER_BY = auto()
 361    ORDER_SIBLINGS_BY = auto()
 362    ORDERED = auto()
 363    ORDINALITY = auto()
 364    OUT = auto()
 365    INOUT = auto()
 366    OUTER = auto()
 367    OVER = auto()
 368    OVERLAPS = auto()
 369    OVERWRITE = auto()
 370    PACKAGE = auto()
 371    PARTITION = auto()
 372    PARTITION_BY = auto()
 373    PERCENT = auto()
 374    PIVOT = auto()
 375    PLACEHOLDER = auto()
 376    POLICY = auto()
 377    POOL = auto()
 378    POSITIONAL = auto()
 379    PRAGMA = auto()
 380    PREWHERE = auto()
 381    PRIMARY_KEY = auto()
 382    PROCEDURE = auto()
 383    PROPERTIES = auto()
 384    PSEUDO_TYPE = auto()
 385    PUT = auto()
 386    QUALIFY = auto()
 387    QUOTE = auto()
 388    QDCOLON = auto()
 389    RANGE = auto()
 390    RECURSIVE = auto()
 391    REFRESH = auto()
 392    RENAME = auto()
 393    REPLACE = auto()
 394    RETURNING = auto()
 395    REVOKE = auto()
 396    REFERENCES = auto()
 397    RIGHT = auto()
 398    RLIKE = auto()
 399    ROLE = auto()
 400    ROLLBACK = auto()
 401    ROLLUP = auto()
 402    ROW = auto()
 403    ROWS = auto()
 404    RULE = auto()
 405    SELECT = auto()
 406    SEMI = auto()
 407    SEPARATOR = auto()
 408    SEQUENCE = auto()
 409    SERDE_PROPERTIES = auto()
 410    SET = auto()
 411    SETTINGS = auto()
 412    SHOW = auto()
 413    SIMILAR_TO = auto()
 414    SOME = auto()
 415    SORT_BY = auto()
 416    SOUNDS_LIKE = auto()
 417    SQL_SECURITY = auto()
 418    START_WITH = auto()
 419    STORAGE_INTEGRATION = auto()
 420    STRAIGHT_JOIN = auto()
 421    STRUCT = auto()
 422    SUMMARIZE = auto()
 423    TABLE_SAMPLE = auto()
 424    TAG = auto()
 425    TEMPORARY = auto()
 426    TOP = auto()
 427    THEN = auto()
 428    TRUE = auto()
 429    TRUNCATE = auto()
 430    TRIGGER = auto()
 431    TYPE = auto()
 432    UNCACHE = auto()
 433    UNION = auto()
 434    UNNEST = auto()
 435    UNPIVOT = auto()
 436    UPDATE = auto()
 437    USE = auto()
 438    USING = auto()
 439    VALUES = auto()
 440    VARIADIC = auto()
 441    VIEW = auto()
 442    SEMANTIC_VIEW = auto()
 443    VOLATILE = auto()
 444    VOLUME = auto()
 445    WHEN = auto()
 446    WHERE = auto()
 447    WINDOW = auto()
 448    WITH = auto()
 449    UNIQUE = auto()
 450    UTC_DATE = auto()
 451    UTC_TIME = auto()
 452    UTC_TIMESTAMP = auto()
 453    VERSION_SNAPSHOT = auto()
 454    TIMESTAMP_SNAPSHOT = auto()
 455    OPTION = auto()
 456    SINK = auto()
 457    SOURCE = auto()
 458    ANALYZE = auto()
 459    NAMESPACE = auto()
 460    EXPORT = auto()
 461
 462    # sentinels
 463    HIVE_TOKEN_STREAM = auto()
 464    SENTINEL = auto()
 465
 466    def __str__(self) -> str:
 467        return f"TokenType.{self.name}"
 468
 469
 470class Token:
 471    # mypyc doesn't expose slots
 472    _attrs: t.ClassVar[tuple[str, ...]] = (
 473        "token_type",
 474        "text",
 475        "line",
 476        "col",
 477        "start",
 478        "end",
 479        "comments",
 480    )
 481    __slots__ = _attrs
 482
 483    @classmethod
 484    def number(cls, number: int) -> Token:
 485        """Returns a NUMBER token with `number` as its text."""
 486        return cls(TokenType.NUMBER, str(number))
 487
 488    @classmethod
 489    def string(cls, string: str) -> Token:
 490        """Returns a STRING token with `string` as its text."""
 491        return cls(TokenType.STRING, string)
 492
 493    @classmethod
 494    def identifier(cls, identifier: str) -> Token:
 495        """Returns an IDENTIFIER token with `identifier` as its text."""
 496        return cls(TokenType.IDENTIFIER, identifier)
 497
 498    @classmethod
 499    def var(cls, var: str) -> Token:
 500        """Returns an VAR token with `var` as its text."""
 501        return cls(TokenType.VAR, var)
 502
 503    def __init__(
 504        self,
 505        token_type: TokenType,
 506        text: str,
 507        line: int = 1,
 508        col: int = 1,
 509        start: int = 0,
 510        end: int = 0,
 511        comments: list[str] | None = None,
 512    ) -> None:
 513        self.token_type = token_type
 514        self.text = text
 515        self.line = line
 516        self.col = col
 517        self.start = start
 518        self.end = end
 519        self.comments = [] if comments is None else comments
 520
 521    def __bool__(self) -> bool:
 522        return self.token_type != TokenType.SENTINEL
 523
 524    def __repr__(self) -> str:
 525        attributes = ", ".join(
 526            f"{k}: TokenType.{self.token_type.name}"
 527            if k == "token_type"
 528            else f"{k}: {getattr(self, k)}"
 529            for k in self._attrs
 530        )
 531        return f"<Token {attributes}>"
 532
 533
 534class TokenizerCore:
 535    __slots__ = (
 536        "sql",
 537        "size",
 538        "tokens",
 539        "_start",
 540        "_current",
 541        "_line",
 542        "_col",
 543        "_comments",
 544        "_char",
 545        "_end",
 546        "_peek",
 547        "_prev_token_line",
 548        "single_tokens",
 549        "keywords",
 550        "quotes",
 551        "format_strings",
 552        "identifiers",
 553        "comments",
 554        "string_escapes",
 555        "byte_string_escapes",
 556        "identifier_escapes",
 557        "escape_follow_chars",
 558        "commands",
 559        "command_prefix_tokens",
 560        "nested_comments",
 561        "hint_start",
 562        "tokens_preceding_hint",
 563        "has_bit_strings",
 564        "has_hex_strings",
 565        "numeric_literals",
 566        "var_single_tokens",
 567        "string_escapes_allowed_in_raw_strings",
 568        "heredoc_tag_is_identifier",
 569        "heredoc_string_alternative",
 570        "keyword_trie",
 571        "numbers_can_be_underscore_separated",
 572        "numbers_can_have_decimals",
 573        "identifiers_can_start_with_digit",
 574        "unescaped_sequences",
 575    )
 576
 577    def __init__(
 578        self,
 579        single_tokens: dict[str, TokenType],
 580        keywords: dict[str, TokenType],
 581        quotes: dict[str, str],
 582        format_strings: dict[str, tuple[str, TokenType]],
 583        identifiers: dict[str, str],
 584        comments: dict[str, str | None],
 585        string_escapes: set[str],
 586        byte_string_escapes: set[str],
 587        identifier_escapes: set[str],
 588        escape_follow_chars: set[str],
 589        commands: set[TokenType],
 590        command_prefix_tokens: set[TokenType],
 591        nested_comments: bool,
 592        hint_start: str,
 593        tokens_preceding_hint: set[TokenType],
 594        has_bit_strings: bool,
 595        has_hex_strings: bool,
 596        numeric_literals: dict[str, str],
 597        var_single_tokens: set[str],
 598        string_escapes_allowed_in_raw_strings: bool,
 599        heredoc_tag_is_identifier: bool,
 600        heredoc_string_alternative: TokenType,
 601        keyword_trie: dict,
 602        numbers_can_be_underscore_separated: bool,
 603        numbers_can_have_decimals: bool,
 604        identifiers_can_start_with_digit: bool,
 605        unescaped_sequences: dict[str, str],
 606    ) -> None:
 607        self.single_tokens = single_tokens
 608        self.keywords = keywords
 609        self.quotes = quotes
 610        self.format_strings = format_strings
 611        self.identifiers = identifiers
 612        self.comments = comments
 613        self.string_escapes = string_escapes
 614        self.byte_string_escapes = byte_string_escapes
 615        self.identifier_escapes = identifier_escapes
 616        self.escape_follow_chars = escape_follow_chars
 617        self.commands = commands
 618        self.command_prefix_tokens = command_prefix_tokens
 619        self.nested_comments = nested_comments
 620        self.hint_start = hint_start
 621        self.tokens_preceding_hint = tokens_preceding_hint
 622        self.has_bit_strings = has_bit_strings
 623        self.has_hex_strings = has_hex_strings
 624        self.numeric_literals = numeric_literals
 625        self.var_single_tokens = var_single_tokens
 626        self.string_escapes_allowed_in_raw_strings = string_escapes_allowed_in_raw_strings
 627        self.heredoc_tag_is_identifier = heredoc_tag_is_identifier
 628        self.heredoc_string_alternative = heredoc_string_alternative
 629        self.keyword_trie = keyword_trie
 630        self.numbers_can_be_underscore_separated = numbers_can_be_underscore_separated
 631        self.numbers_can_have_decimals = numbers_can_have_decimals
 632        self.identifiers_can_start_with_digit = identifiers_can_start_with_digit
 633        self.unescaped_sequences = unescaped_sequences
 634        self.sql = ""
 635        self.size = 0
 636        self.tokens: list[Token] = []
 637        self._start = 0
 638        self._current = 0
 639        self._line = 1
 640        self._col = 0
 641        self._comments: list[str] = []
 642        self._char = ""
 643        self._end = False
 644        self._peek = ""
 645        self._prev_token_line = -1
 646
 647    def reset(self) -> None:
 648        self.sql = ""
 649        self.size = 0
 650        self.tokens = []
 651        self._start = 0
 652        self._current = 0
 653        self._line = 1
 654        self._col = 0
 655        self._comments = []
 656        self._char = ""
 657        self._end = False
 658        self._peek = ""
 659        self._prev_token_line = -1
 660
 661    def tokenize(self, sql: str) -> list[Token]:
 662        """Returns a list of tokens corresponding to the SQL string `sql`."""
 663        self.reset()
 664        self.sql = sql
 665        self.size = len(sql)
 666
 667        try:
 668            self._scan()
 669        except Exception as e:
 670            start = max(self._current - 50, 0)
 671            end = min(self._current + 50, self.size - 1)
 672            context = self.sql[start:end]
 673            raise TokenError(f"Error tokenizing '{context}'") from e
 674
 675        return self.tokens
 676
 677    def _scan(self, check_semicolon: bool = False) -> None:
 678        identifiers = self.identifiers
 679        digit_chars = _DIGIT_CHARS
 680
 681        while self.size and not self._end:
 682            current = self._current
 683
 684            # Skip spaces here rather than iteratively calling advance() for performance reasons
 685            while current < self.size:
 686                char = self.sql[current]
 687
 688                if char == " " or char == "\t":
 689                    current += 1
 690                else:
 691                    break
 692
 693            offset = current - self._current if current > self._current else 1
 694
 695            self._start = current
 696            self._advance(offset)
 697
 698            if not self._char.isspace():
 699                if self._char in digit_chars:
 700                    self._scan_number()
 701                elif self._char in identifiers:
 702                    self._scan_identifier(identifiers[self._char])
 703                else:
 704                    self._scan_keywords()
 705
 706            if check_semicolon and self._peek == ";":
 707                break
 708
 709        if self.tokens and self._comments:
 710            self.tokens[-1].comments.extend(self._comments)
 711
 712    def _chars(self, size: int) -> str:
 713        if size == 1:
 714            return self._char
 715
 716        start = self._current - 1
 717        end = start + size
 718
 719        return self.sql[start:end] if end <= self.size else ""
 720
 721    def _advance(self, i: int = 1, alnum: bool = False) -> None:
 722        char = self._char
 723
 724        if char == "\n" or char == "\r":
 725            # Ensures we don't count an extra line if we get a \r\n line break sequence
 726            if not (char == "\r" and self._peek == "\n"):
 727                self._col = i
 728                self._line += 1
 729        else:
 730            self._col += i
 731
 732        self._current += i
 733        sql = self.sql
 734        size = self.size
 735        self._end = self._current >= size
 736        self._char = sql[self._current - 1]
 737        self._peek = "" if self._end else sql[self._current]
 738
 739        if alnum and self._char.isalnum():
 740            # Cache to local variables instead of attributes for better performance
 741            _col = self._col
 742            _current = self._current
 743            _end = self._end
 744            _peek = self._peek
 745
 746            while _peek.isalnum():
 747                _col += 1
 748                _current += 1
 749                _end = _current >= size
 750                _peek = "" if _end else sql[_current]
 751
 752            self._col = _col
 753            self._current = _current
 754            self._end = _end
 755            self._peek = _peek
 756            self._char = sql[_current - 1]
 757
 758    @property
 759    def _text(self) -> str:
 760        return self.sql[self._start : self._current]
 761
 762    def _add(self, token_type: TokenType, text: str | None = None) -> None:
 763        self._prev_token_line = self._line
 764
 765        if self._comments and token_type == TokenType.SEMICOLON and self.tokens:
 766            self.tokens[-1].comments.extend(self._comments)
 767            self._comments = []
 768
 769        if text is None:
 770            text = self.sql[self._start : self._current]
 771
 772        self.tokens.append(
 773            Token(
 774                token_type,
 775                text=text,
 776                line=self._line,
 777                col=self._col,
 778                start=self._start,
 779                end=self._current - 1,
 780                comments=self._comments,
 781            )
 782        )
 783        self._comments = []
 784
 785        # If we have either a semicolon or a begin token before the command's token, we'll parse
 786        # whatever follows the command's token as a string
 787        if (
 788            token_type in self.commands
 789            and self._peek != ";"
 790            and (len(self.tokens) == 1 or self.tokens[-2].token_type in self.command_prefix_tokens)
 791        ):
 792            start = self._current
 793            tokens = len(self.tokens)
 794            self._scan(check_semicolon=True)
 795            self.tokens = self.tokens[:tokens]
 796            text = self.sql[start : self._current].strip()
 797            if text:
 798                self._add(TokenType.STRING, text)
 799
 800    def _scan_keywords(self) -> None:
 801        sql = self.sql
 802        sql_size = self.size
 803        single_tokens = self.single_tokens
 804        char_upper = _CHAR_UPPER
 805        size = 0
 806        word = None
 807        chars = self._char
 808        char = chars
 809        prev_space = False
 810        skip = False
 811        trie = self.keyword_trie
 812        single_token = char in single_tokens
 813
 814        while chars:
 815            if not skip:
 816                sub = trie.get(char_upper.get(char, char))
 817                if sub is None:
 818                    break
 819                trie = sub
 820                if 0 in trie:
 821                    word = chars
 822
 823            end = self._current + size
 824            size += 1
 825
 826            if end < sql_size:
 827                char = sql[end]
 828                single_token = single_token or char in single_tokens
 829                is_space = char.isspace()
 830
 831                if not is_space or not prev_space:
 832                    if is_space:
 833                        char = " "
 834                    chars += char
 835                    prev_space = is_space
 836                    skip = False
 837                else:
 838                    skip = True
 839            else:
 840                char = ""
 841                break
 842
 843        if word:
 844            if self._scan_string(word):
 845                return
 846            if self._scan_comment(word):
 847                return
 848            if prev_space or single_token or not char:
 849                self._advance(size - 1)
 850                word = word.upper()
 851                self._add(self.keywords[word], text=word)
 852                return
 853
 854        if self._char in single_tokens:
 855            self._add(single_tokens[self._char], text=self._char)
 856            return
 857
 858        self._scan_var()
 859
 860    def _scan_comment(self, comment_start: str) -> bool:
 861        if comment_start not in self.comments:
 862            return False
 863
 864        comment_start_line = self._line
 865        comment_start_size = len(comment_start)
 866        comment_end = self.comments[comment_start]
 867
 868        if comment_end:
 869            # Skip the comment's start delimiter
 870            self._advance(comment_start_size)
 871
 872            comment_count = 1
 873            comment_end_size = len(comment_end)
 874            nested_comments = self.nested_comments
 875
 876            while not self._end:
 877                if self._chars(comment_end_size) == comment_end:
 878                    comment_count -= 1
 879                    if not comment_count:
 880                        break
 881
 882                self._advance(alnum=True)
 883
 884                # Nested comments are allowed by some dialects, e.g. databricks, duckdb, postgres
 885                if (
 886                    nested_comments
 887                    and not self._end
 888                    and self._chars(comment_end_size) == comment_start
 889                ):
 890                    self._advance(comment_start_size)
 891                    comment_count += 1
 892
 893            self._comments.append(self._text[comment_start_size : -comment_end_size + 1])
 894            self._advance(comment_end_size - 1)
 895        else:
 896            _peek = self._peek
 897            while not self._end and _peek != "\n" and _peek != "\r":
 898                self._advance(alnum=True)
 899                _peek = self._peek
 900            self._comments.append(self._text[comment_start_size:])
 901
 902        if (
 903            comment_start == self.hint_start
 904            and self.tokens
 905            and self.tokens[-1].token_type in self.tokens_preceding_hint
 906        ):
 907            self._add(TokenType.HINT)
 908
 909        # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.
 910        # Multiple consecutive comments are preserved by appending them to the current comments list.
 911        if comment_start_line == self._prev_token_line:
 912            self.tokens[-1].comments.extend(self._comments)
 913            self._comments = []
 914            self._prev_token_line = self._line
 915
 916        return True
 917
 918    def _scan_number(self) -> None:
 919        if self._char == "0":
 920            peek = _CHAR_UPPER.get(self._peek, self._peek)
 921            if peek == "B":
 922                return self._scan_bits() if self.has_bit_strings else self._add(TokenType.NUMBER)
 923            elif peek == "X":
 924                return self._scan_hex() if self.has_hex_strings else self._add(TokenType.NUMBER)
 925
 926        decimal = False
 927        scientific = 0
 928        numbers_can_be_underscore_separated = self.numbers_can_be_underscore_separated
 929        single_tokens = self.single_tokens
 930        keywords = self.keywords
 931        numeric_literals = self.numeric_literals
 932        identifiers_can_start_with_digit = self.identifiers_can_start_with_digit
 933
 934        is_underscore_separated: bool = False
 935        number_text: str = ""
 936        numeric_literal: str = ""
 937        numeric_type: TokenType | None = None
 938
 939        while True:
 940            if self._peek in _DIGIT_CHARS:
 941                # Batch consecutive digits: scan ahead to find how many
 942                sql = self.sql
 943                end = self._current + 1
 944                size = self.size
 945                while end < size and sql[end] in _DIGIT_CHARS:
 946                    end += 1
 947                self._advance(end - self._current)
 948            elif self._peek == "." and not decimal:
 949                if (
 950                    self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
 951                ) or not self.numbers_can_have_decimals:
 952                    break
 953                decimal = True
 954                self._advance()
 955            elif self._peek in ("-", "+") and scientific == 1:
 956                # Only consume +/- if followed by a digit
 957                if self._current + 1 < self.size and self.sql[self._current + 1] in _DIGIT_CHARS:
 958                    scientific += 1
 959                    self._advance()
 960                else:
 961                    break
 962            elif _CHAR_UPPER.get(self._peek, self._peek) == "E" and not scientific:
 963                scientific += 1
 964                self._advance()
 965            elif self._peek == "_" and numbers_can_be_underscore_separated:
 966                is_underscore_separated = True
 967                self._advance()
 968            elif self._peek.isidentifier():
 969                number_text = self._text
 970
 971                while self._peek and not self._peek.isspace() and self._peek not in single_tokens:
 972                    numeric_literal += self._peek
 973                    self._advance()
 974
 975                numeric_type = keywords.get(numeric_literals.get(numeric_literal.upper(), ""))
 976
 977                if numeric_type:
 978                    break
 979                elif identifiers_can_start_with_digit:
 980                    return self._add(TokenType.VAR)
 981
 982                self._advance(-len(numeric_literal))
 983                break
 984            else:
 985                break
 986
 987        number_text = number_text or self.sql[self._start : self._current]
 988
 989        # Normalize inputs such as 100_000 to 100000
 990        if is_underscore_separated:
 991            number_text = number_text.replace("_", "")
 992
 993        self._add(TokenType.NUMBER, number_text)
 994
 995        # Normalize inputs such as 123L to 123::BIGINT so that they're parsed as casts
 996        if numeric_type:
 997            self._add(TokenType.DCOLON, "::")
 998            self._add(numeric_type, numeric_literal)
 999
1000    def _scan_bits(self) -> None:
1001        self._advance()
1002        value = self._extract_value()
1003        try:
1004            # If `value` can't be converted to a binary, fallback to tokenizing it as an identifier
1005            int(value, 2)
1006            self._add(TokenType.BIT_STRING, value[2:])  # Drop the 0b
1007        except ValueError:
1008            self._add(TokenType.IDENTIFIER)
1009
1010    def _scan_hex(self) -> None:
1011        self._advance()
1012        value = self._extract_value()
1013        try:
1014            # If `value` can't be converted to a hex, fallback to tokenizing it as an identifier
1015            int(value, 16)
1016            self._add(TokenType.HEX_STRING, value[2:])  # Drop the 0x
1017        except ValueError:
1018            self._add(TokenType.IDENTIFIER)
1019
1020    def _extract_value(self) -> str:
1021        single_tokens = self.single_tokens
1022
1023        while True:
1024            char = self._peek.strip()
1025            if char and char not in single_tokens:
1026                self._advance(alnum=True)
1027            else:
1028                break
1029
1030        return self._text
1031
1032    def _scan_string(self, start: str) -> bool:
1033        base = None
1034        token_type = TokenType.STRING
1035
1036        if start in self.quotes:
1037            end = self.quotes[start]
1038        elif start in self.format_strings:
1039            end, token_type = self.format_strings[start]
1040
1041            if token_type == TokenType.HEX_STRING:
1042                base = 16
1043            elif token_type == TokenType.BIT_STRING:
1044                base = 2
1045            elif token_type == TokenType.HEREDOC_STRING:
1046                self._advance()
1047
1048                if self._char == end:
1049                    tag = ""
1050                else:
1051                    tag = self._extract_string(
1052                        end,
1053                        raw_string=True,
1054                        raise_unmatched=not self.heredoc_tag_is_identifier,
1055                    )
1056
1057                if (
1058                    tag
1059                    and self.heredoc_tag_is_identifier
1060                    and (self._end or tag.isdigit() or any(c.isspace() for c in tag))
1061                ):
1062                    if not self._end:
1063                        self._advance(-1)
1064
1065                    self._advance(-len(tag))
1066                    self._add(self.heredoc_string_alternative)
1067                    return True
1068
1069                end = f"{start}{tag}{end}"
1070        else:
1071            return False
1072
1073        self._advance(len(start))
1074        text = self._extract_string(
1075            end,
1076            escapes=(
1077                self.byte_string_escapes
1078                if token_type == TokenType.BYTE_STRING
1079                else self.string_escapes
1080            ),
1081            raw_string=token_type == TokenType.RAW_STRING,
1082        )
1083
1084        if base and text:
1085            try:
1086                int(text, base)
1087            except Exception:
1088                raise TokenError(
1089                    f"Numeric string contains invalid characters from {self._line}:{self._start}"
1090                )
1091
1092        self._add(token_type, text)
1093        return True
1094
1095    def _scan_identifier(self, identifier_end: str) -> None:
1096        self._advance()
1097        text = self._extract_string(
1098            identifier_end, escapes=self.identifier_escapes | {identifier_end}
1099        )
1100        self._add(TokenType.IDENTIFIER, text)
1101
1102    def _scan_var(self) -> None:
1103        var_single_tokens = self.var_single_tokens
1104        single_tokens = self.single_tokens
1105
1106        while True:
1107            peek = self._peek
1108            if not peek or peek.isspace():
1109                break
1110            if peek not in var_single_tokens and peek in single_tokens:
1111                break
1112            self._advance(alnum=True)
1113
1114        self._add(
1115            TokenType.VAR
1116            if self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER
1117            else self.keywords.get(self.sql[self._start : self._current].upper(), TokenType.VAR)
1118        )
1119
1120    def _extract_string(
1121        self,
1122        delimiter: str,
1123        escapes: set[str] | None = None,
1124        raw_string: bool = False,
1125        raise_unmatched: bool = True,
1126    ) -> str:
1127        text = ""
1128        delim_size = len(delimiter)
1129        escapes = self.string_escapes if escapes is None else escapes
1130        unescaped_sequences = self.unescaped_sequences
1131        escape_follow_chars = self.escape_follow_chars
1132        string_escapes_allowed_in_raw_strings = self.string_escapes_allowed_in_raw_strings
1133        quotes = self.quotes
1134        sql = self.sql
1135
1136        # use str.find() when the string is simple... no \ or other escapes
1137        if delim_size == 1:
1138            pos = self._current - 1
1139            end = sql.find(delimiter, pos)
1140
1141            if (
1142                # the closing delimiter was found
1143                end != -1
1144                # there's no doubled delimiter (e.g. '' escape), or the delimiter isn't an escape char
1145                and (end + 1 >= self.size or sql[end + 1] != delimiter or delimiter not in escapes)
1146                # no backslash in the string that would need escape processing
1147                and (not (unescaped_sequences or "\\" in escapes) or sql.find("\\", pos, end) == -1)
1148            ):
1149                newlines = sql.count("\n", pos, end)
1150                if newlines:
1151                    self._line += newlines
1152                    self._col = end - sql.rfind("\n", pos, end)
1153                else:
1154                    self._col += end - pos
1155
1156                self._current = end + 1
1157                self._end = self._current >= self.size
1158                self._char = sql[end]
1159                self._peek = "" if self._end else sql[self._current]
1160                return sql[pos:end]
1161
1162        while True:
1163            if not raw_string and unescaped_sequences and self._peek and self._char in escapes:
1164                unescaped_sequence = unescaped_sequences.get(self._char + self._peek)
1165                if unescaped_sequence:
1166                    self._advance(2)
1167                    text += unescaped_sequence
1168                    continue
1169
1170            is_valid_custom_escape = (
1171                escape_follow_chars and self._char == "\\" and self._peek not in escape_follow_chars
1172            )
1173
1174            if (
1175                (string_escapes_allowed_in_raw_strings or not raw_string)
1176                and self._char in escapes
1177                and (self._peek == delimiter or self._peek in escapes or is_valid_custom_escape)
1178                and (self._char not in quotes or self._char == self._peek)
1179            ):
1180                if self._peek == delimiter:
1181                    text += self._peek
1182                elif is_valid_custom_escape and self._char != self._peek:
1183                    text += self._peek
1184                else:
1185                    text += self._char + self._peek
1186
1187                if self._current + 1 < self.size:
1188                    self._advance(2)
1189                else:
1190                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._current}")
1191            else:
1192                if self._chars(delim_size) == delimiter:
1193                    if delim_size > 1:
1194                        self._advance(delim_size - 1)
1195                    break
1196
1197                if self._end:
1198                    if not raise_unmatched:
1199                        return text + self._char
1200
1201                    raise TokenError(f"Missing {delimiter} from {self._line}:{self._start}")
1202
1203                current = self._current - 1
1204                self._advance(alnum=True)
1205                text += sql[current : self._current - 1]
1206
1207        return text