15class Athena(Trino):
 16    """
 17    Over the years, it looks like AWS has taken various execution engines, bolted on AWS-specific
 18    modifications and then built the Athena service around them.
 19
 20    Thus, Athena is not simply hosted Trino, it's more like a router that routes SQL queries to an
 21    execution engine depending on the query type.
 22
 23    As at 2024-09-10, assuming your Athena workgroup is configured to use "Athena engine version 3",
 24    the following engines exist:
 25
 26    Hive:
 27     - Accepts mostly the same syntax as Hadoop / Hive
 28     - Uses backticks to quote identifiers
 29     - Has a distinctive DDL syntax (around things like setting table properties, storage locations etc)
 30       that is different from Trino
 31     - Used for *most* DDL, with some exceptions that get routed to the Trino engine instead:
 32        - CREATE [EXTERNAL] TABLE (without AS SELECT)
 33        - ALTER
 34        - DROP
 35
 36    Trino:
 37      - Uses double quotes to quote identifiers
 38      - Used for DDL operations that involve SELECT queries, eg:
 39        - CREATE VIEW / DROP VIEW
 40        - CREATE TABLE... AS SELECT
 41      - Used for DML operations
 42        - SELECT, INSERT, UPDATE, DELETE, MERGE
 43
 44    The SQLGlot Athena dialect tries to identify which engine a query would be routed to and then uses the
 45    tokenizer / parser / generator for that engine. This is unfortunately necessary, as there are certain
 46    incompatibilities between the engines' dialects and thus can't be handled by a single, unifying dialect.
 47
 48    References:
 49    - https://docs.aws.amazon.com/athena/latest/ug/ddl-reference.html
 50    - https://docs.aws.amazon.com/athena/latest/ug/dml-queries-functions-operators.html
 51    """
 52
 53    # This Tokenizer consumes a combination of HiveQL and Trino SQL and then processes the tokens
 54    # to disambiguate which dialect needs to be actually used in order to tokenize correctly.
 55    class Tokenizer(tokens.Tokenizer):
 56        IDENTIFIERS = Trino.Tokenizer.IDENTIFIERS + Hive.Tokenizer.IDENTIFIERS
 57        STRING_ESCAPES = Trino.Tokenizer.STRING_ESCAPES + Hive.Tokenizer.STRING_ESCAPES
 58        HEX_STRINGS = Trino.Tokenizer.HEX_STRINGS + Hive.Tokenizer.HEX_STRINGS
 59        UNICODE_STRINGS = Trino.Tokenizer.UNICODE_STRINGS + Hive.Tokenizer.UNICODE_STRINGS
 60
 61        NUMERIC_LITERALS = {
 62            **Trino.Tokenizer.NUMERIC_LITERALS,
 63            **Hive.Tokenizer.NUMERIC_LITERALS,
 64        }
 65
 66        KEYWORDS = {
 67            **Hive.Tokenizer.KEYWORDS,
 68            **Trino.Tokenizer.KEYWORDS,
 69            "UNLOAD": TokenType.COMMAND,
 70        }
 71
 72        def __init__(self, dialect: DialectType = None) -> None:
 73            super().__init__(dialect=dialect)
 74
 75            self._hive_tokenizer = Hive().tokenizer()
 76            self._trino_tokenizer = _TrinoTokenizer(Trino())
 77
 78        def tokenize(self, sql: str) -> list[Token]:
 79            # DML only needs Trino's tokenizer; possible Hive statements need disambiguation.
 80            try:
 81                trino_tokens: list[Token] | None = self._trino_tokenizer.tokenize(sql)
 82            except TokenError:
 83                trino_tokens = None
 84
 85            if trino_tokens is not None and not _may_be_hive(trino_tokens):
 86                return trino_tokens
 87
 88            try:
 89                tokens = super().tokenize(sql)
 90            except TokenError:
 91                # Combined rules can reject literals valid in only one engine.
 92                if trino_tokens is not None and not _tokenize_as_hive(trino_tokens):
 93                    return trino_tokens
 94
 95                try:
 96                    tokens = self._hive_tokenizer.tokenize(sql)
 97                except TokenError:
 98                    pass
 99                else:
100                    if _tokenize_as_hive(tokens):
101                        return [Token(TokenType.HIVE_TOKEN_STREAM, "")] + tokens
102                raise
103
104            if _tokenize_as_hive(tokens):
105                return [Token(TokenType.HIVE_TOKEN_STREAM, "")] + self._hive_tokenizer.tokenize(sql)
106
107            return trino_tokens if trino_tokens is not None else self._trino_tokenizer.tokenize(sql)
108
109    Parser = AthenaParser
110
111    Generator = AthenaGenerator

 55    class Tokenizer(tokens.Tokenizer):
 56        IDENTIFIERS = Trino.Tokenizer.IDENTIFIERS + Hive.Tokenizer.IDENTIFIERS
 57        STRING_ESCAPES = Trino.Tokenizer.STRING_ESCAPES + Hive.Tokenizer.STRING_ESCAPES
 58        HEX_STRINGS = Trino.Tokenizer.HEX_STRINGS + Hive.Tokenizer.HEX_STRINGS
 59        UNICODE_STRINGS = Trino.Tokenizer.UNICODE_STRINGS + Hive.Tokenizer.UNICODE_STRINGS
 60
 61        NUMERIC_LITERALS = {
 62            **Trino.Tokenizer.NUMERIC_LITERALS,
 63            **Hive.Tokenizer.NUMERIC_LITERALS,
 64        }
 65
 66        KEYWORDS = {
 67            **Hive.Tokenizer.KEYWORDS,
 68            **Trino.Tokenizer.KEYWORDS,
 69            "UNLOAD": TokenType.COMMAND,
 70        }
 71
 72        def __init__(self, dialect: DialectType = None) -> None:
 73            super().__init__(dialect=dialect)
 74
 75            self._hive_tokenizer = Hive().tokenizer()
 76            self._trino_tokenizer = _TrinoTokenizer(Trino())
 77
 78        def tokenize(self, sql: str) -> list[Token]:
 79            # DML only needs Trino's tokenizer; possible Hive statements need disambiguation.
 80            try:
 81                trino_tokens: list[Token] | None = self._trino_tokenizer.tokenize(sql)
 82            except TokenError:
 83                trino_tokens = None
 84
 85            if trino_tokens is not None and not _may_be_hive(trino_tokens):
 86                return trino_tokens
 87
 88            try:
 89                tokens = super().tokenize(sql)
 90            except TokenError:
 91                # Combined rules can reject literals valid in only one engine.
 92                if trino_tokens is not None and not _tokenize_as_hive(trino_tokens):
 93                    return trino_tokens
 94
 95                try:
 96                    tokens = self._hive_tokenizer.tokenize(sql)
 97                except TokenError:
 98                    pass
 99                else:
100                    if _tokenize_as_hive(tokens):
101                        return [Token(TokenType.HIVE_TOKEN_STREAM, "")] + tokens
102                raise
103
104            if _tokenize_as_hive(tokens):
105                return [Token(TokenType.HIVE_TOKEN_STREAM, "")] + self._hive_tokenizer.tokenize(sql)
106
107            return trino_tokens if trino_tokens is not None else self._trino_tokenizer.tokenize(sql)

 23class BigQuery(Dialect):
 24    WEEK_OFFSET = -1
 25    UNNEST_COLUMN_ONLY = True
 26    SUPPORTS_USER_DEFINED_TYPES = False
 27    LOG_BASE_FIRST = False
 28    HEX_LOWERCASE = True
 29    FORCE_EARLY_ALIAS_REF_EXPANSION = True
 30    EXPAND_ONLY_GROUP_ALIAS_REF = True
 31    ORIGINAL_NAME_META_KEY = "bigquery_name"
 32    HEX_STRING_IS_INTEGER_TYPE = True
 33    BYTE_STRING_IS_BYTES_TYPE = True
 34    UUID_IS_STRING_TYPE = True
 35    ANNOTATE_ALL_SCOPES = True
 36    PROJECTION_ALIASES_SHADOW_SOURCE_NAMES = True
 37    TABLES_REFERENCEABLE_AS_COLUMNS = True
 38    TABLE_ALIASES_SHADOW_COLUMN_NAMES = True
 39    SUPPORTS_STRUCT_STAR_EXPANSION = True
 40    EXCLUDES_PSEUDOCOLUMNS_FROM_STAR = True
 41    QUERY_RESULTS_ARE_STRUCTS = True
 42    JSON_EXTRACT_SCALAR_SCALAR_ONLY = True
 43    JSON_PATH_SINGLE_DOT_IS_WILDCARD = True
 44    LEAST_GREATEST_IGNORES_NULLS = False
 45    DEFAULT_NULL_TYPE = exp.DType.BIGINT
 46    PRIORITIZE_NON_LITERAL_TYPES = True
 47    ALIAS_POST_VERSION = False
 48
 49    # https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/string_functions#initcap
 50    INITCAP_DEFAULT_DELIMITER_CHARS = ' \t\n\r\f\v\\[\\](){}/|<>!?@"^#$&~_,.:;*%+\\-'
 51
 52    # https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#case_sensitivity
 53    NORMALIZATION_STRATEGY = NormalizationStrategy.CASE_INSENSITIVE_EXCEPT_TABLES
 54    ASCII_ONLY_NORMALIZATION = True
 55
 56    # bigquery udfs are case sensitive
 57    NORMALIZE_FUNCTIONS = False
 58
 59    # https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_elements_date_time
 60    TIME_MAPPING = {
 61        "%x": "%m/%d/%y",
 62        "%D": "%m/%d/%y",
 63        "%E6S": "%S.%f",
 64        "%e": "%-d",
 65        "%F": "%Y-%m-%d",
 66        "%T": "%H:%M:%S",
 67        "%c": "%a %b %e %H:%M:%S %Y",
 68    }
 69
 70    INVERSE_TIME_MAPPING = {
 71        # Preserve %E6S instead of expanding to %T.%f - since both %E6S & %T.%f are semantically different in BigQuery
 72        # %E6S is semantically different from %T.%f: %E6S works as a single atomic specifier for seconds with microseconds, while %T.%f expands incorrectly and fails to parse.
 73        "%H:%M:%S.%f": "%H:%M:%E6S",
 74    }
 75
 76    FORMAT_MAPPING = {
 77        "dd": "%d",
 78        "DD": "%d",
 79        "mm": "%m",
 80        "MM": "%m",
 81        "mon": "%b",
 82        "MON": "%b",
 83        "month": "%B",
 84        "MONTH": "%B",
 85        "yyyy": "%Y",
 86        "YYYY": "%Y",
 87        "yy": "%y",
 88        "YY": "%y",
 89        "HH": "%I",
 90        "HH12": "%I",
 91        "hh24": "%H",
 92        "HH24": "%H",
 93        "mi": "%M",
 94        "MI": "%M",
 95        "ss": "%S",
 96        "SS": "%S",
 97        "SSSSS": "%f",
 98        "tzh": "%z",
 99        "TZH": "%z",
100    }
101
102    # The _PARTITIONTIME and _PARTITIONDATE pseudo-columns are not returned by a SELECT * statement
103    # https://cloud.google.com/bigquery/docs/querying-partitioned-tables#query_an_ingestion-time_partitioned_table
104    # https://cloud.google.com/bigquery/docs/querying-wildcard-tables#scanning_a_range_of_tables_using_table_suffix
105    # https://cloud.google.com/bigquery/docs/query-cloud-storage-data#query_the_file_name_pseudo-column
106    PSEUDOCOLUMNS = {
107        "_PARTITIONTIME",
108        "_PARTITIONDATE",
109        "_TABLE_SUFFIX",
110        "_FILE_NAME",
111        "_DBT_MAX_PARTITION",
112    }
113
114    # All set operations require either a DISTINCT or ALL specifier
115    SET_OP_DISTINCT_BY_DEFAULT = dict.fromkeys((exp.Except, exp.Intersect, exp.Union), None)
116
117    # https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions#percentile_cont
118    COERCES_TO = {
119        **TypeAnnotator.COERCES_TO,
120        exp.DType.BIGDECIMAL: {exp.DType.DOUBLE},
121    }
122    COERCES_TO[exp.DType.DECIMAL] |= {exp.DType.BIGDECIMAL}
123    COERCES_TO[exp.DType.BIGINT] |= {exp.DType.BIGDECIMAL}
124    COERCES_TO[exp.DType.VARCHAR] |= {
125        exp.DType.DATE,
126        exp.DType.DATETIME,
127        exp.DType.TIME,
128        exp.DType.TIMESTAMP,
129        exp.DType.TIMESTAMPTZ,
130    }
131
132    EXPRESSION_METADATA = EXPRESSION_METADATA.copy()
133
134    class JSONPathTokenizer(jsonpath.JSONPathTokenizer):
135        VAR_TOKENS = {
136            *jsonpath.JSONPathTokenizer.VAR_TOKENS,
137            TokenType.DASH,
138            TokenType.NUMBER,
139        }
140
141    class Tokenizer(tokens.Tokenizer):
142        NUMERIC_ESCAPES = {
143            "x": (16, 2, 2, 0xFF),
144            "X": (16, 2, 2, 0xFF),
145            "u": (16, 4, 4, 0xFFFF),
146            "U": (16, 8, 8, 0x10FFFF),
147            "0": (8, 3, 3, 0xFF),
148        }
149        DROP_UNKNOWN_ESCAPES = True
150        QUOTES = ["'", '"', '"""', "'''"]
151        COMMENTS = ["--", "#", ("/*", "*/")]
152        IDENTIFIERS = ["`"]
153        IDENTIFIER_ESCAPES = ["\\"]
154        STRING_ESCAPES = ["\\"]
155
156        HEX_STRINGS = [("0x", ""), ("0X", "")]
157
158        BYTE_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("b", "B")]
159
160        RAW_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("r", "R")]
161
162        NESTED_COMMENTS = False
163
164        KEYWORDS = {
165            **tokens.Tokenizer.KEYWORDS,
166            "ANY TYPE": TokenType.VARIANT,
167            "BEGIN": TokenType.COMMAND,
168            "BEGIN TRANSACTION": TokenType.BEGIN,
169            "BYTEINT": TokenType.INT,
170            "BYTES": TokenType.BINARY,
171            "CURRENT_DATETIME": TokenType.CURRENT_DATETIME,
172            "DATETIME": TokenType.TIMESTAMP,
173            "DECLARE": TokenType.DECLARE,
174            "ELSEIF": TokenType.COMMAND,
175            "EXCEPTION": TokenType.COMMAND,
176            "EXPORT": TokenType.EXPORT,
177            "FLOAT64": TokenType.DOUBLE,
178            "LOOP": TokenType.COMMAND,
179            "MODEL": TokenType.MODEL,
180            "RECORD": TokenType.STRUCT,
181            "REPEAT": TokenType.COMMAND,
182            "TIMESTAMP": TokenType.TIMESTAMPTZ,
183            "WHILE": TokenType.COMMAND,
184        }
185        KEYWORDS.pop("ISNULL")
186        KEYWORDS.pop("DIV")
187        KEYWORDS.pop("VALUES")
188        KEYWORDS.pop("/*+")
189
190    Parser = BigQueryParser
191
192    Generator = BigQueryGenerator

134    class JSONPathTokenizer(jsonpath.JSONPathTokenizer):
135        VAR_TOKENS = {
136            *jsonpath.JSONPathTokenizer.VAR_TOKENS,
137            TokenType.DASH,
138            TokenType.NUMBER,
139        }

141    class Tokenizer(tokens.Tokenizer):
142        NUMERIC_ESCAPES = {
143            "x": (16, 2, 2, 0xFF),
144            "X": (16, 2, 2, 0xFF),
145            "u": (16, 4, 4, 0xFFFF),
146            "U": (16, 8, 8, 0x10FFFF),
147            "0": (8, 3, 3, 0xFF),
148        }
149        DROP_UNKNOWN_ESCAPES = True
150        QUOTES = ["'", '"', '"""', "'''"]
151        COMMENTS = ["--", "#", ("/*", "*/")]
152        IDENTIFIERS = ["`"]
153        IDENTIFIER_ESCAPES = ["\\"]
154        STRING_ESCAPES = ["\\"]
155
156        HEX_STRINGS = [("0x", ""), ("0X", "")]
157
158        BYTE_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("b", "B")]
159
160        RAW_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("r", "R")]
161
162        NESTED_COMMENTS = False
163
164        KEYWORDS = {
165            **tokens.Tokenizer.KEYWORDS,
166            "ANY TYPE": TokenType.VARIANT,
167            "BEGIN": TokenType.COMMAND,
168            "BEGIN TRANSACTION": TokenType.BEGIN,
169            "BYTEINT": TokenType.INT,
170            "BYTES": TokenType.BINARY,
171            "CURRENT_DATETIME": TokenType.CURRENT_DATETIME,
172            "DATETIME": TokenType.TIMESTAMP,
173            "DECLARE": TokenType.DECLARE,
174            "ELSEIF": TokenType.COMMAND,
175            "EXCEPTION": TokenType.COMMAND,
176            "EXPORT": TokenType.EXPORT,
177            "FLOAT64": TokenType.DOUBLE,
178            "LOOP": TokenType.COMMAND,
179            "MODEL": TokenType.MODEL,
180            "RECORD": TokenType.STRUCT,
181            "REPEAT": TokenType.COMMAND,
182            "TIMESTAMP": TokenType.TIMESTAMPTZ,
183            "WHILE": TokenType.COMMAND,
184        }
185        KEYWORDS.pop("ISNULL")
186        KEYWORDS.pop("DIV")
187        KEYWORDS.pop("VALUES")
188        KEYWORDS.pop("/*+")

 16class ClickHouse(Dialect):
 17    INDEX_OFFSET = 1
 18    NORMALIZE_FUNCTIONS: bool | str = False
 19    NULL_ORDERING = "nulls_are_last"
 20    SUPPORTS_USER_DEFINED_TYPES = False
 21    LOG_BASE_FIRST: bool | None = None
 22    FORCE_EARLY_ALIAS_REF_EXPANSION = True
 23    ORIGINAL_NAME_META_KEY = "clickhouse_name"
 24    NUMBERS_CAN_BE_UNDERSCORE_SEPARATED = True
 25    IDENTIFIERS_CAN_START_WITH_DIGIT = True
 26    HEX_STRING_IS_INTEGER_TYPE = True
 27
 28    # https://github.com/ClickHouse/ClickHouse/issues/33935#issue-1112165779
 29    NORMALIZATION_STRATEGY = NormalizationStrategy.CASE_SENSITIVE
 30
 31    EXPRESSION_METADATA = EXPRESSION_METADATA.copy()
 32
 33    UNESCAPED_SEQUENCES = {
 34        "\\0": "\0",
 35        "\\e": "\x1b",
 36        "\\N": "",
 37        "\\/": "/",
 38        '\\"': '"',
 39        "\\=": "=",
 40        "\\`": "`",
 41        **{"\\" + chr(i): chr(i) for i in range(1, 32)},
 42    }
 43
 44    CREATABLE_KIND_MAPPING = {"DATABASE": "SCHEMA"}
 45
 46    SET_OP_DISTINCT_BY_DEFAULT: dict[type[exp.Expr], bool | None] = {
 47        exp.Except: False,
 48        exp.Intersect: False,
 49        exp.Union: None,
 50    }
 51
 52    def generate_values_aliases(self, expression: exp.Values) -> list[exp.Identifier]:
 53        # Clickhouse allows VALUES to have an embedded structure e.g:
 54        # VALUES('person String, place String', ('Noah', 'Paris'), ...)
 55        # In this case, we don't want to qualify the columns
 56        values = expression.expressions[0].expressions
 57
 58        structure = (
 59            values[0]
 60            if (len(values) > 1 and values[0].is_string and isinstance(values[1], exp.Tuple))
 61            else None
 62        )
 63        if structure:
 64            # Split each column definition into the column name e.g:
 65            # 'person String, place String' -> ['person', 'place']
 66            structure_coldefs = [coldef.strip() for coldef in structure.name.split(",")]
 67            column_aliases = [
 68                exp.to_identifier(coldef.split(" ")[0]) for coldef in structure_coldefs
 69            ]
 70        else:
 71            # Default column aliases in CH are "c1", "c2", etc.
 72            column_aliases = [
 73                exp.to_identifier(f"c{i + 1}") for i in range(len(values[0].expressions))
 74            ]
 75
 76        return column_aliases
 77
 78    class Tokenizer(tokens.Tokenizer):
 79        NUMERIC_ESCAPES = {"x": (16, 2, 2, 0xFF)}
 80        NUMERIC_ESCAPES_ARE_BYTES = True
 81        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
 82        COMMENTS_TERMINATE_AT_NEWLINE_ONLY = True
 83        IDENTIFIERS = ['"', "`"]
 84        IDENTIFIER_ESCAPES = ["\\"]
 85        STRING_ESCAPES = ["'", "\\"]
 86        BIT_STRINGS = [("0b", "")]
 87        HEX_STRINGS = [("0x", ""), ("0X", "")]
 88        HEREDOC_STRINGS = ["$"]
 89
 90        KEYWORDS = {
 91            **tokens.Tokenizer.KEYWORDS,
 92            ".:": TokenType.DOTCOLON,
 93            ".^": TokenType.DOTCARET,
 94            "ATTACH": TokenType.COMMAND,
 95            "DATE32": TokenType.DATE32,
 96            "DETACH": TokenType.DETACH,
 97            "DATETIME64": TokenType.DATETIME64,
 98            "DICTIONARY": TokenType.DICTIONARY,
 99            "DYNAMIC": TokenType.DYNAMIC,
100            "ENUM8": TokenType.ENUM8,
101            "ENUM16": TokenType.ENUM16,
102            "EXCHANGE": TokenType.COMMAND,
103            "FINAL": TokenType.FINAL,
104            "FIXEDSTRING": TokenType.FIXEDSTRING,
105            "FLOAT32": TokenType.FLOAT,
106            "FLOAT64": TokenType.DOUBLE,
107            "GLOBAL": TokenType.GLOBAL,
108            "LOWCARDINALITY": TokenType.LOWCARDINALITY,
109            "MAP": TokenType.MAP,
110            "NESTED": TokenType.NESTED,
111            "NOTHING": TokenType.NOTHING,
112            "SAMPLE": TokenType.TABLE_SAMPLE,
113            "TUPLE": TokenType.STRUCT,
114            "UINT16": TokenType.USMALLINT,
115            "UINT32": TokenType.UINT,
116            "UINT64": TokenType.UBIGINT,
117            "UINT8": TokenType.UTINYINT,
118            "IPV4": TokenType.IPV4,
119            "IPV6": TokenType.IPV6,
120            "PASTE": TokenType.PASTE,
121            "POINT": TokenType.POINT,
122            "PROJECTION": TokenType.PROJECTION,
123            "RING": TokenType.RING,
124            "LINESTRING": TokenType.LINESTRING,
125            "MULTILINESTRING": TokenType.MULTILINESTRING,
126            "POLYGON": TokenType.POLYGON,
127            "MULTIPOLYGON": TokenType.MULTIPOLYGON,
128            "AGGREGATEFUNCTION": TokenType.AGGREGATEFUNCTION,
129            "SIMPLEAGGREGATEFUNCTION": TokenType.SIMPLEAGGREGATEFUNCTION,
130            "SYSTEM": TokenType.COMMAND,
131            "PREWHERE": TokenType.PREWHERE,
132        }
133
134        KEYWORDS.pop("/*+")
135
136        SINGLE_TOKENS = {
137            **tokens.Tokenizer.SINGLE_TOKENS,
138            "$": TokenType.HEREDOC_STRING,
139        }
140
141    Parser = ClickHouseParser
142
143    Generator = ClickHouseGenerator

 78    class Tokenizer(tokens.Tokenizer):
 79        NUMERIC_ESCAPES = {"x": (16, 2, 2, 0xFF)}
 80        NUMERIC_ESCAPES_ARE_BYTES = True
 81        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
 82        COMMENTS_TERMINATE_AT_NEWLINE_ONLY = True
 83        IDENTIFIERS = ['"', "`"]
 84        IDENTIFIER_ESCAPES = ["\\"]
 85        STRING_ESCAPES = ["'", "\\"]
 86        BIT_STRINGS = [("0b", "")]
 87        HEX_STRINGS = [("0x", ""), ("0X", "")]
 88        HEREDOC_STRINGS = ["$"]
 89
 90        KEYWORDS = {
 91            **tokens.Tokenizer.KEYWORDS,
 92            ".:": TokenType.DOTCOLON,
 93            ".^": TokenType.DOTCARET,
 94            "ATTACH": TokenType.COMMAND,
 95            "DATE32": TokenType.DATE32,
 96            "DETACH": TokenType.DETACH,
 97            "DATETIME64": TokenType.DATETIME64,
 98            "DICTIONARY": TokenType.DICTIONARY,
 99            "DYNAMIC": TokenType.DYNAMIC,
100            "ENUM8": TokenType.ENUM8,
101            "ENUM16": TokenType.ENUM16,
102            "EXCHANGE": TokenType.COMMAND,
103            "FINAL": TokenType.FINAL,
104            "FIXEDSTRING": TokenType.FIXEDSTRING,
105            "FLOAT32": TokenType.FLOAT,
106            "FLOAT64": TokenType.DOUBLE,
107            "GLOBAL": TokenType.GLOBAL,
108            "LOWCARDINALITY": TokenType.LOWCARDINALITY,
109            "MAP": TokenType.MAP,
110            "NESTED": TokenType.NESTED,
111            "NOTHING": TokenType.NOTHING,
112            "SAMPLE": TokenType.TABLE_SAMPLE,
113            "TUPLE": TokenType.STRUCT,
114            "UINT16": TokenType.USMALLINT,
115            "UINT32": TokenType.UINT,
116            "UINT64": TokenType.UBIGINT,
117            "UINT8": TokenType.UTINYINT,
118            "IPV4": TokenType.IPV4,
119            "IPV6": TokenType.IPV6,
120            "PASTE": TokenType.PASTE,
121            "POINT": TokenType.POINT,
122            "PROJECTION": TokenType.PROJECTION,
123            "RING": TokenType.RING,
124            "LINESTRING": TokenType.LINESTRING,
125            "MULTILINESTRING": TokenType.MULTILINESTRING,
126            "POLYGON": TokenType.POLYGON,
127            "MULTIPOLYGON": TokenType.MULTIPOLYGON,
128            "AGGREGATEFUNCTION": TokenType.AGGREGATEFUNCTION,
129            "SIMPLEAGGREGATEFUNCTION": TokenType.SIMPLEAGGREGATEFUNCTION,
130            "SYSTEM": TokenType.COMMAND,
131            "PREWHERE": TokenType.PREWHERE,
132        }
133
134        KEYWORDS.pop("/*+")
135
136        SINGLE_TOKENS = {
137            **tokens.Tokenizer.SINGLE_TOKENS,
138            "$": TokenType.HEREDOC_STRING,
139        }