15class Athena(Trino):
 16    """
 17    Over the years, it looks like AWS has taken various execution engines, bolted on AWS-specific
 18    modifications and then built the Athena service around them.
 19
 20    Thus, Athena is not simply hosted Trino, it's more like a router that routes SQL queries to an
 21    execution engine depending on the query type.
 22
 23    As at 2024-09-10, assuming your Athena workgroup is configured to use "Athena engine version 3",
 24    the following engines exist:
 25
 26    Hive:
 27     - Accepts mostly the same syntax as Hadoop / Hive
 28     - Uses backticks to quote identifiers
 29     - Has a distinctive DDL syntax (around things like setting table properties, storage locations etc)
 30       that is different from Trino
 31     - Used for *most* DDL, with some exceptions that get routed to the Trino engine instead:
 32        - CREATE [EXTERNAL] TABLE (without AS SELECT)
 33        - ALTER
 34        - DROP
 35
 36    Trino:
 37      - Uses double quotes to quote identifiers
 38      - Used for DDL operations that involve SELECT queries, eg:
 39        - CREATE VIEW / DROP VIEW
 40        - CREATE TABLE... AS SELECT
 41      - Used for DML operations
 42        - SELECT, INSERT, UPDATE, DELETE, MERGE
 43
 44    The SQLGlot Athena dialect tries to identify which engine a query would be routed to and then uses the
 45    tokenizer / parser / generator for that engine. This is unfortunately necessary, as there are certain
 46    incompatibilities between the engines' dialects and thus can't be handled by a single, unifying dialect.
 47
 48    References:
 49    - https://docs.aws.amazon.com/athena/latest/ug/ddl-reference.html
 50    - https://docs.aws.amazon.com/athena/latest/ug/dml-queries-functions-operators.html
 51    """
 52
 53    # This Tokenizer consumes a combination of HiveQL and Trino SQL and then processes the tokens
 54    # to disambiguate which dialect needs to be actually used in order to tokenize correctly.
 55    class Tokenizer(tokens.Tokenizer):
 56        IDENTIFIERS = Trino.Tokenizer.IDENTIFIERS + Hive.Tokenizer.IDENTIFIERS
 57        STRING_ESCAPES = Trino.Tokenizer.STRING_ESCAPES + Hive.Tokenizer.STRING_ESCAPES
 58        HEX_STRINGS = Trino.Tokenizer.HEX_STRINGS + Hive.Tokenizer.HEX_STRINGS
 59        UNICODE_STRINGS = Trino.Tokenizer.UNICODE_STRINGS + Hive.Tokenizer.UNICODE_STRINGS
 60
 61        NUMERIC_LITERALS = {
 62            **Trino.Tokenizer.NUMERIC_LITERALS,
 63            **Hive.Tokenizer.NUMERIC_LITERALS,
 64        }
 65
 66        KEYWORDS = {
 67            **Hive.Tokenizer.KEYWORDS,
 68            **Trino.Tokenizer.KEYWORDS,
 69            "UNLOAD": TokenType.COMMAND,
 70        }
 71
 72        def __init__(self, dialect: DialectType = None) -> None:
 73            super().__init__(dialect=dialect)
 74
 75            self._hive_tokenizer = Hive().tokenizer()
 76            self._trino_tokenizer = _TrinoTokenizer(Trino())
 77
 78        def tokenize(self, sql: str) -> list[Token]:
 79            # DML only needs Trino's tokenizer; possible Hive statements need disambiguation.
 80            try:
 81                trino_tokens: list[Token] | None = self._trino_tokenizer.tokenize(sql)
 82            except TokenError:
 83                trino_tokens = None
 84
 85            if trino_tokens is not None and not _may_be_hive(trino_tokens):
 86                return trino_tokens
 87
 88            try:
 89                tokens = super().tokenize(sql)
 90            except TokenError:
 91                # Combined rules can reject literals valid in only one engine.
 92                if trino_tokens is not None and not _tokenize_as_hive(trino_tokens):
 93                    return trino_tokens
 94
 95                try:
 96                    tokens = self._hive_tokenizer.tokenize(sql)
 97                except TokenError:
 98                    pass
 99                else:
100                    if _tokenize_as_hive(tokens):
101                        return [Token(TokenType.HIVE_TOKEN_STREAM, "")] + tokens
102                raise
103
104            if _tokenize_as_hive(tokens):
105                return [Token(TokenType.HIVE_TOKEN_STREAM, "")] + self._hive_tokenizer.tokenize(sql)
106
107            return trino_tokens if trino_tokens is not None else self._trino_tokenizer.tokenize(sql)
108
109    Parser = AthenaParser
110
111    Generator = AthenaGenerator

 55    class Tokenizer(tokens.Tokenizer):
 56        IDENTIFIERS = Trino.Tokenizer.IDENTIFIERS + Hive.Tokenizer.IDENTIFIERS
 57        STRING_ESCAPES = Trino.Tokenizer.STRING_ESCAPES + Hive.Tokenizer.STRING_ESCAPES
 58        HEX_STRINGS = Trino.Tokenizer.HEX_STRINGS + Hive.Tokenizer.HEX_STRINGS
 59        UNICODE_STRINGS = Trino.Tokenizer.UNICODE_STRINGS + Hive.Tokenizer.UNICODE_STRINGS
 60
 61        NUMERIC_LITERALS = {
 62            **Trino.Tokenizer.NUMERIC_LITERALS,
 63            **Hive.Tokenizer.NUMERIC_LITERALS,
 64        }
 65
 66        KEYWORDS = {
 67            **Hive.Tokenizer.KEYWORDS,
 68            **Trino.Tokenizer.KEYWORDS,
 69            "UNLOAD": TokenType.COMMAND,
 70        }
 71
 72        def __init__(self, dialect: DialectType = None) -> None:
 73            super().__init__(dialect=dialect)
 74
 75            self._hive_tokenizer = Hive().tokenizer()
 76            self._trino_tokenizer = _TrinoTokenizer(Trino())
 77
 78        def tokenize(self, sql: str) -> list[Token]:
 79            # DML only needs Trino's tokenizer; possible Hive statements need disambiguation.
 80            try:
 81                trino_tokens: list[Token] | None = self._trino_tokenizer.tokenize(sql)
 82            except TokenError:
 83                trino_tokens = None
 84
 85            if trino_tokens is not None and not _may_be_hive(trino_tokens):
 86                return trino_tokens
 87
 88            try:
 89                tokens = super().tokenize(sql)
 90            except TokenError:
 91                # Combined rules can reject literals valid in only one engine.
 92                if trino_tokens is not None and not _tokenize_as_hive(trino_tokens):
 93                    return trino_tokens
 94
 95                try:
 96                    tokens = self._hive_tokenizer.tokenize(sql)
 97                except TokenError:
 98                    pass
 99                else:
100                    if _tokenize_as_hive(tokens):
101                        return [Token(TokenType.HIVE_TOKEN_STREAM, "")] + tokens
102                raise
103
104            if _tokenize_as_hive(tokens):
105                return [Token(TokenType.HIVE_TOKEN_STREAM, "")] + self._hive_tokenizer.tokenize(sql)
106
107            return trino_tokens if trino_tokens is not None else self._trino_tokenizer.tokenize(sql)

 25class BigQuery(Dialect):
 26    WEEK_OFFSET = -1
 27    UNNEST_COLUMN_ONLY = True
 28    SUPPORTS_USER_DEFINED_TYPES = False
 29    LOG_BASE_FIRST = False
 30    HEX_LOWERCASE = True
 31    FORCE_EARLY_ALIAS_REF_EXPANSION = True
 32    EXPAND_ONLY_GROUP_ALIAS_REF = True
 33    ORIGINAL_NAME_META_KEY = "bigquery_name"
 34    HEX_STRING_IS_INTEGER_TYPE = True
 35    BYTE_STRING_IS_BYTES_TYPE = True
 36    UUID_IS_STRING_TYPE = True
 37    ANNOTATE_ALL_SCOPES = True
 38    PROJECTION_ALIASES_SHADOW_SOURCE_NAMES = True
 39    TABLES_REFERENCEABLE_AS_COLUMNS = True
 40    TABLE_ALIASES_SHADOW_COLUMN_NAMES = True
 41    SUPPORTS_STRUCT_STAR_EXPANSION = True
 42    EXCLUDES_PSEUDOCOLUMNS_FROM_STAR = True
 43    QUERY_RESULTS_ARE_STRUCTS = True
 44    JSON_EXTRACT_SCALAR_SCALAR_ONLY = True
 45    JSON_PATH_SINGLE_DOT_IS_WILDCARD = True
 46    LEAST_GREATEST_IGNORES_NULLS = False
 47    DEFAULT_NULL_TYPE = exp.DType.BIGINT
 48    PRIORITIZE_NON_LITERAL_TYPES = True
 49    ALIAS_POST_VERSION = False
 50
 51    # https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/string_functions#initcap
 52    INITCAP_DEFAULT_DELIMITER_CHARS = ' \t\n\r\f\v\\[\\](){}/|<>!?@"^#$&~_,.:;*%+\\-'
 53
 54    # https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#case_sensitivity
 55    NORMALIZATION_STRATEGY = NormalizationStrategy.CASE_INSENSITIVE
 56    ASCII_ONLY_NORMALIZATION = True
 57
 58    # bigquery udfs are case sensitive
 59    NORMALIZE_FUNCTIONS = False
 60
 61    # https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_elements_date_time
 62    TIME_MAPPING = {
 63        "%x": "%m/%d/%y",
 64        "%D": "%m/%d/%y",
 65        "%E6S": "%S.%f",
 66        "%e": "%-d",
 67        "%F": "%Y-%m-%d",
 68        "%T": "%H:%M:%S",
 69        "%c": "%a %b %e %H:%M:%S %Y",
 70    }
 71
 72    INVERSE_TIME_MAPPING = {
 73        # Preserve %E6S instead of expanding to %T.%f - since both %E6S & %T.%f are semantically different in BigQuery
 74        # %E6S is semantically different from %T.%f: %E6S works as a single atomic specifier for seconds with microseconds, while %T.%f expands incorrectly and fails to parse.
 75        "%H:%M:%S.%f": "%H:%M:%E6S",
 76    }
 77
 78    FORMAT_MAPPING = {
 79        "dd": "%d",
 80        "DD": "%d",
 81        "mm": "%m",
 82        "MM": "%m",
 83        "mon": "%b",
 84        "MON": "%b",
 85        "month": "%B",
 86        "MONTH": "%B",
 87        "yyyy": "%Y",
 88        "YYYY": "%Y",
 89        "yy": "%y",
 90        "YY": "%y",
 91        "HH": "%I",
 92        "HH12": "%I",
 93        "hh24": "%H",
 94        "HH24": "%H",
 95        "mi": "%M",
 96        "MI": "%M",
 97        "ss": "%S",
 98        "SS": "%S",
 99        "SSSSS": "%f",
100        "tzh": "%z",
101        "TZH": "%z",
102    }
103
104    # The _PARTITIONTIME and _PARTITIONDATE pseudo-columns are not returned by a SELECT * statement
105    # https://cloud.google.com/bigquery/docs/querying-partitioned-tables#query_an_ingestion-time_partitioned_table
106    # https://cloud.google.com/bigquery/docs/querying-wildcard-tables#scanning_a_range_of_tables_using_table_suffix
107    # https://cloud.google.com/bigquery/docs/query-cloud-storage-data#query_the_file_name_pseudo-column
108    PSEUDOCOLUMNS = {
109        "_PARTITIONTIME",
110        "_PARTITIONDATE",
111        "_TABLE_SUFFIX",
112        "_FILE_NAME",
113        "_DBT_MAX_PARTITION",
114    }
115
116    # All set operations require either a DISTINCT or ALL specifier
117    SET_OP_DISTINCT_BY_DEFAULT = dict.fromkeys((exp.Except, exp.Intersect, exp.Union), None)
118
119    # https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions#percentile_cont
120    COERCES_TO = {
121        **TypeAnnotator.COERCES_TO,
122        exp.DType.BIGDECIMAL: {exp.DType.DOUBLE},
123    }
124    COERCES_TO[exp.DType.DECIMAL] |= {exp.DType.BIGDECIMAL}
125    COERCES_TO[exp.DType.BIGINT] |= {exp.DType.BIGDECIMAL}
126    COERCES_TO[exp.DType.VARCHAR] |= {
127        exp.DType.DATE,
128        exp.DType.DATETIME,
129        exp.DType.TIME,
130        exp.DType.TIMESTAMP,
131        exp.DType.TIMESTAMPTZ,
132    }
133
134    EXPRESSION_METADATA = EXPRESSION_METADATA.copy()
135
136    def normalize_identifier(self, expression: E) -> E:
137        if (
138            isinstance(expression, exp.Identifier)
139            and self.normalization_strategy is NormalizationStrategy.CASE_INSENSITIVE
140        ):
141            parent = expression.parent
142            while isinstance(parent, exp.Dot):
143                parent = parent.parent
144
145            # In BigQuery, CTEs are case-insensitive, but UDF and table names are case-sensitive
146            # by default. The following check uses a heuristic to detect tables based on whether
147            # they are qualified. This should generally be correct, because tables in BigQuery
148            # must be qualified with at least a dataset, unless @@dataset_id is set.
149            case_sensitive = (
150                isinstance(parent, exp.UserDefinedFunction)
151                or (
152                    isinstance(parent, exp.Table)
153                    and parent.db
154                    and (parent.meta_get("quoted_table") or not parent.meta_get("maybe_column"))
155                )
156                or expression.meta_get("is_table")
157            )
158            if not case_sensitive:
159                expression.set("this", expression.this.translate(ASCII_LOWER))
160
161            return t.cast(E, expression)
162
163        return super().normalize_identifier(expression)
164
165    class JSONPathTokenizer(jsonpath.JSONPathTokenizer):
166        VAR_TOKENS = {
167            *jsonpath.JSONPathTokenizer.VAR_TOKENS,
168            TokenType.DASH,
169            TokenType.NUMBER,
170        }
171
172    class Tokenizer(tokens.Tokenizer):
173        NUMERIC_ESCAPES = {
174            "x": (16, 2, 2, 0xFF),
175            "X": (16, 2, 2, 0xFF),
176            "u": (16, 4, 4, 0xFFFF),
177            "U": (16, 8, 8, 0x10FFFF),
178            "0": (8, 3, 3, 0xFF),
179        }
180        DROP_UNKNOWN_ESCAPES = True
181        QUOTES = ["'", '"', '"""', "'''"]
182        COMMENTS = ["--", "#", ("/*", "*/")]
183        IDENTIFIERS = ["`"]
184        STRING_ESCAPES = ["\\"]
185
186        HEX_STRINGS = [("0x", ""), ("0X", "")]
187
188        BYTE_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("b", "B")]
189
190        RAW_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("r", "R")]
191
192        NESTED_COMMENTS = False
193
194        KEYWORDS = {
195            **tokens.Tokenizer.KEYWORDS,
196            "ANY TYPE": TokenType.VARIANT,
197            "BEGIN": TokenType.COMMAND,
198            "BEGIN TRANSACTION": TokenType.BEGIN,
199            "BYTEINT": TokenType.INT,
200            "BYTES": TokenType.BINARY,
201            "CURRENT_DATETIME": TokenType.CURRENT_DATETIME,
202            "DATETIME": TokenType.TIMESTAMP,
203            "DECLARE": TokenType.DECLARE,
204            "ELSEIF": TokenType.COMMAND,
205            "EXCEPTION": TokenType.COMMAND,
206            "EXPORT": TokenType.EXPORT,
207            "FLOAT64": TokenType.DOUBLE,
208            "LOOP": TokenType.COMMAND,
209            "MODEL": TokenType.MODEL,
210            "RECORD": TokenType.STRUCT,
211            "REPEAT": TokenType.COMMAND,
212            "TIMESTAMP": TokenType.TIMESTAMPTZ,
213            "WHILE": TokenType.COMMAND,
214        }
215        KEYWORDS.pop("ISNULL")
216        KEYWORDS.pop("DIV")
217        KEYWORDS.pop("VALUES")
218        KEYWORDS.pop("/*+")
219
220    Parser = BigQueryParser
221
222    Generator = BigQueryGenerator

165    class JSONPathTokenizer(jsonpath.JSONPathTokenizer):
166        VAR_TOKENS = {
167            *jsonpath.JSONPathTokenizer.VAR_TOKENS,
168            TokenType.DASH,
169            TokenType.NUMBER,
170        }

172    class Tokenizer(tokens.Tokenizer):
173        NUMERIC_ESCAPES = {
174            "x": (16, 2, 2, 0xFF),
175            "X": (16, 2, 2, 0xFF),
176            "u": (16, 4, 4, 0xFFFF),
177            "U": (16, 8, 8, 0x10FFFF),
178            "0": (8, 3, 3, 0xFF),
179        }
180        DROP_UNKNOWN_ESCAPES = True
181        QUOTES = ["'", '"', '"""', "'''"]
182        COMMENTS = ["--", "#", ("/*", "*/")]
183        IDENTIFIERS = ["`"]
184        STRING_ESCAPES = ["\\"]
185
186        HEX_STRINGS = [("0x", ""), ("0X", "")]
187
188        BYTE_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("b", "B")]
189
190        RAW_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("r", "R")]
191
192        NESTED_COMMENTS = False
193
194        KEYWORDS = {
195            **tokens.Tokenizer.KEYWORDS,
196            "ANY TYPE": TokenType.VARIANT,
197            "BEGIN": TokenType.COMMAND,
198            "BEGIN TRANSACTION": TokenType.BEGIN,
199            "BYTEINT": TokenType.INT,
200            "BYTES": TokenType.BINARY,
201            "CURRENT_DATETIME": TokenType.CURRENT_DATETIME,
202            "DATETIME": TokenType.TIMESTAMP,
203            "DECLARE": TokenType.DECLARE,
204            "ELSEIF": TokenType.COMMAND,
205            "EXCEPTION": TokenType.COMMAND,
206            "EXPORT": TokenType.EXPORT,
207            "FLOAT64": TokenType.DOUBLE,
208            "LOOP": TokenType.COMMAND,
209            "MODEL": TokenType.MODEL,
210            "RECORD": TokenType.STRUCT,
211            "REPEAT": TokenType.COMMAND,
212            "TIMESTAMP": TokenType.TIMESTAMPTZ,
213            "WHILE": TokenType.COMMAND,
214        }
215        KEYWORDS.pop("ISNULL")
216        KEYWORDS.pop("DIV")
217        KEYWORDS.pop("VALUES")
218        KEYWORDS.pop("/*+")

 16class ClickHouse(Dialect):
 17    INDEX_OFFSET = 1
 18    NORMALIZE_FUNCTIONS: bool | str = False
 19    NULL_ORDERING = "nulls_are_last"
 20    SUPPORTS_USER_DEFINED_TYPES = False
 21    LOG_BASE_FIRST: bool | None = None
 22    FORCE_EARLY_ALIAS_REF_EXPANSION = True
 23    ORIGINAL_NAME_META_KEY = "clickhouse_name"
 24    NUMBERS_CAN_BE_UNDERSCORE_SEPARATED = True
 25    IDENTIFIERS_CAN_START_WITH_DIGIT = True
 26    HEX_STRING_IS_INTEGER_TYPE = True
 27
 28    # https://github.com/ClickHouse/ClickHouse/issues/33935#issue-1112165779
 29    NORMALIZATION_STRATEGY = NormalizationStrategy.CASE_SENSITIVE
 30
 31    EXPRESSION_METADATA = EXPRESSION_METADATA.copy()
 32
 33    UNESCAPED_SEQUENCES = {
 34        "\\0": "\0",
 35        "\\e": "\x1b",
 36        "\\N": "",
 37        "\\/": "/",
 38        '\\"': '"',
 39        "\\=": "=",
 40        "\\`": "`",
 41        **{"\\" + chr(i): chr(i) for i in range(1, 32)},
 42    }
 43
 44    CREATABLE_KIND_MAPPING = {"DATABASE": "SCHEMA"}
 45
 46    SET_OP_DISTINCT_BY_DEFAULT: dict[type[exp.Expr], bool | None] = {
 47        exp.Except: False,
 48        exp.Intersect: False,
 49        exp.Union: None,
 50    }
 51
 52    def generate_values_aliases(self, expression: exp.Values) -> list[exp.Identifier]:
 53        # Clickhouse allows VALUES to have an embedded structure e.g:
 54        # VALUES('person String, place String', ('Noah', 'Paris'), ...)
 55        # In this case, we don't want to qualify the columns
 56        values = expression.expressions[0].expressions
 57
 58        structure = (
 59            values[0]
 60            if (len(values) > 1 and values[0].is_string and isinstance(values[1], exp.Tuple))
 61            else None
 62        )
 63        if structure:
 64            # Split each column definition into the column name e.g:
 65            # 'person String, place String' -> ['person', 'place']
 66            structure_coldefs = [coldef.strip() for coldef in structure.name.split(",")]
 67            column_aliases = [
 68                exp.to_identifier(coldef.split(" ")[0]) for coldef in structure_coldefs
 69            ]
 70        else:
 71            # Default column aliases in CH are "c1", "c2", etc.
 72            column_aliases = [
 73                exp.to_identifier(f"c{i + 1}") for i in range(len(values[0].expressions))
 74            ]
 75
 76        return column_aliases
 77
 78    class Tokenizer(tokens.Tokenizer):
 79        NUMERIC_ESCAPES = {"x": (16, 2, 2, 0xFF)}
 80        NUMERIC_ESCAPES_ARE_BYTES = True
 81        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
 82        COMMENTS_TERMINATE_AT_NEWLINE_ONLY = True
 83        IDENTIFIERS = ['"', "`"]
 84        IDENTIFIER_ESCAPES = ["\\"]
 85        STRING_ESCAPES = ["'", "\\"]
 86        BIT_STRINGS = [("0b", "")]
 87        HEX_STRINGS = [("0x", ""), ("0X", "")]
 88        HEREDOC_STRINGS = ["$"]
 89
 90        KEYWORDS = {
 91            **tokens.Tokenizer.KEYWORDS,
 92            ".:": TokenType.DOTCOLON,
 93            ".^": TokenType.DOTCARET,
 94            "ATTACH": TokenType.COMMAND,
 95            "DATE32": TokenType.DATE32,
 96            "DETACH": TokenType.DETACH,
 97            "DATETIME64": TokenType.DATETIME64,
 98            "DICTIONARY": TokenType.DICTIONARY,
 99            "DYNAMIC": TokenType.DYNAMIC,
100            "ENUM8": TokenType.ENUM8,
101            "ENUM16": TokenType.ENUM16,
102            "EXCHANGE": TokenType.COMMAND,
103            "FINAL": TokenType.FINAL,
104            "FIXEDSTRING": TokenType.FIXEDSTRING,
105            "FLOAT32": TokenType.FLOAT,
106            "FLOAT64": TokenType.DOUBLE,
107            "GLOBAL": TokenType.GLOBAL,
108            "LOWCARDINALITY": TokenType.LOWCARDINALITY,
109            "MAP": TokenType.MAP,
110            "NESTED": TokenType.NESTED,
111            "NOTHING": TokenType.NOTHING,
112            "SAMPLE": TokenType.TABLE_SAMPLE,
113            "TUPLE": TokenType.STRUCT,
114            "UINT16": TokenType.USMALLINT,
115            "UINT32": TokenType.UINT,
116            "UINT64": TokenType.UBIGINT,
117            "UINT8": TokenType.UTINYINT,
118            "IPV4": TokenType.IPV4,
119            "IPV6": TokenType.IPV6,
120            "PASTE": TokenType.PASTE,
121            "POINT": TokenType.POINT,
122            "PROJECTION": TokenType.PROJECTION,
123            "RING": TokenType.RING,
124            "LINESTRING": TokenType.LINESTRING,
125            "MULTILINESTRING": TokenType.MULTILINESTRING,
126            "POLYGON": TokenType.POLYGON,
127            "MULTIPOLYGON": TokenType.MULTIPOLYGON,
128            "AGGREGATEFUNCTION": TokenType.AGGREGATEFUNCTION,
129            "SIMPLEAGGREGATEFUNCTION": TokenType.SIMPLEAGGREGATEFUNCTION,
130            "SYSTEM": TokenType.COMMAND,
131            "PREWHERE": TokenType.PREWHERE,
132        }
133
134        KEYWORDS.pop("/*+")
135
136        SINGLE_TOKENS = {
137            **tokens.Tokenizer.SINGLE_TOKENS,
138            "$": TokenType.HEREDOC_STRING,
139        }
140
141    Parser = ClickHouseParser
142
143    Generator = ClickHouseGenerator