14class Athena(Trino):
15    """
16    Over the years, it looks like AWS has taken various execution engines, bolted on AWS-specific
17    modifications and then built the Athena service around them.
18
19    Thus, Athena is not simply hosted Trino, it's more like a router that routes SQL queries to an
20    execution engine depending on the query type.
21
22    As at 2024-09-10, assuming your Athena workgroup is configured to use "Athena engine version 3",
23    the following engines exist:
24
25    Hive:
26     - Accepts mostly the same syntax as Hadoop / Hive
27     - Uses backticks to quote identifiers
28     - Has a distinctive DDL syntax (around things like setting table properties, storage locations etc)
29       that is different from Trino
30     - Used for *most* DDL, with some exceptions that get routed to the Trino engine instead:
31        - CREATE [EXTERNAL] TABLE (without AS SELECT)
32        - ALTER
33        - DROP
34
35    Trino:
36      - Uses double quotes to quote identifiers
37      - Used for DDL operations that involve SELECT queries, eg:
38        - CREATE VIEW / DROP VIEW
39        - CREATE TABLE... AS SELECT
40      - Used for DML operations
41        - SELECT, INSERT, UPDATE, DELETE, MERGE
42
43    The SQLGlot Athena dialect tries to identify which engine a query would be routed to and then uses the
44    tokenizer / parser / generator for that engine. This is unfortunately necessary, as there are certain
45    incompatibilities between the engines' dialects and thus can't be handled by a single, unifying dialect.
46
47    References:
48    - https://docs.aws.amazon.com/athena/latest/ug/ddl-reference.html
49    - https://docs.aws.amazon.com/athena/latest/ug/dml-queries-functions-operators.html
50    """
51
52    # This Tokenizer consumes a combination of HiveQL and Trino SQL and then processes the tokens
53    # to disambiguate which dialect needs to be actually used in order to tokenize correctly.
54    class Tokenizer(tokens.Tokenizer):
55        IDENTIFIERS = Trino.Tokenizer.IDENTIFIERS + Hive.Tokenizer.IDENTIFIERS
56        STRING_ESCAPES = Trino.Tokenizer.STRING_ESCAPES + Hive.Tokenizer.STRING_ESCAPES
57        HEX_STRINGS = Trino.Tokenizer.HEX_STRINGS + Hive.Tokenizer.HEX_STRINGS
58        UNICODE_STRINGS = Trino.Tokenizer.UNICODE_STRINGS + Hive.Tokenizer.UNICODE_STRINGS
59
60        NUMERIC_LITERALS = {
61            **Trino.Tokenizer.NUMERIC_LITERALS,
62            **Hive.Tokenizer.NUMERIC_LITERALS,
63        }
64
65        KEYWORDS = {
66            **Hive.Tokenizer.KEYWORDS,
67            **Trino.Tokenizer.KEYWORDS,
68            "UNLOAD": TokenType.COMMAND,
69        }
70
71        def __init__(self, dialect: DialectType = None) -> None:
72            super().__init__(dialect=dialect)
73
74            self._hive_tokenizer = Hive().tokenizer()
75            self._trino_tokenizer = _TrinoTokenizer(Trino())
76
77        def tokenize(self, sql: str) -> list[Token]:
78            tokens = super().tokenize(sql)
79
80            if _tokenize_as_hive(tokens):
81                return [Token(TokenType.HIVE_TOKEN_STREAM, "")] + self._hive_tokenizer.tokenize(sql)
82
83            return self._trino_tokenizer.tokenize(sql)
84
85    Parser = AthenaParser
86
87    Generator = AthenaGenerator

54    class Tokenizer(tokens.Tokenizer):
55        IDENTIFIERS = Trino.Tokenizer.IDENTIFIERS + Hive.Tokenizer.IDENTIFIERS
56        STRING_ESCAPES = Trino.Tokenizer.STRING_ESCAPES + Hive.Tokenizer.STRING_ESCAPES
57        HEX_STRINGS = Trino.Tokenizer.HEX_STRINGS + Hive.Tokenizer.HEX_STRINGS
58        UNICODE_STRINGS = Trino.Tokenizer.UNICODE_STRINGS + Hive.Tokenizer.UNICODE_STRINGS
59
60        NUMERIC_LITERALS = {
61            **Trino.Tokenizer.NUMERIC_LITERALS,
62            **Hive.Tokenizer.NUMERIC_LITERALS,
63        }
64
65        KEYWORDS = {
66            **Hive.Tokenizer.KEYWORDS,
67            **Trino.Tokenizer.KEYWORDS,
68            "UNLOAD": TokenType.COMMAND,
69        }
70
71        def __init__(self, dialect: DialectType = None) -> None:
72            super().__init__(dialect=dialect)
73
74            self._hive_tokenizer = Hive().tokenizer()
75            self._trino_tokenizer = _TrinoTokenizer(Trino())
76
77        def tokenize(self, sql: str) -> list[Token]:
78            tokens = super().tokenize(sql)
79
80            if _tokenize_as_hive(tokens):
81                return [Token(TokenType.HIVE_TOKEN_STREAM, "")] + self._hive_tokenizer.tokenize(sql)
82
83            return self._trino_tokenizer.tokenize(sql)

 25class BigQuery(Dialect):
 26    WEEK_OFFSET = -1
 27    UNNEST_COLUMN_ONLY = True
 28    SUPPORTS_USER_DEFINED_TYPES = False
 29    LOG_BASE_FIRST = False
 30    HEX_LOWERCASE = True
 31    FORCE_EARLY_ALIAS_REF_EXPANSION = True
 32    EXPAND_ONLY_GROUP_ALIAS_REF = True
 33    ORIGINAL_NAME_META_KEY = "bigquery_name"
 34    HEX_STRING_IS_INTEGER_TYPE = True
 35    BYTE_STRING_IS_BYTES_TYPE = True
 36    UUID_IS_STRING_TYPE = True
 37    ANNOTATE_ALL_SCOPES = True
 38    PROJECTION_ALIASES_SHADOW_SOURCE_NAMES = True
 39    TABLES_REFERENCEABLE_AS_COLUMNS = True
 40    SUPPORTS_STRUCT_STAR_EXPANSION = True
 41    EXCLUDES_PSEUDOCOLUMNS_FROM_STAR = True
 42    QUERY_RESULTS_ARE_STRUCTS = True
 43    JSON_EXTRACT_SCALAR_SCALAR_ONLY = True
 44    JSON_PATH_SINGLE_DOT_IS_WILDCARD = True
 45    LEAST_GREATEST_IGNORES_NULLS = False
 46    DEFAULT_NULL_TYPE = exp.DType.BIGINT
 47    PRIORITIZE_NON_LITERAL_TYPES = True
 48    ALIAS_POST_VERSION = False
 49
 50    # https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/string_functions#initcap
 51    INITCAP_DEFAULT_DELIMITER_CHARS = ' \t\n\r\f\v\\[\\](){}/|<>!?@"^#$&~_,.:;*%+\\-'
 52
 53    # https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#case_sensitivity
 54    NORMALIZATION_STRATEGY = NormalizationStrategy.CASE_INSENSITIVE
 55    ASCII_ONLY_NORMALIZATION = True
 56
 57    # bigquery udfs are case sensitive
 58    NORMALIZE_FUNCTIONS = False
 59
 60    # https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_elements_date_time
 61    TIME_MAPPING = {
 62        "%x": "%m/%d/%y",
 63        "%D": "%m/%d/%y",
 64        "%E6S": "%S.%f",
 65        "%e": "%-d",
 66        "%F": "%Y-%m-%d",
 67        "%T": "%H:%M:%S",
 68        "%c": "%a %b %e %H:%M:%S %Y",
 69    }
 70
 71    INVERSE_TIME_MAPPING = {
 72        # Preserve %E6S instead of expanding to %T.%f - since both %E6S & %T.%f are semantically different in BigQuery
 73        # %E6S is semantically different from %T.%f: %E6S works as a single atomic specifier for seconds with microseconds, while %T.%f expands incorrectly and fails to parse.
 74        "%H:%M:%S.%f": "%H:%M:%E6S",
 75    }
 76
 77    FORMAT_MAPPING = {
 78        "dd": "%d",
 79        "DD": "%d",
 80        "mm": "%m",
 81        "MM": "%m",
 82        "mon": "%b",
 83        "MON": "%b",
 84        "month": "%B",
 85        "MONTH": "%B",
 86        "yyyy": "%Y",
 87        "YYYY": "%Y",
 88        "yy": "%y",
 89        "YY": "%y",
 90        "HH": "%I",
 91        "HH12": "%I",
 92        "hh24": "%H",
 93        "HH24": "%H",
 94        "mi": "%M",
 95        "MI": "%M",
 96        "ss": "%S",
 97        "SS": "%S",
 98        "SSSSS": "%f",
 99        "tzh": "%z",
100        "TZH": "%z",
101    }
102
103    # The _PARTITIONTIME and _PARTITIONDATE pseudo-columns are not returned by a SELECT * statement
104    # https://cloud.google.com/bigquery/docs/querying-partitioned-tables#query_an_ingestion-time_partitioned_table
105    # https://cloud.google.com/bigquery/docs/querying-wildcard-tables#scanning_a_range_of_tables_using_table_suffix
106    # https://cloud.google.com/bigquery/docs/query-cloud-storage-data#query_the_file_name_pseudo-column
107    PSEUDOCOLUMNS = {
108        "_PARTITIONTIME",
109        "_PARTITIONDATE",
110        "_TABLE_SUFFIX",
111        "_FILE_NAME",
112        "_DBT_MAX_PARTITION",
113    }
114
115    # All set operations require either a DISTINCT or ALL specifier
116    SET_OP_DISTINCT_BY_DEFAULT = dict.fromkeys((exp.Except, exp.Intersect, exp.Union), None)
117
118    # https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions#percentile_cont
119    COERCES_TO = {
120        **TypeAnnotator.COERCES_TO,
121        exp.DType.BIGDECIMAL: {exp.DType.DOUBLE},
122    }
123    COERCES_TO[exp.DType.DECIMAL] |= {exp.DType.BIGDECIMAL}
124    COERCES_TO[exp.DType.BIGINT] |= {exp.DType.BIGDECIMAL}
125    COERCES_TO[exp.DType.VARCHAR] |= {
126        exp.DType.DATE,
127        exp.DType.DATETIME,
128        exp.DType.TIME,
129        exp.DType.TIMESTAMP,
130        exp.DType.TIMESTAMPTZ,
131    }
132
133    EXPRESSION_METADATA = EXPRESSION_METADATA.copy()
134
135    def normalize_identifier(self, expression: E) -> E:
136        if (
137            isinstance(expression, exp.Identifier)
138            and self.normalization_strategy is NormalizationStrategy.CASE_INSENSITIVE
139        ):
140            parent = expression.parent
141            while isinstance(parent, exp.Dot):
142                parent = parent.parent
143
144            # In BigQuery, CTEs are case-insensitive, but UDF and table names are case-sensitive
145            # by default. The following check uses a heuristic to detect tables based on whether
146            # they are qualified. This should generally be correct, because tables in BigQuery
147            # must be qualified with at least a dataset, unless @@dataset_id is set.
148            case_sensitive = (
149                isinstance(parent, exp.UserDefinedFunction)
150                or (
151                    isinstance(parent, exp.Table)
152                    and parent.db
153                    and (parent.meta_get("quoted_table") or not parent.meta_get("maybe_column"))
154                )
155                or expression.meta_get("is_table")
156            )
157            if not case_sensitive:
158                expression.set("this", expression.this.translate(ASCII_LOWER))
159
160            return t.cast(E, expression)
161
162        return super().normalize_identifier(expression)
163
164    class JSONPathTokenizer(jsonpath.JSONPathTokenizer):
165        VAR_TOKENS = {
166            *jsonpath.JSONPathTokenizer.VAR_TOKENS,
167            TokenType.DASH,
168            TokenType.NUMBER,
169        }
170
171    class Tokenizer(tokens.Tokenizer):
172        NUMERIC_ESCAPES = {
173            "x": (16, 2, 2, 0xFF),
174            "X": (16, 2, 2, 0xFF),
175            "u": (16, 4, 4, 0xFFFF),
176            "U": (16, 8, 8, 0x10FFFF),
177            "0": (8, 3, 3, 0xFF),
178        }
179        DROP_UNKNOWN_ESCAPES = True
180        QUOTES = ["'", '"', '"""', "'''"]
181        COMMENTS = ["--", "#", ("/*", "*/")]
182        IDENTIFIERS = ["`"]
183        STRING_ESCAPES = ["\\"]
184
185        HEX_STRINGS = [("0x", ""), ("0X", "")]
186
187        BYTE_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("b", "B")]
188
189        RAW_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("r", "R")]
190
191        NESTED_COMMENTS = False
192
193        KEYWORDS = {
194            **tokens.Tokenizer.KEYWORDS,
195            "ANY TYPE": TokenType.VARIANT,
196            "BEGIN": TokenType.COMMAND,
197            "BEGIN TRANSACTION": TokenType.BEGIN,
198            "BYTEINT": TokenType.INT,
199            "BYTES": TokenType.BINARY,
200            "CURRENT_DATETIME": TokenType.CURRENT_DATETIME,
201            "DATETIME": TokenType.TIMESTAMP,
202            "DECLARE": TokenType.DECLARE,
203            "ELSEIF": TokenType.COMMAND,
204            "EXCEPTION": TokenType.COMMAND,
205            "EXPORT": TokenType.EXPORT,
206            "FLOAT64": TokenType.DOUBLE,
207            "LOOP": TokenType.COMMAND,
208            "MODEL": TokenType.MODEL,
209            "RECORD": TokenType.STRUCT,
210            "REPEAT": TokenType.COMMAND,
211            "TIMESTAMP": TokenType.TIMESTAMPTZ,
212            "WHILE": TokenType.COMMAND,
213        }
214        KEYWORDS.pop("DIV")
215        KEYWORDS.pop("VALUES")
216        KEYWORDS.pop("/*+")
217
218    Parser = BigQueryParser
219
220    Generator = BigQueryGenerator

164    class JSONPathTokenizer(jsonpath.JSONPathTokenizer):
165        VAR_TOKENS = {
166            *jsonpath.JSONPathTokenizer.VAR_TOKENS,
167            TokenType.DASH,
168            TokenType.NUMBER,
169        }

171    class Tokenizer(tokens.Tokenizer):
172        NUMERIC_ESCAPES = {
173            "x": (16, 2, 2, 0xFF),
174            "X": (16, 2, 2, 0xFF),
175            "u": (16, 4, 4, 0xFFFF),
176            "U": (16, 8, 8, 0x10FFFF),
177            "0": (8, 3, 3, 0xFF),
178        }
179        DROP_UNKNOWN_ESCAPES = True
180        QUOTES = ["'", '"', '"""', "'''"]
181        COMMENTS = ["--", "#", ("/*", "*/")]
182        IDENTIFIERS = ["`"]
183        STRING_ESCAPES = ["\\"]
184
185        HEX_STRINGS = [("0x", ""), ("0X", "")]
186
187        BYTE_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("b", "B")]
188
189        RAW_STRINGS = [(prefix + q, q) for q in t.cast(list[str], QUOTES) for prefix in ("r", "R")]
190
191        NESTED_COMMENTS = False
192
193        KEYWORDS = {
194            **tokens.Tokenizer.KEYWORDS,
195            "ANY TYPE": TokenType.VARIANT,
196            "BEGIN": TokenType.COMMAND,
197            "BEGIN TRANSACTION": TokenType.BEGIN,
198            "BYTEINT": TokenType.INT,
199            "BYTES": TokenType.BINARY,
200            "CURRENT_DATETIME": TokenType.CURRENT_DATETIME,
201            "DATETIME": TokenType.TIMESTAMP,
202            "DECLARE": TokenType.DECLARE,
203            "ELSEIF": TokenType.COMMAND,
204            "EXCEPTION": TokenType.COMMAND,
205            "EXPORT": TokenType.EXPORT,
206            "FLOAT64": TokenType.DOUBLE,
207            "LOOP": TokenType.COMMAND,
208            "MODEL": TokenType.MODEL,
209            "RECORD": TokenType.STRUCT,
210            "REPEAT": TokenType.COMMAND,
211            "TIMESTAMP": TokenType.TIMESTAMPTZ,
212            "WHILE": TokenType.COMMAND,
213        }
214        KEYWORDS.pop("DIV")
215        KEYWORDS.pop("VALUES")
216        KEYWORDS.pop("/*+")

 16class ClickHouse(Dialect):
 17    INDEX_OFFSET = 1
 18    NORMALIZE_FUNCTIONS: bool | str = False
 19    NULL_ORDERING = "nulls_are_last"
 20    SUPPORTS_USER_DEFINED_TYPES = False
 21    LOG_BASE_FIRST: bool | None = None
 22    FORCE_EARLY_ALIAS_REF_EXPANSION = True
 23    ORIGINAL_NAME_META_KEY = "clickhouse_name"
 24    NUMBERS_CAN_BE_UNDERSCORE_SEPARATED = True
 25    IDENTIFIERS_CAN_START_WITH_DIGIT = True
 26    HEX_STRING_IS_INTEGER_TYPE = True
 27
 28    # https://github.com/ClickHouse/ClickHouse/issues/33935#issue-1112165779
 29    NORMALIZATION_STRATEGY = NormalizationStrategy.CASE_SENSITIVE
 30
 31    EXPRESSION_METADATA = EXPRESSION_METADATA.copy()
 32
 33    UNESCAPED_SEQUENCES = {
 34        "\\0": "\0",
 35        "\\e": "\x1b",
 36        "\\N": "",
 37        "\\/": "/",
 38        '\\"': '"',
 39        "\\=": "=",
 40        "\\`": "`",
 41        **{"\\" + chr(i): chr(i) for i in range(1, 32)},
 42    }
 43
 44    CREATABLE_KIND_MAPPING = {"DATABASE": "SCHEMA"}
 45
 46    SET_OP_DISTINCT_BY_DEFAULT: dict[type[exp.Expr], bool | None] = {
 47        exp.Except: False,
 48        exp.Intersect: False,
 49        exp.Union: None,
 50    }
 51
 52    def generate_values_aliases(self, expression: exp.Values) -> list[exp.Identifier]:
 53        # Clickhouse allows VALUES to have an embedded structure e.g:
 54        # VALUES('person String, place String', ('Noah', 'Paris'), ...)
 55        # In this case, we don't want to qualify the columns
 56        values = expression.expressions[0].expressions
 57
 58        structure = (
 59            values[0]
 60            if (len(values) > 1 and values[0].is_string and isinstance(values[1], exp.Tuple))
 61            else None
 62        )
 63        if structure:
 64            # Split each column definition into the column name e.g:
 65            # 'person String, place String' -> ['person', 'place']
 66            structure_coldefs = [coldef.strip() for coldef in structure.name.split(",")]
 67            column_aliases = [
 68                exp.to_identifier(coldef.split(" ")[0]) for coldef in structure_coldefs
 69            ]
 70        else:
 71            # Default column aliases in CH are "c1", "c2", etc.
 72            column_aliases = [
 73                exp.to_identifier(f"c{i + 1}") for i in range(len(values[0].expressions))
 74            ]
 75
 76        return column_aliases
 77
 78    class Tokenizer(tokens.Tokenizer):
 79        NUMERIC_ESCAPES = {"x": (16, 2, 2, 0xFF)}
 80        NUMERIC_ESCAPES_ARE_BYTES = True
 81        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
 82        COMMENTS_TERMINATE_AT_NEWLINE_ONLY = True
 83        IDENTIFIERS = ['"', "`"]
 84        IDENTIFIER_ESCAPES = ["\\"]
 85        STRING_ESCAPES = ["'", "\\"]
 86        BIT_STRINGS = [("0b", "")]
 87        HEX_STRINGS = [("0x", ""), ("0X", "")]
 88        HEREDOC_STRINGS = ["$"]
 89
 90        KEYWORDS = {
 91            **tokens.Tokenizer.KEYWORDS,
 92            ".:": TokenType.DOTCOLON,
 93            ".^": TokenType.DOTCARET,
 94            "ATTACH": TokenType.COMMAND,
 95            "DATE32": TokenType.DATE32,
 96            "DETACH": TokenType.DETACH,
 97            "DATETIME64": TokenType.DATETIME64,
 98            "DICTIONARY": TokenType.DICTIONARY,
 99            "DYNAMIC": TokenType.DYNAMIC,
100            "ENUM8": TokenType.ENUM8,
101            "ENUM16": TokenType.ENUM16,
102            "EXCHANGE": TokenType.COMMAND,
103            "FINAL": TokenType.FINAL,
104            "FIXEDSTRING": TokenType.FIXEDSTRING,
105            "FLOAT32": TokenType.FLOAT,
106            "FLOAT64": TokenType.DOUBLE,
107            "GLOBAL": TokenType.GLOBAL,
108            "LOWCARDINALITY": TokenType.LOWCARDINALITY,
109            "MAP": TokenType.MAP,
110            "NESTED": TokenType.NESTED,
111            "NOTHING": TokenType.NOTHING,
112            "SAMPLE": TokenType.TABLE_SAMPLE,
113            "TUPLE": TokenType.STRUCT,
114            "UINT16": TokenType.USMALLINT,
115            "UINT32": TokenType.UINT,
116            "UINT64": TokenType.UBIGINT,
117            "UINT8": TokenType.UTINYINT,
118            "IPV4": TokenType.IPV4,
119            "IPV6": TokenType.IPV6,
120            "POINT": TokenType.POINT,
121            "PROJECTION": TokenType.PROJECTION,
122            "RING": TokenType.RING,
123            "LINESTRING": TokenType.LINESTRING,
124            "MULTILINESTRING": TokenType.MULTILINESTRING,
125            "POLYGON": TokenType.POLYGON,
126            "MULTIPOLYGON": TokenType.MULTIPOLYGON,
127            "AGGREGATEFUNCTION": TokenType.AGGREGATEFUNCTION,
128            "SIMPLEAGGREGATEFUNCTION": TokenType.SIMPLEAGGREGATEFUNCTION,
129            "SYSTEM": TokenType.COMMAND,
130            "PREWHERE": TokenType.PREWHERE,
131        }
132
133        KEYWORDS.pop("/*+")
134
135        SINGLE_TOKENS = {
136            **tokens.Tokenizer.SINGLE_TOKENS,
137            "$": TokenType.HEREDOC_STRING,
138        }
139
140    Parser = ClickHouseParser
141
142    Generator = ClickHouseGenerator

 78    class Tokenizer(tokens.Tokenizer):
 79        NUMERIC_ESCAPES = {"x": (16, 2, 2, 0xFF)}
 80        NUMERIC_ESCAPES_ARE_BYTES = True
 81        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
 82        COMMENTS_TERMINATE_AT_NEWLINE_ONLY = True
 83        IDENTIFIERS = ['"', "`"]
 84        IDENTIFIER_ESCAPES = ["\\"]
 85        STRING_ESCAPES = ["'", "\\"]
 86        BIT_STRINGS = [("0b", "")]
 87        HEX_STRINGS = [("0x", ""), ("0X", "")]
 88        HEREDOC_STRINGS = ["$"]
 89
 90        KEYWORDS = {
 91            **tokens.Tokenizer.KEYWORDS,
 92            ".:": TokenType.DOTCOLON,
 93            ".^": TokenType.DOTCARET,
 94            "ATTACH": TokenType.COMMAND,
 95            "DATE32": TokenType.DATE32,
 96            "DETACH": TokenType.DETACH,
 97            "DATETIME64": TokenType.DATETIME64,
 98            "DICTIONARY": TokenType.DICTIONARY,
 99            "DYNAMIC": TokenType.DYNAMIC,
100            "ENUM8": TokenType.ENUM8,
101            "ENUM16": TokenType.ENUM16,
102            "EXCHANGE": TokenType.COMMAND,
103            "FINAL": TokenType.FINAL,
104            "FIXEDSTRING": TokenType.FIXEDSTRING,
105            "FLOAT32": TokenType.FLOAT,
106            "FLOAT64": TokenType.DOUBLE,
107            "GLOBAL": TokenType.GLOBAL,
108            "LOWCARDINALITY": TokenType.LOWCARDINALITY,
109            "MAP": TokenType.MAP,
110            "NESTED": TokenType.NESTED,
111            "NOTHING": TokenType.NOTHING,
112            "SAMPLE": TokenType.TABLE_SAMPLE,
113            "TUPLE": TokenType.STRUCT,
114            "UINT16": TokenType.USMALLINT,
115            "UINT32": TokenType.UINT,
116            "UINT64": TokenType.UBIGINT,
117            "UINT8": TokenType.UTINYINT,
118            "IPV4": TokenType.IPV4,
119            "IPV6": TokenType.IPV6,
120            "POINT": TokenType.POINT,
121            "PROJECTION": TokenType.PROJECTION,
122            "RING": TokenType.RING,
123            "LINESTRING": TokenType.LINESTRING,
124            "MULTILINESTRING": TokenType.MULTILINESTRING,
125            "POLYGON": TokenType.POLYGON,
126            "MULTIPOLYGON": TokenType.MULTIPOLYGON,
127            "AGGREGATEFUNCTION": TokenType.AGGREGATEFUNCTION,
128            "SIMPLEAGGREGATEFUNCTION": TokenType.SIMPLEAGGREGATEFUNCTION,
129            "SYSTEM": TokenType.COMMAND,
130            "PREWHERE": TokenType.PREWHERE,
131        }
132
133        KEYWORDS.pop("/*+")
134
135        SINGLE_TOKENS = {
136            **tokens.Tokenizer.SINGLE_TOKENS,
137            "$": TokenType.HEREDOC_STRING,
138        }