sqlglot.dialects.hive
1from __future__ import annotations 2 3import typing as t 4 5from sqlglot import exp, generator, parser, tokens, transforms 6from sqlglot.dialects.dialect import ( 7 Dialect, 8 approx_count_distinct_sql, 9 create_with_partitions_sql, 10 format_time_lambda, 11 if_sql, 12 locate_to_strposition, 13 max_or_greatest, 14 min_or_least, 15 no_ilike_sql, 16 no_recursive_cte_sql, 17 no_safe_divide_sql, 18 no_trycast_sql, 19 rename_func, 20 strposition_to_locate_sql, 21 struct_extract_sql, 22 timestrtotime_sql, 23 var_map_sql, 24) 25from sqlglot.helper import seq_get 26from sqlglot.parser import parse_var_map 27from sqlglot.tokens import TokenType 28 29# (FuncType, Multiplier) 30DATE_DELTA_INTERVAL = { 31 "YEAR": ("ADD_MONTHS", 12), 32 "MONTH": ("ADD_MONTHS", 1), 33 "QUARTER": ("ADD_MONTHS", 3), 34 "WEEK": ("DATE_ADD", 7), 35 "DAY": ("DATE_ADD", 1), 36} 37 38TIME_DIFF_FACTOR = { 39 "MILLISECOND": " * 1000", 40 "SECOND": "", 41 "MINUTE": " / 60", 42 "HOUR": " / 3600", 43} 44 45DIFF_MONTH_SWITCH = ("YEAR", "QUARTER", "MONTH") 46 47 48def _add_date_sql(self: generator.Generator, expression: exp.DateAdd | exp.DateSub) -> str: 49 unit = expression.text("unit").upper() 50 func, multiplier = DATE_DELTA_INTERVAL.get(unit, ("DATE_ADD", 1)) 51 52 if isinstance(expression, exp.DateSub): 53 multiplier *= -1 54 55 if expression.expression.is_number: 56 modified_increment = exp.Literal.number(int(expression.text("expression")) * multiplier) 57 else: 58 modified_increment = expression.expression 59 if multiplier != 1: 60 modified_increment = exp.Mul( # type: ignore 61 this=modified_increment, expression=exp.Literal.number(multiplier) 62 ) 63 64 return self.func(func, expression.this, modified_increment) 65 66 67def _date_diff_sql(self: generator.Generator, expression: exp.DateDiff) -> str: 68 unit = expression.text("unit").upper() 69 70 factor = TIME_DIFF_FACTOR.get(unit) 71 if factor is not None: 72 left = self.sql(expression, "this") 73 right = self.sql(expression, "expression") 74 sec_diff = f"UNIX_TIMESTAMP({left}) - UNIX_TIMESTAMP({right})" 75 return f"({sec_diff}){factor}" if factor else sec_diff 76 77 sql_func = "MONTHS_BETWEEN" if unit in DIFF_MONTH_SWITCH else "DATEDIFF" 78 _, multiplier = DATE_DELTA_INTERVAL.get(unit, ("", 1)) 79 multiplier_sql = f" / {multiplier}" if multiplier > 1 else "" 80 diff_sql = f"{sql_func}({self.format_args(expression.this, expression.expression)})" 81 return f"{diff_sql}{multiplier_sql}" 82 83 84def _array_sort(self: generator.Generator, expression: exp.ArraySort) -> str: 85 if expression.expression: 86 self.unsupported("Hive SORT_ARRAY does not support a comparator") 87 return f"SORT_ARRAY({self.sql(expression, 'this')})" 88 89 90def _property_sql(self: generator.Generator, expression: exp.Property) -> str: 91 return f"'{expression.name}'={self.sql(expression, 'value')}" 92 93 94def _str_to_unix(self: generator.Generator, expression: exp.StrToUnix) -> str: 95 return self.func("UNIX_TIMESTAMP", expression.this, _time_format(self, expression)) 96 97 98def _str_to_date(self: generator.Generator, expression: exp.StrToDate) -> str: 99 this = self.sql(expression, "this") 100 time_format = self.format_time(expression) 101 if time_format not in (Hive.time_format, Hive.date_format): 102 this = f"FROM_UNIXTIME(UNIX_TIMESTAMP({this}, {time_format}))" 103 return f"CAST({this} AS DATE)" 104 105 106def _str_to_time(self: generator.Generator, expression: exp.StrToTime) -> str: 107 this = self.sql(expression, "this") 108 time_format = self.format_time(expression) 109 if time_format not in (Hive.time_format, Hive.date_format): 110 this = f"FROM_UNIXTIME(UNIX_TIMESTAMP({this}, {time_format}))" 111 return f"CAST({this} AS TIMESTAMP)" 112 113 114def _time_format( 115 self: generator.Generator, expression: exp.UnixToStr | exp.StrToUnix 116) -> t.Optional[str]: 117 time_format = self.format_time(expression) 118 if time_format == Hive.time_format: 119 return None 120 return time_format 121 122 123def _time_to_str(self: generator.Generator, expression: exp.TimeToStr) -> str: 124 this = self.sql(expression, "this") 125 time_format = self.format_time(expression) 126 return f"DATE_FORMAT({this}, {time_format})" 127 128 129def _to_date_sql(self: generator.Generator, expression: exp.TsOrDsToDate) -> str: 130 this = self.sql(expression, "this") 131 time_format = self.format_time(expression) 132 if time_format and time_format not in (Hive.time_format, Hive.date_format): 133 return f"TO_DATE({this}, {time_format})" 134 return f"TO_DATE({this})" 135 136 137def _index_sql(self: generator.Generator, expression: exp.Index) -> str: 138 this = self.sql(expression, "this") 139 table = self.sql(expression, "table") 140 columns = self.sql(expression, "columns") 141 return f"{this} ON TABLE {table} {columns}" 142 143 144class Hive(Dialect): 145 alias_post_tablesample = True 146 147 time_mapping = { 148 "y": "%Y", 149 "Y": "%Y", 150 "YYYY": "%Y", 151 "yyyy": "%Y", 152 "YY": "%y", 153 "yy": "%y", 154 "MMMM": "%B", 155 "MMM": "%b", 156 "MM": "%m", 157 "M": "%-m", 158 "dd": "%d", 159 "d": "%-d", 160 "HH": "%H", 161 "H": "%-H", 162 "hh": "%I", 163 "h": "%-I", 164 "mm": "%M", 165 "m": "%-M", 166 "ss": "%S", 167 "s": "%-S", 168 "SSSSSS": "%f", 169 "a": "%p", 170 "DD": "%j", 171 "D": "%-j", 172 "E": "%a", 173 "EE": "%a", 174 "EEE": "%a", 175 "EEEE": "%A", 176 } 177 178 date_format = "'yyyy-MM-dd'" 179 dateint_format = "'yyyyMMdd'" 180 time_format = "'yyyy-MM-dd HH:mm:ss'" 181 182 class Tokenizer(tokens.Tokenizer): 183 QUOTES = ["'", '"'] 184 IDENTIFIERS = ["`"] 185 STRING_ESCAPES = ["\\"] 186 ENCODE = "utf-8" 187 IDENTIFIER_CAN_START_WITH_DIGIT = True 188 189 KEYWORDS = { 190 **tokens.Tokenizer.KEYWORDS, 191 "ADD ARCHIVE": TokenType.COMMAND, 192 "ADD ARCHIVES": TokenType.COMMAND, 193 "ADD FILE": TokenType.COMMAND, 194 "ADD FILES": TokenType.COMMAND, 195 "ADD JAR": TokenType.COMMAND, 196 "ADD JARS": TokenType.COMMAND, 197 "MSCK REPAIR": TokenType.COMMAND, 198 "WITH SERDEPROPERTIES": TokenType.SERDE_PROPERTIES, 199 } 200 201 NUMERIC_LITERALS = { 202 "L": "BIGINT", 203 "S": "SMALLINT", 204 "Y": "TINYINT", 205 "D": "DOUBLE", 206 "F": "FLOAT", 207 "BD": "DECIMAL", 208 } 209 210 class Parser(parser.Parser): 211 LOG_DEFAULTS_TO_LN = True 212 STRICT_CAST = False 213 214 FUNCTIONS = { 215 **parser.Parser.FUNCTIONS, # type: ignore 216 "APPROX_COUNT_DISTINCT": exp.ApproxDistinct.from_arg_list, 217 "COLLECT_LIST": exp.ArrayAgg.from_arg_list, 218 "DATE_ADD": lambda args: exp.TsOrDsAdd( 219 this=seq_get(args, 0), 220 expression=seq_get(args, 1), 221 unit=exp.Literal.string("DAY"), 222 ), 223 "DATEDIFF": lambda args: exp.DateDiff( 224 this=exp.TsOrDsToDate(this=seq_get(args, 0)), 225 expression=exp.TsOrDsToDate(this=seq_get(args, 1)), 226 ), 227 "DATE_SUB": lambda args: exp.TsOrDsAdd( 228 this=seq_get(args, 0), 229 expression=exp.Mul( 230 this=seq_get(args, 1), 231 expression=exp.Literal.number(-1), 232 ), 233 unit=exp.Literal.string("DAY"), 234 ), 235 "DATE_FORMAT": lambda args: format_time_lambda(exp.TimeToStr, "hive")( 236 [ 237 exp.TimeStrToTime(this=seq_get(args, 0)), 238 seq_get(args, 1), 239 ] 240 ), 241 "DAY": lambda args: exp.Day(this=exp.TsOrDsToDate(this=seq_get(args, 0))), 242 "FROM_UNIXTIME": format_time_lambda(exp.UnixToStr, "hive", True), 243 "GET_JSON_OBJECT": exp.JSONExtractScalar.from_arg_list, 244 "LOCATE": locate_to_strposition, 245 "MAP": parse_var_map, 246 "MONTH": lambda args: exp.Month(this=exp.TsOrDsToDate.from_arg_list(args)), 247 "PERCENTILE": exp.Quantile.from_arg_list, 248 "PERCENTILE_APPROX": exp.ApproxQuantile.from_arg_list, 249 "COLLECT_SET": exp.SetAgg.from_arg_list, 250 "SIZE": exp.ArraySize.from_arg_list, 251 "SPLIT": exp.RegexpSplit.from_arg_list, 252 "TO_DATE": format_time_lambda(exp.TsOrDsToDate, "hive"), 253 "TO_JSON": exp.JSONFormat.from_arg_list, 254 "UNIX_TIMESTAMP": format_time_lambda(exp.StrToUnix, "hive", True), 255 "YEAR": lambda args: exp.Year(this=exp.TsOrDsToDate.from_arg_list(args)), 256 } 257 258 PROPERTY_PARSERS = { 259 **parser.Parser.PROPERTY_PARSERS, # type: ignore 260 "WITH SERDEPROPERTIES": lambda self: exp.SerdeProperties( 261 expressions=self._parse_wrapped_csv(self._parse_property) 262 ), 263 } 264 265 class Generator(generator.Generator): 266 LIMIT_FETCH = "LIMIT" 267 TABLESAMPLE_WITH_METHOD = False 268 TABLESAMPLE_SIZE_IS_PERCENT = True 269 JOIN_HINTS = False 270 TABLE_HINTS = False 271 272 TYPE_MAPPING = { 273 **generator.Generator.TYPE_MAPPING, # type: ignore 274 exp.DataType.Type.TEXT: "STRING", 275 exp.DataType.Type.DATETIME: "TIMESTAMP", 276 exp.DataType.Type.VARBINARY: "BINARY", 277 exp.DataType.Type.TIMESTAMPTZ: "TIMESTAMP", 278 exp.DataType.Type.BIT: "BOOLEAN", 279 } 280 281 TRANSFORMS = { 282 **generator.Generator.TRANSFORMS, # type: ignore 283 **transforms.UNALIAS_GROUP, # type: ignore 284 **transforms.ELIMINATE_QUALIFY, # type: ignore 285 exp.Select: transforms.preprocess( 286 [transforms.eliminate_qualify, transforms.unnest_to_explode] 287 ), 288 exp.Property: _property_sql, 289 exp.ApproxDistinct: approx_count_distinct_sql, 290 exp.ArrayConcat: rename_func("CONCAT"), 291 exp.ArraySize: rename_func("SIZE"), 292 exp.ArraySort: _array_sort, 293 exp.With: no_recursive_cte_sql, 294 exp.DateAdd: _add_date_sql, 295 exp.DateDiff: _date_diff_sql, 296 exp.DateStrToDate: rename_func("TO_DATE"), 297 exp.DateSub: _add_date_sql, 298 exp.DateToDi: lambda self, e: f"CAST(DATE_FORMAT({self.sql(e, 'this')}, {Hive.dateint_format}) AS INT)", 299 exp.DiToDate: lambda self, e: f"TO_DATE(CAST({self.sql(e, 'this')} AS STRING), {Hive.dateint_format})", 300 exp.FileFormatProperty: lambda self, e: f"STORED AS {self.sql(e, 'this') if isinstance(e.this, exp.InputOutputFormat) else e.name.upper()}", 301 exp.If: if_sql, 302 exp.Index: _index_sql, 303 exp.ILike: no_ilike_sql, 304 exp.JSONExtract: rename_func("GET_JSON_OBJECT"), 305 exp.JSONExtractScalar: rename_func("GET_JSON_OBJECT"), 306 exp.JSONFormat: rename_func("TO_JSON"), 307 exp.Map: var_map_sql, 308 exp.Max: max_or_greatest, 309 exp.Min: min_or_least, 310 exp.VarMap: var_map_sql, 311 exp.Create: create_with_partitions_sql, 312 exp.Quantile: rename_func("PERCENTILE"), 313 exp.ApproxQuantile: rename_func("PERCENTILE_APPROX"), 314 exp.RegexpLike: lambda self, e: self.binary(e, "RLIKE"), 315 exp.RegexpSplit: rename_func("SPLIT"), 316 exp.SafeDivide: no_safe_divide_sql, 317 exp.SchemaCommentProperty: lambda self, e: self.naked_property(e), 318 exp.SetAgg: rename_func("COLLECT_SET"), 319 exp.Split: lambda self, e: f"SPLIT({self.sql(e, 'this')}, CONCAT('\\\\Q', {self.sql(e, 'expression')}))", 320 exp.StrPosition: strposition_to_locate_sql, 321 exp.StrToDate: _str_to_date, 322 exp.StrToTime: _str_to_time, 323 exp.StrToUnix: _str_to_unix, 324 exp.StructExtract: struct_extract_sql, 325 exp.TableFormatProperty: lambda self, e: f"USING {self.sql(e, 'this')}", 326 exp.TimeStrToDate: rename_func("TO_DATE"), 327 exp.TimeStrToTime: timestrtotime_sql, 328 exp.TimeStrToUnix: rename_func("UNIX_TIMESTAMP"), 329 exp.TimeToStr: _time_to_str, 330 exp.TimeToUnix: rename_func("UNIX_TIMESTAMP"), 331 exp.TsOrDiToDi: lambda self, e: f"CAST(SUBSTR(REPLACE(CAST({self.sql(e, 'this')} AS STRING), '-', ''), 1, 8) AS INT)", 332 exp.TsOrDsAdd: lambda self, e: f"DATE_ADD({self.sql(e, 'this')}, {self.sql(e, 'expression')})", 333 exp.TsOrDsToDate: _to_date_sql, 334 exp.TryCast: no_trycast_sql, 335 exp.UnixToStr: lambda self, e: self.func( 336 "FROM_UNIXTIME", e.this, _time_format(self, e) 337 ), 338 exp.UnixToTime: rename_func("FROM_UNIXTIME"), 339 exp.UnixToTimeStr: rename_func("FROM_UNIXTIME"), 340 exp.PartitionedByProperty: lambda self, e: f"PARTITIONED BY {self.sql(e, 'this')}", 341 exp.RowFormatSerdeProperty: lambda self, e: f"ROW FORMAT SERDE {self.sql(e, 'this')}", 342 exp.SerdeProperties: lambda self, e: self.properties(e, prefix="WITH SERDEPROPERTIES"), 343 exp.NumberToStr: rename_func("FORMAT_NUMBER"), 344 exp.LastDateOfMonth: rename_func("LAST_DAY"), 345 exp.National: lambda self, e: self.sql(e, "this"), 346 } 347 348 PROPERTIES_LOCATION = { 349 **generator.Generator.PROPERTIES_LOCATION, # type: ignore 350 exp.FileFormatProperty: exp.Properties.Location.POST_SCHEMA, 351 exp.PartitionedByProperty: exp.Properties.Location.POST_SCHEMA, 352 exp.TableFormatProperty: exp.Properties.Location.POST_SCHEMA, 353 exp.VolatileProperty: exp.Properties.Location.UNSUPPORTED, 354 } 355 356 def arrayagg_sql(self, expression: exp.ArrayAgg) -> str: 357 return self.func( 358 "COLLECT_LIST", 359 expression.this.this if isinstance(expression.this, exp.Order) else expression.this, 360 ) 361 362 def with_properties(self, properties: exp.Properties) -> str: 363 return self.properties( 364 properties, 365 prefix=self.seg("TBLPROPERTIES"), 366 ) 367 368 def datatype_sql(self, expression: exp.DataType) -> str: 369 if ( 370 expression.this in (exp.DataType.Type.VARCHAR, exp.DataType.Type.NVARCHAR) 371 and not expression.expressions 372 ): 373 expression = exp.DataType.build("text") 374 elif expression.this in exp.DataType.TEMPORAL_TYPES: 375 expression = exp.DataType.build(expression.this) 376 377 return super().datatype_sql(expression)
145class Hive(Dialect): 146 alias_post_tablesample = True 147 148 time_mapping = { 149 "y": "%Y", 150 "Y": "%Y", 151 "YYYY": "%Y", 152 "yyyy": "%Y", 153 "YY": "%y", 154 "yy": "%y", 155 "MMMM": "%B", 156 "MMM": "%b", 157 "MM": "%m", 158 "M": "%-m", 159 "dd": "%d", 160 "d": "%-d", 161 "HH": "%H", 162 "H": "%-H", 163 "hh": "%I", 164 "h": "%-I", 165 "mm": "%M", 166 "m": "%-M", 167 "ss": "%S", 168 "s": "%-S", 169 "SSSSSS": "%f", 170 "a": "%p", 171 "DD": "%j", 172 "D": "%-j", 173 "E": "%a", 174 "EE": "%a", 175 "EEE": "%a", 176 "EEEE": "%A", 177 } 178 179 date_format = "'yyyy-MM-dd'" 180 dateint_format = "'yyyyMMdd'" 181 time_format = "'yyyy-MM-dd HH:mm:ss'" 182 183 class Tokenizer(tokens.Tokenizer): 184 QUOTES = ["'", '"'] 185 IDENTIFIERS = ["`"] 186 STRING_ESCAPES = ["\\"] 187 ENCODE = "utf-8" 188 IDENTIFIER_CAN_START_WITH_DIGIT = True 189 190 KEYWORDS = { 191 **tokens.Tokenizer.KEYWORDS, 192 "ADD ARCHIVE": TokenType.COMMAND, 193 "ADD ARCHIVES": TokenType.COMMAND, 194 "ADD FILE": TokenType.COMMAND, 195 "ADD FILES": TokenType.COMMAND, 196 "ADD JAR": TokenType.COMMAND, 197 "ADD JARS": TokenType.COMMAND, 198 "MSCK REPAIR": TokenType.COMMAND, 199 "WITH SERDEPROPERTIES": TokenType.SERDE_PROPERTIES, 200 } 201 202 NUMERIC_LITERALS = { 203 "L": "BIGINT", 204 "S": "SMALLINT", 205 "Y": "TINYINT", 206 "D": "DOUBLE", 207 "F": "FLOAT", 208 "BD": "DECIMAL", 209 } 210 211 class Parser(parser.Parser): 212 LOG_DEFAULTS_TO_LN = True 213 STRICT_CAST = False 214 215 FUNCTIONS = { 216 **parser.Parser.FUNCTIONS, # type: ignore 217 "APPROX_COUNT_DISTINCT": exp.ApproxDistinct.from_arg_list, 218 "COLLECT_LIST": exp.ArrayAgg.from_arg_list, 219 "DATE_ADD": lambda args: exp.TsOrDsAdd( 220 this=seq_get(args, 0), 221 expression=seq_get(args, 1), 222 unit=exp.Literal.string("DAY"), 223 ), 224 "DATEDIFF": lambda args: exp.DateDiff( 225 this=exp.TsOrDsToDate(this=seq_get(args, 0)), 226 expression=exp.TsOrDsToDate(this=seq_get(args, 1)), 227 ), 228 "DATE_SUB": lambda args: exp.TsOrDsAdd( 229 this=seq_get(args, 0), 230 expression=exp.Mul( 231 this=seq_get(args, 1), 232 expression=exp.Literal.number(-1), 233 ), 234 unit=exp.Literal.string("DAY"), 235 ), 236 "DATE_FORMAT": lambda args: format_time_lambda(exp.TimeToStr, "hive")( 237 [ 238 exp.TimeStrToTime(this=seq_get(args, 0)), 239 seq_get(args, 1), 240 ] 241 ), 242 "DAY": lambda args: exp.Day(this=exp.TsOrDsToDate(this=seq_get(args, 0))), 243 "FROM_UNIXTIME": format_time_lambda(exp.UnixToStr, "hive", True), 244 "GET_JSON_OBJECT": exp.JSONExtractScalar.from_arg_list, 245 "LOCATE": locate_to_strposition, 246 "MAP": parse_var_map, 247 "MONTH": lambda args: exp.Month(this=exp.TsOrDsToDate.from_arg_list(args)), 248 "PERCENTILE": exp.Quantile.from_arg_list, 249 "PERCENTILE_APPROX": exp.ApproxQuantile.from_arg_list, 250 "COLLECT_SET": exp.SetAgg.from_arg_list, 251 "SIZE": exp.ArraySize.from_arg_list, 252 "SPLIT": exp.RegexpSplit.from_arg_list, 253 "TO_DATE": format_time_lambda(exp.TsOrDsToDate, "hive"), 254 "TO_JSON": exp.JSONFormat.from_arg_list, 255 "UNIX_TIMESTAMP": format_time_lambda(exp.StrToUnix, "hive", True), 256 "YEAR": lambda args: exp.Year(this=exp.TsOrDsToDate.from_arg_list(args)), 257 } 258 259 PROPERTY_PARSERS = { 260 **parser.Parser.PROPERTY_PARSERS, # type: ignore 261 "WITH SERDEPROPERTIES": lambda self: exp.SerdeProperties( 262 expressions=self._parse_wrapped_csv(self._parse_property) 263 ), 264 } 265 266 class Generator(generator.Generator): 267 LIMIT_FETCH = "LIMIT" 268 TABLESAMPLE_WITH_METHOD = False 269 TABLESAMPLE_SIZE_IS_PERCENT = True 270 JOIN_HINTS = False 271 TABLE_HINTS = False 272 273 TYPE_MAPPING = { 274 **generator.Generator.TYPE_MAPPING, # type: ignore 275 exp.DataType.Type.TEXT: "STRING", 276 exp.DataType.Type.DATETIME: "TIMESTAMP", 277 exp.DataType.Type.VARBINARY: "BINARY", 278 exp.DataType.Type.TIMESTAMPTZ: "TIMESTAMP", 279 exp.DataType.Type.BIT: "BOOLEAN", 280 } 281 282 TRANSFORMS = { 283 **generator.Generator.TRANSFORMS, # type: ignore 284 **transforms.UNALIAS_GROUP, # type: ignore 285 **transforms.ELIMINATE_QUALIFY, # type: ignore 286 exp.Select: transforms.preprocess( 287 [transforms.eliminate_qualify, transforms.unnest_to_explode] 288 ), 289 exp.Property: _property_sql, 290 exp.ApproxDistinct: approx_count_distinct_sql, 291 exp.ArrayConcat: rename_func("CONCAT"), 292 exp.ArraySize: rename_func("SIZE"), 293 exp.ArraySort: _array_sort, 294 exp.With: no_recursive_cte_sql, 295 exp.DateAdd: _add_date_sql, 296 exp.DateDiff: _date_diff_sql, 297 exp.DateStrToDate: rename_func("TO_DATE"), 298 exp.DateSub: _add_date_sql, 299 exp.DateToDi: lambda self, e: f"CAST(DATE_FORMAT({self.sql(e, 'this')}, {Hive.dateint_format}) AS INT)", 300 exp.DiToDate: lambda self, e: f"TO_DATE(CAST({self.sql(e, 'this')} AS STRING), {Hive.dateint_format})", 301 exp.FileFormatProperty: lambda self, e: f"STORED AS {self.sql(e, 'this') if isinstance(e.this, exp.InputOutputFormat) else e.name.upper()}", 302 exp.If: if_sql, 303 exp.Index: _index_sql, 304 exp.ILike: no_ilike_sql, 305 exp.JSONExtract: rename_func("GET_JSON_OBJECT"), 306 exp.JSONExtractScalar: rename_func("GET_JSON_OBJECT"), 307 exp.JSONFormat: rename_func("TO_JSON"), 308 exp.Map: var_map_sql, 309 exp.Max: max_or_greatest, 310 exp.Min: min_or_least, 311 exp.VarMap: var_map_sql, 312 exp.Create: create_with_partitions_sql, 313 exp.Quantile: rename_func("PERCENTILE"), 314 exp.ApproxQuantile: rename_func("PERCENTILE_APPROX"), 315 exp.RegexpLike: lambda self, e: self.binary(e, "RLIKE"), 316 exp.RegexpSplit: rename_func("SPLIT"), 317 exp.SafeDivide: no_safe_divide_sql, 318 exp.SchemaCommentProperty: lambda self, e: self.naked_property(e), 319 exp.SetAgg: rename_func("COLLECT_SET"), 320 exp.Split: lambda self, e: f"SPLIT({self.sql(e, 'this')}, CONCAT('\\\\Q', {self.sql(e, 'expression')}))", 321 exp.StrPosition: strposition_to_locate_sql, 322 exp.StrToDate: _str_to_date, 323 exp.StrToTime: _str_to_time, 324 exp.StrToUnix: _str_to_unix, 325 exp.StructExtract: struct_extract_sql, 326 exp.TableFormatProperty: lambda self, e: f"USING {self.sql(e, 'this')}", 327 exp.TimeStrToDate: rename_func("TO_DATE"), 328 exp.TimeStrToTime: timestrtotime_sql, 329 exp.TimeStrToUnix: rename_func("UNIX_TIMESTAMP"), 330 exp.TimeToStr: _time_to_str, 331 exp.TimeToUnix: rename_func("UNIX_TIMESTAMP"), 332 exp.TsOrDiToDi: lambda self, e: f"CAST(SUBSTR(REPLACE(CAST({self.sql(e, 'this')} AS STRING), '-', ''), 1, 8) AS INT)", 333 exp.TsOrDsAdd: lambda self, e: f"DATE_ADD({self.sql(e, 'this')}, {self.sql(e, 'expression')})", 334 exp.TsOrDsToDate: _to_date_sql, 335 exp.TryCast: no_trycast_sql, 336 exp.UnixToStr: lambda self, e: self.func( 337 "FROM_UNIXTIME", e.this, _time_format(self, e) 338 ), 339 exp.UnixToTime: rename_func("FROM_UNIXTIME"), 340 exp.UnixToTimeStr: rename_func("FROM_UNIXTIME"), 341 exp.PartitionedByProperty: lambda self, e: f"PARTITIONED BY {self.sql(e, 'this')}", 342 exp.RowFormatSerdeProperty: lambda self, e: f"ROW FORMAT SERDE {self.sql(e, 'this')}", 343 exp.SerdeProperties: lambda self, e: self.properties(e, prefix="WITH SERDEPROPERTIES"), 344 exp.NumberToStr: rename_func("FORMAT_NUMBER"), 345 exp.LastDateOfMonth: rename_func("LAST_DAY"), 346 exp.National: lambda self, e: self.sql(e, "this"), 347 } 348 349 PROPERTIES_LOCATION = { 350 **generator.Generator.PROPERTIES_LOCATION, # type: ignore 351 exp.FileFormatProperty: exp.Properties.Location.POST_SCHEMA, 352 exp.PartitionedByProperty: exp.Properties.Location.POST_SCHEMA, 353 exp.TableFormatProperty: exp.Properties.Location.POST_SCHEMA, 354 exp.VolatileProperty: exp.Properties.Location.UNSUPPORTED, 355 } 356 357 def arrayagg_sql(self, expression: exp.ArrayAgg) -> str: 358 return self.func( 359 "COLLECT_LIST", 360 expression.this.this if isinstance(expression.this, exp.Order) else expression.this, 361 ) 362 363 def with_properties(self, properties: exp.Properties) -> str: 364 return self.properties( 365 properties, 366 prefix=self.seg("TBLPROPERTIES"), 367 ) 368 369 def datatype_sql(self, expression: exp.DataType) -> str: 370 if ( 371 expression.this in (exp.DataType.Type.VARCHAR, exp.DataType.Type.NVARCHAR) 372 and not expression.expressions 373 ): 374 expression = exp.DataType.build("text") 375 elif expression.this in exp.DataType.TEMPORAL_TYPES: 376 expression = exp.DataType.build(expression.this) 377 378 return super().datatype_sql(expression)
183 class Tokenizer(tokens.Tokenizer): 184 QUOTES = ["'", '"'] 185 IDENTIFIERS = ["`"] 186 STRING_ESCAPES = ["\\"] 187 ENCODE = "utf-8" 188 IDENTIFIER_CAN_START_WITH_DIGIT = True 189 190 KEYWORDS = { 191 **tokens.Tokenizer.KEYWORDS, 192 "ADD ARCHIVE": TokenType.COMMAND, 193 "ADD ARCHIVES": TokenType.COMMAND, 194 "ADD FILE": TokenType.COMMAND, 195 "ADD FILES": TokenType.COMMAND, 196 "ADD JAR": TokenType.COMMAND, 197 "ADD JARS": TokenType.COMMAND, 198 "MSCK REPAIR": TokenType.COMMAND, 199 "WITH SERDEPROPERTIES": TokenType.SERDE_PROPERTIES, 200 } 201 202 NUMERIC_LITERALS = { 203 "L": "BIGINT", 204 "S": "SMALLINT", 205 "Y": "TINYINT", 206 "D": "DOUBLE", 207 "F": "FLOAT", 208 "BD": "DECIMAL", 209 }
Inherited Members
211 class Parser(parser.Parser): 212 LOG_DEFAULTS_TO_LN = True 213 STRICT_CAST = False 214 215 FUNCTIONS = { 216 **parser.Parser.FUNCTIONS, # type: ignore 217 "APPROX_COUNT_DISTINCT": exp.ApproxDistinct.from_arg_list, 218 "COLLECT_LIST": exp.ArrayAgg.from_arg_list, 219 "DATE_ADD": lambda args: exp.TsOrDsAdd( 220 this=seq_get(args, 0), 221 expression=seq_get(args, 1), 222 unit=exp.Literal.string("DAY"), 223 ), 224 "DATEDIFF": lambda args: exp.DateDiff( 225 this=exp.TsOrDsToDate(this=seq_get(args, 0)), 226 expression=exp.TsOrDsToDate(this=seq_get(args, 1)), 227 ), 228 "DATE_SUB": lambda args: exp.TsOrDsAdd( 229 this=seq_get(args, 0), 230 expression=exp.Mul( 231 this=seq_get(args, 1), 232 expression=exp.Literal.number(-1), 233 ), 234 unit=exp.Literal.string("DAY"), 235 ), 236 "DATE_FORMAT": lambda args: format_time_lambda(exp.TimeToStr, "hive")( 237 [ 238 exp.TimeStrToTime(this=seq_get(args, 0)), 239 seq_get(args, 1), 240 ] 241 ), 242 "DAY": lambda args: exp.Day(this=exp.TsOrDsToDate(this=seq_get(args, 0))), 243 "FROM_UNIXTIME": format_time_lambda(exp.UnixToStr, "hive", True), 244 "GET_JSON_OBJECT": exp.JSONExtractScalar.from_arg_list, 245 "LOCATE": locate_to_strposition, 246 "MAP": parse_var_map, 247 "MONTH": lambda args: exp.Month(this=exp.TsOrDsToDate.from_arg_list(args)), 248 "PERCENTILE": exp.Quantile.from_arg_list, 249 "PERCENTILE_APPROX": exp.ApproxQuantile.from_arg_list, 250 "COLLECT_SET": exp.SetAgg.from_arg_list, 251 "SIZE": exp.ArraySize.from_arg_list, 252 "SPLIT": exp.RegexpSplit.from_arg_list, 253 "TO_DATE": format_time_lambda(exp.TsOrDsToDate, "hive"), 254 "TO_JSON": exp.JSONFormat.from_arg_list, 255 "UNIX_TIMESTAMP": format_time_lambda(exp.StrToUnix, "hive", True), 256 "YEAR": lambda args: exp.Year(this=exp.TsOrDsToDate.from_arg_list(args)), 257 } 258 259 PROPERTY_PARSERS = { 260 **parser.Parser.PROPERTY_PARSERS, # type: ignore 261 "WITH SERDEPROPERTIES": lambda self: exp.SerdeProperties( 262 expressions=self._parse_wrapped_csv(self._parse_property) 263 ), 264 }
Parser consumes a list of tokens produced by the sqlglot.tokens.Tokenizer and produces
a parsed syntax tree.
Arguments:
- error_level: the desired error level. Default: ErrorLevel.RAISE
- error_message_context: determines the amount of context to capture from a query string when displaying the error message (in number of characters). Default: 50.
- index_offset: Index offset for arrays eg ARRAY[0] vs ARRAY[1] as the head of a list. Default: 0
- alias_post_tablesample: If the table alias comes after tablesample. Default: False
- max_errors: Maximum number of error messages to include in a raised ParseError. This is only relevant if error_level is ErrorLevel.RAISE. Default: 3
- null_ordering: Indicates the default null ordering method to use if not explicitly set. Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". Default: "nulls_are_small"
Inherited Members
266 class Generator(generator.Generator): 267 LIMIT_FETCH = "LIMIT" 268 TABLESAMPLE_WITH_METHOD = False 269 TABLESAMPLE_SIZE_IS_PERCENT = True 270 JOIN_HINTS = False 271 TABLE_HINTS = False 272 273 TYPE_MAPPING = { 274 **generator.Generator.TYPE_MAPPING, # type: ignore 275 exp.DataType.Type.TEXT: "STRING", 276 exp.DataType.Type.DATETIME: "TIMESTAMP", 277 exp.DataType.Type.VARBINARY: "BINARY", 278 exp.DataType.Type.TIMESTAMPTZ: "TIMESTAMP", 279 exp.DataType.Type.BIT: "BOOLEAN", 280 } 281 282 TRANSFORMS = { 283 **generator.Generator.TRANSFORMS, # type: ignore 284 **transforms.UNALIAS_GROUP, # type: ignore 285 **transforms.ELIMINATE_QUALIFY, # type: ignore 286 exp.Select: transforms.preprocess( 287 [transforms.eliminate_qualify, transforms.unnest_to_explode] 288 ), 289 exp.Property: _property_sql, 290 exp.ApproxDistinct: approx_count_distinct_sql, 291 exp.ArrayConcat: rename_func("CONCAT"), 292 exp.ArraySize: rename_func("SIZE"), 293 exp.ArraySort: _array_sort, 294 exp.With: no_recursive_cte_sql, 295 exp.DateAdd: _add_date_sql, 296 exp.DateDiff: _date_diff_sql, 297 exp.DateStrToDate: rename_func("TO_DATE"), 298 exp.DateSub: _add_date_sql, 299 exp.DateToDi: lambda self, e: f"CAST(DATE_FORMAT({self.sql(e, 'this')}, {Hive.dateint_format}) AS INT)", 300 exp.DiToDate: lambda self, e: f"TO_DATE(CAST({self.sql(e, 'this')} AS STRING), {Hive.dateint_format})", 301 exp.FileFormatProperty: lambda self, e: f"STORED AS {self.sql(e, 'this') if isinstance(e.this, exp.InputOutputFormat) else e.name.upper()}", 302 exp.If: if_sql, 303 exp.Index: _index_sql, 304 exp.ILike: no_ilike_sql, 305 exp.JSONExtract: rename_func("GET_JSON_OBJECT"), 306 exp.JSONExtractScalar: rename_func("GET_JSON_OBJECT"), 307 exp.JSONFormat: rename_func("TO_JSON"), 308 exp.Map: var_map_sql, 309 exp.Max: max_or_greatest, 310 exp.Min: min_or_least, 311 exp.VarMap: var_map_sql, 312 exp.Create: create_with_partitions_sql, 313 exp.Quantile: rename_func("PERCENTILE"), 314 exp.ApproxQuantile: rename_func("PERCENTILE_APPROX"), 315 exp.RegexpLike: lambda self, e: self.binary(e, "RLIKE"), 316 exp.RegexpSplit: rename_func("SPLIT"), 317 exp.SafeDivide: no_safe_divide_sql, 318 exp.SchemaCommentProperty: lambda self, e: self.naked_property(e), 319 exp.SetAgg: rename_func("COLLECT_SET"), 320 exp.Split: lambda self, e: f"SPLIT({self.sql(e, 'this')}, CONCAT('\\\\Q', {self.sql(e, 'expression')}))", 321 exp.StrPosition: strposition_to_locate_sql, 322 exp.StrToDate: _str_to_date, 323 exp.StrToTime: _str_to_time, 324 exp.StrToUnix: _str_to_unix, 325 exp.StructExtract: struct_extract_sql, 326 exp.TableFormatProperty: lambda self, e: f"USING {self.sql(e, 'this')}", 327 exp.TimeStrToDate: rename_func("TO_DATE"), 328 exp.TimeStrToTime: timestrtotime_sql, 329 exp.TimeStrToUnix: rename_func("UNIX_TIMESTAMP"), 330 exp.TimeToStr: _time_to_str, 331 exp.TimeToUnix: rename_func("UNIX_TIMESTAMP"), 332 exp.TsOrDiToDi: lambda self, e: f"CAST(SUBSTR(REPLACE(CAST({self.sql(e, 'this')} AS STRING), '-', ''), 1, 8) AS INT)", 333 exp.TsOrDsAdd: lambda self, e: f"DATE_ADD({self.sql(e, 'this')}, {self.sql(e, 'expression')})", 334 exp.TsOrDsToDate: _to_date_sql, 335 exp.TryCast: no_trycast_sql, 336 exp.UnixToStr: lambda self, e: self.func( 337 "FROM_UNIXTIME", e.this, _time_format(self, e) 338 ), 339 exp.UnixToTime: rename_func("FROM_UNIXTIME"), 340 exp.UnixToTimeStr: rename_func("FROM_UNIXTIME"), 341 exp.PartitionedByProperty: lambda self, e: f"PARTITIONED BY {self.sql(e, 'this')}", 342 exp.RowFormatSerdeProperty: lambda self, e: f"ROW FORMAT SERDE {self.sql(e, 'this')}", 343 exp.SerdeProperties: lambda self, e: self.properties(e, prefix="WITH SERDEPROPERTIES"), 344 exp.NumberToStr: rename_func("FORMAT_NUMBER"), 345 exp.LastDateOfMonth: rename_func("LAST_DAY"), 346 exp.National: lambda self, e: self.sql(e, "this"), 347 } 348 349 PROPERTIES_LOCATION = { 350 **generator.Generator.PROPERTIES_LOCATION, # type: ignore 351 exp.FileFormatProperty: exp.Properties.Location.POST_SCHEMA, 352 exp.PartitionedByProperty: exp.Properties.Location.POST_SCHEMA, 353 exp.TableFormatProperty: exp.Properties.Location.POST_SCHEMA, 354 exp.VolatileProperty: exp.Properties.Location.UNSUPPORTED, 355 } 356 357 def arrayagg_sql(self, expression: exp.ArrayAgg) -> str: 358 return self.func( 359 "COLLECT_LIST", 360 expression.this.this if isinstance(expression.this, exp.Order) else expression.this, 361 ) 362 363 def with_properties(self, properties: exp.Properties) -> str: 364 return self.properties( 365 properties, 366 prefix=self.seg("TBLPROPERTIES"), 367 ) 368 369 def datatype_sql(self, expression: exp.DataType) -> str: 370 if ( 371 expression.this in (exp.DataType.Type.VARCHAR, exp.DataType.Type.NVARCHAR) 372 and not expression.expressions 373 ): 374 expression = exp.DataType.build("text") 375 elif expression.this in exp.DataType.TEMPORAL_TYPES: 376 expression = exp.DataType.build(expression.this) 377 378 return super().datatype_sql(expression)
Generator interprets the given syntax tree and produces a SQL string as an output.
Arguments:
- time_mapping (dict): the dictionary of custom time mappings in which the key represents a python time format and the output the target time format
- time_trie (trie): a trie of the time_mapping keys
- pretty (bool): if set to True the returned string will be formatted. Default: False.
- quote_start (str): specifies which starting character to use to delimit quotes. Default: '.
- quote_end (str): specifies which ending character to use to delimit quotes. Default: '.
- identifier_start (str): specifies which starting character to use to delimit identifiers. Default: ".
- identifier_end (str): specifies which ending character to use to delimit identifiers. Default: ".
- identify (bool | str): 'always': always quote, 'safe': quote identifiers if they don't contain an upcase, True defaults to always.
- normalize (bool): if set to True all identifiers will lower cased
- string_escape (str): specifies a string escape character. Default: '.
- identifier_escape (str): specifies an identifier escape character. Default: ".
- pad (int): determines padding in a formatted string. Default: 2.
- indent (int): determines the size of indentation in a formatted string. Default: 4.
- unnest_column_only (bool): if true unnest table aliases are considered only as column aliases
- normalize_functions (str): normalize function names, "upper", "lower", or None Default: "upper"
- alias_post_tablesample (bool): if the table alias comes after tablesample Default: False
- unsupported_level (ErrorLevel): determines the generator's behavior when it encounters unsupported expressions. Default ErrorLevel.WARN.
- null_ordering (str): Indicates the default null ordering method to use if not explicitly set. Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". Default: "nulls_are_small"
- max_unsupported (int): Maximum number of unsupported messages to include in a raised UnsupportedError. This is only relevant if unsupported_level is ErrorLevel.RAISE. Default: 3
- leading_comma (bool): if the the comma is leading or trailing in select statements Default: False
- max_text_width: The max number of characters in a segment before creating new lines in pretty mode. The default is on the smaller end because the length only represents a segment and not the true line length. Default: 80
- comments: Whether or not to preserve comments in the output SQL code. Default: True
369 def datatype_sql(self, expression: exp.DataType) -> str: 370 if ( 371 expression.this in (exp.DataType.Type.VARCHAR, exp.DataType.Type.NVARCHAR) 372 and not expression.expressions 373 ): 374 expression = exp.DataType.build("text") 375 elif expression.this in exp.DataType.TEMPORAL_TYPES: 376 expression = exp.DataType.build(expression.this) 377 378 return super().datatype_sql(expression)
Inherited Members
- sqlglot.generator.Generator
- Generator
- generate
- unsupported
- sep
- seg
- pad_comment
- maybe_comment
- wrap
- no_identify
- normalize_func
- indent
- sql
- uncache_sql
- cache_sql
- characterset_sql
- column_sql
- columnposition_sql
- columndef_sql
- columnconstraint_sql
- autoincrementcolumnconstraint_sql
- compresscolumnconstraint_sql
- generatedasidentitycolumnconstraint_sql
- notnullcolumnconstraint_sql
- primarykeycolumnconstraint_sql
- uniquecolumnconstraint_sql
- create_sql
- describe_sql
- prepend_ctes
- with_sql
- cte_sql
- tablealias_sql
- bitstring_sql
- hexstring_sql
- bytestring_sql
- directory_sql
- delete_sql
- drop_sql
- except_sql
- except_op
- fetch_sql
- filter_sql
- hint_sql
- index_sql
- identifier_sql
- inputoutputformat_sql
- national_sql
- partition_sql
- properties_sql
- root_properties
- properties
- locate_properties
- property_sql
- likeproperty_sql
- fallbackproperty_sql
- journalproperty_sql
- freespaceproperty_sql
- afterjournalproperty_sql
- checksumproperty_sql
- mergeblockratioproperty_sql
- datablocksizeproperty_sql
- blockcompressionproperty_sql
- isolatedloadingproperty_sql
- lockingproperty_sql
- withdataproperty_sql
- insert_sql
- intersect_sql
- intersect_op
- introducer_sql
- pseudotype_sql
- onconflict_sql
- returning_sql
- rowformatdelimitedproperty_sql
- table_sql
- tablesample_sql
- pivot_sql
- tuple_sql
- update_sql
- values_sql
- var_sql
- into_sql
- from_sql
- group_sql
- having_sql
- join_sql
- lambda_sql
- lateral_sql
- limit_sql
- offset_sql
- setitem_sql
- set_sql
- pragma_sql
- lock_sql
- literal_sql
- loaddata_sql
- null_sql
- boolean_sql
- order_sql
- cluster_sql
- distribute_sql
- sort_sql
- ordered_sql
- matchrecognize_sql
- query_modifiers
- select_sql
- schema_sql
- star_sql
- structkwarg_sql
- parameter_sql
- sessionparameter_sql
- placeholder_sql
- subquery_sql
- qualify_sql
- union_sql
- union_op
- unnest_sql
- where_sql
- window_sql
- partition_by_sql
- window_spec_sql
- withingroup_sql
- between_sql
- bracket_sql
- all_sql
- any_sql
- exists_sql
- case_sql
- constraint_sql
- extract_sql
- trim_sql
- concat_sql
- check_sql
- foreignkey_sql
- primarykey_sql
- unique_sql
- if_sql
- matchagainst_sql
- jsonkeyvalue_sql
- jsonobject_sql
- in_sql
- in_unnest_op
- interval_sql
- return_sql
- reference_sql
- anonymous_sql
- paren_sql
- neg_sql
- not_sql
- alias_sql
- aliases_sql
- attimezone_sql
- add_sql
- and_sql
- connector_sql
- bitwiseand_sql
- bitwiseleftshift_sql
- bitwisenot_sql
- bitwiseor_sql
- bitwiserightshift_sql
- bitwisexor_sql
- cast_sql
- currentdate_sql
- collate_sql
- command_sql
- comment_sql
- transaction_sql
- commit_sql
- rollback_sql
- altercolumn_sql
- renametable_sql
- altertable_sql
- droppartition_sql
- addconstraint_sql
- distinct_sql
- ignorenulls_sql
- respectnulls_sql
- intdiv_sql
- dpipe_sql
- div_sql
- overlaps_sql
- distance_sql
- dot_sql
- eq_sql
- escape_sql
- glob_sql
- gt_sql
- gte_sql
- ilike_sql
- ilikeany_sql
- is_sql
- like_sql
- likeany_sql
- similarto_sql
- lt_sql
- lte_sql
- mod_sql
- mul_sql
- neq_sql
- nullsafeeq_sql
- nullsafeneq_sql
- or_sql
- slice_sql
- sub_sql
- trycast_sql
- use_sql
- binary
- function_fallback_sql
- func
- format_args
- text_width
- format_time
- expressions
- op_expressions
- naked_property
- set_operation
- tag_sql
- token_sql
- userdefinedfunction_sql
- joinhint_sql
- kwarg_sql
- when_sql
- merge_sql
- tochar_sql