+ Edit on GitHub +

+sqlglot.dialects.spark

+ + + + + View Source + +

  1from __future__ import annotations
+  2
+  3from sqlglot import exp, parser
+  4from sqlglot.dialects.dialect import create_with_partitions_sql, rename_func, trim_sql
+  5from sqlglot.dialects.hive import Hive
+  6from sqlglot.helper import seq_get
+  7
+  8
+  9def _create_sql(self, e):
+ 10    kind = e.args.get("kind")
+ 11    temporary = e.args.get("temporary")
+ 12
+ 13    if kind.upper() == "TABLE" and temporary is True:
+ 14        return f"CREATE TEMPORARY VIEW {self.sql(e, 'this')} AS {self.sql(e, 'expression')}"
+ 15    return create_with_partitions_sql(self, e)
+ 16
+ 17
+ 18def _map_sql(self, expression):
+ 19    keys = self.sql(expression.args["keys"])
+ 20    values = self.sql(expression.args["values"])
+ 21    return f"MAP_FROM_ARRAYS({keys}, {values})"
+ 22
+ 23
+ 24def _str_to_date(self, expression):
+ 25    this = self.sql(expression, "this")
+ 26    time_format = self.format_time(expression)
+ 27    if time_format == Hive.date_format:
+ 28        return f"TO_DATE({this})"
+ 29    return f"TO_DATE({this}, {time_format})"
+ 30
+ 31
+ 32def _unix_to_time(self, expression):
+ 33    scale = expression.args.get("scale")
+ 34    timestamp = self.sql(expression, "this")
+ 35    if scale is None:
+ 36        return f"FROM_UNIXTIME({timestamp})"
+ 37    if scale == exp.UnixToTime.SECONDS:
+ 38        return f"TIMESTAMP_SECONDS({timestamp})"
+ 39    if scale == exp.UnixToTime.MILLIS:
+ 40        return f"TIMESTAMP_MILLIS({timestamp})"
+ 41    if scale == exp.UnixToTime.MICROS:
+ 42        return f"TIMESTAMP_MICROS({timestamp})"
+ 43
+ 44    raise ValueError("Improper scale for timestamp")
+ 45
+ 46
+ 47class Spark(Hive):
+ 48    class Parser(Hive.Parser):
+ 49        FUNCTIONS = {
+ 50            **Hive.Parser.FUNCTIONS,  # type: ignore
+ 51            "MAP_FROM_ARRAYS": exp.Map.from_arg_list,
+ 52            "TO_UNIX_TIMESTAMP": exp.StrToUnix.from_arg_list,
+ 53            "LEFT": lambda args: exp.Substring(
+ 54                this=seq_get(args, 0),
+ 55                start=exp.Literal.number(1),
+ 56                length=seq_get(args, 1),
+ 57            ),
+ 58            "SHIFTLEFT": lambda args: exp.BitwiseLeftShift(
+ 59                this=seq_get(args, 0),
+ 60                expression=seq_get(args, 1),
+ 61            ),
+ 62            "SHIFTRIGHT": lambda args: exp.BitwiseRightShift(
+ 63                this=seq_get(args, 0),
+ 64                expression=seq_get(args, 1),
+ 65            ),
+ 66            "RIGHT": lambda args: exp.Substring(
+ 67                this=seq_get(args, 0),
+ 68                start=exp.Sub(
+ 69                    this=exp.Length(this=seq_get(args, 0)),
+ 70                    expression=exp.Add(this=seq_get(args, 1), expression=exp.Literal.number(1)),
+ 71                ),
+ 72                length=seq_get(args, 1),
+ 73            ),
+ 74            "APPROX_PERCENTILE": exp.ApproxQuantile.from_arg_list,
+ 75            "IIF": exp.If.from_arg_list,
+ 76            "AGGREGATE": exp.Reduce.from_arg_list,
+ 77            "DAYOFWEEK": lambda args: exp.DayOfWeek(
+ 78                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 79            ),
+ 80            "DAYOFMONTH": lambda args: exp.DayOfMonth(
+ 81                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 82            ),
+ 83            "DAYOFYEAR": lambda args: exp.DayOfYear(
+ 84                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 85            ),
+ 86            "WEEKOFYEAR": lambda args: exp.WeekOfYear(
+ 87                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 88            ),
+ 89        }
+ 90
+ 91        FUNCTION_PARSERS = {
+ 92            **parser.Parser.FUNCTION_PARSERS,  # type: ignore
+ 93            "BROADCAST": lambda self: self._parse_join_hint("BROADCAST"),
+ 94            "BROADCASTJOIN": lambda self: self._parse_join_hint("BROADCASTJOIN"),
+ 95            "MAPJOIN": lambda self: self._parse_join_hint("MAPJOIN"),
+ 96            "MERGE": lambda self: self._parse_join_hint("MERGE"),
+ 97            "SHUFFLEMERGE": lambda self: self._parse_join_hint("SHUFFLEMERGE"),
+ 98            "MERGEJOIN": lambda self: self._parse_join_hint("MERGEJOIN"),
+ 99            "SHUFFLE_HASH": lambda self: self._parse_join_hint("SHUFFLE_HASH"),
+100            "SHUFFLE_REPLICATE_NL": lambda self: self._parse_join_hint("SHUFFLE_REPLICATE_NL"),
+101        }
+102
+103        def _parse_add_column(self):
+104            return self._match_text_seq("ADD", "COLUMNS") and self._parse_schema()
+105
+106        def _parse_drop_column(self):
+107            return self._match_text_seq("DROP", "COLUMNS") and self.expression(
+108                exp.Drop,
+109                this=self._parse_schema(),
+110                kind="COLUMNS",
+111            )
+112
+113    class Generator(Hive.Generator):
+114        TYPE_MAPPING = {
+115            **Hive.Generator.TYPE_MAPPING,  # type: ignore
+116            exp.DataType.Type.TINYINT: "BYTE",
+117            exp.DataType.Type.SMALLINT: "SHORT",
+118            exp.DataType.Type.BIGINT: "LONG",
+119        }
+120
+121        PROPERTIES_LOCATION = {
+122            **Hive.Generator.PROPERTIES_LOCATION,  # type: ignore
+123            exp.EngineProperty: exp.Properties.Location.UNSUPPORTED,
+124            exp.AutoIncrementProperty: exp.Properties.Location.UNSUPPORTED,
+125            exp.CharacterSetProperty: exp.Properties.Location.UNSUPPORTED,
+126            exp.CollateProperty: exp.Properties.Location.UNSUPPORTED,
+127        }
+128
+129        TRANSFORMS = {
+130            **Hive.Generator.TRANSFORMS,  # type: ignore
+131            exp.ApproxDistinct: rename_func("APPROX_COUNT_DISTINCT"),
+132            exp.FileFormatProperty: lambda self, e: f"USING {e.name.upper()}",
+133            exp.ArraySum: lambda self, e: f"AGGREGATE({self.sql(e, 'this')}, 0, (acc, x) -> acc + x, acc -> acc)",
+134            exp.BitwiseLeftShift: rename_func("SHIFTLEFT"),
+135            exp.BitwiseRightShift: rename_func("SHIFTRIGHT"),
+136            exp.DateTrunc: rename_func("TRUNC"),
+137            exp.Hint: lambda self, e: f" /*+ {self.expressions(e).strip()} */",
+138            exp.StrToDate: _str_to_date,
+139            exp.StrToTime: lambda self, e: f"TO_TIMESTAMP({self.sql(e, 'this')}, {self.format_time(e)})",
+140            exp.UnixToTime: _unix_to_time,
+141            exp.Create: _create_sql,
+142            exp.Map: _map_sql,
+143            exp.Reduce: rename_func("AGGREGATE"),
+144            exp.StructKwarg: lambda self, e: f"{self.sql(e, 'this')}: {self.sql(e, 'expression')}",
+145            exp.TimestampTrunc: lambda self, e: f"DATE_TRUNC({self.sql(e, 'unit')}, {self.sql(e, 'this')})",
+146            exp.Trim: trim_sql,
+147            exp.VariancePop: rename_func("VAR_POP"),
+148            exp.DateFromParts: rename_func("MAKE_DATE"),
+149            exp.LogicalOr: rename_func("BOOL_OR"),
+150            exp.DayOfWeek: rename_func("DAYOFWEEK"),
+151            exp.DayOfMonth: rename_func("DAYOFMONTH"),
+152            exp.DayOfYear: rename_func("DAYOFYEAR"),
+153            exp.WeekOfYear: rename_func("WEEKOFYEAR"),
+154            exp.AtTimeZone: lambda self, e: f"FROM_UTC_TIMESTAMP({self.sql(e, 'this')}, {self.sql(e, 'zone')})",
+155        }
+156        TRANSFORMS.pop(exp.ArraySort)
+157        TRANSFORMS.pop(exp.ILike)
+158
+159        WRAP_DERIVED_VALUES = False
+160        CREATE_FUNCTION_AS = False
+161
+162        def cast_sql(self, expression: exp.Cast) -> str:
+163            if isinstance(expression.this, exp.Cast) and expression.this.is_type(
+164                exp.DataType.Type.JSON
+165            ):
+166                schema = f"'{self.sql(expression, 'to')}'"
+167                return f"FROM_JSON({self.format_args(self.sql(expression.this, 'this'), schema)})"
+168            if expression.to.is_type(exp.DataType.Type.JSON):
+169                return f"TO_JSON({self.sql(expression, 'this')})"
+170
+171            return super(Spark.Generator, self).cast_sql(expression)
+172
+173    class Tokenizer(Hive.Tokenizer):
+174        HEX_STRINGS = [("X'", "'")]
+

+ + +

+ +

+ + class + Spark(sqlglot.dialects.hive.Hive): + + View Source + +

+ +

 48class Spark(Hive):
+ 49    class Parser(Hive.Parser):
+ 50        FUNCTIONS = {
+ 51            **Hive.Parser.FUNCTIONS,  # type: ignore
+ 52            "MAP_FROM_ARRAYS": exp.Map.from_arg_list,
+ 53            "TO_UNIX_TIMESTAMP": exp.StrToUnix.from_arg_list,
+ 54            "LEFT": lambda args: exp.Substring(
+ 55                this=seq_get(args, 0),
+ 56                start=exp.Literal.number(1),
+ 57                length=seq_get(args, 1),
+ 58            ),
+ 59            "SHIFTLEFT": lambda args: exp.BitwiseLeftShift(
+ 60                this=seq_get(args, 0),
+ 61                expression=seq_get(args, 1),
+ 62            ),
+ 63            "SHIFTRIGHT": lambda args: exp.BitwiseRightShift(
+ 64                this=seq_get(args, 0),
+ 65                expression=seq_get(args, 1),
+ 66            ),
+ 67            "RIGHT": lambda args: exp.Substring(
+ 68                this=seq_get(args, 0),
+ 69                start=exp.Sub(
+ 70                    this=exp.Length(this=seq_get(args, 0)),
+ 71                    expression=exp.Add(this=seq_get(args, 1), expression=exp.Literal.number(1)),
+ 72                ),
+ 73                length=seq_get(args, 1),
+ 74            ),
+ 75            "APPROX_PERCENTILE": exp.ApproxQuantile.from_arg_list,
+ 76            "IIF": exp.If.from_arg_list,
+ 77            "AGGREGATE": exp.Reduce.from_arg_list,
+ 78            "DAYOFWEEK": lambda args: exp.DayOfWeek(
+ 79                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 80            ),
+ 81            "DAYOFMONTH": lambda args: exp.DayOfMonth(
+ 82                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 83            ),
+ 84            "DAYOFYEAR": lambda args: exp.DayOfYear(
+ 85                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 86            ),
+ 87            "WEEKOFYEAR": lambda args: exp.WeekOfYear(
+ 88                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 89            ),
+ 90        }
+ 91
+ 92        FUNCTION_PARSERS = {
+ 93            **parser.Parser.FUNCTION_PARSERS,  # type: ignore
+ 94            "BROADCAST": lambda self: self._parse_join_hint("BROADCAST"),
+ 95            "BROADCASTJOIN": lambda self: self._parse_join_hint("BROADCASTJOIN"),
+ 96            "MAPJOIN": lambda self: self._parse_join_hint("MAPJOIN"),
+ 97            "MERGE": lambda self: self._parse_join_hint("MERGE"),
+ 98            "SHUFFLEMERGE": lambda self: self._parse_join_hint("SHUFFLEMERGE"),
+ 99            "MERGEJOIN": lambda self: self._parse_join_hint("MERGEJOIN"),
+100            "SHUFFLE_HASH": lambda self: self._parse_join_hint("SHUFFLE_HASH"),
+101            "SHUFFLE_REPLICATE_NL": lambda self: self._parse_join_hint("SHUFFLE_REPLICATE_NL"),
+102        }
+103
+104        def _parse_add_column(self):
+105            return self._match_text_seq("ADD", "COLUMNS") and self._parse_schema()
+106
+107        def _parse_drop_column(self):
+108            return self._match_text_seq("DROP", "COLUMNS") and self.expression(
+109                exp.Drop,
+110                this=self._parse_schema(),
+111                kind="COLUMNS",
+112            )
+113
+114    class Generator(Hive.Generator):
+115        TYPE_MAPPING = {
+116            **Hive.Generator.TYPE_MAPPING,  # type: ignore
+117            exp.DataType.Type.TINYINT: "BYTE",
+118            exp.DataType.Type.SMALLINT: "SHORT",
+119            exp.DataType.Type.BIGINT: "LONG",
+120        }
+121
+122        PROPERTIES_LOCATION = {
+123            **Hive.Generator.PROPERTIES_LOCATION,  # type: ignore
+124            exp.EngineProperty: exp.Properties.Location.UNSUPPORTED,
+125            exp.AutoIncrementProperty: exp.Properties.Location.UNSUPPORTED,
+126            exp.CharacterSetProperty: exp.Properties.Location.UNSUPPORTED,
+127            exp.CollateProperty: exp.Properties.Location.UNSUPPORTED,
+128        }
+129
+130        TRANSFORMS = {
+131            **Hive.Generator.TRANSFORMS,  # type: ignore
+132            exp.ApproxDistinct: rename_func("APPROX_COUNT_DISTINCT"),
+133            exp.FileFormatProperty: lambda self, e: f"USING {e.name.upper()}",
+134            exp.ArraySum: lambda self, e: f"AGGREGATE({self.sql(e, 'this')}, 0, (acc, x) -> acc + x, acc -> acc)",
+135            exp.BitwiseLeftShift: rename_func("SHIFTLEFT"),
+136            exp.BitwiseRightShift: rename_func("SHIFTRIGHT"),
+137            exp.DateTrunc: rename_func("TRUNC"),
+138            exp.Hint: lambda self, e: f" /*+ {self.expressions(e).strip()} */",
+139            exp.StrToDate: _str_to_date,
+140            exp.StrToTime: lambda self, e: f"TO_TIMESTAMP({self.sql(e, 'this')}, {self.format_time(e)})",
+141            exp.UnixToTime: _unix_to_time,
+142            exp.Create: _create_sql,
+143            exp.Map: _map_sql,
+144            exp.Reduce: rename_func("AGGREGATE"),
+145            exp.StructKwarg: lambda self, e: f"{self.sql(e, 'this')}: {self.sql(e, 'expression')}",
+146            exp.TimestampTrunc: lambda self, e: f"DATE_TRUNC({self.sql(e, 'unit')}, {self.sql(e, 'this')})",
+147            exp.Trim: trim_sql,
+148            exp.VariancePop: rename_func("VAR_POP"),
+149            exp.DateFromParts: rename_func("MAKE_DATE"),
+150            exp.LogicalOr: rename_func("BOOL_OR"),
+151            exp.DayOfWeek: rename_func("DAYOFWEEK"),
+152            exp.DayOfMonth: rename_func("DAYOFMONTH"),
+153            exp.DayOfYear: rename_func("DAYOFYEAR"),
+154            exp.WeekOfYear: rename_func("WEEKOFYEAR"),
+155            exp.AtTimeZone: lambda self, e: f"FROM_UTC_TIMESTAMP({self.sql(e, 'this')}, {self.sql(e, 'zone')})",
+156        }
+157        TRANSFORMS.pop(exp.ArraySort)
+158        TRANSFORMS.pop(exp.ILike)
+159
+160        WRAP_DERIVED_VALUES = False
+161        CREATE_FUNCTION_AS = False
+162
+163        def cast_sql(self, expression: exp.Cast) -> str:
+164            if isinstance(expression.this, exp.Cast) and expression.this.is_type(
+165                exp.DataType.Type.JSON
+166            ):
+167                schema = f"'{self.sql(expression, 'to')}'"
+168                return f"FROM_JSON({self.format_args(self.sql(expression.this, 'this'), schema)})"
+169            if expression.to.is_type(exp.DataType.Type.JSON):
+170                return f"TO_JSON({self.sql(expression, 'this')})"
+171
+172            return super(Spark.Generator, self).cast_sql(expression)
+173
+174    class Tokenizer(Hive.Tokenizer):
+175        HEX_STRINGS = [("X'", "'")]
+

+ + + + +

+ + Spark() + + +

+ + + + +

Inherited Members

sqlglot.dialects.dialect.Dialect: get_or_raise; format_time; parse; parse_into; generate; transpile; parser; generator

+ +

+ + class + Spark.Parser(sqlglot.dialects.hive.Hive.Parser): + + View Source + +

+ +

 49    class Parser(Hive.Parser):
+ 50        FUNCTIONS = {
+ 51            **Hive.Parser.FUNCTIONS,  # type: ignore
+ 52            "MAP_FROM_ARRAYS": exp.Map.from_arg_list,
+ 53            "TO_UNIX_TIMESTAMP": exp.StrToUnix.from_arg_list,
+ 54            "LEFT": lambda args: exp.Substring(
+ 55                this=seq_get(args, 0),
+ 56                start=exp.Literal.number(1),
+ 57                length=seq_get(args, 1),
+ 58            ),
+ 59            "SHIFTLEFT": lambda args: exp.BitwiseLeftShift(
+ 60                this=seq_get(args, 0),
+ 61                expression=seq_get(args, 1),
+ 62            ),
+ 63            "SHIFTRIGHT": lambda args: exp.BitwiseRightShift(
+ 64                this=seq_get(args, 0),
+ 65                expression=seq_get(args, 1),
+ 66            ),
+ 67            "RIGHT": lambda args: exp.Substring(
+ 68                this=seq_get(args, 0),
+ 69                start=exp.Sub(
+ 70                    this=exp.Length(this=seq_get(args, 0)),
+ 71                    expression=exp.Add(this=seq_get(args, 1), expression=exp.Literal.number(1)),
+ 72                ),
+ 73                length=seq_get(args, 1),
+ 74            ),
+ 75            "APPROX_PERCENTILE": exp.ApproxQuantile.from_arg_list,
+ 76            "IIF": exp.If.from_arg_list,
+ 77            "AGGREGATE": exp.Reduce.from_arg_list,
+ 78            "DAYOFWEEK": lambda args: exp.DayOfWeek(
+ 79                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 80            ),
+ 81            "DAYOFMONTH": lambda args: exp.DayOfMonth(
+ 82                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 83            ),
+ 84            "DAYOFYEAR": lambda args: exp.DayOfYear(
+ 85                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 86            ),
+ 87            "WEEKOFYEAR": lambda args: exp.WeekOfYear(
+ 88                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
+ 89            ),
+ 90        }
+ 91
+ 92        FUNCTION_PARSERS = {
+ 93            **parser.Parser.FUNCTION_PARSERS,  # type: ignore
+ 94            "BROADCAST": lambda self: self._parse_join_hint("BROADCAST"),
+ 95            "BROADCASTJOIN": lambda self: self._parse_join_hint("BROADCASTJOIN"),
+ 96            "MAPJOIN": lambda self: self._parse_join_hint("MAPJOIN"),
+ 97            "MERGE": lambda self: self._parse_join_hint("MERGE"),
+ 98            "SHUFFLEMERGE": lambda self: self._parse_join_hint("SHUFFLEMERGE"),
+ 99            "MERGEJOIN": lambda self: self._parse_join_hint("MERGEJOIN"),
+100            "SHUFFLE_HASH": lambda self: self._parse_join_hint("SHUFFLE_HASH"),
+101            "SHUFFLE_REPLICATE_NL": lambda self: self._parse_join_hint("SHUFFLE_REPLICATE_NL"),
+102        }
+103
+104        def _parse_add_column(self):
+105            return self._match_text_seq("ADD", "COLUMNS") and self._parse_schema()
+106
+107        def _parse_drop_column(self):
+108            return self._match_text_seq("DROP", "COLUMNS") and self.expression(
+109                exp.Drop,
+110                this=self._parse_schema(),
+111                kind="COLUMNS",
+112            )
+

+ + +

Parser consumes a list of tokens produced by the sqlglot.tokens.Tokenizer and produces +a parsed syntax tree.

+ +

Arguments:

+ +

error_level: the desired error level. +Default: ErrorLevel.RAISE
error_message_context: determines the amount of context to capture from a +query string when displaying the error message (in number of characters). +Default: 50.
index_offset: Index offset for arrays eg ARRAY[0] vs ARRAY[1] as the head of a list. +Default: 0
alias_post_tablesample: If the table alias comes after tablesample. +Default: False
max_errors: Maximum number of error messages to include in a raised ParseError. +This is only relevant if error_level is ErrorLevel.RAISE. +Default: 3
null_ordering: Indicates the default null ordering method to use if not explicitly set. +Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". +Default: "nulls_are_small"

+ + +

Inherited Members

sqlglot.parser.Parser: Parser; reset; parse; parse_into; check_errors; raise_error; expression; validate_expression

+ +

+ + class + Spark.Generator(sqlglot.dialects.hive.Hive.Generator): + + View Source + +

+ +

114    class Generator(Hive.Generator):
+115        TYPE_MAPPING = {
+116            **Hive.Generator.TYPE_MAPPING,  # type: ignore
+117            exp.DataType.Type.TINYINT: "BYTE",
+118            exp.DataType.Type.SMALLINT: "SHORT",
+119            exp.DataType.Type.BIGINT: "LONG",
+120        }
+121
+122        PROPERTIES_LOCATION = {
+123            **Hive.Generator.PROPERTIES_LOCATION,  # type: ignore
+124            exp.EngineProperty: exp.Properties.Location.UNSUPPORTED,
+125            exp.AutoIncrementProperty: exp.Properties.Location.UNSUPPORTED,
+126            exp.CharacterSetProperty: exp.Properties.Location.UNSUPPORTED,
+127            exp.CollateProperty: exp.Properties.Location.UNSUPPORTED,
+128        }
+129
+130        TRANSFORMS = {
+131            **Hive.Generator.TRANSFORMS,  # type: ignore
+132            exp.ApproxDistinct: rename_func("APPROX_COUNT_DISTINCT"),
+133            exp.FileFormatProperty: lambda self, e: f"USING {e.name.upper()}",
+134            exp.ArraySum: lambda self, e: f"AGGREGATE({self.sql(e, 'this')}, 0, (acc, x) -> acc + x, acc -> acc)",
+135            exp.BitwiseLeftShift: rename_func("SHIFTLEFT"),
+136            exp.BitwiseRightShift: rename_func("SHIFTRIGHT"),
+137            exp.DateTrunc: rename_func("TRUNC"),
+138            exp.Hint: lambda self, e: f" /*+ {self.expressions(e).strip()} */",
+139            exp.StrToDate: _str_to_date,
+140            exp.StrToTime: lambda self, e: f"TO_TIMESTAMP({self.sql(e, 'this')}, {self.format_time(e)})",
+141            exp.UnixToTime: _unix_to_time,
+142            exp.Create: _create_sql,
+143            exp.Map: _map_sql,
+144            exp.Reduce: rename_func("AGGREGATE"),
+145            exp.StructKwarg: lambda self, e: f"{self.sql(e, 'this')}: {self.sql(e, 'expression')}",
+146            exp.TimestampTrunc: lambda self, e: f"DATE_TRUNC({self.sql(e, 'unit')}, {self.sql(e, 'this')})",
+147            exp.Trim: trim_sql,
+148            exp.VariancePop: rename_func("VAR_POP"),
+149            exp.DateFromParts: rename_func("MAKE_DATE"),
+150            exp.LogicalOr: rename_func("BOOL_OR"),
+151            exp.DayOfWeek: rename_func("DAYOFWEEK"),
+152            exp.DayOfMonth: rename_func("DAYOFMONTH"),
+153            exp.DayOfYear: rename_func("DAYOFYEAR"),
+154            exp.WeekOfYear: rename_func("WEEKOFYEAR"),
+155            exp.AtTimeZone: lambda self, e: f"FROM_UTC_TIMESTAMP({self.sql(e, 'this')}, {self.sql(e, 'zone')})",
+156        }
+157        TRANSFORMS.pop(exp.ArraySort)
+158        TRANSFORMS.pop(exp.ILike)
+159
+160        WRAP_DERIVED_VALUES = False
+161        CREATE_FUNCTION_AS = False
+162
+163        def cast_sql(self, expression: exp.Cast) -> str:
+164            if isinstance(expression.this, exp.Cast) and expression.this.is_type(
+165                exp.DataType.Type.JSON
+166            ):
+167                schema = f"'{self.sql(expression, 'to')}'"
+168                return f"FROM_JSON({self.format_args(self.sql(expression.this, 'this'), schema)})"
+169            if expression.to.is_type(exp.DataType.Type.JSON):
+170                return f"TO_JSON({self.sql(expression, 'this')})"
+171
+172            return super(Spark.Generator, self).cast_sql(expression)
+

+ + +

Generator interprets the given syntax tree and produces a SQL string as an output.

+ +

Arguments:

+ +

time_mapping (dict): the dictionary of custom time mappings in which the key +represents a python time format and the output the target time format
time_trie (trie): a trie of the time_mapping keys
pretty (bool): if set to True the returned string will be formatted. Default: False.
quote_start (str): specifies which starting character to use to delimit quotes. Default: '.
quote_end (str): specifies which ending character to use to delimit quotes. Default: '.
identifier_start (str): specifies which starting character to use to delimit identifiers. Default: ".
identifier_end (str): specifies which ending character to use to delimit identifiers. Default: ".
identify (bool): if set to True all identifiers will be delimited by the corresponding +character.
normalize (bool): if set to True all identifiers will lower cased
string_escape (str): specifies a string escape character. Default: '.
identifier_escape (str): specifies an identifier escape character. Default: ".
pad (int): determines padding in a formatted string. Default: 2.
indent (int): determines the size of indentation in a formatted string. Default: 4.
unnest_column_only (bool): if true unnest table aliases are considered only as column aliases
normalize_functions (str): normalize function names, "upper", "lower", or None +Default: "upper"
alias_post_tablesample (bool): if the table alias comes after tablesample +Default: False
unsupported_level (ErrorLevel): determines the generator's behavior when it encounters +unsupported expressions. Default ErrorLevel.WARN.
null_ordering (str): Indicates the default null ordering method to use if not explicitly set. +Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". +Default: "nulls_are_small"
max_unsupported (int): Maximum number of unsupported messages to include in a raised UnsupportedError. +This is only relevant if unsupported_level is ErrorLevel.RAISE. +Default: 3
leading_comma (bool): if the the comma is leading or trailing in select statements +Default: False
max_text_width: The max number of characters in a segment before creating new lines in pretty mode. +The default is on the smaller end because the length only represents a segment and not the true +line length. +Default: 80
comments: Whether or not to preserve comments in the output SQL code. +Default: True

+ + +

+ +

+ + def + cast_sql(self, expression: sqlglot.expressions.Cast) -> str: + + View Source + +

+ +

163        def cast_sql(self, expression: exp.Cast) -> str:
+164            if isinstance(expression.this, exp.Cast) and expression.this.is_type(
+165                exp.DataType.Type.JSON
+166            ):
+167                schema = f"'{self.sql(expression, 'to')}'"
+168                return f"FROM_JSON({self.format_args(self.sql(expression.this, 'this'), schema)})"
+169            if expression.to.is_type(exp.DataType.Type.JSON):
+170                return f"TO_JSON({self.sql(expression, 'this')})"
+171
+172            return super(Spark.Generator, self).cast_sql(expression)
+

+ + + + +

Inherited Members

sqlglot.generator.Generator: Generator; generate; unsupported; sep; seg; pad_comment; maybe_comment; wrap; no_identify; normalize_func; indent; sql; uncache_sql; cache_sql; characterset_sql; column_sql; columndef_sql; columnconstraint_sql; autoincrementcolumnconstraint_sql; checkcolumnconstraint_sql; commentcolumnconstraint_sql; collatecolumnconstraint_sql; encodecolumnconstraint_sql; defaultcolumnconstraint_sql; generatedasidentitycolumnconstraint_sql; notnullcolumnconstraint_sql; primarykeycolumnconstraint_sql; uniquecolumnconstraint_sql; create_sql; describe_sql; prepend_ctes; with_sql; cte_sql; tablealias_sql; bitstring_sql; hexstring_sql; directory_sql; delete_sql; drop_sql; except_sql; except_op; fetch_sql; filter_sql; hint_sql; index_sql; identifier_sql; national_sql; partition_sql; properties_sql; root_properties; properties; locate_properties; property_sql; likeproperty_sql; fallbackproperty_sql; journalproperty_sql; freespaceproperty_sql; afterjournalproperty_sql; checksumproperty_sql; mergeblockratioproperty_sql; datablocksizeproperty_sql; blockcompressionproperty_sql; isolatedloadingproperty_sql; insert_sql; intersect_sql; intersect_op; introducer_sql; pseudotype_sql; rowformatdelimitedproperty_sql; table_sql; tablesample_sql; pivot_sql; tuple_sql; update_sql; values_sql; var_sql; into_sql; from_sql; group_sql; having_sql; join_sql; lambda_sql; lateral_sql; limit_sql; offset_sql; lock_sql; literal_sql; loaddata_sql; null_sql; boolean_sql; order_sql; cluster_sql; distribute_sql; sort_sql; ordered_sql; matchrecognize_sql; query_modifiers; select_sql; schema_sql; star_sql; structkwarg_sql; parameter_sql; sessionparameter_sql; placeholder_sql; subquery_sql; qualify_sql; union_sql; union_op; unnest_sql; where_sql; window_sql; partition_by_sql; window_spec_sql; withingroup_sql; between_sql; bracket_sql; all_sql; any_sql; exists_sql; case_sql; constraint_sql; extract_sql; trim_sql; concat_sql; check_sql; foreignkey_sql; primarykey_sql; unique_sql; if_sql; in_sql; in_unnest_op; interval_sql; return_sql; reference_sql; anonymous_sql; paren_sql; neg_sql; not_sql; alias_sql; aliases_sql; attimezone_sql; add_sql; and_sql; connector_sql; bitwiseand_sql; bitwiseleftshift_sql; bitwisenot_sql; bitwiseor_sql; bitwiserightshift_sql; bitwisexor_sql; currentdate_sql; collate_sql; command_sql; transaction_sql; commit_sql; rollback_sql; altercolumn_sql; renametable_sql; altertable_sql; droppartition_sql; addconstraint_sql; distinct_sql; ignorenulls_sql; respectnulls_sql; intdiv_sql; dpipe_sql; div_sql; distance_sql; dot_sql; eq_sql; escape_sql; glob_sql; gt_sql; gte_sql; ilike_sql; is_sql; like_sql; similarto_sql; lt_sql; lte_sql; mod_sql; mul_sql; neq_sql; nullsafeeq_sql; nullsafeneq_sql; or_sql; slice_sql; sub_sql; trycast_sql; use_sql; binary; function_fallback_sql; format_args; text_width; format_time; expressions; op_expressions; naked_property; set_operation; tag_sql; token_sql; userdefinedfunction_sql; userdefinedfunctionkwarg_sql; joinhint_sql; kwarg_sql; when_sql; merge_sql
sqlglot.dialects.hive.Hive.Generator: with_properties; datatype_sql

+ +

+ + class + Spark.Tokenizer(sqlglot.dialects.hive.Hive.Tokenizer): + + View Source + +

+ +

174    class Tokenizer(Hive.Tokenizer):
+175        HEX_STRINGS = [("X'", "'")]
+

+ + + + +

Inherited Members

sqlglot.tokens.Tokenizer: Tokenizer; reset; tokenize