Edit on GitHub

sqlglot.dialects.spark

View Source

  1from __future__ import annotations
  2
  3from sqlglot import exp, parser
  4from sqlglot.dialects.dialect import create_with_partitions_sql, rename_func, trim_sql
  5from sqlglot.dialects.hive import Hive
  6from sqlglot.helper import seq_get
  7
  8
  9def _create_sql(self, e):
 10    kind = e.args.get("kind")
 11    properties = e.args.get("properties")
 12
 13    if kind.upper() == "TABLE" and any(
 14        isinstance(prop, exp.TemporaryProperty)
 15        for prop in (properties.expressions if properties else [])
 16    ):
 17        return f"CREATE TEMPORARY VIEW {self.sql(e, 'this')} AS {self.sql(e, 'expression')}"
 18    return create_with_partitions_sql(self, e)
 19
 20
 21def _map_sql(self, expression):
 22    keys = self.sql(expression.args["keys"])
 23    values = self.sql(expression.args["values"])
 24    return f"MAP_FROM_ARRAYS({keys}, {values})"
 25
 26
 27def _str_to_date(self, expression):
 28    this = self.sql(expression, "this")
 29    time_format = self.format_time(expression)
 30    if time_format == Hive.date_format:
 31        return f"TO_DATE({this})"
 32    return f"TO_DATE({this}, {time_format})"
 33
 34
 35def _unix_to_time(self, expression):
 36    scale = expression.args.get("scale")
 37    timestamp = self.sql(expression, "this")
 38    if scale is None:
 39        return f"FROM_UNIXTIME({timestamp})"
 40    if scale == exp.UnixToTime.SECONDS:
 41        return f"TIMESTAMP_SECONDS({timestamp})"
 42    if scale == exp.UnixToTime.MILLIS:
 43        return f"TIMESTAMP_MILLIS({timestamp})"
 44    if scale == exp.UnixToTime.MICROS:
 45        return f"TIMESTAMP_MICROS({timestamp})"
 46
 47    raise ValueError("Improper scale for timestamp")
 48
 49
 50class Spark(Hive):
 51    class Parser(Hive.Parser):
 52        FUNCTIONS = {
 53            **Hive.Parser.FUNCTIONS,  # type: ignore
 54            "MAP_FROM_ARRAYS": exp.Map.from_arg_list,
 55            "TO_UNIX_TIMESTAMP": exp.StrToUnix.from_arg_list,
 56            "LEFT": lambda args: exp.Substring(
 57                this=seq_get(args, 0),
 58                start=exp.Literal.number(1),
 59                length=seq_get(args, 1),
 60            ),
 61            "SHIFTLEFT": lambda args: exp.BitwiseLeftShift(
 62                this=seq_get(args, 0),
 63                expression=seq_get(args, 1),
 64            ),
 65            "SHIFTRIGHT": lambda args: exp.BitwiseRightShift(
 66                this=seq_get(args, 0),
 67                expression=seq_get(args, 1),
 68            ),
 69            "RIGHT": lambda args: exp.Substring(
 70                this=seq_get(args, 0),
 71                start=exp.Sub(
 72                    this=exp.Length(this=seq_get(args, 0)),
 73                    expression=exp.Add(this=seq_get(args, 1), expression=exp.Literal.number(1)),
 74                ),
 75                length=seq_get(args, 1),
 76            ),
 77            "APPROX_PERCENTILE": exp.ApproxQuantile.from_arg_list,
 78            "IIF": exp.If.from_arg_list,
 79            "AGGREGATE": exp.Reduce.from_arg_list,
 80            "DAYOFWEEK": lambda args: exp.DayOfWeek(
 81                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 82            ),
 83            "DAYOFMONTH": lambda args: exp.DayOfMonth(
 84                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 85            ),
 86            "DAYOFYEAR": lambda args: exp.DayOfYear(
 87                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 88            ),
 89            "WEEKOFYEAR": lambda args: exp.WeekOfYear(
 90                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 91            ),
 92            "DATE_TRUNC": lambda args: exp.TimestampTrunc(
 93                this=seq_get(args, 1),
 94                unit=exp.var(seq_get(args, 0)),
 95            ),
 96            "TRUNC": lambda args: exp.DateTrunc(unit=seq_get(args, 1), this=seq_get(args, 0)),
 97        }
 98
 99        FUNCTION_PARSERS = {
100            **parser.Parser.FUNCTION_PARSERS,  # type: ignore
101            "BROADCAST": lambda self: self._parse_join_hint("BROADCAST"),
102            "BROADCASTJOIN": lambda self: self._parse_join_hint("BROADCASTJOIN"),
103            "MAPJOIN": lambda self: self._parse_join_hint("MAPJOIN"),
104            "MERGE": lambda self: self._parse_join_hint("MERGE"),
105            "SHUFFLEMERGE": lambda self: self._parse_join_hint("SHUFFLEMERGE"),
106            "MERGEJOIN": lambda self: self._parse_join_hint("MERGEJOIN"),
107            "SHUFFLE_HASH": lambda self: self._parse_join_hint("SHUFFLE_HASH"),
108            "SHUFFLE_REPLICATE_NL": lambda self: self._parse_join_hint("SHUFFLE_REPLICATE_NL"),
109        }
110
111        def _parse_add_column(self):
112            return self._match_text_seq("ADD", "COLUMNS") and self._parse_schema()
113
114        def _parse_drop_column(self):
115            return self._match_text_seq("DROP", "COLUMNS") and self.expression(
116                exp.Drop,
117                this=self._parse_schema(),
118                kind="COLUMNS",
119            )
120
121    class Generator(Hive.Generator):
122        TYPE_MAPPING = {
123            **Hive.Generator.TYPE_MAPPING,  # type: ignore
124            exp.DataType.Type.TINYINT: "BYTE",
125            exp.DataType.Type.SMALLINT: "SHORT",
126            exp.DataType.Type.BIGINT: "LONG",
127        }
128
129        PROPERTIES_LOCATION = {
130            **Hive.Generator.PROPERTIES_LOCATION,  # type: ignore
131            exp.EngineProperty: exp.Properties.Location.UNSUPPORTED,
132            exp.AutoIncrementProperty: exp.Properties.Location.UNSUPPORTED,
133            exp.CharacterSetProperty: exp.Properties.Location.UNSUPPORTED,
134            exp.CollateProperty: exp.Properties.Location.UNSUPPORTED,
135        }
136
137        TRANSFORMS = {
138            **Hive.Generator.TRANSFORMS,  # type: ignore
139            exp.ApproxDistinct: rename_func("APPROX_COUNT_DISTINCT"),
140            exp.FileFormatProperty: lambda self, e: f"USING {e.name.upper()}",
141            exp.ArraySum: lambda self, e: f"AGGREGATE({self.sql(e, 'this')}, 0, (acc, x) -> acc + x, acc -> acc)",
142            exp.BitwiseLeftShift: rename_func("SHIFTLEFT"),
143            exp.BitwiseRightShift: rename_func("SHIFTRIGHT"),
144            exp.DateTrunc: lambda self, e: self.func("TRUNC", e.this, e.args.get("unit")),
145            exp.Hint: lambda self, e: f" /*+ {self.expressions(e).strip()} */",
146            exp.StrToDate: _str_to_date,
147            exp.StrToTime: lambda self, e: f"TO_TIMESTAMP({self.sql(e, 'this')}, {self.format_time(e)})",
148            exp.UnixToTime: _unix_to_time,
149            exp.Create: _create_sql,
150            exp.Map: _map_sql,
151            exp.Reduce: rename_func("AGGREGATE"),
152            exp.StructKwarg: lambda self, e: f"{self.sql(e, 'this')}: {self.sql(e, 'expression')}",
153            exp.TimestampTrunc: lambda self, e: self.func(
154                "DATE_TRUNC", exp.Literal.string(e.text("unit")), e.this
155            ),
156            exp.Trim: trim_sql,
157            exp.VariancePop: rename_func("VAR_POP"),
158            exp.DateFromParts: rename_func("MAKE_DATE"),
159            exp.LogicalOr: rename_func("BOOL_OR"),
160            exp.LogicalAnd: rename_func("BOOL_AND"),
161            exp.DayOfWeek: rename_func("DAYOFWEEK"),
162            exp.DayOfMonth: rename_func("DAYOFMONTH"),
163            exp.DayOfYear: rename_func("DAYOFYEAR"),
164            exp.WeekOfYear: rename_func("WEEKOFYEAR"),
165            exp.AtTimeZone: lambda self, e: f"FROM_UTC_TIMESTAMP({self.sql(e, 'this')}, {self.sql(e, 'zone')})",
166        }
167        TRANSFORMS.pop(exp.ArraySort)
168        TRANSFORMS.pop(exp.ILike)
169
170        WRAP_DERIVED_VALUES = False
171        CREATE_FUNCTION_RETURN_AS = False
172
173        def cast_sql(self, expression: exp.Cast) -> str:
174            if isinstance(expression.this, exp.Cast) and expression.this.is_type(
175                exp.DataType.Type.JSON
176            ):
177                schema = f"'{self.sql(expression, 'to')}'"
178                return self.func("FROM_JSON", expression.this.this, schema)
179            if expression.to.is_type(exp.DataType.Type.JSON):
180                return self.func("TO_JSON", expression.this)
181
182            return super(Spark.Generator, self).cast_sql(expression)
183
184    class Tokenizer(Hive.Tokenizer):
185        HEX_STRINGS = [("X'", "'")]

class Spark(sqlglot.dialects.hive.Hive): View Source

 51class Spark(Hive):
 52    class Parser(Hive.Parser):
 53        FUNCTIONS = {
 54            **Hive.Parser.FUNCTIONS,  # type: ignore
 55            "MAP_FROM_ARRAYS": exp.Map.from_arg_list,
 56            "TO_UNIX_TIMESTAMP": exp.StrToUnix.from_arg_list,
 57            "LEFT": lambda args: exp.Substring(
 58                this=seq_get(args, 0),
 59                start=exp.Literal.number(1),
 60                length=seq_get(args, 1),
 61            ),
 62            "SHIFTLEFT": lambda args: exp.BitwiseLeftShift(
 63                this=seq_get(args, 0),
 64                expression=seq_get(args, 1),
 65            ),
 66            "SHIFTRIGHT": lambda args: exp.BitwiseRightShift(
 67                this=seq_get(args, 0),
 68                expression=seq_get(args, 1),
 69            ),
 70            "RIGHT": lambda args: exp.Substring(
 71                this=seq_get(args, 0),
 72                start=exp.Sub(
 73                    this=exp.Length(this=seq_get(args, 0)),
 74                    expression=exp.Add(this=seq_get(args, 1), expression=exp.Literal.number(1)),
 75                ),
 76                length=seq_get(args, 1),
 77            ),
 78            "APPROX_PERCENTILE": exp.ApproxQuantile.from_arg_list,
 79            "IIF": exp.If.from_arg_list,
 80            "AGGREGATE": exp.Reduce.from_arg_list,
 81            "DAYOFWEEK": lambda args: exp.DayOfWeek(
 82                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 83            ),
 84            "DAYOFMONTH": lambda args: exp.DayOfMonth(
 85                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 86            ),
 87            "DAYOFYEAR": lambda args: exp.DayOfYear(
 88                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 89            ),
 90            "WEEKOFYEAR": lambda args: exp.WeekOfYear(
 91                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 92            ),
 93            "DATE_TRUNC": lambda args: exp.TimestampTrunc(
 94                this=seq_get(args, 1),
 95                unit=exp.var(seq_get(args, 0)),
 96            ),
 97            "TRUNC": lambda args: exp.DateTrunc(unit=seq_get(args, 1), this=seq_get(args, 0)),
 98        }
 99
100        FUNCTION_PARSERS = {
101            **parser.Parser.FUNCTION_PARSERS,  # type: ignore
102            "BROADCAST": lambda self: self._parse_join_hint("BROADCAST"),
103            "BROADCASTJOIN": lambda self: self._parse_join_hint("BROADCASTJOIN"),
104            "MAPJOIN": lambda self: self._parse_join_hint("MAPJOIN"),
105            "MERGE": lambda self: self._parse_join_hint("MERGE"),
106            "SHUFFLEMERGE": lambda self: self._parse_join_hint("SHUFFLEMERGE"),
107            "MERGEJOIN": lambda self: self._parse_join_hint("MERGEJOIN"),
108            "SHUFFLE_HASH": lambda self: self._parse_join_hint("SHUFFLE_HASH"),
109            "SHUFFLE_REPLICATE_NL": lambda self: self._parse_join_hint("SHUFFLE_REPLICATE_NL"),
110        }
111
112        def _parse_add_column(self):
113            return self._match_text_seq("ADD", "COLUMNS") and self._parse_schema()
114
115        def _parse_drop_column(self):
116            return self._match_text_seq("DROP", "COLUMNS") and self.expression(
117                exp.Drop,
118                this=self._parse_schema(),
119                kind="COLUMNS",
120            )
121
122    class Generator(Hive.Generator):
123        TYPE_MAPPING = {
124            **Hive.Generator.TYPE_MAPPING,  # type: ignore
125            exp.DataType.Type.TINYINT: "BYTE",
126            exp.DataType.Type.SMALLINT: "SHORT",
127            exp.DataType.Type.BIGINT: "LONG",
128        }
129
130        PROPERTIES_LOCATION = {
131            **Hive.Generator.PROPERTIES_LOCATION,  # type: ignore
132            exp.EngineProperty: exp.Properties.Location.UNSUPPORTED,
133            exp.AutoIncrementProperty: exp.Properties.Location.UNSUPPORTED,
134            exp.CharacterSetProperty: exp.Properties.Location.UNSUPPORTED,
135            exp.CollateProperty: exp.Properties.Location.UNSUPPORTED,
136        }
137
138        TRANSFORMS = {
139            **Hive.Generator.TRANSFORMS,  # type: ignore
140            exp.ApproxDistinct: rename_func("APPROX_COUNT_DISTINCT"),
141            exp.FileFormatProperty: lambda self, e: f"USING {e.name.upper()}",
142            exp.ArraySum: lambda self, e: f"AGGREGATE({self.sql(e, 'this')}, 0, (acc, x) -> acc + x, acc -> acc)",
143            exp.BitwiseLeftShift: rename_func("SHIFTLEFT"),
144            exp.BitwiseRightShift: rename_func("SHIFTRIGHT"),
145            exp.DateTrunc: lambda self, e: self.func("TRUNC", e.this, e.args.get("unit")),
146            exp.Hint: lambda self, e: f" /*+ {self.expressions(e).strip()} */",
147            exp.StrToDate: _str_to_date,
148            exp.StrToTime: lambda self, e: f"TO_TIMESTAMP({self.sql(e, 'this')}, {self.format_time(e)})",
149            exp.UnixToTime: _unix_to_time,
150            exp.Create: _create_sql,
151            exp.Map: _map_sql,
152            exp.Reduce: rename_func("AGGREGATE"),
153            exp.StructKwarg: lambda self, e: f"{self.sql(e, 'this')}: {self.sql(e, 'expression')}",
154            exp.TimestampTrunc: lambda self, e: self.func(
155                "DATE_TRUNC", exp.Literal.string(e.text("unit")), e.this
156            ),
157            exp.Trim: trim_sql,
158            exp.VariancePop: rename_func("VAR_POP"),
159            exp.DateFromParts: rename_func("MAKE_DATE"),
160            exp.LogicalOr: rename_func("BOOL_OR"),
161            exp.LogicalAnd: rename_func("BOOL_AND"),
162            exp.DayOfWeek: rename_func("DAYOFWEEK"),
163            exp.DayOfMonth: rename_func("DAYOFMONTH"),
164            exp.DayOfYear: rename_func("DAYOFYEAR"),
165            exp.WeekOfYear: rename_func("WEEKOFYEAR"),
166            exp.AtTimeZone: lambda self, e: f"FROM_UTC_TIMESTAMP({self.sql(e, 'this')}, {self.sql(e, 'zone')})",
167        }
168        TRANSFORMS.pop(exp.ArraySort)
169        TRANSFORMS.pop(exp.ILike)
170
171        WRAP_DERIVED_VALUES = False
172        CREATE_FUNCTION_RETURN_AS = False
173
174        def cast_sql(self, expression: exp.Cast) -> str:
175            if isinstance(expression.this, exp.Cast) and expression.this.is_type(
176                exp.DataType.Type.JSON
177            ):
178                schema = f"'{self.sql(expression, 'to')}'"
179                return self.func("FROM_JSON", expression.this.this, schema)
180            if expression.to.is_type(exp.DataType.Type.JSON):
181                return self.func("TO_JSON", expression.this)
182
183            return super(Spark.Generator, self).cast_sql(expression)
184
185    class Tokenizer(Hive.Tokenizer):
186        HEX_STRINGS = [("X'", "'")]

Inherited Members

sqlglot.dialects.dialect.Dialect: get_or_raise; format_time; parse; parse_into; generate; transpile; tokenize; parser; generator

class Spark.Parser(sqlglot.dialects.hive.Hive.Parser): View Source

 52    class Parser(Hive.Parser):
 53        FUNCTIONS = {
 54            **Hive.Parser.FUNCTIONS,  # type: ignore
 55            "MAP_FROM_ARRAYS": exp.Map.from_arg_list,
 56            "TO_UNIX_TIMESTAMP": exp.StrToUnix.from_arg_list,
 57            "LEFT": lambda args: exp.Substring(
 58                this=seq_get(args, 0),
 59                start=exp.Literal.number(1),
 60                length=seq_get(args, 1),
 61            ),
 62            "SHIFTLEFT": lambda args: exp.BitwiseLeftShift(
 63                this=seq_get(args, 0),
 64                expression=seq_get(args, 1),
 65            ),
 66            "SHIFTRIGHT": lambda args: exp.BitwiseRightShift(
 67                this=seq_get(args, 0),
 68                expression=seq_get(args, 1),
 69            ),
 70            "RIGHT": lambda args: exp.Substring(
 71                this=seq_get(args, 0),
 72                start=exp.Sub(
 73                    this=exp.Length(this=seq_get(args, 0)),
 74                    expression=exp.Add(this=seq_get(args, 1), expression=exp.Literal.number(1)),
 75                ),
 76                length=seq_get(args, 1),
 77            ),
 78            "APPROX_PERCENTILE": exp.ApproxQuantile.from_arg_list,
 79            "IIF": exp.If.from_arg_list,
 80            "AGGREGATE": exp.Reduce.from_arg_list,
 81            "DAYOFWEEK": lambda args: exp.DayOfWeek(
 82                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 83            ),
 84            "DAYOFMONTH": lambda args: exp.DayOfMonth(
 85                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 86            ),
 87            "DAYOFYEAR": lambda args: exp.DayOfYear(
 88                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 89            ),
 90            "WEEKOFYEAR": lambda args: exp.WeekOfYear(
 91                this=exp.TsOrDsToDate(this=seq_get(args, 0)),
 92            ),
 93            "DATE_TRUNC": lambda args: exp.TimestampTrunc(
 94                this=seq_get(args, 1),
 95                unit=exp.var(seq_get(args, 0)),
 96            ),
 97            "TRUNC": lambda args: exp.DateTrunc(unit=seq_get(args, 1), this=seq_get(args, 0)),
 98        }
 99
100        FUNCTION_PARSERS = {
101            **parser.Parser.FUNCTION_PARSERS,  # type: ignore
102            "BROADCAST": lambda self: self._parse_join_hint("BROADCAST"),
103            "BROADCASTJOIN": lambda self: self._parse_join_hint("BROADCASTJOIN"),
104            "MAPJOIN": lambda self: self._parse_join_hint("MAPJOIN"),
105            "MERGE": lambda self: self._parse_join_hint("MERGE"),
106            "SHUFFLEMERGE": lambda self: self._parse_join_hint("SHUFFLEMERGE"),
107            "MERGEJOIN": lambda self: self._parse_join_hint("MERGEJOIN"),
108            "SHUFFLE_HASH": lambda self: self._parse_join_hint("SHUFFLE_HASH"),
109            "SHUFFLE_REPLICATE_NL": lambda self: self._parse_join_hint("SHUFFLE_REPLICATE_NL"),
110        }
111
112        def _parse_add_column(self):
113            return self._match_text_seq("ADD", "COLUMNS") and self._parse_schema()
114
115        def _parse_drop_column(self):
116            return self._match_text_seq("DROP", "COLUMNS") and self.expression(
117                exp.Drop,
118                this=self._parse_schema(),
119                kind="COLUMNS",
120            )

Parser consumes a list of tokens produced by the sqlglot.tokens.Tokenizer and produces a parsed syntax tree.

Arguments:

error_level: the desired error level. Default: ErrorLevel.RAISE
error_message_context: determines the amount of context to capture from a query string when displaying the error message (in number of characters). Default: 50.
index_offset: Index offset for arrays eg ARRAY[0] vs ARRAY[1] as the head of a list. Default: 0
alias_post_tablesample: If the table alias comes after tablesample. Default: False
max_errors: Maximum number of error messages to include in a raised ParseError. This is only relevant if error_level is ErrorLevel.RAISE. Default: 3
null_ordering: Indicates the default null ordering method to use if not explicitly set. Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". Default: "nulls_are_small"

Inherited Members

sqlglot.parser.Parser: Parser; reset; parse; parse_into; check_errors; raise_error; expression; validate_expression

class Spark.Generator(sqlglot.dialects.hive.Hive.Generator): View Source

122    class Generator(Hive.Generator):
123        TYPE_MAPPING = {
124            **Hive.Generator.TYPE_MAPPING,  # type: ignore
125            exp.DataType.Type.TINYINT: "BYTE",
126            exp.DataType.Type.SMALLINT: "SHORT",
127            exp.DataType.Type.BIGINT: "LONG",
128        }
129
130        PROPERTIES_LOCATION = {
131            **Hive.Generator.PROPERTIES_LOCATION,  # type: ignore
132            exp.EngineProperty: exp.Properties.Location.UNSUPPORTED,
133            exp.AutoIncrementProperty: exp.Properties.Location.UNSUPPORTED,
134            exp.CharacterSetProperty: exp.Properties.Location.UNSUPPORTED,
135            exp.CollateProperty: exp.Properties.Location.UNSUPPORTED,
136        }
137
138        TRANSFORMS = {
139            **Hive.Generator.TRANSFORMS,  # type: ignore
140            exp.ApproxDistinct: rename_func("APPROX_COUNT_DISTINCT"),
141            exp.FileFormatProperty: lambda self, e: f"USING {e.name.upper()}",
142            exp.ArraySum: lambda self, e: f"AGGREGATE({self.sql(e, 'this')}, 0, (acc, x) -> acc + x, acc -> acc)",
143            exp.BitwiseLeftShift: rename_func("SHIFTLEFT"),
144            exp.BitwiseRightShift: rename_func("SHIFTRIGHT"),
145            exp.DateTrunc: lambda self, e: self.func("TRUNC", e.this, e.args.get("unit")),
146            exp.Hint: lambda self, e: f" /*+ {self.expressions(e).strip()} */",
147            exp.StrToDate: _str_to_date,
148            exp.StrToTime: lambda self, e: f"TO_TIMESTAMP({self.sql(e, 'this')}, {self.format_time(e)})",
149            exp.UnixToTime: _unix_to_time,
150            exp.Create: _create_sql,
151            exp.Map: _map_sql,
152            exp.Reduce: rename_func("AGGREGATE"),
153            exp.StructKwarg: lambda self, e: f"{self.sql(e, 'this')}: {self.sql(e, 'expression')}",
154            exp.TimestampTrunc: lambda self, e: self.func(
155                "DATE_TRUNC", exp.Literal.string(e.text("unit")), e.this
156            ),
157            exp.Trim: trim_sql,
158            exp.VariancePop: rename_func("VAR_POP"),
159            exp.DateFromParts: rename_func("MAKE_DATE"),
160            exp.LogicalOr: rename_func("BOOL_OR"),
161            exp.LogicalAnd: rename_func("BOOL_AND"),
162            exp.DayOfWeek: rename_func("DAYOFWEEK"),
163            exp.DayOfMonth: rename_func("DAYOFMONTH"),
164            exp.DayOfYear: rename_func("DAYOFYEAR"),
165            exp.WeekOfYear: rename_func("WEEKOFYEAR"),
166            exp.AtTimeZone: lambda self, e: f"FROM_UTC_TIMESTAMP({self.sql(e, 'this')}, {self.sql(e, 'zone')})",
167        }
168        TRANSFORMS.pop(exp.ArraySort)
169        TRANSFORMS.pop(exp.ILike)
170
171        WRAP_DERIVED_VALUES = False
172        CREATE_FUNCTION_RETURN_AS = False
173
174        def cast_sql(self, expression: exp.Cast) -> str:
175            if isinstance(expression.this, exp.Cast) and expression.this.is_type(
176                exp.DataType.Type.JSON
177            ):
178                schema = f"'{self.sql(expression, 'to')}'"
179                return self.func("FROM_JSON", expression.this.this, schema)
180            if expression.to.is_type(exp.DataType.Type.JSON):
181                return self.func("TO_JSON", expression.this)
182
183            return super(Spark.Generator, self).cast_sql(expression)

Generator interprets the given syntax tree and produces a SQL string as an output.

Arguments:

time_mapping (dict): the dictionary of custom time mappings in which the key represents a python time format and the output the target time format
time_trie (trie): a trie of the time_mapping keys
pretty (bool): if set to True the returned string will be formatted. Default: False.
quote_start (str): specifies which starting character to use to delimit quotes. Default: '.
quote_end (str): specifies which ending character to use to delimit quotes. Default: '.
identifier_start (str): specifies which starting character to use to delimit identifiers. Default: ".
identifier_end (str): specifies which ending character to use to delimit identifiers. Default: ".
identify (bool | str): 'always': always quote, 'safe': quote identifiers if they don't contain an upcase, True defaults to always.
normalize (bool): if set to True all identifiers will lower cased
string_escape (str): specifies a string escape character. Default: '.
identifier_escape (str): specifies an identifier escape character. Default: ".
pad (int): determines padding in a formatted string. Default: 2.
indent (int): determines the size of indentation in a formatted string. Default: 4.
unnest_column_only (bool): if true unnest table aliases are considered only as column aliases
normalize_functions (str): normalize function names, "upper", "lower", or None Default: "upper"
alias_post_tablesample (bool): if the table alias comes after tablesample Default: False
unsupported_level (ErrorLevel): determines the generator's behavior when it encounters unsupported expressions. Default ErrorLevel.WARN.
null_ordering (str): Indicates the default null ordering method to use if not explicitly set. Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". Default: "nulls_are_small"
max_unsupported (int): Maximum number of unsupported messages to include in a raised UnsupportedError. This is only relevant if unsupported_level is ErrorLevel.RAISE. Default: 3
leading_comma (bool): if the the comma is leading or trailing in select statements Default: False
max_text_width: The max number of characters in a segment before creating new lines in pretty mode. The default is on the smaller end because the length only represents a segment and not the true line length. Default: 80
comments: Whether or not to preserve comments in the output SQL code. Default: True

def cast_sql(self, expression: sqlglot.expressions.Cast) -> str: View Source

174        def cast_sql(self, expression: exp.Cast) -> str:
175            if isinstance(expression.this, exp.Cast) and expression.this.is_type(
176                exp.DataType.Type.JSON
177            ):
178                schema = f"'{self.sql(expression, 'to')}'"
179                return self.func("FROM_JSON", expression.this.this, schema)
180            if expression.to.is_type(exp.DataType.Type.JSON):
181                return self.func("TO_JSON", expression.this)
182
183            return super(Spark.Generator, self).cast_sql(expression)

Inherited Members

sqlglot.generator.Generator: Generator; generate; unsupported; sep; seg; pad_comment; maybe_comment; wrap; no_identify; normalize_func; indent; sql; uncache_sql; cache_sql; characterset_sql; column_sql; columnposition_sql; columndef_sql; columnconstraint_sql; autoincrementcolumnconstraint_sql; compresscolumnconstraint_sql; generatedasidentitycolumnconstraint_sql; notnullcolumnconstraint_sql; primarykeycolumnconstraint_sql; uniquecolumnconstraint_sql; create_sql; describe_sql; prepend_ctes; with_sql; cte_sql; tablealias_sql; bitstring_sql; hexstring_sql; directory_sql; delete_sql; drop_sql; except_sql; except_op; fetch_sql; filter_sql; hint_sql; index_sql; identifier_sql; national_sql; partition_sql; properties_sql; root_properties; properties; locate_properties; property_sql; likeproperty_sql; fallbackproperty_sql; journalproperty_sql; freespaceproperty_sql; afterjournalproperty_sql; checksumproperty_sql; mergeblockratioproperty_sql; datablocksizeproperty_sql; blockcompressionproperty_sql; isolatedloadingproperty_sql; lockingproperty_sql; withdataproperty_sql; insert_sql; intersect_sql; intersect_op; introducer_sql; pseudotype_sql; returning_sql; rowformatdelimitedproperty_sql; table_sql; tablesample_sql; pivot_sql; tuple_sql; update_sql; values_sql; var_sql; into_sql; from_sql; group_sql; having_sql; join_sql; lambda_sql; lateral_sql; limit_sql; offset_sql; setitem_sql; set_sql; pragma_sql; lock_sql; literal_sql; loaddata_sql; null_sql; boolean_sql; order_sql; cluster_sql; distribute_sql; sort_sql; ordered_sql; matchrecognize_sql; query_modifiers; select_sql; schema_sql; star_sql; structkwarg_sql; parameter_sql; sessionparameter_sql; placeholder_sql; subquery_sql; qualify_sql; union_sql; union_op; unnest_sql; where_sql; window_sql; partition_by_sql; window_spec_sql; withingroup_sql; between_sql; bracket_sql; all_sql; any_sql; exists_sql; case_sql; constraint_sql; extract_sql; trim_sql; concat_sql; check_sql; foreignkey_sql; primarykey_sql; unique_sql; if_sql; matchagainst_sql; jsonkeyvalue_sql; jsonobject_sql; in_sql; in_unnest_op; interval_sql; return_sql; reference_sql; anonymous_sql; paren_sql; neg_sql; not_sql; alias_sql; aliases_sql; attimezone_sql; add_sql; and_sql; connector_sql; bitwiseand_sql; bitwiseleftshift_sql; bitwisenot_sql; bitwiseor_sql; bitwiserightshift_sql; bitwisexor_sql; currentdate_sql; collate_sql; command_sql; comment_sql; transaction_sql; commit_sql; rollback_sql; altercolumn_sql; renametable_sql; altertable_sql; droppartition_sql; addconstraint_sql; distinct_sql; ignorenulls_sql; respectnulls_sql; intdiv_sql; dpipe_sql; div_sql; overlaps_sql; distance_sql; dot_sql; eq_sql; escape_sql; glob_sql; gt_sql; gte_sql; ilike_sql; ilikeany_sql; is_sql; like_sql; likeany_sql; similarto_sql; lt_sql; lte_sql; mod_sql; mul_sql; neq_sql; nullsafeeq_sql; nullsafeneq_sql; or_sql; slice_sql; sub_sql; trycast_sql; use_sql; binary; function_fallback_sql; func; format_args; text_width; format_time; expressions; op_expressions; naked_property; set_operation; tag_sql; token_sql; userdefinedfunction_sql; joinhint_sql; kwarg_sql; when_sql; merge_sql; tochar_sql
sqlglot.dialects.hive.Hive.Generator: arrayagg_sql; with_properties; datatype_sql

class Spark.Tokenizer(sqlglot.dialects.hive.Hive.Tokenizer): View Source

185    class Tokenizer(Hive.Tokenizer):
186        HEX_STRINGS = [("X'", "'")]

Inherited Members

sqlglot.tokens.Tokenizer: reset; tokenize