diff options
Diffstat (limited to 'docs/sqlglot/dialects/spark.html')
| -rw-r--r-- | docs/sqlglot/dialects/spark.html | 561 |
1 files changed, 289 insertions, 272 deletions
diff --git a/docs/sqlglot/dialects/spark.html b/docs/sqlglot/dialects/spark.html index 75d5ac4..ae58e10 100644 --- a/docs/sqlglot/dialects/spark.html +++ b/docs/sqlglot/dialects/spark.html @@ -171,92 +171,99 @@ </span><span id="L-86"><a href="#L-86"><span class="linenos"> 86</span></a> <span class="s2">"WEEKOFYEAR"</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">WeekOfYear</span><span class="p">(</span> </span><span id="L-87"><a href="#L-87"><span class="linenos"> 87</span></a> <span class="n">this</span><span class="o">=</span><span class="n">exp</span><span class="o">.</span><span class="n">TsOrDsToDate</span><span class="p">(</span><span class="n">this</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> </span><span id="L-88"><a href="#L-88"><span class="linenos"> 88</span></a> <span class="p">),</span> -</span><span id="L-89"><a href="#L-89"><span class="linenos"> 89</span></a> <span class="p">}</span> -</span><span id="L-90"><a href="#L-90"><span class="linenos"> 90</span></a> -</span><span id="L-91"><a href="#L-91"><span class="linenos"> 91</span></a> <span class="n">FUNCTION_PARSERS</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="L-92"><a href="#L-92"><span class="linenos"> 92</span></a> <span class="o">**</span><span class="n">parser</span><span class="o">.</span><span class="n">Parser</span><span class="o">.</span><span class="n">FUNCTION_PARSERS</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="L-93"><a href="#L-93"><span class="linenos"> 93</span></a> <span class="s2">"BROADCAST"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCAST"</span><span class="p">),</span> -</span><span id="L-94"><a href="#L-94"><span class="linenos"> 94</span></a> <span class="s2">"BROADCASTJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCASTJOIN"</span><span class="p">),</span> -</span><span id="L-95"><a href="#L-95"><span class="linenos"> 95</span></a> <span class="s2">"MAPJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MAPJOIN"</span><span class="p">),</span> -</span><span id="L-96"><a href="#L-96"><span class="linenos"> 96</span></a> <span class="s2">"MERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGE"</span><span class="p">),</span> -</span><span id="L-97"><a href="#L-97"><span class="linenos"> 97</span></a> <span class="s2">"SHUFFLEMERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLEMERGE"</span><span class="p">),</span> -</span><span id="L-98"><a href="#L-98"><span class="linenos"> 98</span></a> <span class="s2">"MERGEJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGEJOIN"</span><span class="p">),</span> -</span><span id="L-99"><a href="#L-99"><span class="linenos"> 99</span></a> <span class="s2">"SHUFFLE_HASH"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_HASH"</span><span class="p">),</span> -</span><span id="L-100"><a href="#L-100"><span class="linenos">100</span></a> <span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">),</span> -</span><span id="L-101"><a href="#L-101"><span class="linenos">101</span></a> <span class="p">}</span> -</span><span id="L-102"><a href="#L-102"><span class="linenos">102</span></a> -</span><span id="L-103"><a href="#L-103"><span class="linenos">103</span></a> <span class="k">def</span> <span class="nf">_parse_add_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> -</span><span id="L-104"><a href="#L-104"><span class="linenos">104</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"ADD"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">()</span> -</span><span id="L-105"><a href="#L-105"><span class="linenos">105</span></a> -</span><span id="L-106"><a href="#L-106"><span class="linenos">106</span></a> <span class="k">def</span> <span class="nf">_parse_drop_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> -</span><span id="L-107"><a href="#L-107"><span class="linenos">107</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"DROP"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">expression</span><span class="p">(</span> -</span><span id="L-108"><a href="#L-108"><span class="linenos">108</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Drop</span><span class="p">,</span> -</span><span id="L-109"><a href="#L-109"><span class="linenos">109</span></a> <span class="n">this</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">(),</span> -</span><span id="L-110"><a href="#L-110"><span class="linenos">110</span></a> <span class="n">kind</span><span class="o">=</span><span class="s2">"COLUMNS"</span><span class="p">,</span> -</span><span id="L-111"><a href="#L-111"><span class="linenos">111</span></a> <span class="p">)</span> -</span><span id="L-112"><a href="#L-112"><span class="linenos">112</span></a> -</span><span id="L-113"><a href="#L-113"><span class="linenos">113</span></a> <span class="k">class</span> <span class="nc">Generator</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="p">):</span> -</span><span id="L-114"><a href="#L-114"><span class="linenos">114</span></a> <span class="n">TYPE_MAPPING</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="L-115"><a href="#L-115"><span class="linenos">115</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TYPE_MAPPING</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="L-116"><a href="#L-116"><span class="linenos">116</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">TINYINT</span><span class="p">:</span> <span class="s2">"BYTE"</span><span class="p">,</span> -</span><span id="L-117"><a href="#L-117"><span class="linenos">117</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">SMALLINT</span><span class="p">:</span> <span class="s2">"SHORT"</span><span class="p">,</span> -</span><span id="L-118"><a href="#L-118"><span class="linenos">118</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">BIGINT</span><span class="p">:</span> <span class="s2">"LONG"</span><span class="p">,</span> -</span><span id="L-119"><a href="#L-119"><span class="linenos">119</span></a> <span class="p">}</span> -</span><span id="L-120"><a href="#L-120"><span class="linenos">120</span></a> -</span><span id="L-121"><a href="#L-121"><span class="linenos">121</span></a> <span class="n">PROPERTIES_LOCATION</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="L-122"><a href="#L-122"><span class="linenos">122</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">PROPERTIES_LOCATION</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="L-123"><a href="#L-123"><span class="linenos">123</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">EngineProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="L-124"><a href="#L-124"><span class="linenos">124</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AutoIncrementProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="L-125"><a href="#L-125"><span class="linenos">125</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CharacterSetProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="L-126"><a href="#L-126"><span class="linenos">126</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CollateProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="L-127"><a href="#L-127"><span class="linenos">127</span></a> <span class="p">}</span> -</span><span id="L-128"><a href="#L-128"><span class="linenos">128</span></a> -</span><span id="L-129"><a href="#L-129"><span class="linenos">129</span></a> <span class="n">TRANSFORMS</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="L-130"><a href="#L-130"><span class="linenos">130</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TRANSFORMS</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="L-131"><a href="#L-131"><span class="linenos">131</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ApproxDistinct</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"APPROX_COUNT_DISTINCT"</span><span class="p">),</span> -</span><span id="L-132"><a href="#L-132"><span class="linenos">132</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">FileFormatProperty</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"USING </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> -</span><span id="L-133"><a href="#L-133"><span class="linenos">133</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ArraySum</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"AGGREGATE(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, 0, (acc, x) -> acc + x, acc -> acc)"</span><span class="p">,</span> -</span><span id="L-134"><a href="#L-134"><span class="linenos">134</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseLeftShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTLEFT"</span><span class="p">),</span> -</span><span id="L-135"><a href="#L-135"><span class="linenos">135</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseRightShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTRIGHT"</span><span class="p">),</span> -</span><span id="L-136"><a href="#L-136"><span class="linenos">136</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateTrunc</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"TRUNC"</span><span class="p">),</span> -</span><span id="L-137"><a href="#L-137"><span class="linenos">137</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Hint</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">" /*+ </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">expressions</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="si">}</span><span class="s2"> */"</span><span class="p">,</span> -</span><span id="L-138"><a href="#L-138"><span class="linenos">138</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToDate</span><span class="p">:</span> <span class="n">_str_to_date</span><span class="p">,</span> -</span><span id="L-139"><a href="#L-139"><span class="linenos">139</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToTime</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"TO_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">format_time</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> -</span><span id="L-140"><a href="#L-140"><span class="linenos">140</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">UnixToTime</span><span class="p">:</span> <span class="n">_unix_to_time</span><span class="p">,</span> -</span><span id="L-141"><a href="#L-141"><span class="linenos">141</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Create</span><span class="p">:</span> <span class="n">_create_sql</span><span class="p">,</span> -</span><span id="L-142"><a href="#L-142"><span class="linenos">142</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Map</span><span class="p">:</span> <span class="n">_map_sql</span><span class="p">,</span> -</span><span id="L-143"><a href="#L-143"><span class="linenos">143</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Reduce</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"AGGREGATE"</span><span class="p">),</span> -</span><span id="L-144"><a href="#L-144"><span class="linenos">144</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StructKwarg</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'expression'</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> -</span><span id="L-145"><a href="#L-145"><span class="linenos">145</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">TimestampTrunc</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"DATE_TRUNC(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'unit'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> -</span><span id="L-146"><a href="#L-146"><span class="linenos">146</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Trim</span><span class="p">:</span> <span class="n">trim_sql</span><span class="p">,</span> -</span><span id="L-147"><a href="#L-147"><span class="linenos">147</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">VariancePop</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"VAR_POP"</span><span class="p">),</span> -</span><span id="L-148"><a href="#L-148"><span class="linenos">148</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateFromParts</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"MAKE_DATE"</span><span class="p">),</span> -</span><span id="L-149"><a href="#L-149"><span class="linenos">149</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">LogicalOr</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"BOOL_OR"</span><span class="p">),</span> -</span><span id="L-150"><a href="#L-150"><span class="linenos">150</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfWeek</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFWEEK"</span><span class="p">),</span> -</span><span id="L-151"><a href="#L-151"><span class="linenos">151</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfMonth</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFMONTH"</span><span class="p">),</span> -</span><span id="L-152"><a href="#L-152"><span class="linenos">152</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFYEAR"</span><span class="p">),</span> -</span><span id="L-153"><a href="#L-153"><span class="linenos">153</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">WeekOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"WEEKOFYEAR"</span><span class="p">),</span> -</span><span id="L-154"><a href="#L-154"><span class="linenos">154</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AtTimeZone</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"FROM_UTC_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'zone'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> -</span><span id="L-155"><a href="#L-155"><span class="linenos">155</span></a> <span class="p">}</span> -</span><span id="L-156"><a href="#L-156"><span class="linenos">156</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ArraySort</span><span class="p">)</span> -</span><span id="L-157"><a href="#L-157"><span class="linenos">157</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ILike</span><span class="p">)</span> -</span><span id="L-158"><a href="#L-158"><span class="linenos">158</span></a> -</span><span id="L-159"><a href="#L-159"><span class="linenos">159</span></a> <span class="n">WRAP_DERIVED_VALUES</span> <span class="o">=</span> <span class="kc">False</span> -</span><span id="L-160"><a href="#L-160"><span class="linenos">160</span></a> <span class="n">CREATE_FUNCTION_AS</span> <span class="o">=</span> <span class="kc">False</span> -</span><span id="L-161"><a href="#L-161"><span class="linenos">161</span></a> -</span><span id="L-162"><a href="#L-162"><span class="linenos">162</span></a> <span class="k">def</span> <span class="nf">cast_sql</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">expression</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> -</span><span id="L-163"><a href="#L-163"><span class="linenos">163</span></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="ow">and</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span> -</span><span id="L-164"><a href="#L-164"><span class="linenos">164</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span> -</span><span id="L-165"><a href="#L-165"><span class="linenos">165</span></a> <span class="p">):</span> -</span><span id="L-166"><a href="#L-166"><span class="linenos">166</span></a> <span class="n">schema</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"'</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'to'</span><span class="p">)</span><span class="si">}</span><span class="s2">'"</span> -</span><span id="L-167"><a href="#L-167"><span class="linenos">167</span></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"FROM_JSON(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">format_args</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">),</span><span class="w"> </span><span class="n">schema</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span> -</span><span id="L-168"><a href="#L-168"><span class="linenos">168</span></a> <span class="k">if</span> <span class="n">expression</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span><span class="p">):</span> -</span><span id="L-169"><a href="#L-169"><span class="linenos">169</span></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"TO_JSON(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span> -</span><span id="L-170"><a href="#L-170"><span class="linenos">170</span></a> -</span><span id="L-171"><a href="#L-171"><span class="linenos">171</span></a> <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Spark</span><span class="o">.</span><span class="n">Generator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">cast_sql</span><span class="p">(</span><span class="n">expression</span><span class="p">)</span> -</span><span id="L-172"><a href="#L-172"><span class="linenos">172</span></a> -</span><span id="L-173"><a href="#L-173"><span class="linenos">173</span></a> <span class="k">class</span> <span class="nc">Tokenizer</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Tokenizer</span><span class="p">):</span> -</span><span id="L-174"><a href="#L-174"><span class="linenos">174</span></a> <span class="n">HEX_STRINGS</span> <span class="o">=</span> <span class="p">[(</span><span class="s2">"X'"</span><span class="p">,</span> <span class="s2">"'"</span><span class="p">)]</span> +</span><span id="L-89"><a href="#L-89"><span class="linenos"> 89</span></a> <span class="s2">"DATE_TRUNC"</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">TimestampTrunc</span><span class="p">(</span> +</span><span id="L-90"><a href="#L-90"><span class="linenos"> 90</span></a> <span class="n">this</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> +</span><span id="L-91"><a href="#L-91"><span class="linenos"> 91</span></a> <span class="n">unit</span><span class="o">=</span><span class="n">exp</span><span class="o">.</span><span class="n">var</span><span class="p">(</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> +</span><span id="L-92"><a href="#L-92"><span class="linenos"> 92</span></a> <span class="p">),</span> +</span><span id="L-93"><a href="#L-93"><span class="linenos"> 93</span></a> <span class="s2">"TRUNC"</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">DateTrunc</span><span class="p">(</span><span class="n">unit</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">this</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> +</span><span id="L-94"><a href="#L-94"><span class="linenos"> 94</span></a> <span class="p">}</span> +</span><span id="L-95"><a href="#L-95"><span class="linenos"> 95</span></a> +</span><span id="L-96"><a href="#L-96"><span class="linenos"> 96</span></a> <span class="n">FUNCTION_PARSERS</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="L-97"><a href="#L-97"><span class="linenos"> 97</span></a> <span class="o">**</span><span class="n">parser</span><span class="o">.</span><span class="n">Parser</span><span class="o">.</span><span class="n">FUNCTION_PARSERS</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="L-98"><a href="#L-98"><span class="linenos"> 98</span></a> <span class="s2">"BROADCAST"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCAST"</span><span class="p">),</span> +</span><span id="L-99"><a href="#L-99"><span class="linenos"> 99</span></a> <span class="s2">"BROADCASTJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCASTJOIN"</span><span class="p">),</span> +</span><span id="L-100"><a href="#L-100"><span class="linenos">100</span></a> <span class="s2">"MAPJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MAPJOIN"</span><span class="p">),</span> +</span><span id="L-101"><a href="#L-101"><span class="linenos">101</span></a> <span class="s2">"MERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGE"</span><span class="p">),</span> +</span><span id="L-102"><a href="#L-102"><span class="linenos">102</span></a> <span class="s2">"SHUFFLEMERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLEMERGE"</span><span class="p">),</span> +</span><span id="L-103"><a href="#L-103"><span class="linenos">103</span></a> <span class="s2">"MERGEJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGEJOIN"</span><span class="p">),</span> +</span><span id="L-104"><a href="#L-104"><span class="linenos">104</span></a> <span class="s2">"SHUFFLE_HASH"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_HASH"</span><span class="p">),</span> +</span><span id="L-105"><a href="#L-105"><span class="linenos">105</span></a> <span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">),</span> +</span><span id="L-106"><a href="#L-106"><span class="linenos">106</span></a> <span class="p">}</span> +</span><span id="L-107"><a href="#L-107"><span class="linenos">107</span></a> +</span><span id="L-108"><a href="#L-108"><span class="linenos">108</span></a> <span class="k">def</span> <span class="nf">_parse_add_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> +</span><span id="L-109"><a href="#L-109"><span class="linenos">109</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"ADD"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">()</span> +</span><span id="L-110"><a href="#L-110"><span class="linenos">110</span></a> +</span><span id="L-111"><a href="#L-111"><span class="linenos">111</span></a> <span class="k">def</span> <span class="nf">_parse_drop_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> +</span><span id="L-112"><a href="#L-112"><span class="linenos">112</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"DROP"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">expression</span><span class="p">(</span> +</span><span id="L-113"><a href="#L-113"><span class="linenos">113</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Drop</span><span class="p">,</span> +</span><span id="L-114"><a href="#L-114"><span class="linenos">114</span></a> <span class="n">this</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">(),</span> +</span><span id="L-115"><a href="#L-115"><span class="linenos">115</span></a> <span class="n">kind</span><span class="o">=</span><span class="s2">"COLUMNS"</span><span class="p">,</span> +</span><span id="L-116"><a href="#L-116"><span class="linenos">116</span></a> <span class="p">)</span> +</span><span id="L-117"><a href="#L-117"><span class="linenos">117</span></a> +</span><span id="L-118"><a href="#L-118"><span class="linenos">118</span></a> <span class="k">class</span> <span class="nc">Generator</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="p">):</span> +</span><span id="L-119"><a href="#L-119"><span class="linenos">119</span></a> <span class="n">TYPE_MAPPING</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="L-120"><a href="#L-120"><span class="linenos">120</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TYPE_MAPPING</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="L-121"><a href="#L-121"><span class="linenos">121</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">TINYINT</span><span class="p">:</span> <span class="s2">"BYTE"</span><span class="p">,</span> +</span><span id="L-122"><a href="#L-122"><span class="linenos">122</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">SMALLINT</span><span class="p">:</span> <span class="s2">"SHORT"</span><span class="p">,</span> +</span><span id="L-123"><a href="#L-123"><span class="linenos">123</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">BIGINT</span><span class="p">:</span> <span class="s2">"LONG"</span><span class="p">,</span> +</span><span id="L-124"><a href="#L-124"><span class="linenos">124</span></a> <span class="p">}</span> +</span><span id="L-125"><a href="#L-125"><span class="linenos">125</span></a> +</span><span id="L-126"><a href="#L-126"><span class="linenos">126</span></a> <span class="n">PROPERTIES_LOCATION</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="L-127"><a href="#L-127"><span class="linenos">127</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">PROPERTIES_LOCATION</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="L-128"><a href="#L-128"><span class="linenos">128</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">EngineProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="L-129"><a href="#L-129"><span class="linenos">129</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AutoIncrementProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="L-130"><a href="#L-130"><span class="linenos">130</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CharacterSetProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="L-131"><a href="#L-131"><span class="linenos">131</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CollateProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="L-132"><a href="#L-132"><span class="linenos">132</span></a> <span class="p">}</span> +</span><span id="L-133"><a href="#L-133"><span class="linenos">133</span></a> +</span><span id="L-134"><a href="#L-134"><span class="linenos">134</span></a> <span class="n">TRANSFORMS</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="L-135"><a href="#L-135"><span class="linenos">135</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TRANSFORMS</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="L-136"><a href="#L-136"><span class="linenos">136</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ApproxDistinct</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"APPROX_COUNT_DISTINCT"</span><span class="p">),</span> +</span><span id="L-137"><a href="#L-137"><span class="linenos">137</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">FileFormatProperty</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"USING </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> +</span><span id="L-138"><a href="#L-138"><span class="linenos">138</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ArraySum</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"AGGREGATE(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, 0, (acc, x) -> acc + x, acc -> acc)"</span><span class="p">,</span> +</span><span id="L-139"><a href="#L-139"><span class="linenos">139</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseLeftShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTLEFT"</span><span class="p">),</span> +</span><span id="L-140"><a href="#L-140"><span class="linenos">140</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseRightShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTRIGHT"</span><span class="p">),</span> +</span><span id="L-141"><a href="#L-141"><span class="linenos">141</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateTrunc</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"TRUNC"</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"unit"</span><span class="p">)),</span> +</span><span id="L-142"><a href="#L-142"><span class="linenos">142</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Hint</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">" /*+ </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">expressions</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="si">}</span><span class="s2"> */"</span><span class="p">,</span> +</span><span id="L-143"><a href="#L-143"><span class="linenos">143</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToDate</span><span class="p">:</span> <span class="n">_str_to_date</span><span class="p">,</span> +</span><span id="L-144"><a href="#L-144"><span class="linenos">144</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToTime</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"TO_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">format_time</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> +</span><span id="L-145"><a href="#L-145"><span class="linenos">145</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">UnixToTime</span><span class="p">:</span> <span class="n">_unix_to_time</span><span class="p">,</span> +</span><span id="L-146"><a href="#L-146"><span class="linenos">146</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Create</span><span class="p">:</span> <span class="n">_create_sql</span><span class="p">,</span> +</span><span id="L-147"><a href="#L-147"><span class="linenos">147</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Map</span><span class="p">:</span> <span class="n">_map_sql</span><span class="p">,</span> +</span><span id="L-148"><a href="#L-148"><span class="linenos">148</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Reduce</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"AGGREGATE"</span><span class="p">),</span> +</span><span id="L-149"><a href="#L-149"><span class="linenos">149</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StructKwarg</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'expression'</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> +</span><span id="L-150"><a href="#L-150"><span class="linenos">150</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">TimestampTrunc</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span> +</span><span id="L-151"><a href="#L-151"><span class="linenos">151</span></a> <span class="s2">"DATE_TRUNC"</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Literal</span><span class="o">.</span><span class="n">string</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">text</span><span class="p">(</span><span class="s2">"unit"</span><span class="p">)),</span> <span class="n">e</span><span class="o">.</span><span class="n">this</span> +</span><span id="L-152"><a href="#L-152"><span class="linenos">152</span></a> <span class="p">),</span> +</span><span id="L-153"><a href="#L-153"><span class="linenos">153</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Trim</span><span class="p">:</span> <span class="n">trim_sql</span><span class="p">,</span> +</span><span id="L-154"><a href="#L-154"><span class="linenos">154</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">VariancePop</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"VAR_POP"</span><span class="p">),</span> +</span><span id="L-155"><a href="#L-155"><span class="linenos">155</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateFromParts</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"MAKE_DATE"</span><span class="p">),</span> +</span><span id="L-156"><a href="#L-156"><span class="linenos">156</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">LogicalOr</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"BOOL_OR"</span><span class="p">),</span> +</span><span id="L-157"><a href="#L-157"><span class="linenos">157</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfWeek</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFWEEK"</span><span class="p">),</span> +</span><span id="L-158"><a href="#L-158"><span class="linenos">158</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfMonth</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFMONTH"</span><span class="p">),</span> +</span><span id="L-159"><a href="#L-159"><span class="linenos">159</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFYEAR"</span><span class="p">),</span> +</span><span id="L-160"><a href="#L-160"><span class="linenos">160</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">WeekOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"WEEKOFYEAR"</span><span class="p">),</span> +</span><span id="L-161"><a href="#L-161"><span class="linenos">161</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AtTimeZone</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"FROM_UTC_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'zone'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> +</span><span id="L-162"><a href="#L-162"><span class="linenos">162</span></a> <span class="p">}</span> +</span><span id="L-163"><a href="#L-163"><span class="linenos">163</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ArraySort</span><span class="p">)</span> +</span><span id="L-164"><a href="#L-164"><span class="linenos">164</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ILike</span><span class="p">)</span> +</span><span id="L-165"><a href="#L-165"><span class="linenos">165</span></a> +</span><span id="L-166"><a href="#L-166"><span class="linenos">166</span></a> <span class="n">WRAP_DERIVED_VALUES</span> <span class="o">=</span> <span class="kc">False</span> +</span><span id="L-167"><a href="#L-167"><span class="linenos">167</span></a> <span class="n">CREATE_FUNCTION_RETURN_AS</span> <span class="o">=</span> <span class="kc">False</span> +</span><span id="L-168"><a href="#L-168"><span class="linenos">168</span></a> +</span><span id="L-169"><a href="#L-169"><span class="linenos">169</span></a> <span class="k">def</span> <span class="nf">cast_sql</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">expression</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> +</span><span id="L-170"><a href="#L-170"><span class="linenos">170</span></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="ow">and</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span> +</span><span id="L-171"><a href="#L-171"><span class="linenos">171</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span> +</span><span id="L-172"><a href="#L-172"><span class="linenos">172</span></a> <span class="p">):</span> +</span><span id="L-173"><a href="#L-173"><span class="linenos">173</span></a> <span class="n">schema</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"'</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'to'</span><span class="p">)</span><span class="si">}</span><span class="s2">'"</span> +</span><span id="L-174"><a href="#L-174"><span class="linenos">174</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"FROM_JSON"</span><span class="p">,</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">schema</span><span class="p">)</span> +</span><span id="L-175"><a href="#L-175"><span class="linenos">175</span></a> <span class="k">if</span> <span class="n">expression</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span><span class="p">):</span> +</span><span id="L-176"><a href="#L-176"><span class="linenos">176</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"TO_JSON"</span><span class="p">,</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">)</span> +</span><span id="L-177"><a href="#L-177"><span class="linenos">177</span></a> +</span><span id="L-178"><a href="#L-178"><span class="linenos">178</span></a> <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Spark</span><span class="o">.</span><span class="n">Generator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">cast_sql</span><span class="p">(</span><span class="n">expression</span><span class="p">)</span> +</span><span id="L-179"><a href="#L-179"><span class="linenos">179</span></a> +</span><span id="L-180"><a href="#L-180"><span class="linenos">180</span></a> <span class="k">class</span> <span class="nc">Tokenizer</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Tokenizer</span><span class="p">):</span> +</span><span id="L-181"><a href="#L-181"><span class="linenos">181</span></a> <span class="n">HEX_STRINGS</span> <span class="o">=</span> <span class="p">[(</span><span class="s2">"X'"</span><span class="p">,</span> <span class="s2">"'"</span><span class="p">)]</span> </span></pre></div> @@ -314,92 +321,99 @@ </span><span id="Spark-87"><a href="#Spark-87"><span class="linenos"> 87</span></a> <span class="s2">"WEEKOFYEAR"</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">WeekOfYear</span><span class="p">(</span> </span><span id="Spark-88"><a href="#Spark-88"><span class="linenos"> 88</span></a> <span class="n">this</span><span class="o">=</span><span class="n">exp</span><span class="o">.</span><span class="n">TsOrDsToDate</span><span class="p">(</span><span class="n">this</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> </span><span id="Spark-89"><a href="#Spark-89"><span class="linenos"> 89</span></a> <span class="p">),</span> -</span><span id="Spark-90"><a href="#Spark-90"><span class="linenos"> 90</span></a> <span class="p">}</span> -</span><span id="Spark-91"><a href="#Spark-91"><span class="linenos"> 91</span></a> -</span><span id="Spark-92"><a href="#Spark-92"><span class="linenos"> 92</span></a> <span class="n">FUNCTION_PARSERS</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="Spark-93"><a href="#Spark-93"><span class="linenos"> 93</span></a> <span class="o">**</span><span class="n">parser</span><span class="o">.</span><span class="n">Parser</span><span class="o">.</span><span class="n">FUNCTION_PARSERS</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="Spark-94"><a href="#Spark-94"><span class="linenos"> 94</span></a> <span class="s2">"BROADCAST"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCAST"</span><span class="p">),</span> -</span><span id="Spark-95"><a href="#Spark-95"><span class="linenos"> 95</span></a> <span class="s2">"BROADCASTJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCASTJOIN"</span><span class="p">),</span> -</span><span id="Spark-96"><a href="#Spark-96"><span class="linenos"> 96</span></a> <span class="s2">"MAPJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MAPJOIN"</span><span class="p">),</span> -</span><span id="Spark-97"><a href="#Spark-97"><span class="linenos"> 97</span></a> <span class="s2">"MERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGE"</span><span class="p">),</span> -</span><span id="Spark-98"><a href="#Spark-98"><span class="linenos"> 98</span></a> <span class="s2">"SHUFFLEMERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLEMERGE"</span><span class="p">),</span> -</span><span id="Spark-99"><a href="#Spark-99"><span class="linenos"> 99</span></a> <span class="s2">"MERGEJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGEJOIN"</span><span class="p">),</span> -</span><span id="Spark-100"><a href="#Spark-100"><span class="linenos">100</span></a> <span class="s2">"SHUFFLE_HASH"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_HASH"</span><span class="p">),</span> -</span><span id="Spark-101"><a href="#Spark-101"><span class="linenos">101</span></a> <span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">),</span> -</span><span id="Spark-102"><a href="#Spark-102"><span class="linenos">102</span></a> <span class="p">}</span> -</span><span id="Spark-103"><a href="#Spark-103"><span class="linenos">103</span></a> -</span><span id="Spark-104"><a href="#Spark-104"><span class="linenos">104</span></a> <span class="k">def</span> <span class="nf">_parse_add_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> -</span><span id="Spark-105"><a href="#Spark-105"><span class="linenos">105</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"ADD"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">()</span> -</span><span id="Spark-106"><a href="#Spark-106"><span class="linenos">106</span></a> -</span><span id="Spark-107"><a href="#Spark-107"><span class="linenos">107</span></a> <span class="k">def</span> <span class="nf">_parse_drop_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> -</span><span id="Spark-108"><a href="#Spark-108"><span class="linenos">108</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"DROP"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">expression</span><span class="p">(</span> -</span><span id="Spark-109"><a href="#Spark-109"><span class="linenos">109</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Drop</span><span class="p">,</span> -</span><span id="Spark-110"><a href="#Spark-110"><span class="linenos">110</span></a> <span class="n">this</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">(),</span> -</span><span id="Spark-111"><a href="#Spark-111"><span class="linenos">111</span></a> <span class="n">kind</span><span class="o">=</span><span class="s2">"COLUMNS"</span><span class="p">,</span> -</span><span id="Spark-112"><a href="#Spark-112"><span class="linenos">112</span></a> <span class="p">)</span> -</span><span id="Spark-113"><a href="#Spark-113"><span class="linenos">113</span></a> -</span><span id="Spark-114"><a href="#Spark-114"><span class="linenos">114</span></a> <span class="k">class</span> <span class="nc">Generator</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="p">):</span> -</span><span id="Spark-115"><a href="#Spark-115"><span class="linenos">115</span></a> <span class="n">TYPE_MAPPING</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="Spark-116"><a href="#Spark-116"><span class="linenos">116</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TYPE_MAPPING</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="Spark-117"><a href="#Spark-117"><span class="linenos">117</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">TINYINT</span><span class="p">:</span> <span class="s2">"BYTE"</span><span class="p">,</span> -</span><span id="Spark-118"><a href="#Spark-118"><span class="linenos">118</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">SMALLINT</span><span class="p">:</span> <span class="s2">"SHORT"</span><span class="p">,</span> -</span><span id="Spark-119"><a href="#Spark-119"><span class="linenos">119</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">BIGINT</span><span class="p">:</span> <span class="s2">"LONG"</span><span class="p">,</span> -</span><span id="Spark-120"><a href="#Spark-120"><span class="linenos">120</span></a> <span class="p">}</span> -</span><span id="Spark-121"><a href="#Spark-121"><span class="linenos">121</span></a> -</span><span id="Spark-122"><a href="#Spark-122"><span class="linenos">122</span></a> <span class="n">PROPERTIES_LOCATION</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="Spark-123"><a href="#Spark-123"><span class="linenos">123</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">PROPERTIES_LOCATION</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="Spark-124"><a href="#Spark-124"><span class="linenos">124</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">EngineProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="Spark-125"><a href="#Spark-125"><span class="linenos">125</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AutoIncrementProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="Spark-126"><a href="#Spark-126"><span class="linenos">126</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CharacterSetProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="Spark-127"><a href="#Spark-127"><span class="linenos">127</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CollateProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="Spark-128"><a href="#Spark-128"><span class="linenos">128</span></a> <span class="p">}</span> -</span><span id="Spark-129"><a href="#Spark-129"><span class="linenos">129</span></a> -</span><span id="Spark-130"><a href="#Spark-130"><span class="linenos">130</span></a> <span class="n">TRANSFORMS</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="Spark-131"><a href="#Spark-131"><span class="linenos">131</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TRANSFORMS</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="Spark-132"><a href="#Spark-132"><span class="linenos">132</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ApproxDistinct</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"APPROX_COUNT_DISTINCT"</span><span class="p">),</span> -</span><span id="Spark-133"><a href="#Spark-133"><span class="linenos">133</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">FileFormatProperty</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"USING </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> -</span><span id="Spark-134"><a href="#Spark-134"><span class="linenos">134</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ArraySum</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"AGGREGATE(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, 0, (acc, x) -> acc + x, acc -> acc)"</span><span class="p">,</span> -</span><span id="Spark-135"><a href="#Spark-135"><span class="linenos">135</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseLeftShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTLEFT"</span><span class="p">),</span> -</span><span id="Spark-136"><a href="#Spark-136"><span class="linenos">136</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseRightShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTRIGHT"</span><span class="p">),</span> -</span><span id="Spark-137"><a href="#Spark-137"><span class="linenos">137</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateTrunc</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"TRUNC"</span><span class="p">),</span> -</span><span id="Spark-138"><a href="#Spark-138"><span class="linenos">138</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Hint</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">" /*+ </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">expressions</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="si">}</span><span class="s2"> */"</span><span class="p">,</span> -</span><span id="Spark-139"><a href="#Spark-139"><span class="linenos">139</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToDate</span><span class="p">:</span> <span class="n">_str_to_date</span><span class="p">,</span> -</span><span id="Spark-140"><a href="#Spark-140"><span class="linenos">140</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToTime</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"TO_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">format_time</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> -</span><span id="Spark-141"><a href="#Spark-141"><span class="linenos">141</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">UnixToTime</span><span class="p">:</span> <span class="n">_unix_to_time</span><span class="p">,</span> -</span><span id="Spark-142"><a href="#Spark-142"><span class="linenos">142</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Create</span><span class="p">:</span> <span class="n">_create_sql</span><span class="p">,</span> -</span><span id="Spark-143"><a href="#Spark-143"><span class="linenos">143</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Map</span><span class="p">:</span> <span class="n">_map_sql</span><span class="p">,</span> -</span><span id="Spark-144"><a href="#Spark-144"><span class="linenos">144</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Reduce</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"AGGREGATE"</span><span class="p">),</span> -</span><span id="Spark-145"><a href="#Spark-145"><span class="linenos">145</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StructKwarg</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'expression'</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> -</span><span id="Spark-146"><a href="#Spark-146"><span class="linenos">146</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">TimestampTrunc</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"DATE_TRUNC(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'unit'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> -</span><span id="Spark-147"><a href="#Spark-147"><span class="linenos">147</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Trim</span><span class="p">:</span> <span class="n">trim_sql</span><span class="p">,</span> -</span><span id="Spark-148"><a href="#Spark-148"><span class="linenos">148</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">VariancePop</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"VAR_POP"</span><span class="p">),</span> -</span><span id="Spark-149"><a href="#Spark-149"><span class="linenos">149</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateFromParts</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"MAKE_DATE"</span><span class="p">),</span> -</span><span id="Spark-150"><a href="#Spark-150"><span class="linenos">150</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">LogicalOr</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"BOOL_OR"</span><span class="p">),</span> -</span><span id="Spark-151"><a href="#Spark-151"><span class="linenos">151</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfWeek</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFWEEK"</span><span class="p">),</span> -</span><span id="Spark-152"><a href="#Spark-152"><span class="linenos">152</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfMonth</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFMONTH"</span><span class="p">),</span> -</span><span id="Spark-153"><a href="#Spark-153"><span class="linenos">153</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFYEAR"</span><span class="p">),</span> -</span><span id="Spark-154"><a href="#Spark-154"><span class="linenos">154</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">WeekOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"WEEKOFYEAR"</span><span class="p">),</span> -</span><span id="Spark-155"><a href="#Spark-155"><span class="linenos">155</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AtTimeZone</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"FROM_UTC_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'zone'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> -</span><span id="Spark-156"><a href="#Spark-156"><span class="linenos">156</span></a> <span class="p">}</span> -</span><span id="Spark-157"><a href="#Spark-157"><span class="linenos">157</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ArraySort</span><span class="p">)</span> -</span><span id="Spark-158"><a href="#Spark-158"><span class="linenos">158</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ILike</span><span class="p">)</span> -</span><span id="Spark-159"><a href="#Spark-159"><span class="linenos">159</span></a> -</span><span id="Spark-160"><a href="#Spark-160"><span class="linenos">160</span></a> <span class="n">WRAP_DERIVED_VALUES</span> <span class="o">=</span> <span class="kc">False</span> -</span><span id="Spark-161"><a href="#Spark-161"><span class="linenos">161</span></a> <span class="n">CREATE_FUNCTION_AS</span> <span class="o">=</span> <span class="kc">False</span> -</span><span id="Spark-162"><a href="#Spark-162"><span class="linenos">162</span></a> -</span><span id="Spark-163"><a href="#Spark-163"><span class="linenos">163</span></a> <span class="k">def</span> <span class="nf">cast_sql</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">expression</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> -</span><span id="Spark-164"><a href="#Spark-164"><span class="linenos">164</span></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="ow">and</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span> -</span><span id="Spark-165"><a href="#Spark-165"><span class="linenos">165</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span> -</span><span id="Spark-166"><a href="#Spark-166"><span class="linenos">166</span></a> <span class="p">):</span> -</span><span id="Spark-167"><a href="#Spark-167"><span class="linenos">167</span></a> <span class="n">schema</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"'</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'to'</span><span class="p">)</span><span class="si">}</span><span class="s2">'"</span> -</span><span id="Spark-168"><a href="#Spark-168"><span class="linenos">168</span></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"FROM_JSON(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">format_args</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">),</span><span class="w"> </span><span class="n">schema</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span> -</span><span id="Spark-169"><a href="#Spark-169"><span class="linenos">169</span></a> <span class="k">if</span> <span class="n">expression</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span><span class="p">):</span> -</span><span id="Spark-170"><a href="#Spark-170"><span class="linenos">170</span></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"TO_JSON(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span> -</span><span id="Spark-171"><a href="#Spark-171"><span class="linenos">171</span></a> -</span><span id="Spark-172"><a href="#Spark-172"><span class="linenos">172</span></a> <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Spark</span><span class="o">.</span><span class="n">Generator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">cast_sql</span><span class="p">(</span><span class="n">expression</span><span class="p">)</span> -</span><span id="Spark-173"><a href="#Spark-173"><span class="linenos">173</span></a> -</span><span id="Spark-174"><a href="#Spark-174"><span class="linenos">174</span></a> <span class="k">class</span> <span class="nc">Tokenizer</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Tokenizer</span><span class="p">):</span> -</span><span id="Spark-175"><a href="#Spark-175"><span class="linenos">175</span></a> <span class="n">HEX_STRINGS</span> <span class="o">=</span> <span class="p">[(</span><span class="s2">"X'"</span><span class="p">,</span> <span class="s2">"'"</span><span class="p">)]</span> +</span><span id="Spark-90"><a href="#Spark-90"><span class="linenos"> 90</span></a> <span class="s2">"DATE_TRUNC"</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">TimestampTrunc</span><span class="p">(</span> +</span><span id="Spark-91"><a href="#Spark-91"><span class="linenos"> 91</span></a> <span class="n">this</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> +</span><span id="Spark-92"><a href="#Spark-92"><span class="linenos"> 92</span></a> <span class="n">unit</span><span class="o">=</span><span class="n">exp</span><span class="o">.</span><span class="n">var</span><span class="p">(</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> +</span><span id="Spark-93"><a href="#Spark-93"><span class="linenos"> 93</span></a> <span class="p">),</span> +</span><span id="Spark-94"><a href="#Spark-94"><span class="linenos"> 94</span></a> <span class="s2">"TRUNC"</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">DateTrunc</span><span class="p">(</span><span class="n">unit</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">this</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> +</span><span id="Spark-95"><a href="#Spark-95"><span class="linenos"> 95</span></a> <span class="p">}</span> +</span><span id="Spark-96"><a href="#Spark-96"><span class="linenos"> 96</span></a> +</span><span id="Spark-97"><a href="#Spark-97"><span class="linenos"> 97</span></a> <span class="n">FUNCTION_PARSERS</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="Spark-98"><a href="#Spark-98"><span class="linenos"> 98</span></a> <span class="o">**</span><span class="n">parser</span><span class="o">.</span><span class="n">Parser</span><span class="o">.</span><span class="n">FUNCTION_PARSERS</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="Spark-99"><a href="#Spark-99"><span class="linenos"> 99</span></a> <span class="s2">"BROADCAST"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCAST"</span><span class="p">),</span> +</span><span id="Spark-100"><a href="#Spark-100"><span class="linenos">100</span></a> <span class="s2">"BROADCASTJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCASTJOIN"</span><span class="p">),</span> +</span><span id="Spark-101"><a href="#Spark-101"><span class="linenos">101</span></a> <span class="s2">"MAPJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MAPJOIN"</span><span class="p">),</span> +</span><span id="Spark-102"><a href="#Spark-102"><span class="linenos">102</span></a> <span class="s2">"MERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGE"</span><span class="p">),</span> +</span><span id="Spark-103"><a href="#Spark-103"><span class="linenos">103</span></a> <span class="s2">"SHUFFLEMERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLEMERGE"</span><span class="p">),</span> +</span><span id="Spark-104"><a href="#Spark-104"><span class="linenos">104</span></a> <span class="s2">"MERGEJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGEJOIN"</span><span class="p">),</span> +</span><span id="Spark-105"><a href="#Spark-105"><span class="linenos">105</span></a> <span class="s2">"SHUFFLE_HASH"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_HASH"</span><span class="p">),</span> +</span><span id="Spark-106"><a href="#Spark-106"><span class="linenos">106</span></a> <span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">),</span> +</span><span id="Spark-107"><a href="#Spark-107"><span class="linenos">107</span></a> <span class="p">}</span> +</span><span id="Spark-108"><a href="#Spark-108"><span class="linenos">108</span></a> +</span><span id="Spark-109"><a href="#Spark-109"><span class="linenos">109</span></a> <span class="k">def</span> <span class="nf">_parse_add_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> +</span><span id="Spark-110"><a href="#Spark-110"><span class="linenos">110</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"ADD"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">()</span> +</span><span id="Spark-111"><a href="#Spark-111"><span class="linenos">111</span></a> +</span><span id="Spark-112"><a href="#Spark-112"><span class="linenos">112</span></a> <span class="k">def</span> <span class="nf">_parse_drop_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> +</span><span id="Spark-113"><a href="#Spark-113"><span class="linenos">113</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"DROP"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">expression</span><span class="p">(</span> +</span><span id="Spark-114"><a href="#Spark-114"><span class="linenos">114</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Drop</span><span class="p">,</span> +</span><span id="Spark-115"><a href="#Spark-115"><span class="linenos">115</span></a> <span class="n">this</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">(),</span> +</span><span id="Spark-116"><a href="#Spark-116"><span class="linenos">116</span></a> <span class="n">kind</span><span class="o">=</span><span class="s2">"COLUMNS"</span><span class="p">,</span> +</span><span id="Spark-117"><a href="#Spark-117"><span class="linenos">117</span></a> <span class="p">)</span> +</span><span id="Spark-118"><a href="#Spark-118"><span class="linenos">118</span></a> +</span><span id="Spark-119"><a href="#Spark-119"><span class="linenos">119</span></a> <span class="k">class</span> <span class="nc">Generator</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="p">):</span> +</span><span id="Spark-120"><a href="#Spark-120"><span class="linenos">120</span></a> <span class="n">TYPE_MAPPING</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="Spark-121"><a href="#Spark-121"><span class="linenos">121</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TYPE_MAPPING</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="Spark-122"><a href="#Spark-122"><span class="linenos">122</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">TINYINT</span><span class="p">:</span> <span class="s2">"BYTE"</span><span class="p">,</span> +</span><span id="Spark-123"><a href="#Spark-123"><span class="linenos">123</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">SMALLINT</span><span class="p">:</span> <span class="s2">"SHORT"</span><span class="p">,</span> +</span><span id="Spark-124"><a href="#Spark-124"><span class="linenos">124</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">BIGINT</span><span class="p">:</span> <span class="s2">"LONG"</span><span class="p">,</span> +</span><span id="Spark-125"><a href="#Spark-125"><span class="linenos">125</span></a> <span class="p">}</span> +</span><span id="Spark-126"><a href="#Spark-126"><span class="linenos">126</span></a> +</span><span id="Spark-127"><a href="#Spark-127"><span class="linenos">127</span></a> <span class="n">PROPERTIES_LOCATION</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="Spark-128"><a href="#Spark-128"><span class="linenos">128</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">PROPERTIES_LOCATION</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="Spark-129"><a href="#Spark-129"><span class="linenos">129</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">EngineProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="Spark-130"><a href="#Spark-130"><span class="linenos">130</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AutoIncrementProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="Spark-131"><a href="#Spark-131"><span class="linenos">131</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CharacterSetProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="Spark-132"><a href="#Spark-132"><span class="linenos">132</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CollateProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="Spark-133"><a href="#Spark-133"><span class="linenos">133</span></a> <span class="p">}</span> +</span><span id="Spark-134"><a href="#Spark-134"><span class="linenos">134</span></a> +</span><span id="Spark-135"><a href="#Spark-135"><span class="linenos">135</span></a> <span class="n">TRANSFORMS</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="Spark-136"><a href="#Spark-136"><span class="linenos">136</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TRANSFORMS</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="Spark-137"><a href="#Spark-137"><span class="linenos">137</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ApproxDistinct</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"APPROX_COUNT_DISTINCT"</span><span class="p">),</span> +</span><span id="Spark-138"><a href="#Spark-138"><span class="linenos">138</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">FileFormatProperty</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"USING </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> +</span><span id="Spark-139"><a href="#Spark-139"><span class="linenos">139</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ArraySum</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"AGGREGATE(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, 0, (acc, x) -> acc + x, acc -> acc)"</span><span class="p">,</span> +</span><span id="Spark-140"><a href="#Spark-140"><span class="linenos">140</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseLeftShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTLEFT"</span><span class="p">),</span> +</span><span id="Spark-141"><a href="#Spark-141"><span class="linenos">141</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseRightShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTRIGHT"</span><span class="p">),</span> +</span><span id="Spark-142"><a href="#Spark-142"><span class="linenos">142</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateTrunc</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"TRUNC"</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"unit"</span><span class="p">)),</span> +</span><span id="Spark-143"><a href="#Spark-143"><span class="linenos">143</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Hint</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">" /*+ </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">expressions</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="si">}</span><span class="s2"> */"</span><span class="p">,</span> +</span><span id="Spark-144"><a href="#Spark-144"><span class="linenos">144</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToDate</span><span class="p">:</span> <span class="n">_str_to_date</span><span class="p">,</span> +</span><span id="Spark-145"><a href="#Spark-145"><span class="linenos">145</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToTime</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"TO_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">format_time</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> +</span><span id="Spark-146"><a href="#Spark-146"><span class="linenos">146</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">UnixToTime</span><span class="p">:</span> <span class="n">_unix_to_time</span><span class="p">,</span> +</span><span id="Spark-147"><a href="#Spark-147"><span class="linenos">147</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Create</span><span class="p">:</span> <span class="n">_create_sql</span><span class="p">,</span> +</span><span id="Spark-148"><a href="#Spark-148"><span class="linenos">148</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Map</span><span class="p">:</span> <span class="n">_map_sql</span><span class="p">,</span> +</span><span id="Spark-149"><a href="#Spark-149"><span class="linenos">149</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Reduce</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"AGGREGATE"</span><span class="p">),</span> +</span><span id="Spark-150"><a href="#Spark-150"><span class="linenos">150</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StructKwarg</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'expression'</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> +</span><span id="Spark-151"><a href="#Spark-151"><span class="linenos">151</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">TimestampTrunc</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span> +</span><span id="Spark-152"><a href="#Spark-152"><span class="linenos">152</span></a> <span class="s2">"DATE_TRUNC"</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Literal</span><span class="o">.</span><span class="n">string</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">text</span><span class="p">(</span><span class="s2">"unit"</span><span class="p">)),</span> <span class="n">e</span><span class="o">.</span><span class="n">this</span> +</span><span id="Spark-153"><a href="#Spark-153"><span class="linenos">153</span></a> <span class="p">),</span> +</span><span id="Spark-154"><a href="#Spark-154"><span class="linenos">154</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Trim</span><span class="p">:</span> <span class="n">trim_sql</span><span class="p">,</span> +</span><span id="Spark-155"><a href="#Spark-155"><span class="linenos">155</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">VariancePop</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"VAR_POP"</span><span class="p">),</span> +</span><span id="Spark-156"><a href="#Spark-156"><span class="linenos">156</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateFromParts</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"MAKE_DATE"</span><span class="p">),</span> +</span><span id="Spark-157"><a href="#Spark-157"><span class="linenos">157</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">LogicalOr</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"BOOL_OR"</span><span class="p">),</span> +</span><span id="Spark-158"><a href="#Spark-158"><span class="linenos">158</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfWeek</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFWEEK"</span><span class="p">),</span> +</span><span id="Spark-159"><a href="#Spark-159"><span class="linenos">159</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfMonth</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFMONTH"</span><span class="p">),</span> +</span><span id="Spark-160"><a href="#Spark-160"><span class="linenos">160</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFYEAR"</span><span class="p">),</span> +</span><span id="Spark-161"><a href="#Spark-161"><span class="linenos">161</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">WeekOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"WEEKOFYEAR"</span><span class="p">),</span> +</span><span id="Spark-162"><a href="#Spark-162"><span class="linenos">162</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AtTimeZone</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"FROM_UTC_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'zone'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> +</span><span id="Spark-163"><a href="#Spark-163"><span class="linenos">163</span></a> <span class="p">}</span> +</span><span id="Spark-164"><a href="#Spark-164"><span class="linenos">164</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ArraySort</span><span class="p">)</span> +</span><span id="Spark-165"><a href="#Spark-165"><span class="linenos">165</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ILike</span><span class="p">)</span> +</span><span id="Spark-166"><a href="#Spark-166"><span class="linenos">166</span></a> +</span><span id="Spark-167"><a href="#Spark-167"><span class="linenos">167</span></a> <span class="n">WRAP_DERIVED_VALUES</span> <span class="o">=</span> <span class="kc">False</span> +</span><span id="Spark-168"><a href="#Spark-168"><span class="linenos">168</span></a> <span class="n">CREATE_FUNCTION_RETURN_AS</span> <span class="o">=</span> <span class="kc">False</span> +</span><span id="Spark-169"><a href="#Spark-169"><span class="linenos">169</span></a> +</span><span id="Spark-170"><a href="#Spark-170"><span class="linenos">170</span></a> <span class="k">def</span> <span class="nf">cast_sql</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">expression</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> +</span><span id="Spark-171"><a href="#Spark-171"><span class="linenos">171</span></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="ow">and</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span> +</span><span id="Spark-172"><a href="#Spark-172"><span class="linenos">172</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span> +</span><span id="Spark-173"><a href="#Spark-173"><span class="linenos">173</span></a> <span class="p">):</span> +</span><span id="Spark-174"><a href="#Spark-174"><span class="linenos">174</span></a> <span class="n">schema</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"'</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'to'</span><span class="p">)</span><span class="si">}</span><span class="s2">'"</span> +</span><span id="Spark-175"><a href="#Spark-175"><span class="linenos">175</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"FROM_JSON"</span><span class="p">,</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">schema</span><span class="p">)</span> +</span><span id="Spark-176"><a href="#Spark-176"><span class="linenos">176</span></a> <span class="k">if</span> <span class="n">expression</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span><span class="p">):</span> +</span><span id="Spark-177"><a href="#Spark-177"><span class="linenos">177</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"TO_JSON"</span><span class="p">,</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">)</span> +</span><span id="Spark-178"><a href="#Spark-178"><span class="linenos">178</span></a> +</span><span id="Spark-179"><a href="#Spark-179"><span class="linenos">179</span></a> <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Spark</span><span class="o">.</span><span class="n">Generator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">cast_sql</span><span class="p">(</span><span class="n">expression</span><span class="p">)</span> +</span><span id="Spark-180"><a href="#Spark-180"><span class="linenos">180</span></a> +</span><span id="Spark-181"><a href="#Spark-181"><span class="linenos">181</span></a> <span class="k">class</span> <span class="nc">Tokenizer</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Tokenizer</span><span class="p">):</span> +</span><span id="Spark-182"><a href="#Spark-182"><span class="linenos">182</span></a> <span class="n">HEX_STRINGS</span> <span class="o">=</span> <span class="p">[(</span><span class="s2">"X'"</span><span class="p">,</span> <span class="s2">"'"</span><span class="p">)]</span> </span></pre></div> @@ -486,29 +500,34 @@ </span><span id="Spark.Parser-87"><a href="#Spark.Parser-87"><span class="linenos"> 87</span></a> <span class="s2">"WEEKOFYEAR"</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">WeekOfYear</span><span class="p">(</span> </span><span id="Spark.Parser-88"><a href="#Spark.Parser-88"><span class="linenos"> 88</span></a> <span class="n">this</span><span class="o">=</span><span class="n">exp</span><span class="o">.</span><span class="n">TsOrDsToDate</span><span class="p">(</span><span class="n">this</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> </span><span id="Spark.Parser-89"><a href="#Spark.Parser-89"><span class="linenos"> 89</span></a> <span class="p">),</span> -</span><span id="Spark.Parser-90"><a href="#Spark.Parser-90"><span class="linenos"> 90</span></a> <span class="p">}</span> -</span><span id="Spark.Parser-91"><a href="#Spark.Parser-91"><span class="linenos"> 91</span></a> -</span><span id="Spark.Parser-92"><a href="#Spark.Parser-92"><span class="linenos"> 92</span></a> <span class="n">FUNCTION_PARSERS</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="Spark.Parser-93"><a href="#Spark.Parser-93"><span class="linenos"> 93</span></a> <span class="o">**</span><span class="n">parser</span><span class="o">.</span><span class="n">Parser</span><span class="o">.</span><span class="n">FUNCTION_PARSERS</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="Spark.Parser-94"><a href="#Spark.Parser-94"><span class="linenos"> 94</span></a> <span class="s2">"BROADCAST"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCAST"</span><span class="p">),</span> -</span><span id="Spark.Parser-95"><a href="#Spark.Parser-95"><span class="linenos"> 95</span></a> <span class="s2">"BROADCASTJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCASTJOIN"</span><span class="p">),</span> -</span><span id="Spark.Parser-96"><a href="#Spark.Parser-96"><span class="linenos"> 96</span></a> <span class="s2">"MAPJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MAPJOIN"</span><span class="p">),</span> -</span><span id="Spark.Parser-97"><a href="#Spark.Parser-97"><span class="linenos"> 97</span></a> <span class="s2">"MERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGE"</span><span class="p">),</span> -</span><span id="Spark.Parser-98"><a href="#Spark.Parser-98"><span class="linenos"> 98</span></a> <span class="s2">"SHUFFLEMERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLEMERGE"</span><span class="p">),</span> -</span><span id="Spark.Parser-99"><a href="#Spark.Parser-99"><span class="linenos"> 99</span></a> <span class="s2">"MERGEJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGEJOIN"</span><span class="p">),</span> -</span><span id="Spark.Parser-100"><a href="#Spark.Parser-100"><span class="linenos">100</span></a> <span class="s2">"SHUFFLE_HASH"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_HASH"</span><span class="p">),</span> -</span><span id="Spark.Parser-101"><a href="#Spark.Parser-101"><span class="linenos">101</span></a> <span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">),</span> -</span><span id="Spark.Parser-102"><a href="#Spark.Parser-102"><span class="linenos">102</span></a> <span class="p">}</span> -</span><span id="Spark.Parser-103"><a href="#Spark.Parser-103"><span class="linenos">103</span></a> -</span><span id="Spark.Parser-104"><a href="#Spark.Parser-104"><span class="linenos">104</span></a> <span class="k">def</span> <span class="nf">_parse_add_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> -</span><span id="Spark.Parser-105"><a href="#Spark.Parser-105"><span class="linenos">105</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"ADD"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">()</span> -</span><span id="Spark.Parser-106"><a href="#Spark.Parser-106"><span class="linenos">106</span></a> -</span><span id="Spark.Parser-107"><a href="#Spark.Parser-107"><span class="linenos">107</span></a> <span class="k">def</span> <span class="nf">_parse_drop_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> -</span><span id="Spark.Parser-108"><a href="#Spark.Parser-108"><span class="linenos">108</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"DROP"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">expression</span><span class="p">(</span> -</span><span id="Spark.Parser-109"><a href="#Spark.Parser-109"><span class="linenos">109</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Drop</span><span class="p">,</span> -</span><span id="Spark.Parser-110"><a href="#Spark.Parser-110"><span class="linenos">110</span></a> <span class="n">this</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">(),</span> -</span><span id="Spark.Parser-111"><a href="#Spark.Parser-111"><span class="linenos">111</span></a> <span class="n">kind</span><span class="o">=</span><span class="s2">"COLUMNS"</span><span class="p">,</span> -</span><span id="Spark.Parser-112"><a href="#Spark.Parser-112"><span class="linenos">112</span></a> <span class="p">)</span> +</span><span id="Spark.Parser-90"><a href="#Spark.Parser-90"><span class="linenos"> 90</span></a> <span class="s2">"DATE_TRUNC"</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">TimestampTrunc</span><span class="p">(</span> +</span><span id="Spark.Parser-91"><a href="#Spark.Parser-91"><span class="linenos"> 91</span></a> <span class="n">this</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> +</span><span id="Spark.Parser-92"><a href="#Spark.Parser-92"><span class="linenos"> 92</span></a> <span class="n">unit</span><span class="o">=</span><span class="n">exp</span><span class="o">.</span><span class="n">var</span><span class="p">(</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> +</span><span id="Spark.Parser-93"><a href="#Spark.Parser-93"><span class="linenos"> 93</span></a> <span class="p">),</span> +</span><span id="Spark.Parser-94"><a href="#Spark.Parser-94"><span class="linenos"> 94</span></a> <span class="s2">"TRUNC"</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">DateTrunc</span><span class="p">(</span><span class="n">unit</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">this</span><span class="o">=</span><span class="n">seq_get</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> +</span><span id="Spark.Parser-95"><a href="#Spark.Parser-95"><span class="linenos"> 95</span></a> <span class="p">}</span> +</span><span id="Spark.Parser-96"><a href="#Spark.Parser-96"><span class="linenos"> 96</span></a> +</span><span id="Spark.Parser-97"><a href="#Spark.Parser-97"><span class="linenos"> 97</span></a> <span class="n">FUNCTION_PARSERS</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="Spark.Parser-98"><a href="#Spark.Parser-98"><span class="linenos"> 98</span></a> <span class="o">**</span><span class="n">parser</span><span class="o">.</span><span class="n">Parser</span><span class="o">.</span><span class="n">FUNCTION_PARSERS</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="Spark.Parser-99"><a href="#Spark.Parser-99"><span class="linenos"> 99</span></a> <span class="s2">"BROADCAST"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCAST"</span><span class="p">),</span> +</span><span id="Spark.Parser-100"><a href="#Spark.Parser-100"><span class="linenos">100</span></a> <span class="s2">"BROADCASTJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"BROADCASTJOIN"</span><span class="p">),</span> +</span><span id="Spark.Parser-101"><a href="#Spark.Parser-101"><span class="linenos">101</span></a> <span class="s2">"MAPJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MAPJOIN"</span><span class="p">),</span> +</span><span id="Spark.Parser-102"><a href="#Spark.Parser-102"><span class="linenos">102</span></a> <span class="s2">"MERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGE"</span><span class="p">),</span> +</span><span id="Spark.Parser-103"><a href="#Spark.Parser-103"><span class="linenos">103</span></a> <span class="s2">"SHUFFLEMERGE"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLEMERGE"</span><span class="p">),</span> +</span><span id="Spark.Parser-104"><a href="#Spark.Parser-104"><span class="linenos">104</span></a> <span class="s2">"MERGEJOIN"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"MERGEJOIN"</span><span class="p">),</span> +</span><span id="Spark.Parser-105"><a href="#Spark.Parser-105"><span class="linenos">105</span></a> <span class="s2">"SHUFFLE_HASH"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_HASH"</span><span class="p">),</span> +</span><span id="Spark.Parser-106"><a href="#Spark.Parser-106"><span class="linenos">106</span></a> <span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_join_hint</span><span class="p">(</span><span class="s2">"SHUFFLE_REPLICATE_NL"</span><span class="p">),</span> +</span><span id="Spark.Parser-107"><a href="#Spark.Parser-107"><span class="linenos">107</span></a> <span class="p">}</span> +</span><span id="Spark.Parser-108"><a href="#Spark.Parser-108"><span class="linenos">108</span></a> +</span><span id="Spark.Parser-109"><a href="#Spark.Parser-109"><span class="linenos">109</span></a> <span class="k">def</span> <span class="nf">_parse_add_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> +</span><span id="Spark.Parser-110"><a href="#Spark.Parser-110"><span class="linenos">110</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"ADD"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">()</span> +</span><span id="Spark.Parser-111"><a href="#Spark.Parser-111"><span class="linenos">111</span></a> +</span><span id="Spark.Parser-112"><a href="#Spark.Parser-112"><span class="linenos">112</span></a> <span class="k">def</span> <span class="nf">_parse_drop_column</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> +</span><span id="Spark.Parser-113"><a href="#Spark.Parser-113"><span class="linenos">113</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_match_text_seq</span><span class="p">(</span><span class="s2">"DROP"</span><span class="p">,</span> <span class="s2">"COLUMNS"</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">expression</span><span class="p">(</span> +</span><span id="Spark.Parser-114"><a href="#Spark.Parser-114"><span class="linenos">114</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Drop</span><span class="p">,</span> +</span><span id="Spark.Parser-115"><a href="#Spark.Parser-115"><span class="linenos">115</span></a> <span class="n">this</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_schema</span><span class="p">(),</span> +</span><span id="Spark.Parser-116"><a href="#Spark.Parser-116"><span class="linenos">116</span></a> <span class="n">kind</span><span class="o">=</span><span class="s2">"COLUMNS"</span><span class="p">,</span> +</span><span id="Spark.Parser-117"><a href="#Spark.Parser-117"><span class="linenos">117</span></a> <span class="p">)</span> </span></pre></div> @@ -565,65 +584,67 @@ Default: "nulls_are_small"</li> </div> <a class="headerlink" href="#Spark.Generator"></a> - <div class="pdoc-code codehilite"><pre><span></span><span id="Spark.Generator-114"><a href="#Spark.Generator-114"><span class="linenos">114</span></a> <span class="k">class</span> <span class="nc">Generator</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="p">):</span> -</span><span id="Spark.Generator-115"><a href="#Spark.Generator-115"><span class="linenos">115</span></a> <span class="n">TYPE_MAPPING</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="Spark.Generator-116"><a href="#Spark.Generator-116"><span class="linenos">116</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TYPE_MAPPING</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="Spark.Generator-117"><a href="#Spark.Generator-117"><span class="linenos">117</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">TINYINT</span><span class="p">:</span> <span class="s2">"BYTE"</span><span class="p">,</span> -</span><span id="Spark.Generator-118"><a href="#Spark.Generator-118"><span class="linenos">118</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">SMALLINT</span><span class="p">:</span> <span class="s2">"SHORT"</span><span class="p">,</span> -</span><span id="Spark.Generator-119"><a href="#Spark.Generator-119"><span class="linenos">119</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">BIGINT</span><span class="p">:</span> <span class="s2">"LONG"</span><span class="p">,</span> -</span><span id="Spark.Generator-120"><a href="#Spark.Generator-120"><span class="linenos">120</span></a> <span class="p">}</span> -</span><span id="Spark.Generator-121"><a href="#Spark.Generator-121"><span class="linenos">121</span></a> -</span><span id="Spark.Generator-122"><a href="#Spark.Generator-122"><span class="linenos">122</span></a> <span class="n">PROPERTIES_LOCATION</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="Spark.Generator-123"><a href="#Spark.Generator-123"><span class="linenos">123</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">PROPERTIES_LOCATION</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="Spark.Generator-124"><a href="#Spark.Generator-124"><span class="linenos">124</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">EngineProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="Spark.Generator-125"><a href="#Spark.Generator-125"><span class="linenos">125</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AutoIncrementProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="Spark.Generator-126"><a href="#Spark.Generator-126"><span class="linenos">126</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CharacterSetProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="Spark.Generator-127"><a href="#Spark.Generator-127"><span class="linenos">127</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CollateProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> -</span><span id="Spark.Generator-128"><a href="#Spark.Generator-128"><span class="linenos">128</span></a> <span class="p">}</span> -</span><span id="Spark.Generator-129"><a href="#Spark.Generator-129"><span class="linenos">129</span></a> -</span><span id="Spark.Generator-130"><a href="#Spark.Generator-130"><span class="linenos">130</span></a> <span class="n">TRANSFORMS</span> <span class="o">=</span> <span class="p">{</span> -</span><span id="Spark.Generator-131"><a href="#Spark.Generator-131"><span class="linenos">131</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TRANSFORMS</span><span class="p">,</span> <span class="c1"># type: ignore</span> -</span><span id="Spark.Generator-132"><a href="#Spark.Generator-132"><span class="linenos">132</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ApproxDistinct</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"APPROX_COUNT_DISTINCT"</span><span class="p">),</span> -</span><span id="Spark.Generator-133"><a href="#Spark.Generator-133"><span class="linenos">133</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">FileFormatProperty</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"USING </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> -</span><span id="Spark.Generator-134"><a href="#Spark.Generator-134"><span class="linenos">134</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ArraySum</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"AGGREGATE(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, 0, (acc, x) -> acc + x, acc -> acc)"</span><span class="p">,</span> -</span><span id="Spark.Generator-135"><a href="#Spark.Generator-135"><span class="linenos">135</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseLeftShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTLEFT"</span><span class="p">),</span> -</span><span id="Spark.Generator-136"><a href="#Spark.Generator-136"><span class="linenos">136</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseRightShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTRIGHT"</span><span class="p">),</span> -</span><span id="Spark.Generator-137"><a href="#Spark.Generator-137"><span class="linenos">137</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateTrunc</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"TRUNC"</span><span class="p">),</span> -</span><span id="Spark.Generator-138"><a href="#Spark.Generator-138"><span class="linenos">138</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Hint</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">" /*+ </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">expressions</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="si">}</span><span class="s2"> */"</span><span class="p">,</span> -</span><span id="Spark.Generator-139"><a href="#Spark.Generator-139"><span class="linenos">139</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToDate</span><span class="p">:</span> <span class="n">_str_to_date</span><span class="p">,</span> -</span><span id="Spark.Generator-140"><a href="#Spark.Generator-140"><span class="linenos">140</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToTime</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"TO_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">format_time</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> -</span><span id="Spark.Generator-141"><a href="#Spark.Generator-141"><span class="linenos">141</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">UnixToTime</span><span class="p">:</span> <span class="n">_unix_to_time</span><span class="p">,</span> -</span><span id="Spark.Generator-142"><a href="#Spark.Generator-142"><span class="linenos">142</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Create</span><span class="p">:</span> <span class="n">_create_sql</span><span class="p">,</span> -</span><span id="Spark.Generator-143"><a href="#Spark.Generator-143"><span class="linenos">143</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Map</span><span class="p">:</span> <span class="n">_map_sql</span><span class="p">,</span> -</span><span id="Spark.Generator-144"><a href="#Spark.Generator-144"><span class="linenos">144</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Reduce</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"AGGREGATE"</span><span class="p">),</span> -</span><span id="Spark.Generator-145"><a href="#Spark.Generator-145"><span class="linenos">145</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StructKwarg</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'expression'</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> -</span><span id="Spark.Generator-146"><a href="#Spark.Generator-146"><span class="linenos">146</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">TimestampTrunc</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"DATE_TRUNC(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'unit'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> -</span><span id="Spark.Generator-147"><a href="#Spark.Generator-147"><span class="linenos">147</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Trim</span><span class="p">:</span> <span class="n">trim_sql</span><span class="p">,</span> -</span><span id="Spark.Generator-148"><a href="#Spark.Generator-148"><span class="linenos">148</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">VariancePop</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"VAR_POP"</span><span class="p">),</span> -</span><span id="Spark.Generator-149"><a href="#Spark.Generator-149"><span class="linenos">149</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateFromParts</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"MAKE_DATE"</span><span class="p">),</span> -</span><span id="Spark.Generator-150"><a href="#Spark.Generator-150"><span class="linenos">150</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">LogicalOr</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"BOOL_OR"</span><span class="p">),</span> -</span><span id="Spark.Generator-151"><a href="#Spark.Generator-151"><span class="linenos">151</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfWeek</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFWEEK"</span><span class="p">),</span> -</span><span id="Spark.Generator-152"><a href="#Spark.Generator-152"><span class="linenos">152</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfMonth</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFMONTH"</span><span class="p">),</span> -</span><span id="Spark.Generator-153"><a href="#Spark.Generator-153"><span class="linenos">153</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFYEAR"</span><span class="p">),</span> -</span><span id="Spark.Generator-154"><a href="#Spark.Generator-154"><span class="linenos">154</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">WeekOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"WEEKOFYEAR"</span><span class="p">),</span> -</span><span id="Spark.Generator-155"><a href="#Spark.Generator-155"><span class="linenos">155</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AtTimeZone</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"FROM_UTC_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'zone'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> -</span><span id="Spark.Generator-156"><a href="#Spark.Generator-156"><span class="linenos">156</span></a> <span class="p">}</span> -</span><span id="Spark.Generator-157"><a href="#Spark.Generator-157"><span class="linenos">157</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ArraySort</span><span class="p">)</span> -</span><span id="Spark.Generator-158"><a href="#Spark.Generator-158"><span class="linenos">158</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ILike</span><span class="p">)</span> -</span><span id="Spark.Generator-159"><a href="#Spark.Generator-159"><span class="linenos">159</span></a> -</span><span id="Spark.Generator-160"><a href="#Spark.Generator-160"><span class="linenos">160</span></a> <span class="n">WRAP_DERIVED_VALUES</span> <span class="o">=</span> <span class="kc">False</span> -</span><span id="Spark.Generator-161"><a href="#Spark.Generator-161"><span class="linenos">161</span></a> <span class="n">CREATE_FUNCTION_AS</span> <span class="o">=</span> <span class="kc">False</span> -</span><span id="Spark.Generator-162"><a href="#Spark.Generator-162"><span class="linenos">162</span></a> -</span><span id="Spark.Generator-163"><a href="#Spark.Generator-163"><span class="linenos">163</span></a> <span class="k">def</span> <span class="nf">cast_sql</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">expression</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> -</span><span id="Spark.Generator-164"><a href="#Spark.Generator-164"><span class="linenos">164</span></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="ow">and</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span> -</span><span id="Spark.Generator-165"><a href="#Spark.Generator-165"><span class="linenos">165</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span> -</span><span id="Spark.Generator-166"><a href="#Spark.Generator-166"><span class="linenos">166</span></a> <span class="p">):</span> -</span><span id="Spark.Generator-167"><a href="#Spark.Generator-167"><span class="linenos">167</span></a> <span class="n">schema</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"'</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'to'</span><span class="p">)</span><span class="si">}</span><span class="s2">'"</span> -</span><span id="Spark.Generator-168"><a href="#Spark.Generator-168"><span class="linenos">168</span></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"FROM_JSON(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">format_args</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">),</span><span class="w"> </span><span class="n">schema</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span> -</span><span id="Spark.Generator-169"><a href="#Spark.Generator-169"><span class="linenos">169</span></a> <span class="k">if</span> <span class="n">expression</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span><span class="p">):</span> -</span><span id="Spark.Generator-170"><a href="#Spark.Generator-170"><span class="linenos">170</span></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"TO_JSON(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span> -</span><span id="Spark.Generator-171"><a href="#Spark.Generator-171"><span class="linenos">171</span></a> -</span><span id="Spark.Generator-172"><a href="#Spark.Generator-172"><span class="linenos">172</span></a> <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Spark</span><span class="o">.</span><span class="n">Generator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">cast_sql</span><span class="p">(</span><span class="n">expression</span><span class="p">)</span> + <div class="pdoc-code codehilite"><pre><span></span><span id="Spark.Generator-119"><a href="#Spark.Generator-119"><span class="linenos">119</span></a> <span class="k">class</span> <span class="nc">Generator</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="p">):</span> +</span><span id="Spark.Generator-120"><a href="#Spark.Generator-120"><span class="linenos">120</span></a> <span class="n">TYPE_MAPPING</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="Spark.Generator-121"><a href="#Spark.Generator-121"><span class="linenos">121</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TYPE_MAPPING</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="Spark.Generator-122"><a href="#Spark.Generator-122"><span class="linenos">122</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">TINYINT</span><span class="p">:</span> <span class="s2">"BYTE"</span><span class="p">,</span> +</span><span id="Spark.Generator-123"><a href="#Spark.Generator-123"><span class="linenos">123</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">SMALLINT</span><span class="p">:</span> <span class="s2">"SHORT"</span><span class="p">,</span> +</span><span id="Spark.Generator-124"><a href="#Spark.Generator-124"><span class="linenos">124</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">BIGINT</span><span class="p">:</span> <span class="s2">"LONG"</span><span class="p">,</span> +</span><span id="Spark.Generator-125"><a href="#Spark.Generator-125"><span class="linenos">125</span></a> <span class="p">}</span> +</span><span id="Spark.Generator-126"><a href="#Spark.Generator-126"><span class="linenos">126</span></a> +</span><span id="Spark.Generator-127"><a href="#Spark.Generator-127"><span class="linenos">127</span></a> <span class="n">PROPERTIES_LOCATION</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="Spark.Generator-128"><a href="#Spark.Generator-128"><span class="linenos">128</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">PROPERTIES_LOCATION</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="Spark.Generator-129"><a href="#Spark.Generator-129"><span class="linenos">129</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">EngineProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="Spark.Generator-130"><a href="#Spark.Generator-130"><span class="linenos">130</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AutoIncrementProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="Spark.Generator-131"><a href="#Spark.Generator-131"><span class="linenos">131</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CharacterSetProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="Spark.Generator-132"><a href="#Spark.Generator-132"><span class="linenos">132</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">CollateProperty</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Properties</span><span class="o">.</span><span class="n">Location</span><span class="o">.</span><span class="n">UNSUPPORTED</span><span class="p">,</span> +</span><span id="Spark.Generator-133"><a href="#Spark.Generator-133"><span class="linenos">133</span></a> <span class="p">}</span> +</span><span id="Spark.Generator-134"><a href="#Spark.Generator-134"><span class="linenos">134</span></a> +</span><span id="Spark.Generator-135"><a href="#Spark.Generator-135"><span class="linenos">135</span></a> <span class="n">TRANSFORMS</span> <span class="o">=</span> <span class="p">{</span> +</span><span id="Spark.Generator-136"><a href="#Spark.Generator-136"><span class="linenos">136</span></a> <span class="o">**</span><span class="n">Hive</span><span class="o">.</span><span class="n">Generator</span><span class="o">.</span><span class="n">TRANSFORMS</span><span class="p">,</span> <span class="c1"># type: ignore</span> +</span><span id="Spark.Generator-137"><a href="#Spark.Generator-137"><span class="linenos">137</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ApproxDistinct</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"APPROX_COUNT_DISTINCT"</span><span class="p">),</span> +</span><span id="Spark.Generator-138"><a href="#Spark.Generator-138"><span class="linenos">138</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">FileFormatProperty</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"USING </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> +</span><span id="Spark.Generator-139"><a href="#Spark.Generator-139"><span class="linenos">139</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">ArraySum</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"AGGREGATE(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, 0, (acc, x) -> acc + x, acc -> acc)"</span><span class="p">,</span> +</span><span id="Spark.Generator-140"><a href="#Spark.Generator-140"><span class="linenos">140</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseLeftShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTLEFT"</span><span class="p">),</span> +</span><span id="Spark.Generator-141"><a href="#Spark.Generator-141"><span class="linenos">141</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">BitwiseRightShift</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"SHIFTRIGHT"</span><span class="p">),</span> +</span><span id="Spark.Generator-142"><a href="#Spark.Generator-142"><span class="linenos">142</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateTrunc</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"TRUNC"</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"unit"</span><span class="p">)),</span> +</span><span id="Spark.Generator-143"><a href="#Spark.Generator-143"><span class="linenos">143</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Hint</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">" /*+ </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">expressions</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="si">}</span><span class="s2"> */"</span><span class="p">,</span> +</span><span id="Spark.Generator-144"><a href="#Spark.Generator-144"><span class="linenos">144</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToDate</span><span class="p">:</span> <span class="n">_str_to_date</span><span class="p">,</span> +</span><span id="Spark.Generator-145"><a href="#Spark.Generator-145"><span class="linenos">145</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StrToTime</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"TO_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">format_time</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> +</span><span id="Spark.Generator-146"><a href="#Spark.Generator-146"><span class="linenos">146</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">UnixToTime</span><span class="p">:</span> <span class="n">_unix_to_time</span><span class="p">,</span> +</span><span id="Spark.Generator-147"><a href="#Spark.Generator-147"><span class="linenos">147</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Create</span><span class="p">:</span> <span class="n">_create_sql</span><span class="p">,</span> +</span><span id="Spark.Generator-148"><a href="#Spark.Generator-148"><span class="linenos">148</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Map</span><span class="p">:</span> <span class="n">_map_sql</span><span class="p">,</span> +</span><span id="Spark.Generator-149"><a href="#Spark.Generator-149"><span class="linenos">149</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Reduce</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"AGGREGATE"</span><span class="p">),</span> +</span><span id="Spark.Generator-150"><a href="#Spark.Generator-150"><span class="linenos">150</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">StructKwarg</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'expression'</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> +</span><span id="Spark.Generator-151"><a href="#Spark.Generator-151"><span class="linenos">151</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">TimestampTrunc</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span> +</span><span id="Spark.Generator-152"><a href="#Spark.Generator-152"><span class="linenos">152</span></a> <span class="s2">"DATE_TRUNC"</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Literal</span><span class="o">.</span><span class="n">string</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">text</span><span class="p">(</span><span class="s2">"unit"</span><span class="p">)),</span> <span class="n">e</span><span class="o">.</span><span class="n">this</span> +</span><span id="Spark.Generator-153"><a href="#Spark.Generator-153"><span class="linenos">153</span></a> <span class="p">),</span> +</span><span id="Spark.Generator-154"><a href="#Spark.Generator-154"><span class="linenos">154</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">Trim</span><span class="p">:</span> <span class="n">trim_sql</span><span class="p">,</span> +</span><span id="Spark.Generator-155"><a href="#Spark.Generator-155"><span class="linenos">155</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">VariancePop</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"VAR_POP"</span><span class="p">),</span> +</span><span id="Spark.Generator-156"><a href="#Spark.Generator-156"><span class="linenos">156</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DateFromParts</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"MAKE_DATE"</span><span class="p">),</span> +</span><span id="Spark.Generator-157"><a href="#Spark.Generator-157"><span class="linenos">157</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">LogicalOr</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"BOOL_OR"</span><span class="p">),</span> +</span><span id="Spark.Generator-158"><a href="#Spark.Generator-158"><span class="linenos">158</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfWeek</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFWEEK"</span><span class="p">),</span> +</span><span id="Spark.Generator-159"><a href="#Spark.Generator-159"><span class="linenos">159</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfMonth</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFMONTH"</span><span class="p">),</span> +</span><span id="Spark.Generator-160"><a href="#Spark.Generator-160"><span class="linenos">160</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DayOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"DAYOFYEAR"</span><span class="p">),</span> +</span><span id="Spark.Generator-161"><a href="#Spark.Generator-161"><span class="linenos">161</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">WeekOfYear</span><span class="p">:</span> <span class="n">rename_func</span><span class="p">(</span><span class="s2">"WEEKOFYEAR"</span><span class="p">),</span> +</span><span id="Spark.Generator-162"><a href="#Spark.Generator-162"><span class="linenos">162</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">AtTimeZone</span><span class="p">:</span> <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"FROM_UTC_TIMESTAMP(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">e</span><span class="p">,</span><span class="w"> </span><span class="s1">'zone'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">,</span> +</span><span id="Spark.Generator-163"><a href="#Spark.Generator-163"><span class="linenos">163</span></a> <span class="p">}</span> +</span><span id="Spark.Generator-164"><a href="#Spark.Generator-164"><span class="linenos">164</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ArraySort</span><span class="p">)</span> +</span><span id="Spark.Generator-165"><a href="#Spark.Generator-165"><span class="linenos">165</span></a> <span class="n">TRANSFORMS</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">ILike</span><span class="p">)</span> +</span><span id="Spark.Generator-166"><a href="#Spark.Generator-166"><span class="linenos">166</span></a> +</span><span id="Spark.Generator-167"><a href="#Spark.Generator-167"><span class="linenos">167</span></a> <span class="n">WRAP_DERIVED_VALUES</span> <span class="o">=</span> <span class="kc">False</span> +</span><span id="Spark.Generator-168"><a href="#Spark.Generator-168"><span class="linenos">168</span></a> <span class="n">CREATE_FUNCTION_RETURN_AS</span> <span class="o">=</span> <span class="kc">False</span> +</span><span id="Spark.Generator-169"><a href="#Spark.Generator-169"><span class="linenos">169</span></a> +</span><span id="Spark.Generator-170"><a href="#Spark.Generator-170"><span class="linenos">170</span></a> <span class="k">def</span> <span class="nf">cast_sql</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">expression</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> +</span><span id="Spark.Generator-171"><a href="#Spark.Generator-171"><span class="linenos">171</span></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="ow">and</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span> +</span><span id="Spark.Generator-172"><a href="#Spark.Generator-172"><span class="linenos">172</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span> +</span><span id="Spark.Generator-173"><a href="#Spark.Generator-173"><span class="linenos">173</span></a> <span class="p">):</span> +</span><span id="Spark.Generator-174"><a href="#Spark.Generator-174"><span class="linenos">174</span></a> <span class="n">schema</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"'</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'to'</span><span class="p">)</span><span class="si">}</span><span class="s2">'"</span> +</span><span id="Spark.Generator-175"><a href="#Spark.Generator-175"><span class="linenos">175</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"FROM_JSON"</span><span class="p">,</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">schema</span><span class="p">)</span> +</span><span id="Spark.Generator-176"><a href="#Spark.Generator-176"><span class="linenos">176</span></a> <span class="k">if</span> <span class="n">expression</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span><span class="p">):</span> +</span><span id="Spark.Generator-177"><a href="#Spark.Generator-177"><span class="linenos">177</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"TO_JSON"</span><span class="p">,</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">)</span> +</span><span id="Spark.Generator-178"><a href="#Spark.Generator-178"><span class="linenos">178</span></a> +</span><span id="Spark.Generator-179"><a href="#Spark.Generator-179"><span class="linenos">179</span></a> <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Spark</span><span class="o">.</span><span class="n">Generator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">cast_sql</span><span class="p">(</span><span class="n">expression</span><span class="p">)</span> </span></pre></div> @@ -683,16 +704,16 @@ Default: True</li> </div> <a class="headerlink" href="#Spark.Generator.cast_sql"></a> - <div class="pdoc-code codehilite"><pre><span></span><span id="Spark.Generator.cast_sql-163"><a href="#Spark.Generator.cast_sql-163"><span class="linenos">163</span></a> <span class="k">def</span> <span class="nf">cast_sql</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">expression</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> -</span><span id="Spark.Generator.cast_sql-164"><a href="#Spark.Generator.cast_sql-164"><span class="linenos">164</span></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="ow">and</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span> -</span><span id="Spark.Generator.cast_sql-165"><a href="#Spark.Generator.cast_sql-165"><span class="linenos">165</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span> -</span><span id="Spark.Generator.cast_sql-166"><a href="#Spark.Generator.cast_sql-166"><span class="linenos">166</span></a> <span class="p">):</span> -</span><span id="Spark.Generator.cast_sql-167"><a href="#Spark.Generator.cast_sql-167"><span class="linenos">167</span></a> <span class="n">schema</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"'</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'to'</span><span class="p">)</span><span class="si">}</span><span class="s2">'"</span> -</span><span id="Spark.Generator.cast_sql-168"><a href="#Spark.Generator.cast_sql-168"><span class="linenos">168</span></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"FROM_JSON(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">format_args</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">),</span><span class="w"> </span><span class="n">schema</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span> -</span><span id="Spark.Generator.cast_sql-169"><a href="#Spark.Generator.cast_sql-169"><span class="linenos">169</span></a> <span class="k">if</span> <span class="n">expression</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span><span class="p">):</span> -</span><span id="Spark.Generator.cast_sql-170"><a href="#Spark.Generator.cast_sql-170"><span class="linenos">170</span></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"TO_JSON(</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'this'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span> -</span><span id="Spark.Generator.cast_sql-171"><a href="#Spark.Generator.cast_sql-171"><span class="linenos">171</span></a> -</span><span id="Spark.Generator.cast_sql-172"><a href="#Spark.Generator.cast_sql-172"><span class="linenos">172</span></a> <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Spark</span><span class="o">.</span><span class="n">Generator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">cast_sql</span><span class="p">(</span><span class="n">expression</span><span class="p">)</span> + <div class="pdoc-code codehilite"><pre><span></span><span id="Spark.Generator.cast_sql-170"><a href="#Spark.Generator.cast_sql-170"><span class="linenos">170</span></a> <span class="k">def</span> <span class="nf">cast_sql</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">expression</span><span class="p">:</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> +</span><span id="Spark.Generator.cast_sql-171"><a href="#Spark.Generator.cast_sql-171"><span class="linenos">171</span></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">Cast</span><span class="p">)</span> <span class="ow">and</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span> +</span><span id="Spark.Generator.cast_sql-172"><a href="#Spark.Generator.cast_sql-172"><span class="linenos">172</span></a> <span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span> +</span><span id="Spark.Generator.cast_sql-173"><a href="#Spark.Generator.cast_sql-173"><span class="linenos">173</span></a> <span class="p">):</span> +</span><span id="Spark.Generator.cast_sql-174"><a href="#Spark.Generator.cast_sql-174"><span class="linenos">174</span></a> <span class="n">schema</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"'</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="n">expression</span><span class="p">,</span><span class="w"> </span><span class="s1">'to'</span><span class="p">)</span><span class="si">}</span><span class="s2">'"</span> +</span><span id="Spark.Generator.cast_sql-175"><a href="#Spark.Generator.cast_sql-175"><span class="linenos">175</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"FROM_JSON"</span><span class="p">,</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="o">.</span><span class="n">this</span><span class="p">,</span> <span class="n">schema</span><span class="p">)</span> +</span><span id="Spark.Generator.cast_sql-176"><a href="#Spark.Generator.cast_sql-176"><span class="linenos">176</span></a> <span class="k">if</span> <span class="n">expression</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">is_type</span><span class="p">(</span><span class="n">exp</span><span class="o">.</span><span class="n">DataType</span><span class="o">.</span><span class="n">Type</span><span class="o">.</span><span class="n">JSON</span><span class="p">):</span> +</span><span id="Spark.Generator.cast_sql-177"><a href="#Spark.Generator.cast_sql-177"><span class="linenos">177</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="s2">"TO_JSON"</span><span class="p">,</span> <span class="n">expression</span><span class="o">.</span><span class="n">this</span><span class="p">)</span> +</span><span id="Spark.Generator.cast_sql-178"><a href="#Spark.Generator.cast_sql-178"><span class="linenos">178</span></a> +</span><span id="Spark.Generator.cast_sql-179"><a href="#Spark.Generator.cast_sql-179"><span class="linenos">179</span></a> <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Spark</span><span class="o">.</span><span class="n">Generator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">cast_sql</span><span class="p">(</span><span class="n">expression</span><span class="p">)</span> </span></pre></div> @@ -722,11 +743,6 @@ Default: True</li> <dd id="Spark.Generator.columndef_sql" class="function"><a href="../generator.html#Generator.columndef_sql">columndef_sql</a></dd> <dd id="Spark.Generator.columnconstraint_sql" class="function"><a href="../generator.html#Generator.columnconstraint_sql">columnconstraint_sql</a></dd> <dd id="Spark.Generator.autoincrementcolumnconstraint_sql" class="function"><a href="../generator.html#Generator.autoincrementcolumnconstraint_sql">autoincrementcolumnconstraint_sql</a></dd> - <dd id="Spark.Generator.checkcolumnconstraint_sql" class="function"><a href="../generator.html#Generator.checkcolumnconstraint_sql">checkcolumnconstraint_sql</a></dd> - <dd id="Spark.Generator.commentcolumnconstraint_sql" class="function"><a href="../generator.html#Generator.commentcolumnconstraint_sql">commentcolumnconstraint_sql</a></dd> - <dd id="Spark.Generator.collatecolumnconstraint_sql" class="function"><a href="../generator.html#Generator.collatecolumnconstraint_sql">collatecolumnconstraint_sql</a></dd> - <dd id="Spark.Generator.encodecolumnconstraint_sql" class="function"><a href="../generator.html#Generator.encodecolumnconstraint_sql">encodecolumnconstraint_sql</a></dd> - <dd id="Spark.Generator.defaultcolumnconstraint_sql" class="function"><a href="../generator.html#Generator.defaultcolumnconstraint_sql">defaultcolumnconstraint_sql</a></dd> <dd id="Spark.Generator.generatedasidentitycolumnconstraint_sql" class="function"><a href="../generator.html#Generator.generatedasidentitycolumnconstraint_sql">generatedasidentitycolumnconstraint_sql</a></dd> <dd id="Spark.Generator.notnullcolumnconstraint_sql" class="function"><a href="../generator.html#Generator.notnullcolumnconstraint_sql">notnullcolumnconstraint_sql</a></dd> <dd id="Spark.Generator.primarykeycolumnconstraint_sql" class="function"><a href="../generator.html#Generator.primarykeycolumnconstraint_sql">primarykeycolumnconstraint_sql</a></dd> @@ -766,6 +782,7 @@ Default: True</li> <dd id="Spark.Generator.datablocksizeproperty_sql" class="function"><a href="../generator.html#Generator.datablocksizeproperty_sql">datablocksizeproperty_sql</a></dd> <dd id="Spark.Generator.blockcompressionproperty_sql" class="function"><a href="../generator.html#Generator.blockcompressionproperty_sql">blockcompressionproperty_sql</a></dd> <dd id="Spark.Generator.isolatedloadingproperty_sql" class="function"><a href="../generator.html#Generator.isolatedloadingproperty_sql">isolatedloadingproperty_sql</a></dd> + <dd id="Spark.Generator.lockingproperty_sql" class="function"><a href="../generator.html#Generator.lockingproperty_sql">lockingproperty_sql</a></dd> <dd id="Spark.Generator.insert_sql" class="function"><a href="../generator.html#Generator.insert_sql">insert_sql</a></dd> <dd id="Spark.Generator.intersect_sql" class="function"><a href="../generator.html#Generator.intersect_sql">intersect_sql</a></dd> <dd id="Spark.Generator.intersect_op" class="function"><a href="../generator.html#Generator.intersect_op">intersect_op</a></dd> @@ -895,6 +912,7 @@ Default: True</li> <dd id="Spark.Generator.use_sql" class="function"><a href="../generator.html#Generator.use_sql">use_sql</a></dd> <dd id="Spark.Generator.binary" class="function"><a href="../generator.html#Generator.binary">binary</a></dd> <dd id="Spark.Generator.function_fallback_sql" class="function"><a href="../generator.html#Generator.function_fallback_sql">function_fallback_sql</a></dd> + <dd id="Spark.Generator.func" class="function"><a href="../generator.html#Generator.func">func</a></dd> <dd id="Spark.Generator.format_args" class="function"><a href="../generator.html#Generator.format_args">format_args</a></dd> <dd id="Spark.Generator.text_width" class="function"><a href="../generator.html#Generator.text_width">text_width</a></dd> <dd id="Spark.Generator.format_time" class="function"><a href="../generator.html#Generator.format_time">format_time</a></dd> @@ -905,7 +923,6 @@ Default: True</li> <dd id="Spark.Generator.tag_sql" class="function"><a href="../generator.html#Generator.tag_sql">tag_sql</a></dd> <dd id="Spark.Generator.token_sql" class="function"><a href="../generator.html#Generator.token_sql">token_sql</a></dd> <dd id="Spark.Generator.userdefinedfunction_sql" class="function"><a href="../generator.html#Generator.userdefinedfunction_sql">userdefinedfunction_sql</a></dd> - <dd id="Spark.Generator.userdefinedfunctionkwarg_sql" class="function"><a href="../generator.html#Generator.userdefinedfunctionkwarg_sql">userdefinedfunctionkwarg_sql</a></dd> <dd id="Spark.Generator.joinhint_sql" class="function"><a href="../generator.html#Generator.joinhint_sql">joinhint_sql</a></dd> <dd id="Spark.Generator.kwarg_sql" class="function"><a href="../generator.html#Generator.kwarg_sql">kwarg_sql</a></dd> <dd id="Spark.Generator.when_sql" class="function"><a href="../generator.html#Generator.when_sql">when_sql</a></dd> @@ -931,8 +948,8 @@ Default: True</li> </div> <a class="headerlink" href="#Spark.Tokenizer"></a> - <div class="pdoc-code codehilite"><pre><span></span><span id="Spark.Tokenizer-174"><a href="#Spark.Tokenizer-174"><span class="linenos">174</span></a> <span class="k">class</span> <span class="nc">Tokenizer</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Tokenizer</span><span class="p">):</span> -</span><span id="Spark.Tokenizer-175"><a href="#Spark.Tokenizer-175"><span class="linenos">175</span></a> <span class="n">HEX_STRINGS</span> <span class="o">=</span> <span class="p">[(</span><span class="s2">"X'"</span><span class="p">,</span> <span class="s2">"'"</span><span class="p">)]</span> + <div class="pdoc-code codehilite"><pre><span></span><span id="Spark.Tokenizer-181"><a href="#Spark.Tokenizer-181"><span class="linenos">181</span></a> <span class="k">class</span> <span class="nc">Tokenizer</span><span class="p">(</span><span class="n">Hive</span><span class="o">.</span><span class="n">Tokenizer</span><span class="p">):</span> +</span><span id="Spark.Tokenizer-182"><a href="#Spark.Tokenizer-182"><span class="linenos">182</span></a> <span class="n">HEX_STRINGS</span> <span class="o">=</span> <span class="p">[(</span><span class="s2">"X'"</span><span class="p">,</span> <span class="s2">"'"</span><span class="p">)]</span> </span></pre></div> |