[Link].
OutOfMemorySparkException: Size of broadcasted table
far exceeds estimates and exceeds limit of [Link]=4294967296.
You can disable broadcasts for this query using set
[Link]=-1
---------------------------------------------------------------------------
Py4JJavaError Traceback (most recent call last)
127 if submission:
128 submission.submit_status('In Progress', stage=stage)
--> 129 method()
130 submission.submit_status('Completed', stage=stage)
131 else:
/local_disk0/spark-d72471c9-08db-4f0d-b38c-8cb06cc4fc63/userFiles-87519a89-ae11-
4133-92d0-28f1df2ea516/[Link]/integrated_dim_product_unharmonised.py in
process()
37
38 output_df = transform(source_df, current_df, epos_product_lookup_df)
---> 39 load(output_df)
40
41 log_info(__name__, "process", "finish::success")
/local_disk0/spark-d72471c9-08db-4f0d-b38c-8cb06cc4fc63/userFiles-87519a89-ae11-
4133-92d0-28f1df2ea516/[Link]/integrated_dim_product_unharmonised.py in
load(df)
85
86 # don't write output unless df has values in
---> 87 if [Link](1):
88 [Link]('overwrite').parquet(output_path)
89
/databricks/spark/python/pyspark/sql/[Link] in head(self, n)
1744 rs = [Link](1)
1745 return rs[0] if rs else None
-> 1746 return [Link](n)
1747
1748 def first(self):
/databricks/spark/python/pyspark/sql/[Link] in take(self, num)
767 [Row(age=2, name='Alice'), Row(age=5, name='Bob')]
768 """
--> 769 return [Link](num).collect()
770
771 def tail(self, num):
/databricks/spark/python/pyspark/sql/[Link] in collect(self)
713 # Default path used in OSS Spark / for non-DF-ACL clusters:
714 with SCCallSiteSync(self._sc) as css:
--> 715 sock_info = self._jdf.collectToPython()
716 return list(_load_from_socket(sock_info,
BatchedSerializer(PickleSerializer())))
717
/databricks/spark/python/lib/[Link]/py4j/java_gateway.py in
__call__(self, *args)
1302
1303 answer = self.gateway_client.send_command(command)
-> 1304 return_value = get_return_value(
1305 answer, self.gateway_client, self.target_id, [Link])
1306
/databricks/spark/python/pyspark/sql/[Link] in deco(*a, **kw)
115 def deco(*a, **kw):
116 try:
--> 117 return f(*a, **kw)
118 except [Link].Py4JJavaError as e:
119 converted = convert_exception(e.java_exception)
/databricks/spark/python/lib/[Link]/py4j/[Link] in
get_return_value(answer, gateway_client, target_id, name)
324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
325 if answer[1] == REFERENCE_TYPE:
--> 326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
328 format(target_id, ".", name), value)
Py4JJavaError: An error occurred while calling [Link].
: [Link]: Size of broadcasted
table far exceeds estimates and exceeds limit of
[Link]=4294967296. You can disable broadcasts for this query
using set [Link]=-1
at [Link].
$anonfun$relationFuture$1([Link])
at [Link]([Link])
at [Link]$.
$anonfun$withThreadLocalCaptured$4([Link])
at [Link]([Link])
at [Link]$.
$anonfun$withThreadLocalCaptured$3([Link])
at [Link]([Link])
at [Link]$.
$anonfun$withThreadLocalCaptured$2([Link])
at
[Link]$.withOptimisticTransaction(SQLExecution
.scala:465)
at [Link]$.
$anonfun$withThreadLocalCaptured$1([Link])
at
[Link]$[Link]([Link])
at [Link].
$anonfun$run$1([Link])
at [Link].java8.JFunction0$mcV$[Link](JFunction0$mcV$[Link])
at
[Link](Spark
[Link])
at
[Link]$
([Link])
at
[Link](Spa
[Link])
at
[Link](SparkThreadLoca
[Link])
at
[Link]([Link])
at
[Link]$[Link]([Link])
at [Link]([Link])