ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

PySpark: replacing class manifest by class tag for Scala 2.10.2 inside rdd.py

Browse source
This commit is contained in:
Andre Schumacher 2013-08-30 15:00:42 -07:00
parent ad61349c8e
commit 96571c2524
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
python/pyspark/rdd.py
View file

@ -731,13 +731,13 @@ class PipelinedRDD(RDD):
[x._jbroadcast for x in self.ctx._pickled_broadcast_vars],
self.ctx._gateway._gateway_client)
self.ctx._pickled_broadcast_vars.clear()
class_manifest = self._prev_jrdd.classManifest()
class_tag = self._prev_jrdd.classTag()
env = copy.copy(self.ctx.environment)
env['PYTHONPATH'] = os.environ.get("PYTHONPATH", "")
env = MapConverter().convert(env, self.ctx._gateway._gateway_client)
python_rdd = self.ctx._jvm.PythonRDD(self._prev_jrdd.rdd(),
pipe_command, env, self.preservesPartitioning, self.ctx.pythonExec,
broadcast_vars, self.ctx._javaAccumulator, class_manifest)
broadcast_vars, self.ctx._javaAccumulator, class_tag)
self._jrdd_val = python_rdd.asJavaRDD()
return self._jrdd_val