PySpark: replacing class manifest by class tag for Scala 2.10.2 inside rdd.py
This commit is contained in:
parent
ad61349c8e
commit
96571c2524
|
@ -731,13 +731,13 @@ class PipelinedRDD(RDD):
|
||||||
[x._jbroadcast for x in self.ctx._pickled_broadcast_vars],
|
[x._jbroadcast for x in self.ctx._pickled_broadcast_vars],
|
||||||
self.ctx._gateway._gateway_client)
|
self.ctx._gateway._gateway_client)
|
||||||
self.ctx._pickled_broadcast_vars.clear()
|
self.ctx._pickled_broadcast_vars.clear()
|
||||||
class_manifest = self._prev_jrdd.classManifest()
|
class_tag = self._prev_jrdd.classTag()
|
||||||
env = copy.copy(self.ctx.environment)
|
env = copy.copy(self.ctx.environment)
|
||||||
env['PYTHONPATH'] = os.environ.get("PYTHONPATH", "")
|
env['PYTHONPATH'] = os.environ.get("PYTHONPATH", "")
|
||||||
env = MapConverter().convert(env, self.ctx._gateway._gateway_client)
|
env = MapConverter().convert(env, self.ctx._gateway._gateway_client)
|
||||||
python_rdd = self.ctx._jvm.PythonRDD(self._prev_jrdd.rdd(),
|
python_rdd = self.ctx._jvm.PythonRDD(self._prev_jrdd.rdd(),
|
||||||
pipe_command, env, self.preservesPartitioning, self.ctx.pythonExec,
|
pipe_command, env, self.preservesPartitioning, self.ctx.pythonExec,
|
||||||
broadcast_vars, self.ctx._javaAccumulator, class_manifest)
|
broadcast_vars, self.ctx._javaAccumulator, class_tag)
|
||||||
self._jrdd_val = python_rdd.asJavaRDD()
|
self._jrdd_val = python_rdd.asJavaRDD()
|
||||||
return self._jrdd_val
|
return self._jrdd_val
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue