From eae79dd31ba7ae6f7a40459a5b146b538ef42d3f Mon Sep 17 00:00:00 2001 From: Xinrong Meng Date: Tue, 13 Jul 2021 12:07:05 +0900 Subject: [PATCH] [SPARK-36104][PYTHON] Manage InternalField in DataTypeOps.neg/abs ### What changes were proposed in this pull request? Manage InternalField for DataTypeOps.neg/abs. ### Why are the changes needed? The spark data type and nullability must be the same as the original when DataTypeOps.neg/abs. We should manage InternalField for this case. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Unit tests. Closes #33307 from xinrong-databricks/internalField. Authored-by: Xinrong Meng Signed-off-by: Hyukjin Kwon (cherry picked from commit 5afc27f899db7881b69302b94ff996c6f2305ad1) Signed-off-by: Hyukjin Kwon --- python/pyspark/pandas/data_type_ops/num_ops.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/pyspark/pandas/data_type_ops/num_ops.py b/python/pyspark/pandas/data_type_ops/num_ops.py index 16cb30526d..94bbd7c2a4 100644 --- a/python/pyspark/pandas/data_type_ops/num_ops.py +++ b/python/pyspark/pandas/data_type_ops/num_ops.py @@ -123,10 +123,12 @@ class NumericOps(DataTypeOps): return column_op(rmod)(left, right) def neg(self, operand: IndexOpsLike) -> IndexOpsLike: - return cast(IndexOpsLike, column_op(Column.__neg__)(operand)) + return operand._with_new_scol(-operand.spark.column, field=operand._internal.data_fields[0]) def abs(self, operand: IndexOpsLike) -> IndexOpsLike: - return cast(IndexOpsLike, column_op(F.abs)(operand)) + return operand._with_new_scol( + F.abs(operand.spark.column), field=operand._internal.data_fields[0] + ) def lt(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex: return column_op(Column.__lt__)(left, right)