ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
spark-instrumented-optimizer/python/pyspark/pandas/tests/test_series.py
Yikun Jiang 44b7931936 [SPARK-35176][PYTHON] Standardize input validation error type 
### What changes were proposed in this pull request?
This PR corrects some exception type when the function input params are failed to validate due to TypeError.
In order to convenient to review, there are 3 commits in this PR:
- Standardize input validation error type on sql
- Standardize input validation error type on ml
- Standardize input validation error type on pandas

### Why are the changes needed?
As suggestion from Python exception doc [1]: "Raised when an operation or function is applied to an object of inappropriate type.", but there are many Value error are raised in some pyspark code, this patch fix them.

[1] https://docs.python.org/3/library/exceptions.html#TypeError

Note that: this patch only addresses the exsiting some wrong raise type for input validation, the input validation decorator/framework which mentioned in [SPARK-35176](https://issues.apache.org/jira/browse/SPARK-35176), would be submited in a speparated patch.

### Does this PR introduce _any_ user-facing change?
Yes, code can raise the right TypeError instead of ValueError.

### How was this patch tested?
Existing test case and UT

Closes #32368 from Yikun/SPARK-35176.

Authored-by: Yikun Jiang <yikunkero@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
2021-05-03 15:34:24 +09:00

2918 lines
117 KiB
Python
Raw Blame History

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import unittest
from collections import defaultdict
from distutils.version import LooseVersion
import inspect
from itertools import product
from datetime import datetime, timedelta
import numpy as np
import pandas as pd
from pyspark.ml.linalg import SparseVector
from pyspark.sql import functions as F
from pyspark import pandas as ps
from pyspark.testing.pandasutils import (
have_tabulate,
PandasOnSparkTestCase,
SPARK_CONF_ARROW_ENABLED,
tabulate_requirement_message,
)
from pyspark.testing.sqlutils import SQLTestUtils
from pyspark.pandas.exceptions import PandasNotImplementedError
from pyspark.pandas.missing.series import MissingPandasLikeSeries
from pyspark.pandas.typedef.typehints import (
extension_dtypes,
extension_dtypes_available,
extension_float_dtypes_available,
extension_object_dtypes_available,
)
class SeriesTest(PandasOnSparkTestCase, SQLTestUtils):
@property
def pser(self):
return pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
@property
def kser(self):
return ps.from_pandas(self.pser)
def test_series_ops(self):
pser = self.pser
kser = self.kser
self.assert_eq(kser + 1, pser + 1)
self.assert_eq(1 + kser, 1 + pser)
self.assert_eq(kser + 1 + 10 * kser, pser + 1 + 10 * pser)
self.assert_eq(kser + 1 + 10 * kser.index, pser + 1 + 10 * pser.index)
self.assert_eq(kser.index + 1 + 10 * kser, pser.index + 1 + 10 * pser)
def test_series_tuple_name(self):
pser = self.pser
pser.name = ("x", "a")
kser = ps.from_pandas(pser)
self.assert_eq(kser, pser)
self.assert_eq(kser.name, pser.name)
pser.name = ("y", "z")
kser.name = ("y", "z")
self.assert_eq(kser, pser)
self.assert_eq(kser.name, pser.name)
def test_repr_cache_invalidation(self):
# If there is any cache, inplace operations should invalidate it.
s = ps.range(10)["id"]
s.__repr__()
s.rename("a", inplace=True)
self.assertEqual(s.__repr__(), s.rename("a").__repr__())
def _check_extension(self, kser, pser):
if LooseVersion("1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.2.2"):
self.assert_eq(kser, pser, check_exact=False)
self.assertTrue(isinstance(kser.dtype, extension_dtypes))
else:
self.assert_eq(kser, pser)
@unittest.skipIf(not extension_dtypes_available, "pandas extension dtypes are not available")
def test_extension_dtypes(self):
for pser in [
pd.Series([1, 2, None, 4], dtype="Int8"),
pd.Series([1, 2, None, 4], dtype="Int16"),
pd.Series([1, 2, None, 4], dtype="Int32"),
pd.Series([1, 2, None, 4], dtype="Int64"),
]:
kser = ps.from_pandas(pser)
self._check_extension(kser, pser)
self._check_extension(kser + F.lit(1).cast("byte"), pser + 1)
self._check_extension(kser + kser, pser + pser)
@unittest.skipIf(
not extension_object_dtypes_available, "pandas extension object dtypes are not available"
)
def test_extension_object_dtypes(self):
# string
pser = pd.Series(["a", None, "c", "d"], dtype="string")
kser = ps.from_pandas(pser)
self._check_extension(kser, pser)
# boolean
pser = pd.Series([True, False, True, None], dtype="boolean")
kser = ps.from_pandas(pser)
self._check_extension(kser, pser)
self._check_extension(kser & kser, pser & pser)
self._check_extension(kser | kser, pser | pser)
@unittest.skipIf(
not extension_float_dtypes_available, "pandas extension float dtypes are not available"
)
def test_extension_float_dtypes(self):
for pser in [
pd.Series([1.0, 2.0, None, 4.0], dtype="Float32"),
pd.Series([1.0, 2.0, None, 4.0], dtype="Float64"),
]:
kser = ps.from_pandas(pser)
self._check_extension(kser, pser)
self._check_extension(kser + 1, pser + 1)
self._check_extension(kser + kser, pser + pser)
def test_empty_series(self):
pser_a = pd.Series([], dtype="i1")
pser_b = pd.Series([], dtype="str")
self.assert_eq(ps.from_pandas(pser_a), pser_a)
kser_b = ps.from_pandas(pser_b)
self.assert_eq(kser_b, pser_b)
with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
self.assert_eq(ps.from_pandas(pser_a), pser_a)
self.assert_eq(ps.from_pandas(pser_b), pser_b)
def test_all_null_series(self):
pser_a = pd.Series([None, None, None], dtype="float64")
pser_b = pd.Series([None, None, None], dtype="str")
self.assert_eq(ps.from_pandas(pser_a), pser_a)
kser_b = ps.from_pandas(pser_b)
self.assert_eq(kser_b, pser_b)
with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
self.assert_eq(ps.from_pandas(pser_a), pser_a)
self.assert_eq(ps.from_pandas(pser_b), pser_b)
def test_head(self):
kser = self.kser
pser = self.pser
self.assert_eq(kser.head(3), pser.head(3))
self.assert_eq(kser.head(0), pser.head(0))
self.assert_eq(kser.head(-3), pser.head(-3))
self.assert_eq(kser.head(-10), pser.head(-10))
def test_last(self):
with self.assertRaises(TypeError):
self.kser.last("1D")
index = pd.date_range("2018-04-09", periods=4, freq="2D")
pser = pd.Series([1, 2, 3, 4], index=index)
kser = ps.from_pandas(pser)
self.assert_eq(kser.last("1D"), pser.last("1D"))
def test_first(self):
with self.assertRaises(TypeError):
self.kser.first("1D")
index = pd.date_range("2018-04-09", periods=4, freq="2D")
pser = pd.Series([1, 2, 3, 4], index=index)
kser = ps.from_pandas(pser)
self.assert_eq(kser.first("1D"), pser.first("1D"))
def test_rename(self):
pser = pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
kser = ps.from_pandas(pser)
pser.name = "renamed"
kser.name = "renamed"
self.assertEqual(kser.name, "renamed")
self.assert_eq(kser, pser)
pser.name = None
kser.name = None
self.assertEqual(kser.name, None)
self.assert_eq(kser, pser)
pidx = pser.index
kidx = kser.index
pidx.name = "renamed"
kidx.name = "renamed"
self.assertEqual(kidx.name, "renamed")
self.assert_eq(kidx, pidx)
expected_error_message = "Series.name must be a hashable type"
with self.assertRaisesRegex(TypeError, expected_error_message):
kser.name = ["renamed"]
with self.assertRaisesRegex(TypeError, expected_error_message):
kser.name = ["0", "1"]
with self.assertRaisesRegex(TypeError, expected_error_message):
ps.Series([1, 2, 3], name=["0", "1"])
def test_rename_method(self):
# Series name
pser = pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
kser = ps.from_pandas(pser)
self.assert_eq(kser.rename("y"), pser.rename("y"))
self.assertEqual(kser.name, "x") # no mutation
self.assert_eq(kser.rename(), pser.rename())
self.assert_eq((kser.rename("y") + 1).head(), (pser.rename("y") + 1).head())
kser.rename("z", inplace=True)
pser.rename("z", inplace=True)
self.assertEqual(kser.name, "z")
self.assert_eq(kser, pser)
expected_error_message = "Series.name must be a hashable type"
with self.assertRaisesRegex(TypeError, expected_error_message):
kser.rename(["0", "1"])
# Series index
# pser = pd.Series(['a', 'b', 'c', 'd', 'e', 'f', 'g'], name='x')
# kser = ps.from_pandas(s)
# TODO: index
# res = kser.rename(lambda x: x ** 2)
# self.assert_eq(res, pser.rename(lambda x: x ** 2))
# res = kser.rename(pser)
# self.assert_eq(res, pser.rename(pser))
# res = kser.rename(kser)
# self.assert_eq(res, pser.rename(pser))
# res = kser.rename(lambda x: x**2, inplace=True)
# self.assertis(res, kser)
# s.rename(lambda x: x**2, inplace=True)
# self.assert_eq(kser, pser)
def test_rename_axis(self):
index = pd.Index(["A", "B", "C"], name="index")
pser = pd.Series([1.0, 2.0, 3.0], index=index, name="name")
kser = ps.from_pandas(pser)
self.assert_eq(
pser.rename_axis("index2").sort_index(), kser.rename_axis("index2").sort_index(),
)
self.assert_eq(
(pser + 1).rename_axis("index2").sort_index(),
(kser + 1).rename_axis("index2").sort_index(),
)
pser2 = pser.copy()
kser2 = kser.copy()
pser2.rename_axis("index2", inplace=True)
kser2.rename_axis("index2", inplace=True)
self.assert_eq(pser2.sort_index(), kser2.sort_index())
self.assertRaises(ValueError, lambda: kser.rename_axis(["index2", "index3"]))
self.assertRaises(TypeError, lambda: kser.rename_axis(mapper=["index2"], index=["index3"]))
# index/columns parameters and dict_like/functions mappers introduced in pandas 0.24.0
if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
self.assert_eq(
pser.rename_axis(index={"index": "index2", "missing": "index4"}).sort_index(),
kser.rename_axis(index={"index": "index2", "missing": "index4"}).sort_index(),
)
self.assert_eq(
pser.rename_axis(index=str.upper).sort_index(),
kser.rename_axis(index=str.upper).sort_index(),
)
else:
expected = kser
expected.index.name = "index2"
result = kser.rename_axis(index={"index": "index2", "missing": "index4"}).sort_index()
self.assert_eq(expected, result)
expected = kser
expected.index.name = "INDEX"
result = kser.rename_axis(index=str.upper).sort_index()
self.assert_eq(expected, result)
index = pd.MultiIndex.from_tuples(
[("A", "B"), ("C", "D"), ("E", "F")], names=["index1", "index2"]
)
pser = pd.Series([1.0, 2.0, 3.0], index=index, name="name")
kser = ps.from_pandas(pser)
self.assert_eq(
pser.rename_axis(["index3", "index4"]).sort_index(),
kser.rename_axis(["index3", "index4"]).sort_index(),
)
self.assertRaises(ValueError, lambda: kser.rename_axis(["index3", "index4", "index5"]))
# index/columns parameters and dict_like/functions mappers introduced in pandas 0.24.0
if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
self.assert_eq(
pser.rename_axis(
index={"index1": "index3", "index2": "index4", "missing": "index5"}
).sort_index(),
kser.rename_axis(
index={"index1": "index3", "index2": "index4", "missing": "index5"}
).sort_index(),
)
self.assert_eq(
pser.rename_axis(index=str.upper).sort_index(),
kser.rename_axis(index=str.upper).sort_index(),
)
else:
expected = kser
expected.index.names = ["index3", "index4"]
result = kser.rename_axis(
index={"index1": "index3", "index2": "index4", "missing": "index5"}
).sort_index()
self.assert_eq(expected, result)
expected.index.names = ["INDEX1", "INDEX2"]
result = kser.rename_axis(index=str.upper).sort_index()
self.assert_eq(expected, result)
def test_or(self):
pdf = pd.DataFrame(
{
"left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
"right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
}
)
kdf = ps.from_pandas(pdf)
self.assert_eq(kdf["left"] | kdf["right"], pdf["left"] | pdf["right"])
self.assert_eq(kdf["left"] | True, pdf["left"] | True)
self.assert_eq(kdf["left"] | False, pdf["left"] | False)
self.assert_eq(kdf["left"] | None, pdf["left"] | None)
self.assert_eq(True | kdf["right"], True | pdf["right"])
self.assert_eq(False | kdf["right"], False | pdf["right"])
self.assert_eq(None | kdf["right"], None | pdf["right"])
@unittest.skipIf(
not extension_object_dtypes_available, "pandas extension object dtypes are not available"
)
def test_or_extenstion_dtypes(self):
pdf = pd.DataFrame(
{
"left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
"right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
}
).astype("boolean")
kdf = ps.from_pandas(pdf)
self._check_extension(kdf["left"] | kdf["right"], pdf["left"] | pdf["right"])
self._check_extension(kdf["left"] | True, pdf["left"] | True)
self._check_extension(kdf["left"] | False, pdf["left"] | False)
self._check_extension(kdf["left"] | pd.NA, pdf["left"] | pd.NA)
self._check_extension(True | kdf["right"], True | pdf["right"])
self._check_extension(False | kdf["right"], False | pdf["right"])
self._check_extension(pd.NA | kdf["right"], pd.NA | pdf["right"])
def test_and(self):
pdf = pd.DataFrame(
{
"left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
"right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
}
)
kdf = ps.from_pandas(pdf)
self.assert_eq(kdf["left"] & kdf["right"], pdf["left"] & pdf["right"])
self.assert_eq(kdf["left"] & True, pdf["left"] & True)
self.assert_eq(kdf["left"] & False, pdf["left"] & False)
self.assert_eq(kdf["left"] & None, pdf["left"] & None)
self.assert_eq(True & kdf["right"], True & pdf["right"])
self.assert_eq(False & kdf["right"], False & pdf["right"])
self.assert_eq(None & kdf["right"], None & pdf["right"])
@unittest.skipIf(
not extension_object_dtypes_available, "pandas extension object dtypes are not available"
)
def test_and_extenstion_dtypes(self):
pdf = pd.DataFrame(
{
"left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
"right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
}
).astype("boolean")
kdf = ps.from_pandas(pdf)
self._check_extension(kdf["left"] & kdf["right"], pdf["left"] & pdf["right"])
self._check_extension(kdf["left"] & True, pdf["left"] & True)
self._check_extension(kdf["left"] & False, pdf["left"] & False)
self._check_extension(kdf["left"] & pd.NA, pdf["left"] & pd.NA)
self._check_extension(True & kdf["right"], True & pdf["right"])
self._check_extension(False & kdf["right"], False & pdf["right"])
self._check_extension(pd.NA & kdf["right"], pd.NA & pdf["right"])
def test_to_numpy(self):
pser = pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
kser = ps.from_pandas(pser)
self.assert_eq(kser.to_numpy(), pser.values)
def test_isin(self):
pser = pd.Series(["lama", "cow", "lama", "beetle", "lama", "hippo"], name="animal")
kser = ps.from_pandas(pser)
self.assert_eq(kser.isin(["cow", "lama"]), pser.isin(["cow", "lama"]))
self.assert_eq(kser.isin(np.array(["cow", "lama"])), pser.isin(np.array(["cow", "lama"])))
self.assert_eq(kser.isin({"cow"}), pser.isin({"cow"}))
msg = "only list-like objects are allowed to be passed to isin()"
with self.assertRaisesRegex(TypeError, msg):
kser.isin(1)
def test_drop_duplicates(self):
pdf = pd.DataFrame({"animal": ["lama", "cow", "lama", "beetle", "lama", "hippo"]})
kdf = ps.from_pandas(pdf)
pser = pdf.animal
kser = kdf.animal
self.assert_eq(kser.drop_duplicates().sort_index(), pser.drop_duplicates().sort_index())
self.assert_eq(
kser.drop_duplicates(keep="last").sort_index(),
pser.drop_duplicates(keep="last").sort_index(),
)
# inplace
kser.drop_duplicates(keep=False, inplace=True)
pser.drop_duplicates(keep=False, inplace=True)
self.assert_eq(kser.sort_index(), pser.sort_index())
self.assert_eq(kdf, pdf)
def test_reindex(self):
index = ["A", "B", "C", "D", "E"]
pser = pd.Series([1.0, 2.0, 3.0, 4.0, None], index=index, name="x")
kser = ps.from_pandas(pser)
self.assert_eq(pser, kser)
self.assert_eq(
pser.reindex(["A", "B"]).sort_index(), kser.reindex(["A", "B"]).sort_index(),
)
self.assert_eq(
pser.reindex(["A", "B", "2", "3"]).sort_index(),
kser.reindex(["A", "B", "2", "3"]).sort_index(),
)
self.assert_eq(
pser.reindex(["A", "E", "2"], fill_value=0).sort_index(),
kser.reindex(["A", "E", "2"], fill_value=0).sort_index(),
)
self.assertRaises(TypeError, lambda: kser.reindex(index=123))
def test_reindex_like(self):
data = [1.0, 2.0, None]
index = pd.Index(["A", "B", "C"], name="index1")
pser = pd.Series(data=data, index=index, name="name1")
kser = ps.from_pandas(pser)
# Reindexing single Index on single Index
data2 = [3.0, None, 4.0]
index2 = pd.Index(["A", "C", "D"], name="index2")
pser2 = pd.Series(data=data2, index=index2, name="name2")
kser2 = ps.from_pandas(pser2)
self.assert_eq(
pser.reindex_like(pser2).sort_index(), kser.reindex_like(kser2).sort_index(),
)
self.assert_eq(
(pser + 1).reindex_like(pser2).sort_index(),
(kser + 1).reindex_like(kser2).sort_index(),
)
# Reindexing MultiIndex on single Index
index2 = pd.MultiIndex.from_tuples(
[("A", "G"), ("C", "D"), ("I", "J")], names=["index3", "index4"]
)
pser2 = pd.Series(data=data2, index=index2, name="name2")
kser2 = ps.from_pandas(pser2)
self.assert_eq(
pser.reindex_like(pser2).sort_index(), kser.reindex_like(kser2).sort_index(),
)
self.assertRaises(TypeError, lambda: kser.reindex_like(index2))
self.assertRaises(AssertionError, lambda: kser2.reindex_like(kser))
# Reindexing MultiIndex on MultiIndex
index = pd.MultiIndex.from_tuples(
[("A", "B"), ("C", "D"), ("E", "F")], names=["index1", "index2"]
)
pser = pd.Series(data=data, index=index, name="name1")
kser = ps.from_pandas(pser)
self.assert_eq(
pser.reindex_like(pser2).sort_index(), kser.reindex_like(kser2).sort_index(),
)
# Reindexing with DataFrame
index2 = pd.MultiIndex.from_tuples(
[("A", "B"), ("C", "D"), ("E", "F")], names=["name3", "name4"]
)
pdf = pd.DataFrame(data=data, index=index2)
kdf = ps.from_pandas(pdf)
self.assert_eq(
pser.reindex_like(pdf).sort_index(), kser.reindex_like(kdf).sort_index(),
)
def test_fillna(self):
pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6], "y": [np.nan, 2, 3, 4, np.nan, 6]})
kdf = ps.from_pandas(pdf)
pser = pdf.x
kser = kdf.x
self.assert_eq(kser.fillna(0), pser.fillna(0))
self.assert_eq(kser.fillna(np.nan).fillna(0), pser.fillna(np.nan).fillna(0))
kser.fillna(0, inplace=True)
pser.fillna(0, inplace=True)
self.assert_eq(kser, pser)
self.assert_eq(kdf, pdf)
# test considering series does not have NA/NaN values
kser.fillna(0, inplace=True)
pser.fillna(0, inplace=True)
self.assert_eq(kser, pser)
kser = kdf.x.rename("y")
pser = pdf.x.rename("y")
kser.fillna(0, inplace=True)
pser.fillna(0, inplace=True)
self.assert_eq(kser.head(), pser.head())
pser = pd.Series([1, 2, 3, 4, 5, 6], name="x")
kser = ps.from_pandas(pser)
pser.loc[3] = np.nan
kser.loc[3] = np.nan
self.assert_eq(kser.fillna(0), pser.fillna(0))
self.assert_eq(kser.fillna(method="ffill"), pser.fillna(method="ffill"))
self.assert_eq(kser.fillna(method="bfill"), pser.fillna(method="bfill"))
# inplace fillna on non-nullable column
pdf = pd.DataFrame({"a": [1, 2, None], "b": [1, 2, 3]})
kdf = ps.from_pandas(pdf)
pser = pdf.b
kser = kdf.b
self.assert_eq(kser.fillna(0), pser.fillna(0))
self.assert_eq(kser.fillna(np.nan).fillna(0), pser.fillna(np.nan).fillna(0))
kser.fillna(0, inplace=True)
pser.fillna(0, inplace=True)
self.assert_eq(kser, pser)
self.assert_eq(kdf, pdf)
def test_dropna(self):
pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6]})
kdf = ps.from_pandas(pdf)
pser = pdf.x
kser = kdf.x
self.assert_eq(kser.dropna(), pser.dropna())
pser.dropna(inplace=True)
kser.dropna(inplace=True)
self.assert_eq(kser, pser)
self.assert_eq(kdf, pdf)
def test_nunique(self):
pser = pd.Series([1, 2, 1, np.nan])
kser = ps.from_pandas(pser)
# Assert NaNs are dropped by default
nunique_result = kser.nunique()
self.assertEqual(nunique_result, 2)
self.assert_eq(nunique_result, pser.nunique())
# Assert including NaN values
nunique_result = kser.nunique(dropna=False)
self.assertEqual(nunique_result, 3)
self.assert_eq(nunique_result, pser.nunique(dropna=False))
# Assert approximate counts
self.assertEqual(ps.Series(range(100)).nunique(approx=True), 103)
self.assertEqual(ps.Series(range(100)).nunique(approx=True, rsd=0.01), 100)
def test_value_counts(self):
# this is also containing test for Index & MultiIndex
pser = pd.Series(
[1, 2, 1, 3, 3, np.nan, 1, 4, 2, np.nan, 3, np.nan, 3, 1, 3],
index=[1, 2, 1, 3, 3, np.nan, 1, 4, 2, np.nan, 3, np.nan, 3, 1, 3],
name="x",
)
kser = ps.from_pandas(pser)
exp = pser.value_counts()
res = kser.value_counts()
self.assertEqual(res.name, exp.name)
self.assert_eq(res, exp)
self.assert_eq(kser.value_counts(normalize=True), pser.value_counts(normalize=True))
self.assert_eq(kser.value_counts(ascending=True), pser.value_counts(ascending=True))
self.assert_eq(
kser.value_counts(normalize=True, dropna=False),
pser.value_counts(normalize=True, dropna=False),
)
self.assert_eq(
kser.value_counts(ascending=True, dropna=False),
pser.value_counts(ascending=True, dropna=False),
)
self.assert_eq(
kser.index.value_counts(normalize=True), pser.index.value_counts(normalize=True)
)
self.assert_eq(
kser.index.value_counts(ascending=True), pser.index.value_counts(ascending=True)
)
self.assert_eq(
kser.index.value_counts(normalize=True, dropna=False),
pser.index.value_counts(normalize=True, dropna=False),
)
self.assert_eq(
kser.index.value_counts(ascending=True, dropna=False),
pser.index.value_counts(ascending=True, dropna=False),
)
with self.assertRaisesRegex(
NotImplementedError, "value_counts currently does not support bins"
):
kser.value_counts(bins=3)
pser.name = "index"
kser.name = "index"
self.assert_eq(kser.value_counts(), pser.value_counts())
# Series from DataFrame
pdf = pd.DataFrame({"a": [2, 2, 3], "b": [None, 1, None]})
kdf = ps.from_pandas(pdf)
self.assert_eq(kdf.a.value_counts(normalize=True), pdf.a.value_counts(normalize=True))
self.assert_eq(kdf.a.value_counts(ascending=True), pdf.a.value_counts(ascending=True))
self.assert_eq(
kdf.a.value_counts(normalize=True, dropna=False),
pdf.a.value_counts(normalize=True, dropna=False),
)
self.assert_eq(
kdf.a.value_counts(ascending=True, dropna=False),
pdf.a.value_counts(ascending=True, dropna=False),
)
self.assert_eq(
kser.index.value_counts(normalize=True), pser.index.value_counts(normalize=True)
)
self.assert_eq(
kser.index.value_counts(ascending=True), pser.index.value_counts(ascending=True)
)
self.assert_eq(
kser.index.value_counts(normalize=True, dropna=False),
pser.index.value_counts(normalize=True, dropna=False),
)
self.assert_eq(
kser.index.value_counts(ascending=True, dropna=False),
pser.index.value_counts(ascending=True, dropna=False),
)
# Series with NaN index
pser = pd.Series([3, 2, 3, 1, 2, 3], index=[2.0, None, 5.0, 5.0, None, 5.0])
kser = ps.from_pandas(pser)
self.assert_eq(kser.value_counts(normalize=True), pser.value_counts(normalize=True))
self.assert_eq(kser.value_counts(ascending=True), pser.value_counts(ascending=True))
self.assert_eq(
kser.value_counts(normalize=True, dropna=False),
pser.value_counts(normalize=True, dropna=False),
)
self.assert_eq(
kser.value_counts(ascending=True, dropna=False),
pser.value_counts(ascending=True, dropna=False),
)
self.assert_eq(
kser.index.value_counts(normalize=True), pser.index.value_counts(normalize=True)
)
self.assert_eq(
kser.index.value_counts(ascending=True), pser.index.value_counts(ascending=True)
)
self.assert_eq(
kser.index.value_counts(normalize=True, dropna=False),
pser.index.value_counts(normalize=True, dropna=False),
)
self.assert_eq(
kser.index.value_counts(ascending=True, dropna=False),
pser.index.value_counts(ascending=True, dropna=False),
)
# Series with MultiIndex
pser.index = pd.MultiIndex.from_tuples(
[("x", "a"), ("x", "b"), ("y", "c"), ("x", "a"), ("y", "c"), ("x", "a")]
)
kser = ps.from_pandas(pser)
self.assert_eq(kser.value_counts(normalize=True), pser.value_counts(normalize=True))
self.assert_eq(kser.value_counts(ascending=True), pser.value_counts(ascending=True))
self.assert_eq(
kser.value_counts(normalize=True, dropna=False),
pser.value_counts(normalize=True, dropna=False),
)
self.assert_eq(
kser.value_counts(ascending=True, dropna=False),
pser.value_counts(ascending=True, dropna=False),
)
# FIXME: MultiIndex.value_counts returns wrong indices.
self.assert_eq(
kser.index.value_counts(normalize=True),
pser.index.value_counts(normalize=True),
almost=True,
)
self.assert_eq(
kser.index.value_counts(ascending=True),
pser.index.value_counts(ascending=True),
almost=True,
)
self.assert_eq(
kser.index.value_counts(normalize=True, dropna=False),
pser.index.value_counts(normalize=True, dropna=False),
almost=True,
)
self.assert_eq(
kser.index.value_counts(ascending=True, dropna=False),
pser.index.value_counts(ascending=True, dropna=False),
almost=True,
)
# Series with MultiIndex some of index has NaN
pser.index = pd.MultiIndex.from_tuples(
[("x", "a"), ("x", None), ("y", "c"), ("x", "a"), ("y", "c"), ("x", "a")]
)
kser = ps.from_pandas(pser)
self.assert_eq(kser.value_counts(normalize=True), pser.value_counts(normalize=True))
self.assert_eq(kser.value_counts(ascending=True), pser.value_counts(ascending=True))
self.assert_eq(
kser.value_counts(normalize=True, dropna=False),
pser.value_counts(normalize=True, dropna=False),
)
self.assert_eq(
kser.value_counts(ascending=True, dropna=False),
pser.value_counts(ascending=True, dropna=False),
)
# FIXME: MultiIndex.value_counts returns wrong indices.
self.assert_eq(
kser.index.value_counts(normalize=True),
pser.index.value_counts(normalize=True),
almost=True,
)
self.assert_eq(
kser.index.value_counts(ascending=True),
pser.index.value_counts(ascending=True),
almost=True,
)
self.assert_eq(
kser.index.value_counts(normalize=True, dropna=False),
pser.index.value_counts(normalize=True, dropna=False),
almost=True,
)
self.assert_eq(
kser.index.value_counts(ascending=True, dropna=False),
pser.index.value_counts(ascending=True, dropna=False),
almost=True,
)
# Series with MultiIndex some of index is NaN.
# This test only available for pandas >= 0.24.
if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
pser.index = pd.MultiIndex.from_tuples(
[("x", "a"), None, ("y", "c"), ("x", "a"), ("y", "c"), ("x", "a")]
)
kser = ps.from_pandas(pser)
self.assert_eq(kser.value_counts(normalize=True), pser.value_counts(normalize=True))
self.assert_eq(kser.value_counts(ascending=True), pser.value_counts(ascending=True))
self.assert_eq(
kser.value_counts(normalize=True, dropna=False),
pser.value_counts(normalize=True, dropna=False),
)
self.assert_eq(
kser.value_counts(ascending=True, dropna=False),
pser.value_counts(ascending=True, dropna=False),
)
# FIXME: MultiIndex.value_counts returns wrong indices.
self.assert_eq(
kser.index.value_counts(normalize=True),
pser.index.value_counts(normalize=True),
almost=True,
)
self.assert_eq(
kser.index.value_counts(ascending=True),
pser.index.value_counts(ascending=True),
almost=True,
)
self.assert_eq(
kser.index.value_counts(normalize=True, dropna=False),
pser.index.value_counts(normalize=True, dropna=False),
almost=True,
)
self.assert_eq(
kser.index.value_counts(ascending=True, dropna=False),
pser.index.value_counts(ascending=True, dropna=False),
almost=True,
)
def test_nsmallest(self):
sample_lst = [1, 2, 3, 4, np.nan, 6]
pser = pd.Series(sample_lst, name="x")
kser = ps.Series(sample_lst, name="x")
self.assert_eq(kser.nsmallest(n=3), pser.nsmallest(n=3))
self.assert_eq(kser.nsmallest(), pser.nsmallest())
self.assert_eq((kser + 1).nsmallest(), (pser + 1).nsmallest())
def test_nlargest(self):
sample_lst = [1, 2, 3, 4, np.nan, 6]
pser = pd.Series(sample_lst, name="x")
kser = ps.Series(sample_lst, name="x")
self.assert_eq(kser.nlargest(n=3), pser.nlargest(n=3))
self.assert_eq(kser.nlargest(), pser.nlargest())
self.assert_eq((kser + 1).nlargest(), (pser + 1).nlargest())
def test_isnull(self):
pser = pd.Series([1, 2, 3, 4, np.nan, 6], name="x")
kser = ps.from_pandas(pser)
self.assert_eq(kser.notnull(), pser.notnull())
self.assert_eq(kser.isnull(), pser.isnull())
pser = self.pser
kser = self.kser
self.assert_eq(kser.notnull(), pser.notnull())
self.assert_eq(kser.isnull(), pser.isnull())
def test_all(self):
for pser in [
pd.Series([True, True], name="x"),
pd.Series([True, False], name="x"),
pd.Series([0, 1], name="x"),
pd.Series([1, 2, 3], name="x"),
pd.Series([True, True, None], name="x"),
pd.Series([True, False, None], name="x"),
pd.Series([], name="x"),
pd.Series([np.nan], name="x"),
]:
kser = ps.from_pandas(pser)
self.assert_eq(kser.all(), pser.all())
pser = pd.Series([1, 2, 3, 4], name="x")
kser = ps.from_pandas(pser)
self.assert_eq((kser % 2 == 0).all(), (pser % 2 == 0).all())
with self.assertRaisesRegex(
NotImplementedError, 'axis should be either 0 or "index" currently.'
):
kser.all(axis=1)
def test_any(self):
for pser in [
pd.Series([False, False], name="x"),
pd.Series([True, False], name="x"),
pd.Series([0, 1], name="x"),
pd.Series([1, 2, 3], name="x"),
pd.Series([True, True, None], name="x"),
pd.Series([True, False, None], name="x"),
pd.Series([], name="x"),
pd.Series([np.nan], name="x"),
]:
kser = ps.from_pandas(pser)
self.assert_eq(kser.any(), pser.any())
pser = pd.Series([1, 2, 3, 4], name="x")
kser = ps.from_pandas(pser)
self.assert_eq((kser % 2 == 0).any(), (pser % 2 == 0).any())
with self.assertRaisesRegex(
NotImplementedError, 'axis should be either 0 or "index" currently.'
):
kser.any(axis=1)
def test_reset_index(self):
pdf = pd.DataFrame({"foo": [1, 2, 3, 4]}, index=pd.Index(["a", "b", "c", "d"], name="idx"))
kdf = ps.from_pandas(pdf)
pser = pdf.foo
kser = kdf.foo
self.assert_eq(kser.reset_index(), pser.reset_index())
self.assert_eq(kser.reset_index(name="values"), pser.reset_index(name="values"))
self.assert_eq(kser.reset_index(drop=True), pser.reset_index(drop=True))
# inplace
kser.reset_index(drop=True, inplace=True)
pser.reset_index(drop=True, inplace=True)
self.assert_eq(kser, pser)
self.assert_eq(kdf, pdf)
def test_reset_index_with_default_index_types(self):
pser = pd.Series([1, 2, 3], name="0", index=np.random.rand(3))
kser = ps.from_pandas(pser)
with ps.option_context("compute.default_index_type", "sequence"):
self.assert_eq(kser.reset_index(), pser.reset_index())
with ps.option_context("compute.default_index_type", "distributed-sequence"):
# the order might be changed.
self.assert_eq(kser.reset_index().sort_index(), pser.reset_index())
with ps.option_context("compute.default_index_type", "distributed"):
# the index is different.
self.assert_eq(
kser.reset_index().to_pandas().reset_index(drop=True), pser.reset_index()
)
def test_index_to_series_reset_index(self):
def check(kser, pser):
self.assert_eq(kser.reset_index(), pser.reset_index())
self.assert_eq(kser.reset_index(drop=True), pser.reset_index(drop=True))
pser.reset_index(drop=True, inplace=True)
kser.reset_index(drop=True, inplace=True)
self.assert_eq(kser, pser)
pdf = pd.DataFrame(
{"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
index=np.random.rand(9),
)
kdf = ps.from_pandas(pdf)
check(kdf.index.to_series(), pdf.index.to_series())
check(kdf.index.to_series(name="a"), pdf.index.to_series(name="a"))
check(kdf.index.to_series(name=("x", "a")), pdf.index.to_series(name=("x", "a")))
def test_sort_values(self):
pdf = pd.DataFrame({"x": [1, 2, 3, 4, 5, None, 7]})
kdf = ps.from_pandas(pdf)
pser = pdf.x
kser = kdf.x
self.assert_eq(kser.sort_values(), pser.sort_values())
self.assert_eq(kser.sort_values(ascending=False), pser.sort_values(ascending=False))
self.assert_eq(kser.sort_values(na_position="first"), pser.sort_values(na_position="first"))
self.assertRaises(ValueError, lambda: kser.sort_values(na_position="invalid"))
# inplace
# pandas raises an exception when the Series is derived from DataFrame
kser.sort_values(inplace=True)
self.assert_eq(kser, pser.sort_values())
self.assert_eq(kdf, pdf)
pser = pdf.x.copy()
kser = kdf.x.copy()
kser.sort_values(inplace=True)
pser.sort_values(inplace=True)
self.assert_eq(kser, pser)
self.assert_eq(kdf, pdf)
def test_sort_index(self):
pdf = pd.DataFrame({"x": [2, 1, np.nan]}, index=["b", "a", np.nan])
kdf = ps.from_pandas(pdf)
pser = pdf.x
kser = kdf.x
# Assert invalid parameters
self.assertRaises(NotImplementedError, lambda: kser.sort_index(axis=1))
self.assertRaises(NotImplementedError, lambda: kser.sort_index(kind="mergesort"))
self.assertRaises(ValueError, lambda: kser.sort_index(na_position="invalid"))
# Assert default behavior without parameters
self.assert_eq(kser.sort_index(), pser.sort_index())
# Assert sorting descending
self.assert_eq(kser.sort_index(ascending=False), pser.sort_index(ascending=False))
# Assert sorting NA indices first
self.assert_eq(kser.sort_index(na_position="first"), pser.sort_index(na_position="first"))
# Assert sorting inplace
# pandas sorts pdf.x by the index and update the column only
# when the Series is derived from DataFrame.
kser.sort_index(inplace=True)
self.assert_eq(kser, pser.sort_index())
self.assert_eq(kdf, pdf)
pser = pdf.x.copy()
kser = kdf.x.copy()
kser.sort_index(inplace=True)
pser.sort_index(inplace=True)
self.assert_eq(kser, pser)
self.assert_eq(kdf, pdf)
# Assert multi-indices
pser = pd.Series(range(4), index=[["b", "b", "a", "a"], [1, 0, 1, 0]], name="0")
kser = ps.from_pandas(pser)
self.assert_eq(kser.sort_index(), pser.sort_index())
self.assert_eq(kser.sort_index(level=[1, 0]), pser.sort_index(level=[1, 0]))
self.assert_eq(kser.reset_index().sort_index(), pser.reset_index().sort_index())
def test_to_datetime(self):
pser = pd.Series(["3/11/2000", "3/12/2000", "3/13/2000"] * 100)
kser = ps.from_pandas(pser)
self.assert_eq(
pd.to_datetime(pser, infer_datetime_format=True),
ps.to_datetime(kser, infer_datetime_format=True),
)
def test_missing(self):
kser = self.kser
missing_functions = inspect.getmembers(MissingPandasLikeSeries, inspect.isfunction)
unsupported_functions = [
name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
]
for name in unsupported_functions:
with self.assertRaisesRegex(
PandasNotImplementedError,
"method.*Series.*{}.*not implemented( yet\\.|\\. .+)".format(name),
):
getattr(kser, name)()
deprecated_functions = [
name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
]
for name in deprecated_functions:
with self.assertRaisesRegex(
PandasNotImplementedError, "method.*Series.*{}.*is deprecated".format(name)
):
getattr(kser, name)()
missing_properties = inspect.getmembers(
MissingPandasLikeSeries, lambda o: isinstance(o, property)
)
unsupported_properties = [
name
for (name, type_) in missing_properties
if type_.fget.__name__ == "unsupported_property"
]
for name in unsupported_properties:
with self.assertRaisesRegex(
PandasNotImplementedError,
"property.*Series.*{}.*not implemented( yet\\.|\\. .+)".format(name),
):
getattr(kser, name)
deprecated_properties = [
name
for (name, type_) in missing_properties
if type_.fget.__name__ == "deprecated_property"
]
for name in deprecated_properties:
with self.assertRaisesRegex(
PandasNotImplementedError, "property.*Series.*{}.*is deprecated".format(name)
):
getattr(kser, name)
def test_clip(self):
pser = pd.Series([0, 2, 4], index=np.random.rand(3))
kser = ps.from_pandas(pser)
# Assert list-like values are not accepted for 'lower' and 'upper'
msg = "List-like value are not supported for 'lower' and 'upper' at the moment"
with self.assertRaises(TypeError, msg=msg):
kser.clip(lower=[1])
with self.assertRaises(TypeError, msg=msg):
kser.clip(upper=[1])
# Assert no lower or upper
self.assert_eq(kser.clip(), pser.clip())
# Assert lower only
self.assert_eq(kser.clip(1), pser.clip(1))
# Assert upper only
self.assert_eq(kser.clip(upper=3), pser.clip(upper=3))
# Assert lower and upper
self.assert_eq(kser.clip(1, 3), pser.clip(1, 3))
# Assert behavior on string values
str_kser = ps.Series(["a", "b", "c"])
self.assert_eq(str_kser.clip(1, 3), str_kser)
def test_compare(self):
if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
pser = pd.Series([1, 2])
kser = ps.from_pandas(pser)
res_kdf = kser.compare(kser)
self.assertTrue(res_kdf.empty)
self.assert_eq(res_kdf.columns, pd.Index(["self", "other"]))
self.assert_eq(pser.compare(pser + 1).sort_index(), kser.compare(kser + 1).sort_index())
pser = pd.Series([1, 2], index=["x", "y"])
kser = ps.from_pandas(pser)
self.assert_eq(pser.compare(pser + 1).sort_index(), kser.compare(kser + 1).sort_index())
else:
kser = ps.Series([1, 2])
res_kdf = kser.compare(kser)
self.assertTrue(res_kdf.empty)
self.assert_eq(res_kdf.columns, pd.Index(["self", "other"]))
expected = ps.DataFrame([[1, 2], [2, 3]], columns=["self", "other"])
self.assert_eq(expected, kser.compare(kser + 1).sort_index())
kser = ps.Series([1, 2], index=["x", "y"])
expected = ps.DataFrame([[1, 2], [2, 3]], index=["x", "y"], columns=["self", "other"])
self.assert_eq(expected, kser.compare(kser + 1).sort_index())
def test_is_unique(self):
# We can't use pandas' is_unique for comparison. pandas 0.23 ignores None
pser = pd.Series([1, 2, 2, None, None])
kser = ps.from_pandas(pser)
self.assertEqual(False, kser.is_unique)
self.assertEqual(False, (kser + 1).is_unique)
pser = pd.Series([1, None, None])
kser = ps.from_pandas(pser)
self.assertEqual(False, kser.is_unique)
self.assertEqual(False, (kser + 1).is_unique)
pser = pd.Series([1])
kser = ps.from_pandas(pser)
self.assertEqual(pser.is_unique, kser.is_unique)
self.assertEqual((pser + 1).is_unique, (kser + 1).is_unique)
pser = pd.Series([1, 1, 1])
kser = ps.from_pandas(pser)
self.assertEqual(pser.is_unique, kser.is_unique)
self.assertEqual((pser + 1).is_unique, (kser + 1).is_unique)
def test_to_list(self):
self.assert_eq(self.kser.tolist(), self.pser.tolist())
def test_append(self):
pser1 = pd.Series([1, 2, 3], name="0")
pser2 = pd.Series([4, 5, 6], name="0")
pser3 = pd.Series([4, 5, 6], index=[3, 4, 5], name="0")
kser1 = ps.from_pandas(pser1)
kser2 = ps.from_pandas(pser2)
kser3 = ps.from_pandas(pser3)
self.assert_eq(kser1.append(kser2), pser1.append(pser2))
self.assert_eq(kser1.append(kser3), pser1.append(pser3))
self.assert_eq(
kser1.append(kser2, ignore_index=True), pser1.append(pser2, ignore_index=True)
)
kser1.append(kser3, verify_integrity=True)
msg = "Indices have overlapping values"
with self.assertRaises(ValueError, msg=msg):
kser1.append(kser2, verify_integrity=True)
def test_map(self):
pser = pd.Series(["cat", "dog", None, "rabbit"])
kser = ps.from_pandas(pser)
# Currently Koalas doesn't return NaN as pandas does.
self.assert_eq(kser.map({}), pser.map({}).replace({pd.np.nan: None}))
d = defaultdict(lambda: "abc")
self.assertTrue("abc" in repr(kser.map(d)))
self.assert_eq(kser.map(d), pser.map(d))
def tomorrow(date) -> datetime:
return date + timedelta(days=1)
pser = pd.Series([datetime(2019, 10, 24)])
kser = ps.from_pandas(pser)
self.assert_eq(kser.map(tomorrow), pser.map(tomorrow))
def test_add_prefix(self):
pser = pd.Series([1, 2, 3, 4], name="0")
kser = ps.from_pandas(pser)
self.assert_eq(pser.add_prefix("item_"), kser.add_prefix("item_"))
pser = pd.Series(
[1, 2, 3],
name="0",
index=pd.MultiIndex.from_tuples([("A", "X"), ("A", "Y"), ("B", "X")]),
)
kser = ps.from_pandas(pser)
self.assert_eq(pser.add_prefix("item_"), kser.add_prefix("item_"))
def test_add_suffix(self):
pser = pd.Series([1, 2, 3, 4], name="0")
kser = ps.from_pandas(pser)
self.assert_eq(pser.add_suffix("_item"), kser.add_suffix("_item"))
pser = pd.Series(
[1, 2, 3],
name="0",
index=pd.MultiIndex.from_tuples([("A", "X"), ("A", "Y"), ("B", "X")]),
)
kser = ps.from_pandas(pser)
self.assert_eq(pser.add_suffix("_item"), kser.add_suffix("_item"))
def test_cummin(self):
pser = pd.Series([1.0, None, 0.0, 4.0, 9.0])
kser = ps.from_pandas(pser)
self.assert_eq(pser.cummin(), kser.cummin())
self.assert_eq(pser.cummin(skipna=False), kser.cummin(skipna=False))
self.assert_eq(pser.cummin().sum(), kser.cummin().sum())
# with reversed index
pser.index = [4, 3, 2, 1, 0]
kser = ps.from_pandas(pser)
self.assert_eq(pser.cummin(), kser.cummin())
self.assert_eq(pser.cummin(skipna=False), kser.cummin(skipna=False))
def test_cummax(self):
pser = pd.Series([1.0, None, 0.0, 4.0, 9.0])
kser = ps.from_pandas(pser)
self.assert_eq(pser.cummax(), kser.cummax())
self.assert_eq(pser.cummax(skipna=False), kser.cummax(skipna=False))
self.assert_eq(pser.cummax().sum(), kser.cummax().sum())
# with reversed index
pser.index = [4, 3, 2, 1, 0]
kser = ps.from_pandas(pser)
self.assert_eq(pser.cummax(), kser.cummax())
self.assert_eq(pser.cummax(skipna=False), kser.cummax(skipna=False))
def test_cumsum(self):
pser = pd.Series([1.0, None, 0.0, 4.0, 9.0])
kser = ps.from_pandas(pser)
self.assert_eq(pser.cumsum(), kser.cumsum())
self.assert_eq(pser.cumsum(skipna=False), kser.cumsum(skipna=False))
self.assert_eq(pser.cumsum().sum(), kser.cumsum().sum())
# with reversed index
pser.index = [4, 3, 2, 1, 0]
kser = ps.from_pandas(pser)
self.assert_eq(pser.cumsum(), kser.cumsum())
self.assert_eq(pser.cumsum(skipna=False), kser.cumsum(skipna=False))
# bool
pser = pd.Series([True, True, False, True])
kser = ps.from_pandas(pser)
self.assert_eq(pser.cumsum().astype(int), kser.cumsum())
self.assert_eq(pser.cumsum(skipna=False).astype(int), kser.cumsum(skipna=False))
def test_cumprod(self):
pser = pd.Series([1.0, None, 1.0, 4.0, 9.0])
kser = ps.from_pandas(pser)
self.assert_eq(pser.cumprod(), kser.cumprod())
self.assert_eq(pser.cumprod(skipna=False), kser.cumprod(skipna=False))
self.assert_eq(pser.cumprod().sum(), kser.cumprod().sum())
# with integer type
pser = pd.Series([1, 10, 1, 4, 9])
kser = ps.from_pandas(pser)
self.assert_eq(pser.cumprod(), kser.cumprod())
self.assert_eq(pser.cumprod(skipna=False), kser.cumprod(skipna=False))
self.assert_eq(pser.cumprod().sum(), kser.cumprod().sum())
# with reversed index
pser.index = [4, 3, 2, 1, 0]
kser = ps.from_pandas(pser)
self.assert_eq(pser.cumprod(), kser.cumprod())
self.assert_eq(pser.cumprod(skipna=False), kser.cumprod(skipna=False))
# including zero
pser = pd.Series([1, 2, 0, 3])
kser = ps.from_pandas(pser)
self.assert_eq(pser.cumprod(), kser.cumprod())
self.assert_eq(pser.cumprod(skipna=False), kser.cumprod(skipna=False))
# including negative values
pser = pd.Series([1, -1, -2])
kser = ps.from_pandas(pser)
self.assert_eq(pser.cumprod(), kser.cumprod())
self.assert_eq(pser.cumprod(skipna=False), kser.cumprod(skipna=False))
# bool
pser = pd.Series([True, True, False, True])
kser = ps.from_pandas(pser)
self.assert_eq(pser.cumprod(), kser.cumprod())
self.assert_eq(pser.cumprod(skipna=False).astype(int), kser.cumprod(skipna=False))
def test_median(self):
with self.assertRaisesRegex(TypeError, "accuracy must be an integer; however"):
ps.Series([24.0, 21.0, 25.0, 33.0, 26.0]).median(accuracy="a")
def test_rank(self):
pser = pd.Series([1, 2, 3, 1], name="x")
kser = ps.from_pandas(pser)
self.assert_eq(pser.rank(), kser.rank().sort_index())
self.assert_eq(pser.rank().sum(), kser.rank().sum())
self.assert_eq(pser.rank(ascending=False), kser.rank(ascending=False).sort_index())
self.assert_eq(pser.rank(method="min"), kser.rank(method="min").sort_index())
self.assert_eq(pser.rank(method="max"), kser.rank(method="max").sort_index())
self.assert_eq(pser.rank(method="first"), kser.rank(method="first").sort_index())
self.assert_eq(pser.rank(method="dense"), kser.rank(method="dense").sort_index())
msg = "method must be one of 'average', 'min', 'max', 'first', 'dense'"
with self.assertRaisesRegex(ValueError, msg):
kser.rank(method="nothing")
def test_round(self):
pser = pd.Series([0.028208, 0.038683, 0.877076], name="x")
kser = ps.from_pandas(pser)
self.assert_eq(pser.round(2), kser.round(2))
msg = "decimals must be an integer"
with self.assertRaisesRegex(TypeError, msg):
kser.round(1.5)
def test_quantile(self):
pser = pd.Series([])
kser = ps.from_pandas(pser)
self.assert_eq(kser.quantile(0.5), pser.quantile(0.5))
self.assert_eq(kser.quantile([0.25, 0.5, 0.75]), pser.quantile([0.25, 0.5, 0.75]))
with self.assertRaisesRegex(TypeError, "accuracy must be an integer; however"):
ps.Series([24.0, 21.0, 25.0, 33.0, 26.0]).quantile(accuracy="a")
with self.assertRaisesRegex(TypeError, "q must be a float or an array of floats;"):
ps.Series([24.0, 21.0, 25.0, 33.0, 26.0]).quantile(q="a")
with self.assertRaisesRegex(TypeError, "q must be a float or an array of floats;"):
ps.Series([24.0, 21.0, 25.0, 33.0, 26.0]).quantile(q=["a"])
with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
ps.Series(["a", "b", "c"]).quantile()
with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
ps.Series(["a", "b", "c"]).quantile([0.25, 0.5, 0.75])
def test_idxmax(self):
pser = pd.Series(data=[1, 4, 5], index=["A", "B", "C"])
kser = ps.Series(pser)
self.assertEqual(kser.idxmax(), pser.idxmax())
self.assertEqual(kser.idxmax(skipna=False), pser.idxmax(skipna=False))
index = pd.MultiIndex.from_arrays(
[["a", "a", "b", "b"], ["c", "d", "e", "f"]], names=("first", "second")
)
pser = pd.Series(data=[1, 2, 4, 5], index=index)
kser = ps.Series(pser)
self.assertEqual(kser.idxmax(), pser.idxmax())
self.assertEqual(kser.idxmax(skipna=False), pser.idxmax(skipna=False))
kser = ps.Series([])
with self.assertRaisesRegex(ValueError, "an empty sequence"):
kser.idxmax()
pser = pd.Series([1, 100, None, 100, 1, 100], index=[10, 3, 5, 2, 1, 8])
kser = ps.Series(pser)
self.assertEqual(kser.idxmax(), pser.idxmax())
self.assertEqual(repr(kser.idxmax(skipna=False)), repr(pser.idxmax(skipna=False)))
def test_idxmin(self):
pser = pd.Series(data=[1, 4, 5], index=["A", "B", "C"])
kser = ps.Series(pser)
self.assertEqual(kser.idxmin(), pser.idxmin())
self.assertEqual(kser.idxmin(skipna=False), pser.idxmin(skipna=False))
index = pd.MultiIndex.from_arrays(
[["a", "a", "b", "b"], ["c", "d", "e", "f"]], names=("first", "second")
)
pser = pd.Series(data=[1, 2, 4, 5], index=index)
kser = ps.Series(pser)
self.assertEqual(kser.idxmin(), pser.idxmin())
self.assertEqual(kser.idxmin(skipna=False), pser.idxmin(skipna=False))
kser = ps.Series([])
with self.assertRaisesRegex(ValueError, "an empty sequence"):
kser.idxmin()
pser = pd.Series([1, 100, None, 100, 1, 100], index=[10, 3, 5, 2, 1, 8])
kser = ps.Series(pser)
self.assertEqual(kser.idxmin(), pser.idxmin())
self.assertEqual(repr(kser.idxmin(skipna=False)), repr(pser.idxmin(skipna=False)))
def test_shift(self):
pser = pd.Series([10, 20, 15, 30, 45], name="x")
kser = ps.Series(pser)
self.assert_eq(kser.shift(2), pser.shift(2))
self.assert_eq(kser.shift().shift(-1), pser.shift().shift(-1))
self.assert_eq(kser.shift().sum(), pser.shift().sum())
if LooseVersion(pd.__version__) < LooseVersion("0.24.2"):
self.assert_eq(kser.shift(periods=2), pser.shift(periods=2))
else:
self.assert_eq(kser.shift(periods=2, fill_value=0), pser.shift(periods=2, fill_value=0))
with self.assertRaisesRegex(TypeError, "periods should be an int; however"):
kser.shift(periods=1.5)
def test_diff(self):
pser = pd.Series([10, 20, 15, 30, 45], name="x")
kser = ps.Series(pser)
self.assert_eq(kser.diff(2), pser.diff(2))
self.assert_eq(kser.diff().diff(-1), pser.diff().diff(-1))
self.assert_eq(kser.diff().sum(), pser.diff().sum())
def _test_numeric_astype(self, pser):
kser = ps.Series(pser)
self.assert_eq(kser.astype(int), pser.astype(int))
self.assert_eq(kser.astype(np.int), pser.astype(np.int))
self.assert_eq(kser.astype(np.int8), pser.astype(np.int8))
self.assert_eq(kser.astype(np.int16), pser.astype(np.int16))
self.assert_eq(kser.astype(np.int32), pser.astype(np.int32))
self.assert_eq(kser.astype(np.int64), pser.astype(np.int64))
self.assert_eq(kser.astype(np.byte), pser.astype(np.byte))
self.assert_eq(kser.astype("int"), pser.astype("int"))
self.assert_eq(kser.astype("int8"), pser.astype("int8"))
self.assert_eq(kser.astype("int16"), pser.astype("int16"))
self.assert_eq(kser.astype("int32"), pser.astype("int32"))
self.assert_eq(kser.astype("int64"), pser.astype("int64"))
self.assert_eq(kser.astype("b"), pser.astype("b"))
self.assert_eq(kser.astype("byte"), pser.astype("byte"))
self.assert_eq(kser.astype("i"), pser.astype("i"))
self.assert_eq(kser.astype("long"), pser.astype("long"))
self.assert_eq(kser.astype("short"), pser.astype("short"))
self.assert_eq(kser.astype(np.float), pser.astype(np.float))
self.assert_eq(kser.astype(np.float32), pser.astype(np.float32))
self.assert_eq(kser.astype(np.float64), pser.astype(np.float64))
self.assert_eq(kser.astype("float"), pser.astype("float"))
self.assert_eq(kser.astype("float32"), pser.astype("float32"))
self.assert_eq(kser.astype("float64"), pser.astype("float64"))
self.assert_eq(kser.astype("double"), pser.astype("double"))
self.assert_eq(kser.astype("f"), pser.astype("f"))
self.assert_eq(kser.astype(bool), pser.astype(bool))
self.assert_eq(kser.astype("bool"), pser.astype("bool"))
self.assert_eq(kser.astype("?"), pser.astype("?"))
self.assert_eq(kser.astype(np.unicode_), pser.astype(np.unicode_))
self.assert_eq(kser.astype("str"), pser.astype("str"))
self.assert_eq(kser.astype("U"), pser.astype("U"))
if extension_dtypes_available:
from pandas import Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype
self._check_extension(kser.astype("Int8"), pser.astype("Int8"))
self._check_extension(kser.astype("Int16"), pser.astype("Int16"))
self._check_extension(kser.astype("Int32"), pser.astype("Int32"))
self._check_extension(kser.astype("Int64"), pser.astype("Int64"))
self._check_extension(kser.astype(Int8Dtype()), pser.astype(Int8Dtype()))
self._check_extension(kser.astype(Int16Dtype()), pser.astype(Int16Dtype()))
self._check_extension(kser.astype(Int32Dtype()), pser.astype(Int32Dtype()))
self._check_extension(kser.astype(Int64Dtype()), pser.astype(Int64Dtype()))
if extension_object_dtypes_available:
from pandas import StringDtype
if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
self._check_extension(kser.astype("string"), pser.astype("string"))
self._check_extension(kser.astype(StringDtype()), pser.astype(StringDtype()))
else:
self._check_extension(
kser.astype("string"),
pd.Series(["10", "20", "15", "30", "45"], name="x", dtype="string"),
)
self._check_extension(
kser.astype(StringDtype()),
pd.Series(["10", "20", "15", "30", "45"], name="x", dtype=StringDtype()),
)
if extension_float_dtypes_available:
from pandas import Float32Dtype, Float64Dtype
self._check_extension(kser.astype("Float32"), pser.astype("Float32"))
self._check_extension(kser.astype("Float64"), pser.astype("Float64"))
self._check_extension(kser.astype(Float32Dtype()), pser.astype(Float32Dtype()))
self._check_extension(kser.astype(Float64Dtype()), pser.astype(Float64Dtype()))
def test_astype(self):
psers = [pd.Series([10, 20, 15, 30, 45], name="x")]
if extension_dtypes_available:
psers.append(pd.Series([10, 20, 15, 30, 45], name="x", dtype="Int64"))
if extension_float_dtypes_available:
psers.append(pd.Series([10, 20, 15, 30, 45], name="x", dtype="Float64"))
for pser in psers:
self._test_numeric_astype(pser)
pser = pd.Series([10, 20, 15, 30, 45, None, np.nan], name="x")
kser = ps.Series(pser)
self.assert_eq(kser.astype(bool), pser.astype(bool))
self.assert_eq(kser.astype(str), pser.astype(str))
pser = pd.Series(["hi", "hi ", " ", " \t", "", None], name="x")
kser = ps.Series(pser)
self.assert_eq(kser.astype(bool), pser.astype(bool))
if LooseVersion("1.1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.1.4"):
# a pandas bug: https://github.com/databricks/koalas/pull/1818#issuecomment-703961980
self.assert_eq(kser.astype(str).tolist(), ["hi", "hi ", " ", " \t", "", "None"])
else:
self.assert_eq(kser.astype(str), pser.astype(str))
self.assert_eq(kser.str.strip().astype(bool), pser.str.strip().astype(bool))
if extension_object_dtypes_available:
from pandas import StringDtype
self._check_extension(kser.astype("string"), pser.astype("string"))
self._check_extension(kser.astype(StringDtype()), pser.astype(StringDtype()))
pser = pd.Series([True, False, None], name="x")
kser = ps.Series(pser)
self.assert_eq(kser.astype(bool), pser.astype(bool))
# Comment out the below test cause because pandas returns `None` or `nan` randomly
# self.assert_eq(kser.astype(str), pser.astype(str))
if extension_object_dtypes_available:
from pandas import BooleanDtype, StringDtype
self._check_extension(kser.astype("boolean"), pser.astype("boolean"))
self._check_extension(kser.astype(BooleanDtype()), pser.astype(BooleanDtype()))
if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
self._check_extension(kser.astype("string"), pser.astype("string"))
self._check_extension(kser.astype(StringDtype()), pser.astype(StringDtype()))
else:
self._check_extension(
kser.astype("string"),
pd.Series(["True", "False", None], name="x", dtype="string"),
)
self._check_extension(
kser.astype(StringDtype()),
pd.Series(["True", "False", None], name="x", dtype=StringDtype()),
)
pser = pd.Series(["2020-10-27 00:00:01", None], name="x")
kser = ps.Series(pser)
self.assert_eq(kser.astype(np.datetime64), pser.astype(np.datetime64))
self.assert_eq(kser.astype("datetime64[ns]"), pser.astype("datetime64[ns]"))
self.assert_eq(kser.astype("M"), pser.astype("M"))
self.assert_eq(kser.astype("M").astype(str), pser.astype("M").astype(str))
# Comment out the below test cause because pandas returns `NaT` or `nan` randomly
# self.assert_eq(kser.astype("M").dt.date.astype(str), pser.astype("M").dt.date.astype(str))
if extension_object_dtypes_available:
from pandas import StringDtype
self._check_extension(
kser.astype("M").astype("string"), pser.astype("M").astype("string")
)
self._check_extension(
kser.astype("M").astype(StringDtype()), pser.astype("M").astype(StringDtype())
)
with self.assertRaisesRegex(TypeError, "not understood"):
kser.astype("int63")
def test_aggregate(self):
pser = pd.Series([10, 20, 15, 30, 45], name="x")
kser = ps.Series(pser)
msg = "func must be a string or list of strings"
with self.assertRaisesRegex(TypeError, msg):
kser.aggregate({"x": ["min", "max"]})
msg = (
"If the given function is a list, it " "should only contains function names as strings."
)
with self.assertRaisesRegex(ValueError, msg):
kser.aggregate(["min", max])
def test_drop(self):
pser = pd.Series([10, 20, 15, 30, 45], name="x")
kser = ps.Series(pser)
self.assert_eq(kser.drop(1), pser.drop(1))
self.assert_eq(kser.drop([1, 4]), pser.drop([1, 4]))
msg = "Need to specify at least one of 'labels' or 'index'"
with self.assertRaisesRegex(ValueError, msg):
kser.drop()
self.assertRaises(KeyError, lambda: kser.drop((0, 1)))
# For MultiIndex
midx = pd.MultiIndex(
[["lama", "cow", "falcon"], ["speed", "weight", "length"]],
[[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
)
pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
kser = ps.from_pandas(pser)
self.assert_eq(kser.drop("lama"), pser.drop("lama"))
self.assert_eq(kser.drop(labels="weight", level=1), pser.drop(labels="weight", level=1))
self.assert_eq(kser.drop(("lama", "weight")), pser.drop(("lama", "weight")))
self.assert_eq(
kser.drop([("lama", "speed"), ("falcon", "weight")]),
pser.drop([("lama", "speed"), ("falcon", "weight")]),
)
self.assert_eq(kser.drop({"lama": "speed"}), pser.drop({"lama": "speed"}))
msg = "'level' should be less than the number of indexes"
with self.assertRaisesRegex(ValueError, msg):
kser.drop(labels="weight", level=2)
msg = (
"If the given index is a list, it "
"should only contains names as all tuples or all non tuples "
"that contain index names"
)
with self.assertRaisesRegex(ValueError, msg):
kser.drop(["lama", ["cow", "falcon"]])
msg = "Cannot specify both 'labels' and 'index'"
with self.assertRaisesRegex(ValueError, msg):
kser.drop("lama", index="cow")
msg = r"'Key length \(2\) exceeds index depth \(3\)'"
with self.assertRaisesRegex(KeyError, msg):
kser.drop(("lama", "speed", "x"))
def test_pop(self):
midx = pd.MultiIndex(
[["lama", "cow", "falcon"], ["speed", "weight", "length"]],
[[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
)
pdf = pd.DataFrame({"x": [45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3]}, index=midx)
kdf = ps.from_pandas(pdf)
pser = pdf.x
kser = kdf.x
self.assert_eq(kser.pop(("lama", "speed")), pser.pop(("lama", "speed")))
self.assert_eq(kser, pser)
self.assert_eq(kdf, pdf)
msg = r"'Key length \(3\) exceeds index depth \(2\)'"
with self.assertRaisesRegex(KeyError, msg):
kser.pop(("lama", "speed", "x"))
def test_replace(self):
pser = pd.Series([10, 20, 15, 30, np.nan], name="x")
kser = ps.Series(pser)
self.assert_eq(kser.replace(), pser.replace())
self.assert_eq(kser.replace({}), pser.replace({}))
self.assert_eq(kser.replace(np.nan, 45), pser.replace(np.nan, 45))
self.assert_eq(kser.replace([10, 15], 45), pser.replace([10, 15], 45))
self.assert_eq(kser.replace((10, 15), 45), pser.replace((10, 15), 45))
self.assert_eq(kser.replace([10, 15], [45, 50]), pser.replace([10, 15], [45, 50]))
self.assert_eq(kser.replace((10, 15), (45, 50)), pser.replace((10, 15), (45, 50)))
msg = "'to_replace' should be one of str, list, tuple, dict, int, float"
with self.assertRaisesRegex(TypeError, msg):
kser.replace(ps.range(5))
msg = "Replacement lists must match in length. Expecting 3 got 2"
with self.assertRaisesRegex(ValueError, msg):
kser.replace([10, 20, 30], [1, 2])
msg = "replace currently not support for regex"
with self.assertRaisesRegex(NotImplementedError, msg):
kser.replace(r"^1.$", regex=True)
def test_xs(self):
midx = pd.MultiIndex(
[["a", "b", "c"], ["lama", "cow", "falcon"], ["speed", "weight", "length"]],
[[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
)
pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
kser = ps.from_pandas(pser)
self.assert_eq(kser.xs(("a", "lama", "speed")), pser.xs(("a", "lama", "speed")))
def test_duplicates(self):
psers = {
"test on texts": pd.Series(
["lama", "cow", "lama", "beetle", "lama", "hippo"], name="animal"
),
"test on numbers": pd.Series([1, 1, 2, 4, 3]),
}
keeps = ["first", "last", False]
for (msg, pser), keep in product(psers.items(), keeps):
with self.subTest(msg, keep=keep):
kser = ps.Series(pser)
self.assert_eq(
pser.drop_duplicates(keep=keep).sort_values(),
kser.drop_duplicates(keep=keep).sort_values(),
)
def test_update(self):
pser = pd.Series([10, 20, 15, 30, 45], name="x")
kser = ps.Series(pser)
msg = "'other' must be a Series"
with self.assertRaisesRegex(TypeError, msg):
kser.update(10)
def test_where(self):
pser1 = pd.Series([0, 1, 2, 3, 4])
kser1 = ps.from_pandas(pser1)
self.assert_eq(pser1.where(pser1 > 3), kser1.where(kser1 > 3).sort_index())
def test_mask(self):
pser1 = pd.Series([0, 1, 2, 3, 4])
kser1 = ps.from_pandas(pser1)
self.assert_eq(pser1.mask(pser1 > 3), kser1.mask(kser1 > 3).sort_index())
def test_truncate(self):
pser1 = pd.Series([10, 20, 30, 40, 50, 60, 70], index=[1, 2, 3, 4, 5, 6, 7])
kser1 = ps.Series(pser1)
pser2 = pd.Series([10, 20, 30, 40, 50, 60, 70], index=[7, 6, 5, 4, 3, 2, 1])
kser2 = ps.Series(pser2)
self.assert_eq(kser1.truncate(), pser1.truncate())
self.assert_eq(kser1.truncate(before=2), pser1.truncate(before=2))
self.assert_eq(kser1.truncate(after=5), pser1.truncate(after=5))
self.assert_eq(kser1.truncate(copy=False), pser1.truncate(copy=False))
self.assert_eq(kser1.truncate(2, 5, copy=False), pser1.truncate(2, 5, copy=False))
# The bug for these tests has been fixed in pandas 1.1.0.
if LooseVersion(pd.__version__) >= LooseVersion("1.1.0"):
self.assert_eq(kser2.truncate(4, 6), pser2.truncate(4, 6))
self.assert_eq(kser2.truncate(4, 6, copy=False), pser2.truncate(4, 6, copy=False))
else:
expected_kser = ps.Series([20, 30, 40], index=[6, 5, 4])
self.assert_eq(kser2.truncate(4, 6), expected_kser)
self.assert_eq(kser2.truncate(4, 6, copy=False), expected_kser)
kser = ps.Series([10, 20, 30, 40, 50, 60, 70], index=[1, 2, 3, 4, 3, 2, 1])
msg = "truncate requires a sorted index"
with self.assertRaisesRegex(ValueError, msg):
kser.truncate()
kser = ps.Series([10, 20, 30, 40, 50, 60, 70], index=[1, 2, 3, 4, 5, 6, 7])
msg = "Truncate: 2 must be after 5"
with self.assertRaisesRegex(ValueError, msg):
kser.truncate(5, 2)
def test_getitem(self):
pser = pd.Series([10, 20, 15, 30, 45], ["A", "A", "B", "C", "D"])
kser = ps.Series(pser)
self.assert_eq(kser["A"], pser["A"])
self.assert_eq(kser["B"], pser["B"])
self.assert_eq(kser[kser > 15], pser[pser > 15])
# for MultiIndex
midx = pd.MultiIndex(
[["a", "b", "c"], ["lama", "cow", "falcon"], ["speed", "weight", "length"]],
[[0, 0, 0, 0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 0, 0, 0, 1, 2, 0, 1, 2]],
)
pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], name="0", index=midx)
kser = ps.Series(pser)
self.assert_eq(kser["a"], pser["a"])
self.assert_eq(kser["a", "lama"], pser["a", "lama"])
self.assert_eq(kser[kser > 1.5], pser[pser > 1.5])
msg = r"'Key length \(4\) exceeds index depth \(3\)'"
with self.assertRaisesRegex(KeyError, msg):
kser[("a", "lama", "speed", "x")]
def test_keys(self):
midx = pd.MultiIndex(
[["lama", "cow", "falcon"], ["speed", "weight", "length"]],
[[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
)
pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
kser = ps.from_pandas(pser)
self.assert_eq(kser.keys(), pser.keys())
def test_index(self):
# to check setting name of Index properly.
idx = pd.Index([1, 2, 3, 4, 5, 6, 7, 8, 9])
pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=idx)
kser = ps.from_pandas(pser)
kser.name = "koalas"
pser.name = "koalas"
self.assert_eq(kser.index.name, pser.index.name)
# for check setting names of MultiIndex properly.
kser.names = ["hello", "koalas"]
pser.names = ["hello", "koalas"]
self.assert_eq(kser.index.names, pser.index.names)
def test_pct_change(self):
pser = pd.Series([90, 91, 85], index=[2, 4, 1])
kser = ps.from_pandas(pser)
self.assert_eq(kser.pct_change(), pser.pct_change(), check_exact=False)
self.assert_eq(kser.pct_change().sum(), pser.pct_change().sum(), almost=True)
self.assert_eq(kser.pct_change(periods=2), pser.pct_change(periods=2), check_exact=False)
self.assert_eq(kser.pct_change(periods=-1), pser.pct_change(periods=-1), check_exact=False)
self.assert_eq(kser.pct_change(periods=-100000000), pser.pct_change(periods=-100000000))
self.assert_eq(kser.pct_change(periods=100000000), pser.pct_change(periods=100000000))
# for MultiIndex
midx = pd.MultiIndex(
[["lama", "cow", "falcon"], ["speed", "weight", "length"]],
[[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
)
pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
kser = ps.from_pandas(pser)
self.assert_eq(kser.pct_change(), pser.pct_change(), check_exact=False)
self.assert_eq(kser.pct_change().sum(), pser.pct_change().sum(), almost=True)
self.assert_eq(kser.pct_change(periods=2), pser.pct_change(periods=2), check_exact=False)
self.assert_eq(kser.pct_change(periods=-1), pser.pct_change(periods=-1), check_exact=False)
self.assert_eq(kser.pct_change(periods=-100000000), pser.pct_change(periods=-100000000))
self.assert_eq(kser.pct_change(periods=100000000), pser.pct_change(periods=100000000))
def test_axes(self):
pser = pd.Series([90, 91, 85], index=[2, 4, 1])
kser = ps.from_pandas(pser)
self.assert_eq(kser.axes, pser.axes)
# for MultiIndex
midx = pd.MultiIndex(
[["lama", "cow", "falcon"], ["speed", "weight", "length"]],
[[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
)
pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
kser = ps.from_pandas(pser)
self.assert_eq(kser.axes, pser.axes)
def test_udt(self):
sparse_values = {0: 0.1, 1: 1.1}
sparse_vector = SparseVector(len(sparse_values), sparse_values)
pser = pd.Series([sparse_vector])
kser = ps.from_pandas(pser)
self.assert_eq(kser, pser)
def test_repeat(self):
pser = pd.Series(["a", "b", "c"], name="0", index=np.random.rand(3))
kser = ps.from_pandas(pser)
self.assert_eq(kser.repeat(3).sort_index(), pser.repeat(3).sort_index())
self.assert_eq(kser.repeat(0).sort_index(), pser.repeat(0).sort_index())
self.assertRaises(ValueError, lambda: kser.repeat(-1))
self.assertRaises(TypeError, lambda: kser.repeat("abc"))
pdf = pd.DataFrame({"a": ["a", "b", "c"], "rep": [10, 20, 30]}, index=np.random.rand(3))
kdf = ps.from_pandas(pdf)
self.assert_eq(kdf.a.repeat(kdf.rep).sort_index(), pdf.a.repeat(pdf.rep).sort_index())
def test_take(self):
pser = pd.Series([100, 200, 300, 400, 500], name="Koalas")
kser = ps.from_pandas(pser)
self.assert_eq(kser.take([0, 2, 4]).sort_values(), pser.take([0, 2, 4]).sort_values())
self.assert_eq(
kser.take(range(0, 5, 2)).sort_values(), pser.take(range(0, 5, 2)).sort_values()
)
self.assert_eq(kser.take([-4, -2, 0]).sort_values(), pser.take([-4, -2, 0]).sort_values())
self.assert_eq(
kser.take(range(-2, 1, 2)).sort_values(), pser.take(range(-2, 1, 2)).sort_values()
)
# Checking the type of indices.
self.assertRaises(TypeError, lambda: kser.take(1))
self.assertRaises(TypeError, lambda: kser.take("1"))
self.assertRaises(TypeError, lambda: kser.take({1, 2}))
self.assertRaises(TypeError, lambda: kser.take({1: None, 2: None}))
def test_divmod(self):
pser = pd.Series([100, None, 300, None, 500], name="Koalas")
kser = ps.from_pandas(pser)
if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
kdiv, kmod = kser.divmod(-100)
pdiv, pmod = pser.divmod(-100)
self.assert_eq(kdiv, pdiv)
self.assert_eq(kmod, pmod)
kdiv, kmod = kser.divmod(100)
pdiv, pmod = pser.divmod(100)
self.assert_eq(kdiv, pdiv)
self.assert_eq(kmod, pmod)
elif LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
kdiv, kmod = kser.divmod(-100)
pdiv, pmod = pser.floordiv(-100), pser.mod(-100)
self.assert_eq(kdiv, pdiv)
self.assert_eq(kmod, pmod)
kdiv, kmod = kser.divmod(100)
pdiv, pmod = pser.floordiv(100), pser.mod(100)
self.assert_eq(kdiv, pdiv)
self.assert_eq(kmod, pmod)
def test_rdivmod(self):
pser = pd.Series([100, None, 300, None, 500])
kser = ps.from_pandas(pser)
if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
krdiv, krmod = kser.rdivmod(-100)
prdiv, prmod = pser.rdivmod(-100)
self.assert_eq(krdiv, prdiv)
self.assert_eq(krmod, prmod)
krdiv, krmod = kser.rdivmod(100)
prdiv, prmod = pser.rdivmod(100)
self.assert_eq(krdiv, prdiv)
self.assert_eq(krmod, prmod)
elif LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
krdiv, krmod = kser.rdivmod(-100)
prdiv, prmod = pser.rfloordiv(-100), pser.rmod(-100)
self.assert_eq(krdiv, prdiv)
self.assert_eq(krmod, prmod)
krdiv, krmod = kser.rdivmod(100)
prdiv, prmod = pser.rfloordiv(100), pser.rmod(100)
self.assert_eq(krdiv, prdiv)
self.assert_eq(krmod, prmod)
def test_mod(self):
pser = pd.Series([100, None, -300, None, 500, -700], name="Koalas")
kser = ps.from_pandas(pser)
self.assert_eq(kser.mod(-150), pser.mod(-150))
self.assert_eq(kser.mod(0), pser.mod(0))
self.assert_eq(kser.mod(150), pser.mod(150))
pdf = pd.DataFrame({"a": [100, None, -300, None, 500, -700], "b": [150] * 6})
kdf = ps.from_pandas(pdf)
self.assert_eq(kdf.a.mod(kdf.b), pdf.a.mod(pdf.b))
def test_mode(self):
pser = pd.Series([0, 0, 1, 1, 1, np.nan, np.nan, np.nan])
kser = ps.from_pandas(pser)
self.assert_eq(kser.mode(), pser.mode())
if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
# The `dropna` argument is added in pandas 0.24.
self.assert_eq(
kser.mode(dropna=False).sort_values().reset_index(drop=True),
pser.mode(dropna=False).sort_values().reset_index(drop=True),
)
pser.name = "x"
kser = ps.from_pandas(pser)
self.assert_eq(kser.mode(), pser.mode())
if LooseVersion(pd.__version__) >= LooseVersion("0.24"):
# The `dropna` argument is added in pandas 0.24.
self.assert_eq(
kser.mode(dropna=False).sort_values().reset_index(drop=True),
pser.mode(dropna=False).sort_values().reset_index(drop=True),
)
def test_rmod(self):
pser = pd.Series([100, None, -300, None, 500, -700], name="Koalas")
kser = ps.from_pandas(pser)
self.assert_eq(kser.rmod(-150), pser.rmod(-150))
self.assert_eq(kser.rmod(0), pser.rmod(0))
self.assert_eq(kser.rmod(150), pser.rmod(150))
pdf = pd.DataFrame({"a": [100, None, -300, None, 500, -700], "b": [150] * 6})
kdf = ps.from_pandas(pdf)
self.assert_eq(kdf.a.rmod(kdf.b), pdf.a.rmod(pdf.b))
def test_asof(self):
pser = pd.Series([1, 2, np.nan, 4], index=[10, 20, 30, 40], name="Koalas")
kser = ps.from_pandas(pser)
self.assert_eq(kser.asof(20), pser.asof(20))
self.assert_eq(kser.asof([5, 20]).sort_index(), pser.asof([5, 20]).sort_index())
self.assert_eq(kser.asof(100), pser.asof(100))
self.assert_eq(repr(kser.asof(-100)), repr(pser.asof(-100)))
self.assert_eq(kser.asof([-100, 100]).sort_index(), pser.asof([-100, 100]).sort_index())
# where cannot be an Index, Series or a DataFrame
self.assertRaises(ValueError, lambda: kser.asof(ps.Index([-100, 100])))
self.assertRaises(ValueError, lambda: kser.asof(ps.Series([-100, 100])))
self.assertRaises(ValueError, lambda: kser.asof(ps.DataFrame({"A": [1, 2, 3]})))
# asof is not supported for a MultiIndex
pser.index = pd.MultiIndex.from_tuples([("x", "a"), ("x", "b"), ("y", "c"), ("y", "d")])
kser = ps.from_pandas(pser)
self.assertRaises(ValueError, lambda: kser.asof(20))
# asof requires a sorted index (More precisely, should be a monotonic increasing)
kser = ps.Series([1, 2, np.nan, 4], index=[10, 30, 20, 40], name="Koalas")
self.assertRaises(ValueError, lambda: kser.asof(20))
kser = ps.Series([1, 2, np.nan, 4], index=[40, 30, 20, 10], name="Koalas")
self.assertRaises(ValueError, lambda: kser.asof(20))
pidx = pd.DatetimeIndex(["2013-12-31", "2014-01-02", "2014-01-03"])
pser = pd.Series([1, 2, np.nan], index=pidx)
kser = ps.from_pandas(pser)
self.assert_eq(kser.asof("2014-01-01"), pser.asof("2014-01-01"))
self.assert_eq(kser.asof("2014-01-02"), pser.asof("2014-01-02"))
self.assert_eq(repr(kser.asof("1999-01-02")), repr(pser.asof("1999-01-02")))
def test_squeeze(self):
# Single value
pser = pd.Series([90])
kser = ps.from_pandas(pser)
self.assert_eq(kser.squeeze(), pser.squeeze())
# Single value with MultiIndex
midx = pd.MultiIndex.from_tuples([("a", "b", "c")])
pser = pd.Series([90], index=midx)
kser = ps.from_pandas(pser)
self.assert_eq(kser.squeeze(), pser.squeeze())
# Multiple values
pser = pd.Series([90, 91, 85])
kser = ps.from_pandas(pser)
self.assert_eq(kser.squeeze(), pser.squeeze())
# Multiple values with MultiIndex
midx = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
pser = pd.Series([90, 91, 85], index=midx)
kser = ps.from_pandas(pser)
self.assert_eq(kser.squeeze(), pser.squeeze())
def test_swaplevel(self):
# MultiIndex with two levels
arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
pidx = pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
pser = pd.Series(["a", "b", "c", "d"], index=pidx)
kser = ps.from_pandas(pser)
self.assert_eq(pser.swaplevel(), kser.swaplevel())
self.assert_eq(pser.swaplevel(0, 1), kser.swaplevel(0, 1))
self.assert_eq(pser.swaplevel(1, 1), kser.swaplevel(1, 1))
self.assert_eq(pser.swaplevel("number", "color"), kser.swaplevel("number", "color"))
# MultiIndex with more than two levels
arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"], ["l", "m", "s", "xs"]]
pidx = pd.MultiIndex.from_arrays(arrays, names=("number", "color", "size"))
pser = pd.Series(["a", "b", "c", "d"], index=pidx)
kser = ps.from_pandas(pser)
self.assert_eq(pser.swaplevel(), kser.swaplevel())
self.assert_eq(pser.swaplevel(0, 1), kser.swaplevel(0, 1))
self.assert_eq(pser.swaplevel(0, 2), kser.swaplevel(0, 2))
self.assert_eq(pser.swaplevel(1, 2), kser.swaplevel(1, 2))
self.assert_eq(pser.swaplevel(1, 1), kser.swaplevel(1, 1))
self.assert_eq(pser.swaplevel(-1, -2), kser.swaplevel(-1, -2))
self.assert_eq(pser.swaplevel("number", "color"), kser.swaplevel("number", "color"))
self.assert_eq(pser.swaplevel("number", "size"), kser.swaplevel("number", "size"))
self.assert_eq(pser.swaplevel("color", "size"), kser.swaplevel("color", "size"))
# Error conditions
self.assertRaises(AssertionError, lambda: ps.Series([1, 2]).swaplevel())
self.assertRaises(IndexError, lambda: kser.swaplevel(0, 9))
self.assertRaises(KeyError, lambda: kser.swaplevel("not_number", "color"))
self.assertRaises(AssertionError, lambda: kser.swaplevel(copy=False))
def test_swapaxes(self):
pser = pd.Series([1, 2, 3], index=["x", "y", "z"], name="ser")
kser = ps.from_pandas(pser)
self.assert_eq(kser.swapaxes(0, 0), pser.swapaxes(0, 0))
self.assert_eq(kser.swapaxes("index", "index"), pser.swapaxes("index", "index"))
self.assert_eq((kser + 1).swapaxes(0, 0), (pser + 1).swapaxes(0, 0))
self.assertRaises(AssertionError, lambda: kser.swapaxes(0, 1, copy=False))
self.assertRaises(ValueError, lambda: kser.swapaxes(0, 1))
self.assertRaises(ValueError, lambda: kser.swapaxes("index", "columns"))
def test_div_zero_and_nan(self):
pser = pd.Series([100, None, -300, None, 500, -700, np.inf, -np.inf], name="Koalas")
kser = ps.from_pandas(pser)
self.assert_eq(pser.div(0), kser.div(0))
self.assert_eq(pser.truediv(0), kser.truediv(0))
self.assert_eq(pser / 0, kser / 0)
self.assert_eq(pser.div(np.nan), kser.div(np.nan))
self.assert_eq(pser.truediv(np.nan), kser.truediv(np.nan))
self.assert_eq(pser / np.nan, kser / np.nan)
# floordiv has different behavior in pandas > 1.0.0 when divide by 0
if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
self.assert_eq(pser.floordiv(0), kser.floordiv(0))
self.assert_eq(pser // 0, kser // 0)
else:
result = pd.Series(
[np.inf, np.nan, -np.inf, np.nan, np.inf, -np.inf, np.inf, -np.inf], name="Koalas"
)
self.assert_eq(kser.floordiv(0), result)
self.assert_eq(kser // 0, result)
self.assert_eq(pser.floordiv(np.nan), kser.floordiv(np.nan))
def test_mad(self):
pser = pd.Series([1, 2, 3, 4], name="Koalas")
kser = ps.from_pandas(pser)
self.assert_eq(pser.mad(), kser.mad())
pser = pd.Series([None, -2, 5, 10, 50, np.nan, -20], name="Koalas")
kser = ps.from_pandas(pser)
self.assert_eq(pser.mad(), kser.mad())
pmidx = pd.MultiIndex.from_tuples(
[("a", "1"), ("a", "2"), ("b", "1"), ("b", "2"), ("c", "1")]
)
pser = pd.Series([1, 2, 3, 4, 5], name="Koalas")
pser.index = pmidx
kser = ps.from_pandas(pser)
self.assert_eq(pser.mad(), kser.mad())
pmidx = pd.MultiIndex.from_tuples(
[("a", "1"), ("a", "2"), ("b", "1"), ("b", "2"), ("c", "1")]
)
pser = pd.Series([None, -2, 5, 50, np.nan], name="Koalas")
pser.index = pmidx
kser = ps.from_pandas(pser)
self.assert_eq(pser.mad(), kser.mad())
def test_to_frame(self):
pser = pd.Series(["a", "b", "c"])
kser = ps.from_pandas(pser)
self.assert_eq(pser.to_frame(name="a"), kser.to_frame(name="a"))
# for MultiIndex
midx = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
pser = pd.Series(["a", "b", "c"], index=midx)
kser = ps.from_pandas(pser)
self.assert_eq(pser.to_frame(name="a"), kser.to_frame(name="a"))
def test_shape(self):
pser = pd.Series(["a", "b", "c"])
kser = ps.from_pandas(pser)
self.assert_eq(pser.shape, kser.shape)
# for MultiIndex
midx = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
pser = pd.Series(["a", "b", "c"], index=midx)
kser = ps.from_pandas(pser)
self.assert_eq(pser.shape, kser.shape)
@unittest.skipIf(not have_tabulate, tabulate_requirement_message)
def test_to_markdown(self):
pser = pd.Series(["elk", "pig", "dog", "quetzal"], name="animal")
kser = ps.from_pandas(pser)
# `to_markdown()` is supported in pandas >= 1.0.0 since it's newly added in pandas 1.0.0.
if LooseVersion(pd.__version__) < LooseVersion("1.0.0"):
self.assertRaises(NotImplementedError, lambda: kser.to_markdown())
else:
self.assert_eq(pser.to_markdown(), kser.to_markdown())
def test_unstack(self):
pser = pd.Series(
[10, -2, 4, 7],
index=pd.MultiIndex.from_tuples(
[("one", "a", "z"), ("one", "b", "x"), ("two", "a", "c"), ("two", "b", "v")],
names=["A", "B", "C"],
),
)
kser = ps.from_pandas(pser)
levels = [-3, -2, -1, 0, 1, 2]
for level in levels:
pandas_result = pser.unstack(level=level)
koalas_result = kser.unstack(level=level).sort_index()
self.assert_eq(pandas_result, koalas_result)
self.assert_eq(pandas_result.index.names, koalas_result.index.names)
self.assert_eq(pandas_result.columns.names, koalas_result.columns.names)
# non-numeric datatypes
pser = pd.Series(
list("abcd"), index=pd.MultiIndex.from_product([["one", "two"], ["a", "b"]])
)
kser = ps.from_pandas(pser)
levels = [-2, -1, 0, 1]
for level in levels:
pandas_result = pser.unstack(level=level)
koalas_result = kser.unstack(level=level).sort_index()
self.assert_eq(pandas_result, koalas_result)
self.assert_eq(pandas_result.index.names, koalas_result.index.names)
self.assert_eq(pandas_result.columns.names, koalas_result.columns.names)
# Exceeding the range of level
self.assertRaises(IndexError, lambda: kser.unstack(level=3))
self.assertRaises(IndexError, lambda: kser.unstack(level=-4))
# Only support for MultiIndex
kser = ps.Series([10, -2, 4, 7])
self.assertRaises(ValueError, lambda: kser.unstack())
def test_item(self):
kser = ps.Series([10, 20])
self.assertRaises(ValueError, lambda: kser.item())
def test_filter(self):
pser = pd.Series([0, 1, 2], index=["one", "two", "three"])
kser = ps.from_pandas(pser)
self.assert_eq(pser.filter(items=["one", "three"]), kser.filter(items=["one", "three"]))
self.assert_eq(pser.filter(regex="e$"), kser.filter(regex="e$"))
self.assert_eq(pser.filter(like="hre"), kser.filter(like="hre"))
with self.assertRaisesRegex(ValueError, "Series does not support columns axis."):
kser.filter(like="hre", axis=1)
# for MultiIndex
midx = pd.MultiIndex.from_tuples([("one", "x"), ("two", "y"), ("three", "z")])
pser = pd.Series([0, 1, 2], index=midx)
kser = ps.from_pandas(pser)
self.assert_eq(
pser.filter(items=[("one", "x"), ("three", "z")]),
kser.filter(items=[("one", "x"), ("three", "z")]),
)
with self.assertRaisesRegex(TypeError, "Unsupported type list"):
kser.filter(items=[["one", "x"], ("three", "z")])
with self.assertRaisesRegex(ValueError, "The item should not be empty."):
kser.filter(items=[(), ("three", "z")])
def test_abs(self):
pser = pd.Series([-2, -1, 0, 1])
kser = ps.from_pandas(pser)
self.assert_eq(abs(kser), abs(pser))
self.assert_eq(np.abs(kser), np.abs(pser))
def test_bfill(self):
pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6], "y": [np.nan, 2, 3, 4, np.nan, 6]})
kdf = ps.from_pandas(pdf)
pser = pdf.x
kser = kdf.x
self.assert_eq(kser.bfill(), pser.bfill())
self.assert_eq(kser.bfill()[0], pser.bfill()[0])
kser.bfill(inplace=True)
pser.bfill(inplace=True)
self.assert_eq(kser, pser)
self.assert_eq(kser[0], pser[0])
self.assert_eq(kdf, pdf)
def test_ffill(self):
pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6], "y": [np.nan, 2, 3, 4, np.nan, 6]})
kdf = ps.from_pandas(pdf)
pser = pdf.x
kser = kdf.x
self.assert_eq(kser.ffill(), pser.ffill())
self.assert_eq(kser.ffill()[4], pser.ffill()[4])
kser.ffill(inplace=True)
pser.ffill(inplace=True)
self.assert_eq(kser, pser)
self.assert_eq(kser[4], pser[4])
self.assert_eq(kdf, pdf)
def test_iteritems(self):
pser = pd.Series(["A", "B", "C"])
kser = ps.from_pandas(pser)
for (p_name, p_items), (k_name, k_items) in zip(pser.iteritems(), kser.iteritems()):
self.assert_eq(p_name, k_name)
self.assert_eq(p_items, k_items)
def test_droplevel(self):
# droplevel is new in pandas 0.24.0
if LooseVersion(pd.__version__) >= LooseVersion("0.24.0"):
pser = pd.Series(
[1, 2, 3],
index=pd.MultiIndex.from_tuples(
[("x", "a", "q"), ("x", "b", "w"), ("y", "c", "e")],
names=["level_1", "level_2", "level_3"],
),
)
kser = ps.from_pandas(pser)
self.assert_eq(pser.droplevel(0), kser.droplevel(0))
self.assert_eq(pser.droplevel("level_1"), kser.droplevel("level_1"))
self.assert_eq(pser.droplevel(-1), kser.droplevel(-1))
self.assert_eq(pser.droplevel([0]), kser.droplevel([0]))
self.assert_eq(pser.droplevel(["level_1"]), kser.droplevel(["level_1"]))
self.assert_eq(pser.droplevel((0,)), kser.droplevel((0,)))
self.assert_eq(pser.droplevel(("level_1",)), kser.droplevel(("level_1",)))
self.assert_eq(pser.droplevel([0, 2]), kser.droplevel([0, 2]))
self.assert_eq(
pser.droplevel(["level_1", "level_3"]), kser.droplevel(["level_1", "level_3"])
)
self.assert_eq(pser.droplevel((1, 2)), kser.droplevel((1, 2)))
self.assert_eq(
pser.droplevel(("level_2", "level_3")), kser.droplevel(("level_2", "level_3"))
)
with self.assertRaisesRegex(KeyError, "Level {0, 1, 2} not found"):
kser.droplevel({0, 1, 2})
with self.assertRaisesRegex(KeyError, "Level level_100 not found"):
kser.droplevel(["level_1", "level_100"])
with self.assertRaisesRegex(
IndexError, "Too many levels: Index has only 3 levels, not 11"
):
kser.droplevel(10)
with self.assertRaisesRegex(
IndexError,
"Too many levels: Index has only 3 levels, -10 is not a valid level number",
):
kser.droplevel(-10)
with self.assertRaisesRegex(
ValueError,
"Cannot remove 3 levels from an index with 3 levels: "
"at least one level must be left.",
):
kser.droplevel([0, 1, 2])
with self.assertRaisesRegex(
ValueError,
"Cannot remove 5 levels from an index with 3 levels: "
"at least one level must be left.",
):
kser.droplevel([1, 1, 1, 1, 1])
# Tupled names
pser.index.names = [("a", "1"), ("b", "2"), ("c", "3")]
kser = ps.from_pandas(pser)
self.assert_eq(
pser.droplevel([("a", "1"), ("c", "3")]), kser.droplevel([("a", "1"), ("c", "3")])
)
def test_dot(self):
pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
kdf = ps.from_pandas(pdf)
self.assert_eq((kdf["b"] * 10).dot(kdf["a"]), (pdf["b"] * 10).dot(pdf["a"]))
self.assert_eq((kdf["b"] * 10).dot(kdf), (pdf["b"] * 10).dot(pdf))
self.assert_eq((kdf["b"] * 10).dot(kdf + 1), (pdf["b"] * 10).dot(pdf + 1))
def test_tail(self):
pser = pd.Series(range(1000), name="Koalas")
kser = ps.from_pandas(pser)
self.assert_eq(pser.tail(), kser.tail())
self.assert_eq(pser.tail(10), kser.tail(10))
self.assert_eq(pser.tail(-990), kser.tail(-990))
self.assert_eq(pser.tail(0), kser.tail(0))
self.assert_eq(pser.tail(1001), kser.tail(1001))
self.assert_eq(pser.tail(-1001), kser.tail(-1001))
self.assert_eq((pser + 1).tail(), (kser + 1).tail())
self.assert_eq((pser + 1).tail(10), (kser + 1).tail(10))
self.assert_eq((pser + 1).tail(-990), (kser + 1).tail(-990))
self.assert_eq((pser + 1).tail(0), (kser + 1).tail(0))
self.assert_eq((pser + 1).tail(1001), (kser + 1).tail(1001))
self.assert_eq((pser + 1).tail(-1001), (kser + 1).tail(-1001))
with self.assertRaisesRegex(TypeError, "bad operand type for unary -: 'str'"):
kser.tail("10")
def test_product(self):
pser = pd.Series([10, 20, 30, 40, 50])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(), kser.prod())
# Containing NA values
pser = pd.Series([10, np.nan, 30, np.nan, 50])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(), kser.prod(), almost=True)
# All-NA values
pser = pd.Series([np.nan, np.nan, np.nan])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(), kser.prod())
# Empty Series
pser = pd.Series([])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(), kser.prod())
# Boolean Series
pser = pd.Series([True, True, True])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(), kser.prod())
pser = pd.Series([False, False, False])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(), kser.prod())
pser = pd.Series([True, False, True])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(), kser.prod())
# With `min_count` parameter
pser = pd.Series([10, 20, 30, 40, 50])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(min_count=5), kser.prod(min_count=5))
self.assert_eq(pser.prod(min_count=6), kser.prod(min_count=6))
pser = pd.Series([10, np.nan, 30, np.nan, 50])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(min_count=3), kser.prod(min_count=3), almost=True)
self.assert_eq(pser.prod(min_count=4), kser.prod(min_count=4))
pser = pd.Series([np.nan, np.nan, np.nan])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(min_count=1), kser.prod(min_count=1))
pser = pd.Series([])
kser = ps.from_pandas(pser)
self.assert_eq(pser.prod(min_count=1), kser.prod(min_count=1))
with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
ps.Series(["a", "b", "c"]).prod()
with self.assertRaisesRegex(
TypeError, "Could not convert datetime64\\[ns\\] \\(timestamp\\) to numeric"
):
ps.Series([pd.Timestamp("2016-01-01") for _ in range(3)]).prod()
def test_hasnans(self):
# BooleanType
pser = pd.Series([True, False, True, True])
kser = ps.from_pandas(pser)
self.assert_eq(pser.hasnans, kser.hasnans)
pser = pd.Series([True, False, np.nan, True])
kser = ps.from_pandas(pser)
self.assert_eq(pser.hasnans, kser.hasnans)
# TimestampType
pser = pd.Series([pd.Timestamp("2020-07-30") for _ in range(3)])
kser = ps.from_pandas(pser)
self.assert_eq(pser.hasnans, kser.hasnans)
pser = pd.Series([pd.Timestamp("2020-07-30"), np.nan, pd.Timestamp("2020-07-30")])
kser = ps.from_pandas(pser)
self.assert_eq(pser.hasnans, kser.hasnans)
def test_last_valid_index(self):
pser = pd.Series([250, 1.5, 320, 1, 0.3, None, None, None, None])
kser = ps.from_pandas(pser)
self.assert_eq(pser.last_valid_index(), kser.last_valid_index())
# MultiIndex columns
midx = pd.MultiIndex(
[["lama", "cow", "falcon"], ["speed", "weight", "length"]],
[[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
)
pser.index = midx
kser = ps.from_pandas(pser)
self.assert_eq(pser.last_valid_index(), kser.last_valid_index())
# Empty Series
pser = pd.Series([])
kser = ps.from_pandas(pser)
self.assert_eq(pser.last_valid_index(), kser.last_valid_index())
def test_first_valid_index(self):
# Empty Series
pser = pd.Series([])
kser = ps.from_pandas(pser)
self.assert_eq(pser.first_valid_index(), kser.first_valid_index())
def test_factorize(self):
pser = pd.Series(["a", "b", "a", "b"])
kser = ps.from_pandas(pser)
pcodes, puniques = pser.factorize(sort=True)
kcodes, kuniques = kser.factorize()
self.assert_eq(pcodes.tolist(), kcodes.to_list())
self.assert_eq(puniques, kuniques)
pser = pd.Series([5, 1, 5, 1])
kser = ps.from_pandas(pser)
pcodes, puniques = (pser + 1).factorize(sort=True)
kcodes, kuniques = (kser + 1).factorize()
self.assert_eq(pcodes.tolist(), kcodes.to_list())
self.assert_eq(puniques, kuniques)
pser = pd.Series(["a", "b", "a", "b"], name="ser", index=["w", "x", "y", "z"])
kser = ps.from_pandas(pser)
pcodes, puniques = pser.factorize(sort=True)
kcodes, kuniques = kser.factorize()
self.assert_eq(pcodes.tolist(), kcodes.to_list())
self.assert_eq(puniques, kuniques)
pser = pd.Series(
["a", "b", "a", "b"], index=pd.MultiIndex.from_arrays([[4, 3, 2, 1], [1, 2, 3, 4]])
)
kser = ps.from_pandas(pser)
pcodes, puniques = pser.factorize(sort=True)
kcodes, kuniques = kser.factorize()
self.assert_eq(pcodes.tolist(), kcodes.to_list())
self.assert_eq(puniques, kuniques)
#
# Deals with None and np.nan
#
pser = pd.Series(["a", "b", "a", np.nan])
kser = ps.from_pandas(pser)
pcodes, puniques = pser.factorize(sort=True)
kcodes, kuniques = kser.factorize()
self.assert_eq(pcodes.tolist(), kcodes.to_list())
self.assert_eq(puniques, kuniques)
pser = pd.Series([1, None, 3, 2, 1])
kser = ps.from_pandas(pser)
pcodes, puniques = pser.factorize(sort=True)
kcodes, kuniques = kser.factorize()
self.assert_eq(pcodes.tolist(), kcodes.to_list())
self.assert_eq(puniques, kuniques)
pser = pd.Series(["a", None, "a"])
kser = ps.from_pandas(pser)
pcodes, puniques = pser.factorize(sort=True)
kcodes, kuniques = kser.factorize()
self.assert_eq(pcodes.tolist(), kcodes.to_list())
self.assert_eq(puniques, kuniques)
pser = pd.Series([None, np.nan])
kser = ps.from_pandas(pser)
pcodes, puniques = pser.factorize()
kcodes, kuniques = kser.factorize()
self.assert_eq(pcodes, kcodes.to_list())
# pandas: Float64Index([], dtype='float64')
self.assert_eq(pd.Index([]), kuniques)
pser = pd.Series([np.nan, np.nan])
kser = ps.from_pandas(pser)
pcodes, puniques = pser.factorize()
kcodes, kuniques = kser.factorize()
self.assert_eq(pcodes, kcodes.to_list())
# pandas: Float64Index([], dtype='float64')
self.assert_eq(pd.Index([]), kuniques)
#
# Deals with na_sentinel
#
# pandas >= 1.1.2 support na_sentinel=None
# pandas >= 0.24 support na_sentinel not to be -1
#
pd_below_1_1_2 = LooseVersion(pd.__version__) < LooseVersion("1.1.2")
pd_below_0_24 = LooseVersion(pd.__version__) < LooseVersion("0.24")
pser = pd.Series(["a", "b", "a", np.nan, None])
kser = ps.from_pandas(pser)
pcodes, puniques = pser.factorize(sort=True, na_sentinel=-2)
kcodes, kuniques = kser.factorize(na_sentinel=-2)
self.assert_eq([0, 1, 0, -2, -2] if pd_below_0_24 else pcodes.tolist(), kcodes.to_list())
self.assert_eq(puniques, kuniques)
pcodes, puniques = pser.factorize(sort=True, na_sentinel=2)
kcodes, kuniques = kser.factorize(na_sentinel=2)
self.assert_eq([0, 1, 0, 2, 2] if pd_below_0_24 else pcodes.tolist(), kcodes.to_list())
self.assert_eq(puniques, kuniques)
if not pd_below_1_1_2:
pcodes, puniques = pser.factorize(sort=True, na_sentinel=None)
kcodes, kuniques = kser.factorize(na_sentinel=None)
self.assert_eq(pcodes.tolist(), kcodes.to_list())
# puniques is Index(['a', 'b', nan], dtype='object')
self.assert_eq(ps.Index(["a", "b", None]), kuniques)
kser = ps.Series([1, 2, np.nan, 4, 5]) # Arrow takes np.nan as null
kser.loc[3] = np.nan # Spark takes np.nan as NaN
kcodes, kuniques = kser.factorize(na_sentinel=None)
pcodes, puniques = kser.to_pandas().factorize(sort=True, na_sentinel=None)
self.assert_eq(pcodes.tolist(), kcodes.to_list())
self.assert_eq(puniques, kuniques)
def test_pad(self):
pser = pd.Series([np.nan, 2, 3, 4, np.nan, 6], name="x")
kser = ps.from_pandas(pser)
if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
self.assert_eq(pser.pad(), kser.pad())
# Test `inplace=True`
pser.pad(inplace=True)
kser.pad(inplace=True)
self.assert_eq(pser, kser)
else:
expected = ps.Series([np.nan, 2, 3, 4, 4, 6], name="x")
self.assert_eq(expected, kser.pad())
# Test `inplace=True`
kser.pad(inplace=True)
self.assert_eq(expected, kser)
def test_explode(self):
if LooseVersion(pd.__version__) >= LooseVersion("0.25"):
pser = pd.Series([[1, 2, 3], [], None, [3, 4]])
kser = ps.from_pandas(pser)
self.assert_eq(pser.explode(), kser.explode(), almost=True)
# MultiIndex
pser.index = pd.MultiIndex.from_tuples([("a", "w"), ("b", "x"), ("c", "y"), ("d", "z")])
kser = ps.from_pandas(pser)
self.assert_eq(pser.explode(), kser.explode(), almost=True)
# non-array type Series
pser = pd.Series([1, 2, 3, 4])
kser = ps.from_pandas(pser)
self.assert_eq(pser.explode(), kser.explode())
else:
pser = pd.Series([[1, 2, 3], [], None, [3, 4]])
kser = ps.from_pandas(pser)
expected = pd.Series([1.0, 2.0, 3.0, None, None, 3.0, 4.0], index=[0, 0, 0, 1, 2, 3, 3])
self.assert_eq(kser.explode(), expected)
# MultiIndex
pser.index = pd.MultiIndex.from_tuples([("a", "w"), ("b", "x"), ("c", "y"), ("d", "z")])
kser = ps.from_pandas(pser)
expected = pd.Series(
[1.0, 2.0, 3.0, None, None, 3.0, 4.0],
index=pd.MultiIndex.from_tuples(
[
("a", "w"),
("a", "w"),
("a", "w"),
("b", "x"),
("c", "y"),
("d", "z"),
("d", "z"),
]
),
)
self.assert_eq(kser.explode(), expected)
# non-array type Series
pser = pd.Series([1, 2, 3, 4])
kser = ps.from_pandas(pser)
expected = pser
self.assert_eq(kser.explode(), expected)
def test_argsort(self):
# Without null values
pser = pd.Series([0, -100, 50, 100, 20], index=["A", "B", "C", "D", "E"])
kser = ps.from_pandas(pser)
self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
# MultiIndex
pser.index = pd.MultiIndex.from_tuples(
[("a", "v"), ("b", "w"), ("c", "x"), ("d", "y"), ("e", "z")]
)
kser = ps.from_pandas(pser)
self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
# With name
pser.name = "Koalas"
kser = ps.from_pandas(pser)
self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
# Series from Index
pidx = pd.Index([4.0, -6.0, 2.0, -100.0, 11.0, 20.0, 1.0, -99.0])
kidx = ps.from_pandas(pidx)
self.assert_eq(
pidx.to_series().argsort().sort_index(), kidx.to_series().argsort().sort_index()
)
self.assert_eq(
(-pidx.to_series()).argsort().sort_index(), (-kidx.to_series()).argsort().sort_index()
)
# Series from Index with name
pidx.name = "Koalas"
kidx = ps.from_pandas(pidx)
self.assert_eq(
pidx.to_series().argsort().sort_index(), kidx.to_series().argsort().sort_index()
)
self.assert_eq(
(-pidx.to_series()).argsort().sort_index(), (-kidx.to_series()).argsort().sort_index()
)
# Series from DataFrame
pdf = pd.DataFrame({"A": [4.0, -6.0, 2.0, np.nan, -100.0, 11.0, 20.0, np.nan, 1.0, -99.0]})
kdf = ps.from_pandas(pdf)
self.assert_eq(pdf.A.argsort().sort_index(), kdf.A.argsort().sort_index())
self.assert_eq((-pdf.A).argsort().sort_index(), (-kdf.A).argsort().sort_index())
# With null values
pser = pd.Series([0, -100, np.nan, 100, np.nan], index=["A", "B", "C", "D", "E"])
kser = ps.from_pandas(pser)
self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
# MultiIndex with null values
pser.index = pd.MultiIndex.from_tuples(
[("a", "v"), ("b", "w"), ("c", "x"), ("d", "y"), ("e", "z")]
)
kser = ps.from_pandas(pser)
self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
# With name with null values
pser.name = "Koalas"
kser = ps.from_pandas(pser)
self.assert_eq(pser.argsort().sort_index(), kser.argsort().sort_index())
self.assert_eq((-pser).argsort().sort_index(), (-kser).argsort().sort_index())
# Series from Index with null values
pidx = pd.Index([4.0, -6.0, 2.0, np.nan, -100.0, 11.0, 20.0, np.nan, 1.0, -99.0])
kidx = ps.from_pandas(pidx)
self.assert_eq(
pidx.to_series().argsort().sort_index(), kidx.to_series().argsort().sort_index()
)
self.assert_eq(
(-pidx.to_series()).argsort().sort_index(), (-kidx.to_series()).argsort().sort_index()
)
# Series from Index with name with null values
pidx.name = "Koalas"
kidx = ps.from_pandas(pidx)
self.assert_eq(
pidx.to_series().argsort().sort_index(), kidx.to_series().argsort().sort_index()
)
self.assert_eq(
(-pidx.to_series()).argsort().sort_index(), (-kidx.to_series()).argsort().sort_index()
)
# Series from DataFrame with null values
pdf = pd.DataFrame({"A": [4.0, -6.0, 2.0, np.nan, -100.0, 11.0, 20.0, np.nan, 1.0, -99.0]})
kdf = ps.from_pandas(pdf)
self.assert_eq(pdf.A.argsort().sort_index(), kdf.A.argsort().sort_index())
self.assert_eq((-pdf.A).argsort().sort_index(), (-kdf.A).argsort().sort_index())
def test_argmin_argmax(self):
pser = pd.Series(
{
"Corn Flakes": 100.0,
"Almond Delight": 110.0,
"Cinnamon Toast Crunch": 120.0,
"Cocoa Puff": 110.0,
"Expensive Flakes": 120.0,
"Cheap Flakes": 100.0,
},
name="Koalas",
)
kser = ps.from_pandas(pser)
if LooseVersion(pd.__version__) >= LooseVersion("1.0"):
self.assert_eq(pser.argmin(), kser.argmin())
self.assert_eq(pser.argmax(), kser.argmax())
# MultiIndex
pser.index = pd.MultiIndex.from_tuples(
[("a", "t"), ("b", "u"), ("c", "v"), ("d", "w"), ("e", "x"), ("f", "u")]
)
kser = ps.from_pandas(pser)
self.assert_eq(pser.argmin(), kser.argmin())
self.assert_eq(pser.argmax(), kser.argmax())
# Null Series
self.assert_eq(pd.Series([np.nan]).argmin(), ps.Series([np.nan]).argmin())
self.assert_eq(pd.Series([np.nan]).argmax(), ps.Series([np.nan]).argmax())
else:
self.assert_eq(pser.values.argmin(), kser.argmin())
self.assert_eq(pser.values.argmax(), kser.argmax())
# MultiIndex
pser.index = pd.MultiIndex.from_tuples(
[("a", "t"), ("b", "u"), ("c", "v"), ("d", "w"), ("e", "x"), ("f", "u")]
)
kser = ps.from_pandas(pser)
self.assert_eq(pser.values.argmin(), kser.argmin())
self.assert_eq(pser.values.argmax(), kser.argmax())
# Null Series
self.assert_eq(-1, ps.Series([np.nan]).argmin())
self.assert_eq(-1, ps.Series([np.nan]).argmax())
with self.assertRaisesRegex(ValueError, "attempt to get argmin of an empty sequence"):
ps.Series([]).argmin()
with self.assertRaisesRegex(ValueError, "attempt to get argmax of an empty sequence"):
ps.Series([]).argmax()
def test_backfill(self):
pser = pd.Series([np.nan, 2, 3, 4, np.nan, 6], name="x")
kser = ps.from_pandas(pser)
if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
self.assert_eq(pser.backfill(), kser.backfill())
# Test `inplace=True`
pser.backfill(inplace=True)
kser.backfill(inplace=True)
self.assert_eq(pser, kser)
else:
expected = ps.Series([2.0, 2.0, 3.0, 4.0, 6.0, 6.0], name="x")
self.assert_eq(expected, kser.backfill())
# Test `inplace=True`
kser.backfill(inplace=True)
self.assert_eq(expected, kser)
def test_align(self):
pdf = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
kdf = ps.from_pandas(pdf)
for join in ["outer", "inner", "left", "right"]:
for axis in [None, 0]:
kser_l, kser_r = kdf.a.align(kdf.b, join=join, axis=axis)
pser_l, pser_r = pdf.a.align(pdf.b, join=join, axis=axis)
self.assert_eq(kser_l, pser_l)
self.assert_eq(kser_r, pser_r)
kser_l, kdf_r = kdf.b.align(kdf[["b", "a"]], join=join, axis=axis)
pser_l, pdf_r = pdf.b.align(pdf[["b", "a"]], join=join, axis=axis)
self.assert_eq(kser_l, pser_l)
self.assert_eq(kdf_r, pdf_r)
self.assertRaises(ValueError, lambda: kdf.a.align(kdf.b, axis=1))
def test_pow_and_rpow(self):
pser = pd.Series([1, 2, np.nan])
kser = ps.from_pandas(pser)
self.assert_eq(pser.pow(np.nan), kser.pow(np.nan))
self.assert_eq(pser ** np.nan, kser ** np.nan)
self.assert_eq(pser.rpow(np.nan), kser.rpow(np.nan))
self.assert_eq(1 ** pser, 1 ** kser)
def test_between_time(self):
idx = pd.date_range("2018-04-09", periods=4, freq="1D20min")
pser = pd.Series([1, 2, 3, 4], index=idx)
kser = ps.from_pandas(pser)
self.assert_eq(
pser.between_time("0:15", "0:45").sort_index(),
kser.between_time("0:15", "0:45").sort_index(),
)
pser.index.name = "ts"
kser = ps.from_pandas(pser)
self.assert_eq(
pser.between_time("0:15", "0:45").sort_index(),
kser.between_time("0:15", "0:45").sort_index(),
)
pser.index.name = "index"
kser = ps.from_pandas(pser)
self.assert_eq(
pser.between_time("0:15", "0:45").sort_index(),
kser.between_time("0:15", "0:45").sort_index(),
)
def test_at_time(self):
idx = pd.date_range("2018-04-09", periods=4, freq="1D20min")
pser = pd.Series([1, 2, 3, 4], index=idx)
kser = ps.from_pandas(pser)
self.assert_eq(
pser.at_time("0:20").sort_index(), kser.at_time("0:20").sort_index(),
)
pser.index.name = "ts"
kser = ps.from_pandas(pser)
self.assert_eq(
pser.at_time("0:20").sort_index(), kser.at_time("0:20").sort_index(),
)
pser.index.name = "index"
kser = ps.from_pandas(pser)
self.assert_eq(
pser.at_time("0:20").sort_index(), kser.at_time("0:20").sort_index(),
)
if __name__ == "__main__":
from pyspark.pandas.tests.test_series import * # noqa: F401
try:
import xmlrunner # type: ignore[import]
testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
except ImportError:
testRunner = None
unittest.main(testRunner=testRunner, verbosity=2)
Reference in a new issue View git blame Copy permalink