Merge remote-tracking branch 'upstream/main' into string-dtype-object

Author: jorisvandenbossche

doc/source/whatsnew/v3.0.0.rst

@@ -589,7 +589,7 @@ I/O
 
 Period
 ^^^^^^
--
+- Fixed error message when passing invalid period alias to :meth:`PeriodIndex.to_timestamp` (:issue:`58974`)
 -
 
 Plotting

pandas/_libs/tslibs/offsets.pyx

@@ -4752,20 +4752,16 @@ def _validate_to_offset_alias(alias: str, is_period: bool) -> None:
                 alias.lower() not in {"s", "ms", "us", "ns"} and
                 alias.upper().split("-")[0].endswith(("S", "E"))):
             raise ValueError(INVALID_FREQ_ERR_MSG.format(alias))
-    if (is_period and
-            alias.upper() in c_OFFSET_TO_PERIOD_FREQSTR and
-            alias != "ms" and
-            alias.upper().split("-")[0].endswith(("S", "E"))):
-        if (alias.upper().startswith("B") or
-                alias.upper().startswith("S") or
-                alias.upper().startswith("C")):
-            raise ValueError(INVALID_FREQ_ERR_MSG.format(alias))
-        else:
-            alias_msg = "".join(alias.upper().split("E", 1))
-            raise ValueError(
-                f"for Period, please use \'{alias_msg}\' "
-                f"instead of \'{alias}\'"
-            )
+    if (
+        is_period and
+        alias in c_OFFSET_TO_PERIOD_FREQSTR and
+        alias != c_OFFSET_TO_PERIOD_FREQSTR[alias]
+    ):
+        alias_msg = c_OFFSET_TO_PERIOD_FREQSTR.get(alias)
+        raise ValueError(
+            f"for Period, please use \'{alias_msg}\' "
+            f"instead of \'{alias}\'"
+        )
 
 
 # TODO: better name?

pandas/conftest.py

@@ -1297,7 +1297,6 @@ def nullable_string_dtype(request):
         "python",
         "python_numpy",
         pytest.param("pyarrow", marks=td.skip_if_no("pyarrow")),
-        pytest.param("pyarrow_numpy", marks=td.skip_if_no("pyarrow")),
     ]
 )
 def string_storage(request):
@@ -1306,7 +1305,24 @@ def string_storage(request):
 
     * 'python'
     * 'pyarrow'
-    * 'pyarrow_numpy'
+    """
+    return request.param
+
+
+@pytest.fixture(
+    params=[
+        ("python", pd.NA),
+        pytest.param(("pyarrow", pd.NA), marks=td.skip_if_no("pyarrow")),
+        pytest.param(("pyarrow", np.nan), marks=td.skip_if_no("pyarrow")),
+    ]
+)
+def string_dtype_arguments(request):
+    """
+    Parametrized fixture for StringDtype storage and na_value.
+
+    * 'python' + pd.NA
+    * 'pyarrow' + pd.NA
+    * 'pyarrow' + np.nan
     """
     return request.param
 

pandas/core/config_init.py

@@ -14,6 +14,7 @@
 
 from collections.abc import Callable
 import os
+from typing import Any
 
 import pandas._config.config as cf
 from pandas._config.config import (
@@ -455,14 +456,27 @@ def is_terminal() -> bool:
     ``future.infer_string`` is set to True.
 """
 
+
+def is_valid_string_storage(value: Any) -> None:
+    legal_values = ["python", "pyarrow"]
+    if value not in legal_values:
+        msg = "Value must be one of python|pyarrow"
+        if value == "pyarrow_numpy":
+            # TODO: we can remove extra message after 3.0
+            msg += (
+                ". 'pyarrow_numpy' was specified, but this option should be "
+                "enabled using pandas.options.future.infer_string instead"
+            )
+        raise ValueError(msg)
+
+
 with cf.config_prefix("mode"):
     cf.register_option(
         "string_storage",
         "python",
         string_storage_doc,
-        validator=is_one_of_factory(
-            ["python", "pyarrow", "python_numpy", "pyarrow_numpy"]
-        ),
+        # validator=is_one_of_factory(["python", "pyarrow"]),
+        validator=is_valid_string_storage,
     )
 
 

pandas/tests/arrays/string_/test_string.py

@@ -24,9 +24,10 @@
 
 
 @pytest.fixture
-def dtype(string_storage):
-    """Fixture giving StringDtype from parametrized 'string_storage'"""
-    return pd.StringDtype(storage=string_storage)
+def dtype(string_dtype_arguments):
+    """Fixture giving StringDtype from parametrized storage and na_value arguments"""
+    storage, na_value = string_dtype_arguments
+    return pd.StringDtype(storage=storage, na_value=na_value)
 
 
 @pytest.fixture
@@ -521,50 +522,34 @@ def test_arrow_array(dtype):
     assert arr.equals(expected)
 
 
-@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False)
+@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)")
 @pytest.mark.filterwarnings("ignore:Passing a BlockManager:DeprecationWarning")
-def test_arrow_roundtrip(dtype, string_storage2, request, using_infer_string):
+def test_arrow_roundtrip(dtype, string_storage, using_infer_string):
     # roundtrip possible from arrow 1.0.0
     pa = pytest.importorskip("pyarrow")
 
-    if using_infer_string and string_storage2 not in ("python_numpy", "pyarrow_numpy"):
-        request.applymarker(
-            pytest.mark.xfail(
-                reason="infer_string takes precedence over string storage"
-            )
-        )
-
     data = pd.array(["a", "b", None], dtype=dtype)
     df = pd.DataFrame({"a": data})
     table = pa.table(df)
     if dtype.storage == "python":
         assert table.field("a").type == "string"
     else:
         assert table.field("a").type == "large_string"
-    with pd.option_context("string_storage", string_storage2):
+    with pd.option_context("string_storage", string_storage):
         result = table.to_pandas()
     assert isinstance(result["a"].dtype, pd.StringDtype)
-    expected = df.astype(f"string[{string_storage2}]")
+    expected = df.astype(f"string[{string_storage}]")
     tm.assert_frame_equal(result, expected)
     # ensure the missing value is represented by NA and not np.nan or None
     assert result.loc[2, "a"] is result["a"].dtype.na_value
 
 
-@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)", strict=False)
+@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)")
 @pytest.mark.filterwarnings("ignore:Passing a BlockManager:DeprecationWarning")
-def test_arrow_load_from_zero_chunks(
-    dtype, string_storage2, request, using_infer_string
-):
+def test_arrow_load_from_zero_chunks(dtype, string_storage, using_infer_string):
     # GH-41040
     pa = pytest.importorskip("pyarrow")
 
-    if using_infer_string and string_storage2 != "pyarrow_numpy":
-        request.applymarker(
-            pytest.mark.xfail(
-                reason="infer_string takes precedence over string storage"
-            )
-        )
-
     data = pd.array([], dtype=dtype)
     df = pd.DataFrame({"a": data})
     table = pa.table(df)
@@ -574,10 +559,10 @@ def test_arrow_load_from_zero_chunks(
         assert table.field("a").type == "large_string"
     # Instantiate the same table with no chunks at all
     table = pa.table([pa.chunked_array([], type=pa.string())], schema=table.schema)
-    with pd.option_context("string_storage", string_storage2):
+    with pd.option_context("string_storage", string_storage):
         result = table.to_pandas()
     assert isinstance(result["a"].dtype, pd.StringDtype)
-    expected = df.astype(f"string[{string_storage2}]")
+    expected = df.astype(f"string[{string_storage}]")
     tm.assert_frame_equal(result, expected)
 
 

pandas/tests/arrays/string_/test_string_arrow.py

@@ -27,16 +27,18 @@ def test_eq_all_na():
 
 
 def test_config(string_storage, request, using_infer_string):
-    if using_infer_string and string_storage in ("python_numpy", "pyarrow_numpy"):
-        request.applymarker(pytest.mark.xfail(reason="infer string takes precedence"))
-    if string_storage in ("pyarrow_numpy", "python_numpy"):
+    if using_infer_string and string_storage == "python":
+        # python string storage with na_value=NaN is not yet implemented
         request.applymarker(pytest.mark.xfail(reason="TODO(infer_string)"))
+
     with pd.option_context("string_storage", string_storage):
         assert StringDtype().storage == string_storage
         result = pd.array(["a", "b"])
         assert result.dtype.storage == string_storage
 
-    dtype = StringDtype(string_storage)
+    dtype = StringDtype(
+        string_storage, na_value=np.nan if using_infer_string else pd.NA
+    )
     expected = dtype.construct_array_type()._from_sequence(["a", "b"], dtype=dtype)
     tm.assert_equal(result, expected)
 

pandas/tests/extension/test_string.py

@@ -59,8 +59,9 @@ def chunked(request):
 
 
 @pytest.fixture
-def dtype(string_storage):
-    return StringDtype(storage=string_storage)
+def dtype(string_dtype_arguments):
+    storage, na_value = string_dtype_arguments
+    return StringDtype(storage=storage, na_value=na_value)
 
 
 @pytest.fixture

pandas/tests/frame/methods/test_astype.py

@@ -897,3 +897,12 @@ def test_astype_to_string_not_modifying_input(string_storage, val):
     with option_context("mode.string_storage", string_storage):
         df.astype("string")
     tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.parametrize("val", [None, 1, 1.5, np.nan, NaT])
+def test_astype_to_string_dtype_not_modifying_input(any_string_dtype, val):
+    # GH#51073 - variant of the above test with explicit dtype instances
+    df = DataFrame({"a": ["a", "b", val]})
+    expected = df.copy()
+    df.astype(any_string_dtype)
+    tm.assert_frame_equal(df, expected)

pandas/tests/frame/methods/test_convert_dtypes.py

@@ -10,6 +10,8 @@
 
 
 class TestConvertDtypes:
+    # TODO convert_dtypes should not use NaN variant of string dtype, but always NA
+    @pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)")
     @pytest.mark.parametrize(
         "convert_integer, expected", [(False, np.dtype("int32")), (True, "Int32")]
     )
@@ -18,9 +20,6 @@ def test_convert_dtypes(
     ):
         # Specific types are tested in tests/series/test_dtypes.py
         # Just check that it works for DataFrame here
-        if using_infer_string:
-            string_storage = "pyarrow_numpy"
-
         df = pd.DataFrame(
             {
                 "a": pd.Series([1, 2, 3], dtype=np.dtype("int32")),

pandas/tests/indexes/period/methods/test_to_timestamp.py

@@ -140,3 +140,10 @@ def test_to_timestamp_1703(self):
 
         result = index.to_timestamp()
         assert result[0] == Timestamp("1/1/2012")
+
+
+def test_ms_to_timestamp_error_message():
+    # https://github.com/pandas-dev/pandas/issues/58974#issuecomment-2164265446
+    ix = period_range("2000", periods=3, freq="M")
+    with pytest.raises(ValueError, match="for Period, please use 'M' instead of 'MS'"):
+        ix.to_timestamp("MS")