Merge remote-tracking branch 'upstream/main' into string-dtype-object

jorisvandenbossche · jorisvandenbossche · commit fdf1454bcba9 · 2024-08-07T09:42:55.000+02:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -16,6 +16,7 @@ jobs:
       - run:
           name: Install Environment and Run Tests
           shell: /bin/bash -exo pipefail
+          # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
           command: |
             MINI_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-aarch64.sh"
             wget -q $MINI_URL -O Miniforge3.sh
@@ -33,6 +34,7 @@ jobs:
             fi
             python -m pip install --no-build-isolation -ve . --config-settings=setup-args="--werror"
             PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
+            sudo apt-get update && sudo apt-get install -y libegl1 libopengl0
             ci/run_tests.sh
   test-linux-musl:
     docker:
@@ -89,8 +91,8 @@ jobs:
           name: Build aarch64 wheels
           no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that
           command: |
-            pip3 install cibuildwheel==2.18.1
-            cibuildwheel --prerelease-pythons --output-dir wheelhouse
+            pip3 install cibuildwheel==2.20.0
+            cibuildwheel --output-dir wheelhouse
 
           environment:
             CIBW_BUILD: << parameters.cibw-build >>
diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
@@ -51,6 +51,11 @@ jobs:
     # TODO: The doctests have to be run first right now, since the Cython doctests only work
     # with pandas installed in non-editable mode
     # This can be removed once pytest-cython doesn't require C extensions to be installed inplace
+
+    - name: Extra installs
+      # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
+      run: sudo apt-get update && sudo apt-get install -y libegl1 libopengl0
+
     - name: Run doctests
       run: cd ci && ./code_checks.sh doctests
       if: ${{ steps.build.outcome == 'success' && always() }}
diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml
@@ -46,6 +46,10 @@ jobs:
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 
+    - name: Extra installs
+      # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
+      run: sudo apt-get update && sudo apt-get install -y libegl1 libopengl0
+
     - name: Test website
       run: python -m pytest web/
 
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -134,8 +134,8 @@ jobs:
         fetch-depth: 0
 
     - name: Extra installs
-      run: sudo apt-get update && sudo apt-get install -y ${{ matrix.extra_apt }}
-      if: ${{ matrix.extra_apt }}
+      # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
+      run: sudo apt-get update && sudo apt-get install -y libegl1 libopengl0 ${{ matrix.extra_apt || ''}}
 
     - name: Generate extra locales
       # These extra locales will be available for locale.setlocale() calls in tests
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -158,11 +158,10 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.19.2
+        uses: pypa/cibuildwheel@v2.20.0
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
-          CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
           CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend || 'pip' }}
           CIBW_PLATFORM: ${{ matrix.buildplat[1] == 'pyodide_wasm32' && 'pyodide' || 'auto' }}
diff --git a/ci/code_checks.sh b/ci/code_checks.sh
@@ -70,7 +70,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         --format=actions \
         -i ES01 `# For now it is ok if docstrings are missing the extended summary` \
         -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
-        -i "pandas.MultiIndex.get_level_values SA01" \
         -i "pandas.MultiIndex.get_loc_level PR07" \
         -i "pandas.MultiIndex.names SA01" \
         -i "pandas.MultiIndex.reorder_levels RT03,SA01" \
@@ -165,9 +164,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.str.center RT03,SA01" \
         -i "pandas.Series.str.decode PR07,RT03,SA01" \
         -i "pandas.Series.str.encode PR07,RT03,SA01" \
-        -i "pandas.Series.str.find RT03" \
-        -i "pandas.Series.str.fullmatch RT03" \
-        -i "pandas.Series.str.get RT03,SA01" \
         -i "pandas.Series.str.index RT03" \
         -i "pandas.Series.str.ljust RT03,SA01" \
         -i "pandas.Series.str.lower RT03" \
@@ -177,7 +173,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.str.partition RT03" \
         -i "pandas.Series.str.repeat SA01" \
         -i "pandas.Series.str.replace SA01" \
-        -i "pandas.Series.str.rfind RT03" \
         -i "pandas.Series.str.rindex RT03" \
         -i "pandas.Series.str.rjust RT03,SA01" \
         -i "pandas.Series.str.rpartition RT03" \
@@ -267,9 +262,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.api.extensions.ExtensionArray.tolist RT03,SA01" \
         -i "pandas.api.extensions.ExtensionArray.unique RT03,SA01" \
         -i "pandas.api.extensions.ExtensionArray.view SA01" \
-        -i "pandas.api.indexers.VariableOffsetWindowIndexer PR01,SA01" \
         -i "pandas.api.interchange.from_dataframe RT03,SA01" \
-        -i "pandas.api.types.is_any_real_numeric_dtype SA01" \
         -i "pandas.api.types.is_bool PR01,SA01" \
         -i "pandas.api.types.is_bool_dtype SA01" \
         -i "pandas.api.types.is_categorical_dtype SA01" \
@@ -296,7 +289,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.api.types.is_period_dtype SA01" \
         -i "pandas.api.types.is_re PR07,SA01" \
         -i "pandas.api.types.is_re_compilable PR07,SA01" \
-        -i "pandas.api.types.is_sparse SA01" \
         -i "pandas.api.types.is_timedelta64_ns_dtype SA01" \
         -i "pandas.api.types.pandas_dtype PR07,RT03,SA01" \
         -i "pandas.arrays.ArrowExtensionArray PR07,SA01" \
diff --git a/doc/source/user_guide/basics.rst b/doc/source/user_guide/basics.rst
@@ -155,16 +155,6 @@ speedups. ``numexpr`` uses smart chunking, caching, and multiple cores. ``bottle
 a set of specialized cython routines that are especially fast when dealing with arrays that have
 ``nans``.
 
-Here is a sample (using 100 column x 100,000 row ``DataFrames``):
-
-.. csv-table::
-    :header: "Operation", "0.11.0 (ms)", "Prior Version (ms)", "Ratio to Prior"
-    :widths: 25, 25, 25, 25
-
-    ``df1 > df2``, 13.32, 125.35,  0.1063
-    ``df1 * df2``, 21.71,  36.63,  0.5928
-    ``df1 + df2``, 22.04,  36.50,  0.6039
-
 You are highly encouraged to install both libraries. See the section
 :ref:`Recommended Dependencies <install.recommended_dependencies>` for more installation info.
 
diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst
@@ -547,6 +547,7 @@ Strings
 
 Interval
 ^^^^^^^^
+- :meth:`Index.is_monotonic_decreasing`, :meth:`Index.is_monotonic_increasing`, and :meth:`Index.is_unique` could incorrectly be ``False`` for an ``Index`` created from a slice of another ``Index``. (:issue:`57911`)
 - Bug in :func:`interval_range` where start and end numeric types were always cast to 64 bit (:issue:`57268`)
 -
 
@@ -609,6 +610,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrameGroupBy.agg` that raises ``AttributeError`` when there is dictionary input and duplicated columns, instead of returning a DataFrame with the aggregation of all duplicate columns. (:issue:`55041`)
 - Bug in :meth:`DataFrameGroupBy.apply` that was returning a completely empty DataFrame when all return values of ``func`` were ``None`` instead of returning an empty DataFrame with the original columns and dtypes. (:issue:`57775`)
 - Bug in :meth:`DataFrameGroupBy.apply` with ``as_index=False`` that was returning :class:`MultiIndex` instead of returning :class:`Index`. (:issue:`58291`)
+- Bug in :meth:`DataFrameGroupBy.cumsum` and :meth:`DataFrameGroupBy.cumprod` where ``numeric_only`` parameter was passed indirectly through kwargs instead of passing directly. (:issue:`58811`)
 - Bug in :meth:`DataFrameGroupBy.cumsum` where it did not return the correct dtype when the label contained ``None``. (:issue:`58811`)
 - Bug in :meth:`DataFrameGroupby.transform` and :meth:`SeriesGroupby.transform` with a reducer and ``observed=False`` that coerces dtype to float when there are unobserved categories. (:issue:`55326`)
 - Bug in :meth:`Rolling.apply` where the applied function could be called on fewer than ``min_period`` periods if ``method="table"``. (:issue:`58868`)
diff --git a/pandas/_libs/index.pyx b/pandas/_libs/index.pyx
@@ -252,14 +252,24 @@ cdef class IndexEngine:
         return self.sizeof()
 
     cpdef _update_from_sliced(self, IndexEngine other, reverse: bool):
-        self.unique = other.unique
-        self.need_unique_check = other.need_unique_check
+        if other.unique:
+            self.unique = other.unique
+            self.need_unique_check = other.need_unique_check
+
         if not other.need_monotonic_check and (
                 other.is_monotonic_increasing or other.is_monotonic_decreasing):
-            self.need_monotonic_check = other.need_monotonic_check
-            # reverse=True means the index has been reversed
-            self.monotonic_inc = other.monotonic_dec if reverse else other.monotonic_inc
-            self.monotonic_dec = other.monotonic_inc if reverse else other.monotonic_dec
+            self.need_monotonic_check = 0
+            if len(self.values) > 0 and self.values[0] != self.values[-1]:
+                # reverse=True means the index has been reversed
+                if reverse:
+                    self.monotonic_inc = other.monotonic_dec
+                    self.monotonic_dec = other.monotonic_inc
+                else:
+                    self.monotonic_inc = other.monotonic_inc
+                    self.monotonic_dec = other.monotonic_dec
+            else:
+                self.monotonic_inc = 1
+                self.monotonic_dec = 1
 
     @property
     def is_unique(self) -> bool:
@@ -882,14 +892,24 @@ cdef class SharedEngine:
         pass
 
     cpdef _update_from_sliced(self, ExtensionEngine other, reverse: bool):
-        self.unique = other.unique
-        self.need_unique_check = other.need_unique_check
+        if other.unique:
+            self.unique = other.unique
+            self.need_unique_check = other.need_unique_check
+
         if not other.need_monotonic_check and (
                 other.is_monotonic_increasing or other.is_monotonic_decreasing):
-            self.need_monotonic_check = other.need_monotonic_check
-            # reverse=True means the index has been reversed
-            self.monotonic_inc = other.monotonic_dec if reverse else other.monotonic_inc
-            self.monotonic_dec = other.monotonic_inc if reverse else other.monotonic_dec
+            self.need_monotonic_check = 0
+            if len(self.values) > 0 and self.values[0] != self.values[-1]:
+                # reverse=True means the index has been reversed
+                if reverse:
+                    self.monotonic_inc = other.monotonic_dec
+                    self.monotonic_dec = other.monotonic_inc
+                else:
+                    self.monotonic_inc = other.monotonic_inc
+                    self.monotonic_dec = other.monotonic_dec
+            else:
+                self.monotonic_inc = 1
+                self.monotonic_dec = 1
 
     @property
     def is_unique(self) -> bool:
diff --git a/pandas/core/arrays/string_arrow.py b/pandas/core/arrays/string_arrow.py
@@ -697,9 +697,9 @@ def _reduce(
         self, name: str, *, skipna: bool = True, keepdims: bool = False, **kwargs
     ):
         if name in ["any", "all"]:
-            if not skipna and name == "all":
-                nas = pc.invert(pc.is_null(self._pa_array))
-                arr = pc.and_kleene(nas, pc.not_equal(self._pa_array, ""))
+            if not skipna:
+                nas = pc.is_null(self._pa_array)
+                arr = pc.or_kleene(nas, pc.not_equal(self._pa_array, ""))
             else:
                 arr = pc.not_equal(self._pa_array, "")
             return ArrowExtensionArray(arr)._reduce(
diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py
@@ -187,6 +187,10 @@ def is_sparse(arr) -> bool:
     bool
         Whether or not the array-like is a pandas sparse array.
 
+    See Also
+    --------
+    api.types.SparseDtype : The dtype object for pandas sparse arrays.
+
     Examples
     --------
     Returns `True` if the parameter is a 1-D pandas sparse array.
@@ -1181,6 +1185,12 @@ def is_any_real_numeric_dtype(arr_or_dtype) -> bool:
     boolean
         Whether or not the array or dtype is of a real number dtype.
 
+    See Also
+    --------
+    is_numeric_dtype : Check if a dtype is numeric.
+    is_complex_dtype : Check if a dtype is complex.
+    is_bool_dtype : Check if a dtype is boolean.
+
     Examples
     --------
     >>> from pandas.api.types import is_any_real_numeric_dtype
diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py
@@ -2437,7 +2437,8 @@ def std(
         Parameters
         ----------
         ddof : int, default 1
-            Degrees of freedom.
+            Delta Degrees of Freedom. The divisor used in calculations is ``N - ddof``,
+            where ``N`` represents the number of elements.
 
         engine : str, default None
             * ``'cython'`` : Runs the operation through C-extensions from cython.
@@ -4681,12 +4682,14 @@ def rank(
     @final
     @Substitution(name="groupby")
     @Substitution(see_also=_common_see_also)
-    def cumprod(self, *args, **kwargs) -> NDFrameT:
+    def cumprod(self, numeric_only: bool = False, *args, **kwargs) -> NDFrameT:
         """
         Cumulative product for each group.
 
         Parameters
         ----------
+        numeric_only : bool, default False
+            Include only float, int, boolean columns.
         *args : tuple
             Positional arguments to be passed to `func`.
         **kwargs : dict
@@ -4734,18 +4737,20 @@ def cumprod(self, *args, **kwargs) -> NDFrameT:
         horse  16  10
         bull    6   9
         """
-        nv.validate_groupby_func("cumprod", args, kwargs, ["numeric_only", "skipna"])
-        return self._cython_transform("cumprod", **kwargs)
+        nv.validate_groupby_func("cumprod", args, kwargs, ["skipna"])
+        return self._cython_transform("cumprod", numeric_only, **kwargs)
 
     @final
     @Substitution(name="groupby")
     @Substitution(see_also=_common_see_also)
-    def cumsum(self, *args, **kwargs) -> NDFrameT:
+    def cumsum(self, numeric_only: bool = False, *args, **kwargs) -> NDFrameT:
         """
         Cumulative sum for each group.
 
         Parameters
         ----------
+        numeric_only : bool, default False
+            Include only float, int, boolean columns.
         *args : tuple
             Positional arguments to be passed to `func`.
         **kwargs : dict
@@ -4793,8 +4798,8 @@ def cumsum(self, *args, **kwargs) -> NDFrameT:
         gorilla  10   7
         lion      6   9
         """
-        nv.validate_groupby_func("cumsum", args, kwargs, ["numeric_only", "skipna"])
-        return self._cython_transform("cumsum", **kwargs)
+        nv.validate_groupby_func("cumsum", args, kwargs, ["skipna"])
+        return self._cython_transform("cumsum", numeric_only, **kwargs)
 
     @final
     @Substitution(name="groupby")
diff --git a/pandas/core/indexers/objects.py b/pandas/core/indexers/objects.py
@@ -167,6 +167,31 @@ class VariableOffsetWindowIndexer(BaseIndexer):
     """
     Calculate window boundaries based on a non-fixed offset such as a BusinessDay.
 
+    Parameters
+    ----------
+    index_array : np.ndarray, default 0
+        Array-like structure specifying the indices for data points.
+        This parameter is currently not used.
+
+    window_size : int, optional, default 0
+        Specifies the number of data points in each window.
+        This parameter is currently not used.
+
+    index : DatetimeIndex, optional
+        ``DatetimeIndex`` of the labels of each observation.
+
+    offset : BaseOffset, optional
+        ``DateOffset`` representing the size of the window.
+
+    **kwargs
+        Additional keyword arguments passed to the parent class ``BaseIndexer``.
+
+    See Also
+    --------
+    api.indexers.BaseIndexer : Base class for all indexers.
+    DataFrame.rolling : Rolling window calculations on DataFrames.
+    offsets : Module providing various time offset classes.
+
     Examples
     --------
     >>> from pandas.api.indexers import VariableOffsetWindowIndexer
diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py
@@ -1783,6 +1783,16 @@ def get_level_values(self, level) -> Index:  # type: ignore[override]
         Return vector of label values for requested level.
 
         Length of returned vector is equal to the length of the index.
+        The `get_level_values` method is a crucial utility for extracting
+        specific level values from a `MultiIndex`. This function is particularly
+        useful when working with multi-level data, allowing you to isolate
+        and manipulate individual levels without having to deal with the
+        complexity of the entire `MultiIndex` structure. It seamlessly handles
+        both integer and string-based level access, providing flexibility in
+        how you can interact with the data. Additionally, this method ensures
+        that the returned `Index` maintains the integrity of the original data,
+        even when missing values are present, by appropriately casting the
+        result to a suitable data type.
 
         Parameters
         ----------
@@ -1796,6 +1806,13 @@ def get_level_values(self, level) -> Index:  # type: ignore[override]
             Values is a level of this MultiIndex converted to
             a single :class:`Index` (or subclass thereof).
 
+        See Also
+        --------
+        MultiIndex : A multi-level, or hierarchical, index object for pandas objects.
+        Index : Immutable sequence used for indexing and alignment.
+        MultiIndex.remove_unused_levels : Create new MultiIndex from current that
+            removes unused levels.
+
         Notes
         -----
         If the level contains missing values, the result may be casted to
diff --git a/pandas/core/strings/accessor.py b/pandas/core/strings/accessor.py
diff --git a/pandas/tests/groupby/test_api.py b/pandas/tests/groupby/test_api.py
diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py
diff --git a/pandas/tests/reductions/test_reductions.py b/pandas/tests/reductions/test_reductions.py
