Merge remote-tracking branch 'upstream/2.1.x' into auto-backport-of-p…

…r-55227-on-2.1.x

.github/workflows/unit-tests.yml

@@ -236,7 +236,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -274,7 +274,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
 
@@ -347,7 +347,7 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
+          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps
           python -m pip list
 

.github/workflows/wheels.yml

@@ -168,7 +168,7 @@ jobs:
         shell: pwsh
         run: |
           $TST_CMD = @"
-          python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17;
+          python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0;
           python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
           python -c `'import pandas as pd; pd.test(extra_args=[\"`\"--no-strict-data-files`\"\", \"`\"-m not clipboard and not single_cpu and not slow and not network and not db`\"\"])`';
           "@

ci/deps/actions-310.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-311-downstream_compat.yaml

@@ -15,7 +15,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-311-numpydev.yaml

@@ -18,7 +18,6 @@ dependencies:
   # causes an InternalError within pytest
   - pytest-xdist>=2.2.0, <3
   - hypothesis>=6.46.1
-  - pytest-asyncio>=0.17.0
 
   # pandas dependencies
   - python-dateutil

ci/deps/actions-311-pyarrownightly.yaml

@@ -15,7 +15,6 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=2.2.0
   - hypothesis>=6.46.1
-  - pytest-asyncio>=0.17.0
 
   # required dependencies
   - python-dateutil

ci/deps/actions-311.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-39-minimum_versions.yaml

@@ -16,7 +16,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-39.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-pypy-39.yaml

@@ -16,7 +16,6 @@ dependencies:
   # test dependencies
   - pytest>=7.3.2
   - pytest-cov
-  - pytest-asyncio>=0.17.0
   - pytest-xdist>=2.2.0
   - hypothesis>=6.46.1
 

ci/deps/circle-310-arm64.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/meta.yaml

@@ -64,7 +64,6 @@ test:
   requires:
     - pip
     - pytest >=7.3.2
-    - pytest-asyncio >=0.17.0
     - pytest-xdist >=2.2.0
     - pytest-cov
     - hypothesis >=6.46.1

environment.yml

@@ -16,7 +16,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - coverage
 
   # required dependencies

pandas/tests/arrays/categorical/test_warnings.py

@@ -1,19 +1,16 @@
 import pytest
 
-from pandas.util._test_decorators import async_mark
-
 import pandas._testing as tm
 
 
 class TestCategoricalWarnings:
-    @async_mark()
-    async def test_tab_complete_warning(self, ip):
+    def test_tab_complete_warning(self, ip):
         # https://github.com/pandas-dev/pandas/issues/16409
         pytest.importorskip("IPython", minversion="6.0.0")
         from IPython.core.completer import provisionalcompleter
 
         code = "import pandas as pd; c = pd.Categorical([])"
-        await ip.run_code(code)
+        ip.run_cell(code)
 
         # GH 31324 newer jedi version raises Deprecation warning;
         #  appears resolved 2021-02-02

pandas/tests/frame/test_api.py

@@ -7,8 +7,6 @@
 
 from pandas._config.config import option_context
 
-from pandas.util._test_decorators import async_mark
-
 import pandas as pd
 from pandas import (
     DataFrame,
@@ -288,8 +286,7 @@ def _check_f(base, f):
         f = lambda x: x.rename({1: "foo"}, inplace=True)
         _check_f(d.copy(), f)
 
-    @async_mark()
-    async def test_tab_complete_warning(self, ip, frame_or_series):
+    def test_tab_complete_warning(self, ip, frame_or_series):
         # GH 16409
         pytest.importorskip("IPython", minversion="6.0.0")
         from IPython.core.completer import provisionalcompleter
@@ -299,8 +296,7 @@ async def test_tab_complete_warning(self, ip, frame_or_series):
         else:
             code = "from pandas import Series; obj = Series(dtype=object)"
 
-        await ip.run_code(code)
-
+        ip.run_cell(code)
         # GH 31324 newer jedi version raises Deprecation warning;
         #  appears resolved 2021-02-02
         with tm.assert_produces_warning(None, raise_on_extra_warnings=False):

pandas/tests/indexes/test_base.py

@@ -10,7 +10,6 @@
 
 from pandas.compat import IS64
 from pandas.errors import InvalidIndexError
-from pandas.util._test_decorators import async_mark
 
 from pandas.core.dtypes.common import (
     is_any_real_numeric_dtype,
@@ -1218,14 +1217,13 @@ def test_cached_properties_not_settable(self):
         with pytest.raises(AttributeError, match="Can't set attribute"):
             index.is_unique = False
 
-    @async_mark()
-    async def test_tab_complete_warning(self, ip):
+    def test_tab_complete_warning(self, ip):
         # https://github.com/pandas-dev/pandas/issues/16409
         pytest.importorskip("IPython", minversion="6.0.0")
         from IPython.core.completer import provisionalcompleter
 
         code = "import pandas as pd; idx = pd.Index([1, 2])"
-        await ip.run_code(code)
+        ip.run_cell(code)
 
         # GH 31324 newer jedi version raises Deprecation warning;
         #  appears resolved 2021-02-02

pandas/tests/interchange/test_impl.py

@@ -4,6 +4,10 @@
 import pytest
 
 from pandas._libs.tslibs import iNaT
+from pandas.compat import (
+    is_ci_environment,
+    is_platform_windows,
+)
 import pandas.util._test_decorators as td
 
 import pandas as pd
@@ -310,11 +314,21 @@ def test_datetimetzdtype(tz, unit):
     tm.assert_frame_equal(df, from_dataframe(df.__dataframe__()))
 
 
-def test_interchange_from_non_pandas_tz_aware():
+def test_interchange_from_non_pandas_tz_aware(request):
     # GH 54239, 54287
     pa = pytest.importorskip("pyarrow", "11.0.0")
     import pyarrow.compute as pc
 
+    if is_platform_windows() and is_ci_environment():
+        mark = pytest.mark.xfail(
+            raises=pa.ArrowInvalid,
+            reason=(
+                "TODO: Set ARROW_TIMEZONE_DATABASE environment variable "
+                "on CI to path to the tzdata for pyarrow."
+            ),
+        )
+        request.node.add_marker(mark)
+
     arr = pa.array([datetime(2020, 1, 1), None, datetime(2020, 1, 2)])
     arr = pc.assume_timezone(arr, "Asia/Kathmandu")
     table = pa.table({"arr": arr})

pandas/tests/io/parser/common/test_common_basic.py

@@ -86,7 +86,9 @@ def test_read_csv_local(all_parsers, csv1):
 
     fname = prefix + str(os.path.abspath(csv1))
     result = parser.read_csv(fname, index_col=0, parse_dates=True)
-
+    # TODO: make unit check more specific
+    if parser.engine == "pyarrow":
+        result.index = result.index.as_unit("ns")
     expected = DataFrame(
         [
             [0.980269, 3.685731, -0.364216805298, -1.159738],
@@ -177,7 +179,9 @@ def test_read_csv_low_memory_no_rows_with_index(all_parsers):
 def test_read_csv_dataframe(all_parsers, csv1):
     parser = all_parsers
     result = parser.read_csv(csv1, index_col=0, parse_dates=True)
-
+    # TODO: make unit check more specific
+    if parser.engine == "pyarrow":
+        result.index = result.index.as_unit("ns")
     expected = DataFrame(
         [
             [0.980269, 3.685731, -0.364216805298, -1.159738],

pandas/tests/io/parser/test_parse_dates.py

@@ -979,12 +979,15 @@ def test_parse_dates_custom_euro_format(all_parsers, kwargs):
             )
 
 
-def test_parse_tz_aware(all_parsers, request):
+def test_parse_tz_aware(all_parsers):
     # See gh-1693
     parser = all_parsers
     data = "Date,x\n2012-06-13T01:39:00Z,0.5"
 
     result = parser.read_csv(StringIO(data), index_col=0, parse_dates=True)
+    # TODO: make unit check more specific
+    if parser.engine == "pyarrow":
+        result.index = result.index.as_unit("ns")
     expected = DataFrame(
         {"x": [0.5]}, index=Index([Timestamp("2012-06-13 01:39:00+00:00")], name="Date")
     )
@@ -2231,6 +2234,9 @@ def test_parse_dates_arrow_engine(all_parsers):
 2000-01-01 00:00:01,1"""
 
     result = parser.read_csv(StringIO(data), parse_dates=["a"])
+    # TODO: make unit check more specific
+    if parser.engine == "pyarrow":
+        result["a"] = result["a"].dt.as_unit("ns")
     expected = DataFrame(
         {
             "a": [

pandas/tests/resample/test_resampler_grouper.py

@@ -4,7 +4,6 @@
 import pytest
 
 from pandas.compat import is_platform_windows
-from pandas.util._test_decorators import async_mark
 
 import pandas as pd
 from pandas import (
@@ -26,8 +25,7 @@ def test_frame():
     )
 
 
-@async_mark()
-async def test_tab_complete_ipython6_warning(ip):
+def test_tab_complete_ipython6_warning(ip):
     from IPython.core.completer import provisionalcompleter
 
     code = dedent(
@@ -37,7 +35,7 @@ async def test_tab_complete_ipython6_warning(ip):
     rs = s.resample("D")
     """
     )
-    await ip.run_code(code)
+    ip.run_cell(code)
 
     # GH 31324 newer jedi version raises Deprecation warning;
     #  appears resolved 2021-02-02

pandas/util/_test_decorators.py

@@ -38,11 +38,11 @@ def test_foo():
 
 if TYPE_CHECKING:
     from pandas._typing import F
+
 from pandas.compat import (
     IS64,
     is_platform_windows,
 )
-from pandas.compat._optional import import_optional_dependency
 
 from pandas.core.computation.expressions import (
     NUMEXPR_INSTALLED,
@@ -214,16 +214,6 @@ def documented_fixture(fixture):
     return documented_fixture
 
 
-def async_mark():
-    try:
-        import_optional_dependency("pytest_asyncio")
-        async_mark = pytest.mark.asyncio
-    except ImportError:
-        async_mark = pytest.mark.skip(reason="Missing dependency pytest-asyncio")
-
-    return async_mark
-
-
 def mark_array_manager_not_yet_implemented(request) -> None:
     mark = pytest.mark.xfail(reason="Not yet implemented for ArrayManager")
     request.node.add_marker(mark)

pyproject.toml

@@ -61,7 +61,7 @@ repository = 'https://github.com/pandas-dev/pandas'
 matplotlib = "pandas:plotting._matplotlib"
 
 [project.optional-dependencies]
-test = ['hypothesis>=6.46.1', 'pytest>=7.3.2', 'pytest-xdist>=2.2.0', 'pytest-asyncio>=0.17.0']
+test = ['hypothesis>=6.46.1', 'pytest>=7.3.2', 'pytest-xdist>=2.2.0']
 performance = ['bottleneck>=1.3.4', 'numba>=0.55.2', 'numexpr>=2.8.0']
 computation = ['scipy>=1.8.1', 'xarray>=2022.03.0']
 fss = ['fsspec>=2022.05.0']
@@ -109,7 +109,6 @@ all = ['beautifulsoup4>=4.11.1',
        'pyreadstat>=1.1.5',
        'pytest>=7.3.2',
        'pytest-xdist>=2.2.0',
-       'pytest-asyncio>=0.17.0',
        'pyxlsb>=1.0.9',
        'qtpy>=2.2.0',
        'scipy>=1.8.1',
@@ -151,7 +150,7 @@ setup = ['--vsenv'] # For Windows
 skip = "cp36-* cp37-* cp38-* pp* *_i686 *_ppc64le *_s390x *-musllinux_aarch64"
 build-verbosity = "3"
 environment = {LDFLAGS="-Wl,--strip-all"}
-test-requires = "hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17"
+test-requires = "hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0"
 test-command = """
   PANDAS_CI='1' python -c 'import pandas as pd; \
   pd.test(extra_args=["-m not clipboard and not single_cpu and not slow and not network and not db", "-n 2", "--no-strict-data-files"]); \
@@ -514,7 +513,6 @@ markers = [
   "arm_slow: mark a test as slow for arm64 architecture",
   "arraymanager: mark a test to run with ArrayManager enabled",
 ]
-asyncio_mode = "strict"
 
 [tool.mypy]
 # Import discovery

requirements-dev.txt

@@ -9,7 +9,6 @@ meson-python==0.13.1
 pytest>=7.3.2
 pytest-cov
 pytest-xdist>=2.2.0
-pytest-asyncio>=0.17.0
 coverage
 python-dateutil
 numpy<2

scripts/tests/data/deps_expected_random.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=2.2.0
   - psutil
-  - pytest-asyncio>=0.17.0
   - boto3
 
   # required dependencies