Merge remote-tracking branch 'upstream/main' into fix-ci

Author: jorisvandenbossche

.github/workflows/unit-tests.yml

@@ -86,7 +86,6 @@ jobs:
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: 'auto'
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
-      NPY_PROMOTION_STATE: ${{ matrix.env_file == 'actions-311-numpydev.yaml' && 'weak' || 'legacy' }}
       # Clipboard tests
       QT_QPA_PLATFORM: offscreen
       REMOVE_PYARROW: ${{ matrix.name == 'Future infer strings (without pyarrow)' && '1' || '0' }}

ci/deps/actions-311-pyarrownightly.yaml

@@ -18,7 +18,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil
-  - numpy<2
+  - numpy
   - pip
 
   - pip:

pandas/tests/dtypes/test_dtypes.py

@@ -1243,3 +1243,12 @@ def test_loc_setitem_empty_labels_no_dtype_conversion():
 
     assert df.a.dtype == "int64"
     tm.assert_frame_equal(df, expected)
+
+
+def test_categorical_nan_no_dtype_conversion():
+    # GH 43996
+
+    df = pd.DataFrame({"a": Categorical([np.nan], [1]), "b": [1]})
+    expected = pd.DataFrame({"a": Categorical([1], [1]), "b": [1]})
+    df.loc[0, "a"] = np.array([1])
+    tm.assert_frame_equal(df, expected)

pandas/tests/io/test_parquet.py

@@ -17,7 +17,6 @@
     pa_version_under13p0,
     pa_version_under15p0,
     pa_version_under17p0,
-    pa_version_under18p0,
 )
 
 import pandas as pd
@@ -974,21 +973,9 @@ def test_timestamp_nanoseconds(self, pa):
         df = pd.DataFrame({"a": pd.date_range("2017-01-01", freq="1ns", periods=10)})
         check_round_trip(df, pa, write_kwargs={"version": ver})
 
-    def test_timezone_aware_index(self, request, pa, timezone_aware_date_list):
+    def test_timezone_aware_index(self, pa, timezone_aware_date_list):
         pytest.importorskip("pyarrow", "11.0.0")
 
-        if (
-            timezone_aware_date_list.tzinfo != datetime.timezone.utc
-            and pa_version_under18p0
-        ):
-            request.applymarker(
-                pytest.mark.xfail(
-                    reason=(
-                        "pyarrow returns pytz.FixedOffset while pandas "
-                        "constructs datetime.timezone https://github.com/pandas-dev/pandas/issues/37286"
-                    )
-                )
-            )
         idx = 5 * [timezone_aware_date_list]
         df = pd.DataFrame(index=idx, data={"index_as_col": idx})
 
@@ -1005,6 +992,18 @@ def test_timezone_aware_index(self, request, pa, timezone_aware_date_list):
         expected = df[:]
         if pa_version_under11p0:
             expected.index = expected.index.as_unit("ns")
+        if timezone_aware_date_list.tzinfo != datetime.timezone.utc:
+            # pyarrow returns pytz.FixedOffset while pandas constructs datetime.timezone
+            # https://github.com/pandas-dev/pandas/issues/37286
+            try:
+                import pytz
+            except ImportError:
+                pass
+            else:
+                offset = df.index.tz.utcoffset(timezone_aware_date_list)
+                tz = pytz.FixedOffset(offset.total_seconds() / 60)
+                expected.index = expected.index.tz_convert(tz)
+                expected["index_as_col"] = expected["index_as_col"].dt.tz_convert(tz)
         check_round_trip(df, pa, check_dtype=False, expected=expected)
 
     def test_filter_row_groups(self, pa):