BUG: ArrowExtensionArray silently truncated fractional values set from a list into an integer dtype

@@ -320,7 +320,7 @@ ExtensionArray ^^^^^^^^^^^^^^ - Bug in :func:`read_csv` and other parsing of strings into a decimal :class:`ArrowDtype` raising ``ArrowInvalid`` instead of parsing the values (:issue:`69838`) - Bug in :meth:`Series.mode` and :meth:`DataFrame.mode` with ``dropna=False`` raising ``TypeError`` for ``"boolean"`` dtype (:issue:`70487`)-- Bug in :meth:`Series.replace`, :meth:`Series.where`, :meth:`Series.fillna`, item assignment and related methods on a PyArrow-backed integer dtype silently truncating a fractional value (``replace(2, 1.5)`` stored ``1``); these now raise ``TypeError``, while :meth:`Index.where`, :meth:`Index.putmask` and :meth:`Index.fillna` upcast, matching the nullable integer dtypes (:issue:`68638`)+- Bug in :meth:`Series.replace`, :meth:`Series.where`, :meth:`Series.fillna`, item assignment and related methods on a PyArrow-backed integer dtype silently truncating a fractional value, including one passed in a list (``replace(2, 1.5)`` stored ``1``); these now raise ``TypeError``, while :meth:`Index.where`, :meth:`Index.putmask` and :meth:`Index.fillna` upcast, matching the nullable integer dtypes (:issue:`68638`, :issue:`71076`) - Bug in arithmetic and logical operations on PyArrow-backed data raising ``pyarrow.ArrowInvalid`` or ``pyarrow.ArrowTypeError`` instead of ``TypeError`` when the other operand contains objects PyArrow cannot convert (:issue:`62682`) - Bug in comparisons between PyArrow-backed data and objects PyArrow cannot convert raising ``pyarrow.ArrowInvalid`` or ``pyarrow.ArrowTypeError``; ``==``/``!=`` now compare elementwise, ordered comparisons raise ``TypeError``, and a missing value on either side propagates instead of answering ``False`` (:issue:`62682`) 

@@ -3470,10 +3470,12 @@ def _validate_setitem_value(self, value):         pa_type = self._pa_array.type         if pa.types.is_integer(pa_type):             # GH#68638 pyarrow truncates the fractional part instead of refusing-            #  the value; the masked dtypes raise+            #  the value; the masked dtypes raise. Its safe cast catches a numpy+            #  array but not a list, so list-likes are checked here too             value = lib.item_from_zerodim(value)-            if _truncates_to_integer(value):-                raise TypeError(f"Invalid value '{value!s}' for dtype '{self.dtype}'")+            bad = _first_truncated_to_integer(value)+            if bad is not None:+                raise TypeError(f"Invalid value '{bad!s}' for dtype '{self.dtype}'")         self_temporal = _is_temporal_pa_type(pa_type)         if self_temporal is not None:             value_temporal = _is_temporal_value(value)@@ -4844,6 +4846,61 @@ def _truncates_to_integer(value) -> bool:     return False  +def _may_truncate_to_integer(values) -> bool:+    """+    Whether ``values`` could hold anything an integer-typed pyarrow column would+    truncate. Cheaper than inspecting the entries. See GH#71076.+    """+    return lib.infer_dtype(values, skipna=True) not in (+        "integer",+        "boolean",+        "string",+        "empty",+    )+++def _first_truncated_to_integer(value):+    """+    The first entry of ``value`` an integer-typed pyarrow column would truncate,+    or None if there is none. See GH#71076.+    """+    if not is_list_like(value) or isinstance(value, (pa.Array, pa.ChunkedArray)):+        # a pyarrow container is cast, not converted, so pyarrow rejects a lossy+        #  value itself; see test_setitem_lossy_float_pyarrow_container_raises+        return value if _truncates_to_integer(value) else None++    dtype = getattr(value, "dtype", None)+    if getattr(dtype, "kind", None) in ("i", "u", "b"):+        # skip materializing a container whose own dtype has nothing to drop+        return None+    if isinstance(value, (list, tuple)) and not _may_truncate_to_integer(value):+        # same, for a container that has no dtype to ask+        return None++    if isinstance(value, np.ndarray):+        arr = value+    else:+        try:+            arr = np.asarray(value)+        except ValueError:+            # ragged, handled the way _box_pa_array handles it+            arr = construct_1d_object_array_from_listlike(value)+    if arr.dtype.kind == "f":+        lossy = ~isna(arr) & ((arr != np.trunc(arr)) | np.isinf(arr))+        return arr[lossy].flat[0] if lossy.any() else None+    if arr.dtype.kind != "O":+        # pyarrow refuses every other numpy dtype that could have one+        return None+    flat = arr.ravel()+    if not _may_truncate_to_integer(flat):+        return None+    for item in flat:+        item = lib.item_from_zerodim(item)+        if _truncates_to_integer(item):+            return item+    return None++ def transpose_homogeneous_pyarrow(     arrays: Sequence[ArrowExtensionArray], ) -> list[ArrowExtensionArray]:

@@ -637,12 +637,13 @@ def test_replace_lossy_float_raises(box):         lambda ser: ser.mask([False, True, False], 1.5),         lambda ser: ser.fillna(1.5),         lambda ser: ser.clip(lower=1.5),+        lambda ser: ser.update(pd.Series([1.5])),         lambda ser: ser.reindex([0, 1, 5], fill_value=1.5),     ],-    ids=["where", "mask", "fillna", "clip", "reindex"],+    ids=["where", "mask", "fillna", "clip", "update", "reindex"], ) def test_lossy_float_raises(op):-    # GH#68638 methods that pass a scalar through _validate_setitem_value+    # GH#68638 callers that route through _validate_setitem_value     ser = pd.Series([1, None, 3], dtype="int64[pyarrow]")     with pytest.raises(TypeError, match="Invalid value '1.5'"):         op(ser)@@ -657,23 +658,47 @@ def test_lossy_float_raises(op):         Fraction(3, 2),         float("inf"),         Decimal("Infinity"),+        [9.5, 8.5],+        (9.5, 8.5),+        np.array([9.5, 8.5]),+        np.array([np.inf, 1.0]),+        [Decimal("1.5"), Decimal("2")],+        [Fraction(3, 2), 8],     ], ) def test_setitem_lossy_float_raises(value):     # GH#68638 pyarrow truncated a fractional value; a non-finite one raised-    #  ArrowInvalid+    #  ArrowInvalid. GH#71076 pa.array's safe cast catches a numpy array but+    #  truncates a list     arr = pd.array([1, 2, 3], dtype="int64[pyarrow]")     with pytest.raises(TypeError, match="Invalid value"):-        arr[1] = value+        arr[:2] = value     assert list(arr) == [1, 2, 3]  -@pytest.mark.parametrize("value", [4, 4.0, Decimal("4"), np.array(4), Fraction(8, 2)])+@pytest.mark.parametrize(+    "box",+    [pa.array, lambda values: pa.chunked_array([values])],+    ids=["array", "chunked"],+)+def test_setitem_lossy_float_pyarrow_container_raises(box):+    # GH#71076 a pyarrow container is skipped by the check because pyarrow's own+    #  cast already refuses it, so it raises ArrowInvalid rather than TypeError+    arr = pd.array([1, 2, 3], dtype="int64[pyarrow]")+    with pytest.raises(pa.ArrowInvalid, match="truncated"):+        arr[:2] = box([9.5, 8.5])+    assert list(arr) == [1, 2, 3]+++@pytest.mark.parametrize(+    "value",+    [4, 4.0, Decimal("4"), np.array(4), Fraction(8, 2), [4, 4.0], np.array([4.0, 4.0])],+) def test_setitem_integral_value_still_accepted(value):     # GH#68638 only a fractional value is refused     arr = pd.array([1, 2, 3], dtype="int64[pyarrow]")-    arr[1] = value-    assert arr[1] == 4+    arr[:2] = value+    assert list(arr) == [4, 4, 3]   @pytest.mark.parametrize(@@ -701,6 +726,13 @@ def test_index_fillna_lossy_float_upcasts():     tm.assert_index_equal(result, expected)  +def test_fillna_lossy_float_list_like_raises():+    # GH#71076 a list-like fill value is checked too, not only a scalar+    arr = pd.array([1, None, 3], dtype="int64[pyarrow]")+    with pytest.raises(TypeError, match="Invalid value '8.5'"):+        arr.fillna([9.0, 8.5, 7.0])++ def test_arrow_string_accepts_pyarrow_string_scalar():     # GH#71075 Series.where stored a character code of the string     arr = pd.array(["a", None], dtype="string[pyarrow]")