BUG: read_csv pyarrow engine silently wrapped out-of-range integer dtype (GH#38013)

@@ -227,6 +227,7 @@ I/O - Bug in :func:`read_csv` silently misparsing non-ASCII ``thousands`` and ``decimal`` markers with the C engine; these now fall back to the Python engine with a :class:`errors.ParserWarning`, or raise ``ValueError`` if ``engine="c"`` is explicitly specified (:issue:`71158`) - Bug in :func:`read_csv` where a ``dtype`` given for a ``parse_dates`` column undid the date parsing, returning strings, objects, integers or categories instead of datetimes (:issue:`57512`) - Bug in :func:`read_csv` with ``engine="c"`` and ``comment`` not skipping a line containing only whitespace before the comment character (:issue:`55916`)+- Bug in :func:`read_csv` with ``engine="pyarrow"`` silently wrapping values outside the range of the requested integer ``dtype`` instead of raising (:issue:`38013`) - Bug in :func:`read_csv` with ``engine="python"`` and ``index_col`` reading a date like ``"20180101"`` as nanoseconds since the epoch when ``dtype`` is a pyarrow timestamp dtype (:issue:`24542`) - Bug in :func:`read_csv`, :func:`read_excel`, :func:`read_fwf` and :func:`read_html` returning wrong column labels and treating header rows as data when ``header`` was a list not in increasing order, such as ``header=[1, 0]`` (:issue:`47011`) - Bug in :func:`read_csv`, :meth:`DataFrame.to_csv` and other I/O functions raising ``URLError`` for ``sftp://`` URLs instead of reading or writing them with ``fsspec`` (:issue:`46765`)

@@ -26,6 +26,8 @@ from pandas.errors import Pandas4Warning  from pandas.core.dtypes.common import pandas_dtype+from pandas.core.dtypes.dtypes import BaseMaskedDtype+from pandas.core.dtypes.missing import isna  import pandas as pd @@ -42,6 +44,7 @@     from pandas._typing import (         DtypeArg,         DtypeBackend,+        DtypeObj,     )      from pandas.core.dtypes.base import ExtensionDtype@@ -213,10 +216,12 @@ def _post_convert_dtypes(             dtype = pandas_dtype(dtype)          try:-            df = df.astype(dtype)+            casted = df.astype(dtype)         except TypeError as err:             # GH#44901 reraise to keep api consistent             raise ValueError(str(err)) from err+        validate_integer_casts(df, casted)+        df = casted          # GH#56136 IntegerDtype was used above to avoid lossy float64         #  conversion in pyarrow; convert back to numpy now that the data@@ -271,6 +276,61 @@ def _post_convert_dtypes(     return df  +def _numpy_dtype(dtype: DtypeObj) -> np.dtype | None:+    # the numpy dtype underlying a column's dtype, if any+    if isinstance(dtype, np.dtype):+        return dtype+    if isinstance(dtype, pd.SparseDtype):+        return dtype.subtype+    if isinstance(dtype, (BaseMaskedDtype, pd.ArrowDtype)):+        return dtype.numpy_dtype+    return None+++def _can_wrap(old: DtypeObj, new: DtypeObj) -> bool:+    """+    Whether casting from ``old`` to ``new`` can wrap around an integer value.+    """+    old_np = _numpy_dtype(old)+    new_np = _numpy_dtype(new)+    if old_np is None or new_np is None:+        return False+    if new_np.kind not in "iu" or old_np.kind not in "iuf":+        return False+    # NA cannot wrap, so e.g. Int64 -> int64 is as safe as int64 -> int64+    return not np.can_cast(old_np, new_np)+++def validate_integer_casts(original: pd.DataFrame, casted: pd.DataFrame) -> None:+    """+    Apply validate_integer_cast to each column of a DataFrame cast.+    """+    for i, (old, new) in enumerate(zip(original.dtypes, casted.dtypes, strict=True)):+        # check dtypes first; iloc on every column is slow for wide frames+        if _can_wrap(old, new):+            validate_integer_cast(original.iloc[:, i], new)+++def validate_integer_cast(original: pd.Series, dtype: DtypeObj) -> None:+    """+    Raise if casting a column to an integer ``dtype`` wrapped around (GH#38013).++    Discarding a float's fractional part is allowed, as in the python engine.+    """+    if not _can_wrap(original.dtype, dtype):+        return+    lo, hi = original.min(), original.max()+    if isna(lo):+        return+    info = np.iinfo(_numpy_dtype(dtype))+    # int() truncates exactly, so e.g. 2.0**63 does not equal int64 max+    if int(lo) < info.min or int(hi) > info.max:+        raise ValueError(+            f"cannot safely convert passed user dtype of {dtype} for "+            f"{_numpy_dtype(original.dtype)} dtyped data in column {original.name}"+        )++ def _maybe_convert_string_to_object(     data: pd.Series | pd.Index, ) -> pd.Series | pd.Index | None:

@@ -30,7 +30,11 @@  from pandas.core.arrays.arrow.array import to_pyarrow_type -from pandas.io._util import arrow_table_to_pandas+from pandas.io._util import (+    arrow_table_to_pandas,+    validate_integer_cast,+    validate_integer_casts,+) from pandas.io.common import mangle_dupe_names from pandas.io.parsers.base_parser import (     ParserBase,@@ -436,7 +440,9 @@ def _finalize_index(self, frame: DataFrame, multi_index_named: bool) -> DataFram                             and pandas_dtype(new_dtype).kind != "M"                             and parse_as_dates                         ):-                            frame[key] = frame[key].astype(new_dtype)+                            casted = frame[key].astype(new_dtype)+                            validate_integer_cast(frame[key], casted.dtype)+                            frame[key] = casted                         del self.dtype[key]              if self.dtype is not None and not isinstance(self.dtype, dict):@@ -495,10 +501,12 @@ def _finalize_dtype(                     if col not in parse_dates_cols                 }             try:-                frame = frame.astype(self.dtype)+                casted = frame.astype(self.dtype)             except TypeError as err:                 # GH#44901 reraise to keep api consistent                 raise ValueError(str(err)) from err+            validate_integer_casts(frame, casted)+            frame = casted         return frame      def _finalize_pandas_output(

@@ -1051,7 +1051,6 @@ def test_leading_zeros_preserved_with_dtype_dict(all_parsers):     tm.assert_frame_equal(result, expected)  -@xfail_pyarrow  # pyarrow engine casts the parsed frame; wraps, or raises ValueError @pytest.mark.parametrize("dtype_backend", [lib.no_default, "numpy_nullable", "pyarrow"]) @pytest.mark.parametrize(     "dtype, err, msg",@@ -1069,19 +1068,25 @@ def test_out_of_range_integer_dtype_raises(all_parsers, dtype, err, msg, dtype_b     if dtype_backend == "pyarrow":         pytest.importorskip("pyarrow")     parser = all_parsers+    if parser.engine == "pyarrow":+        # this engine raises ValueError for the nullable dtypes too+        err = ValueError+        msg = "cannot safely (cast non-equivalent|convert passed user dtype)"     data = "x\n-1\n257\n"     with pytest.raises(err, match=msg):         parser.read_csv(StringIO(data), dtype={"x": dtype}, dtype_backend=dtype_backend)  -@xfail_pyarrow  # pyarrow engine casts the parsed frame; wraps, or raises ValueError @pytest.mark.parametrize(     "data, dtype",     [         # only fits uint64, so the c engine falls back to _try_uint64         ("x\n18446744073709551615\n1\n", "uint8"),         # the requested dtype is the int64 that overflowed into the fallback         ("x\n18446744073709551615\n1\n", "int64"),+        # one past the int64 max; the pyarrow engine reads it as a float+        #  whose cast to int64 can saturate rather than wrap+        ("x\n9223372036854775808\n", "int64"),         # parses as float rather than int, which both engines must check too         ("x\n300.0\n1.0\n", "uint8"),         ("x\n257.0\n1.0\n", "int8"),@@ -1103,3 +1108,69 @@ def test_wrap_object_columns_only_wraps_object():     result = wrap_object_columns({"a": ints, "b": strings}, dtype, pd.RangeIndex(2))     assert result["a"] is ints     tm.assert_series_equal(result["b"], pd.Series(strings, dtype=object))+++@pytest.mark.parametrize("dtype_backend", [lib.no_default, "numpy_nullable", "pyarrow"])+@pytest.mark.parametrize("index_col", [None, 0])+def test_out_of_range_int32_dtype_raises(all_parsers, index_col, dtype_backend):+    # GH#38013 with a non-default dtype_backend, the pyarrow engine casts+    #  index columns separately+    if dtype_backend == "pyarrow":+        pytest.importorskip("pyarrow")+    parser = all_parsers+    data = "x,y\n10000000000,0\n"+    with pytest.raises(ValueError, match="cannot safely convert passed user dtype"):+        parser.read_csv(+            StringIO(data),+            dtype={"x": "int32"},+            index_col=index_col,+            dtype_backend=dtype_backend,+        )+++@pytest.mark.parametrize(+    "data, dtype, dtype_backend, expected",+    [+        ("x\n1e16\n", "int64[pyarrow]", lib.no_default, 10**16),+        ("x\n1e16\n", "Int64", "pyarrow", 10**16),+        ("x\n9223372036854775808\n", "uint64[pyarrow]", lib.no_default, 2**63),+    ],+)+def test_in_range_integer_above_float_precision(+    all_parsers, data, dtype, dtype_backend, expected+):+    # GH#38013 in-range values above 2**53 must not be rejected+    pytest.importorskip("pyarrow")+    parser = all_parsers+    result = parser.read_csv(+        StringIO(data), dtype={"x": dtype}, dtype_backend=dtype_backend+    )+    assert result["x"].tolist() == [expected]+++def test_sparse_integer_dtype(pyarrow_parser_only):+    # GH#38013 the overflow check covers sparse dtypes by their subtype+    parser = pyarrow_parser_only+    result = parser.read_csv(+        StringIO("x\n1\n300\n"), dtype={"x": pd.SparseDtype("int64")}+    )+    expected = pd.DataFrame({"x": pd.arrays.SparseArray([1, 300], dtype="int64")})+    tm.assert_frame_equal(result, expected)+    with pytest.raises(ValueError, match="cannot safely convert passed user dtype"):+        parser.read_csv(StringIO("x\n1\n300\n"), dtype={"x": pd.SparseDtype("int8")})+++def test_integer_dtype_truncates_float(pyarrow_parser_only):+    # GH#38013 dropping the fractional part is not a wrap, as with the python+    #  engine+    parser = pyarrow_parser_only+    result = parser.read_csv(StringIO("x\n1.5\n127.9\n-128.9\n"), dtype={"x": "int8"})+    assert result["x"].tolist() == [1, 127, -128]+++def test_uint64_max_read_as_float_raises(pyarrow_parser_only):+    # GH#38013 a pyarrow inference limitation, not desired behavior: pyarrow+    #  reads this value as the float 2**64, which is out of range for uint64+    parser = pyarrow_parser_only+    with pytest.raises(ValueError, match="cannot safely convert passed user dtype"):+        parser.read_csv(StringIO("x\n18446744073709551615\n"), dtype={"x": "uint64"})