pandas-dev/pandas · #71457

BUG: interval_range loses precision for large integer endpoints

brenobarreto · merged Oct 23, 20263 files · 105 + / 4 −
doc/source/whatsnew/v3.2.0.rst1 + / 0 −
@@ -197,6 +197,7 @@ Interval ^^^^^^^^ - Bug in :class:`IntervalIndex` set operations with an empty :class:`RangeIndex`, and in :class:`IntervalIndex` construction or ``astype("interval")`` from a :class:`RangeIndex` or non-interval extension array, raising an uninformative ``TypeError`` (:issue:`68343`) - Bug in :func:`concat` and :meth:`Index.append` with :class:`IntervalDtype` not retaining an extension dtype subtype, either casting it to a NumPy dtype or raising ``TypeError`` (:issue:`64297`)+- Bug in :func:`interval_range` with large integer ``start`` or ``end`` silently returning too few intervals, or empty or wrongly spaced intervals, due to float precision loss (:issue:`71288`) - Bug in :meth:`Series.shift` and :meth:`DataFrame.shift` with an :class:`IntervalDtype` column not retaining the subtype, either changing it or raising ``TypeError`` (:issue:`69922`)  Indexing
pandas/core/indexes/interval.py43 + / 4 −
@@ -1670,6 +1670,28 @@ def _is_type_compatible(a, b) -> bool:     )  +def _integer_breaks(start, end, freq, dtype: np.dtype) -> np.ndarray | None:+    """+    Helper for interval_range to generate integer breaks with exact arithmetic.++    np.arange with a float ``stop`` and np.linspace both compute in float64, which+    can't represent every integer above 2**53, and the float nudge used to include+    ``end`` is lost for large ``end``. Returns None if the inputs aren't all integers+    or don't fit in ``dtype``, so the caller can fall back to the float path.+    """+    if not all(is_integer(x) for x in (start, end, freq)) or dtype.kind not in "iu":+        return None+    start, end, freq = int(start), int(end), int(freq)+    info = np.iinfo(dtype)+    if freq <= 0 or start < info.min or end > info.max:+        return None+    # count the breaks with Python ints; numpy's own length computation for+    # arange is a float division, so give it a stop that's an exact multiple+    # of freq away from start+    periods = max((end - start) // freq + 1, 0)+    return np.arange(start, start + periods * freq, freq, dtype=dtype)++ @set_module("pandas") def interval_range(     start=None,@@ -1831,9 +1853,13 @@ def interval_range(                 or isinstance(freq, (float, np.floating))             ):                 dtype = np.dtype("float64")-            # 0.1 ensures we capture end-            breaks = np.arange(start, end + (freq * 0.1), freq)-            breaks = maybe_downcast_numeric(breaks, dtype)+            int_breaks = _integer_breaks(start, end, freq, dtype)+            if int_breaks is not None:+                breaks = int_breaks+            else:+                # 0.1 ensures we capture end+                breaks = np.arange(start, end + (freq * 0.1), freq)+                breaks = maybe_downcast_numeric(breaks, dtype)         else:             # compute the period/start/end if unspecified (at most one)             if periods is None:@@ -1843,7 +1869,20 @@ def interval_range(             elif end is None:                 end = start + (periods - 1) * freq -            breaks = np.linspace(start, end, periods)+            step = freq+            if step is None and periods > 1 and is_integer(start) and is_integer(end):+                # start/end/periods with evenly spaced integer breaks; use+                # Python ints so the span of NumPy integers can't overflow+                span = int(end) - int(start)+                if span % (periods - 1) == 0:+                    step = span // (periods - 1)+            int_breaks = None+            if step is not None:+                int_breaks = _integer_breaks(start, end, step, dtype)+            if int_breaks is not None:+                breaks = int_breaks+            else:+                breaks = np.linspace(start, end, periods)         if all(is_integer(x) for x in com.not_none(start, end, freq)):             # np.linspace always produces float output             breaks = maybe_downcast_numeric(breaks, dtype)
pandas/tests/indexes/interval/test_interval_range.py61 + / 0 −
@@ -159,6 +159,67 @@ def test_no_invalid_float_truncation(self, start, end, freq):         result = pd.interval_range(start=start, end=end, periods=4, freq=freq)         tm.assert_index_equal(result, expected) +    @pytest.mark.parametrize("freq", [1, 3, 1000])+    @pytest.mark.parametrize(+        "start",+        [+            2**50 - 1,+            2**50,+            2**52 + 1,+            2**53,+            2**53 + 1,+            2**62 + 3,+            -(2**62) - 3,+            np.iinfo(np.int64).max - 4000,+        ],+    )+    def test_large_integer_endpoints(self, start, freq):+        # GH#71288 float arithmetic dropped the last break from ~2**50 and+        # rounded the breaks above 2**53+        breaks = np.array([start + i * freq for i in range(5)], dtype=np.int64)+        expected = pd.IntervalIndex.from_breaks(breaks)+        end = int(breaks[-1])++        result = pd.interval_range(start=start, end=end, freq=freq)+        tm.assert_index_equal(result, expected)++        result = pd.interval_range(start=start, periods=4, freq=freq)+        tm.assert_index_equal(result, expected)++        result = pd.interval_range(end=end, periods=4, freq=freq)+        tm.assert_index_equal(result, expected)++        result = pd.interval_range(start=start, end=end, periods=4)+        tm.assert_index_equal(result, expected)++    def test_large_integer_early_truncation(self):+        # GH#71288+        start = 2**53 + 1+        result = pd.interval_range(start=start, end=start + 10, freq=3)+        expected = pd.IntervalIndex.from_breaks(+            np.array([start, start + 3, start + 6, start + 9], dtype=np.int64)+        )+        tm.assert_index_equal(result, expected)++    def test_large_numpy_integer_span_with_periods(self):+        # GH#71288 end - start overflows int64 for NumPy integer endpoints+        start = np.int64(-(2**62) - 1)+        end = np.int64(2**62 + 1)+        result = pd.interval_range(start=start, end=end, periods=2)+        expected = pd.IntervalIndex.from_breaks(+            np.array([start, 0, end], dtype=np.int64)+        )+        tm.assert_index_equal(result, expected)++    def test_large_unsigned_endpoints(self):+        # GH#71288+        start = np.uint64(2**63)+        result = pd.interval_range(start=start, end=start + np.uint64(4))+        expected = pd.IntervalIndex.from_breaks(+            np.array([2**63 + i for i in range(5)], dtype=np.uint64)+        )+        tm.assert_index_equal(result, expected)+     @pytest.mark.parametrize(         "start, mid, end",         [