pandas-dev/pandas · #71457
BUG: interval_range loses precision for large integer endpoints
doc/source/whatsnew/v3.2.0.rst1 + / 0 −
@@ -197,6 +197,7 @@ Interval ^^^^^^^^ - Bug in :class:`IntervalIndex` set operations with an empty :class:`RangeIndex`, and in :class:`IntervalIndex` construction or ``astype("interval")`` from a :class:`RangeIndex` or non-interval extension array, raising an uninformative ``TypeError`` (:issue:`68343`) - Bug in :func:`concat` and :meth:`Index.append` with :class:`IntervalDtype` not retaining an extension dtype subtype, either casting it to a NumPy dtype or raising ``TypeError`` (:issue:`64297`)+- Bug in :func:`interval_range` with large integer ``start`` or ``end`` silently returning too few intervals, or empty or wrongly spaced intervals, due to float precision loss (:issue:`71288`) - Bug in :meth:`Series.shift` and :meth:`DataFrame.shift` with an :class:`IntervalDtype` column not retaining the subtype, either changing it or raising ``TypeError`` (:issue:`69922`) Indexingpandas/core/indexes/interval.py43 + / 4 −
@@ -1670,6 +1670,28 @@ def _is_type_compatible(a, b) -> bool: ) +def _integer_breaks(start, end, freq, dtype: np.dtype) -> np.ndarray | None:+ """+ Helper for interval_range to generate integer breaks with exact arithmetic.++ np.arange with a float ``stop`` and np.linspace both compute in float64, which+ can't represent every integer above 2**53, and the float nudge used to include+ ``end`` is lost for large ``end``. Returns None if the inputs aren't all integers+ or don't fit in ``dtype``, so the caller can fall back to the float path.+ """+ if not all(is_integer(x) for x in (start, end, freq)) or dtype.kind not in "iu":+ return None+ start, end, freq = int(start), int(end), int(freq)+ info = np.iinfo(dtype)+ if freq <= 0 or start < info.min or end > info.max:+ return None+ # count the breaks with Python ints; numpy's own length computation for+ # arange is a float division, so give it a stop that's an exact multiple+ # of freq away from start+ periods = max((end - start) // freq + 1, 0)+ return np.arange(start, start + periods * freq, freq, dtype=dtype)++ @set_module("pandas") def interval_range( start=None,@@ -1831,9 +1853,13 @@ def interval_range( or isinstance(freq, (float, np.floating)) ): dtype = np.dtype("float64")- # 0.1 ensures we capture end- breaks = np.arange(start, end + (freq * 0.1), freq)- breaks = maybe_downcast_numeric(breaks, dtype)+ int_breaks = _integer_breaks(start, end, freq, dtype)+ if int_breaks is not None:+ breaks = int_breaks+ else:+ # 0.1 ensures we capture end+ breaks = np.arange(start, end + (freq * 0.1), freq)+ breaks = maybe_downcast_numeric(breaks, dtype) else: # compute the period/start/end if unspecified (at most one) if periods is None:@@ -1843,7 +1869,20 @@ def interval_range( elif end is None: end = start + (periods - 1) * freq - breaks = np.linspace(start, end, periods)+ step = freq+ if step is None and periods > 1 and is_integer(start) and is_integer(end):+ # start/end/periods with evenly spaced integer breaks; use+ # Python ints so the span of NumPy integers can't overflow+ span = int(end) - int(start)+ if span % (periods - 1) == 0:+ step = span // (periods - 1)+ int_breaks = None+ if step is not None:+ int_breaks = _integer_breaks(start, end, step, dtype)+ if int_breaks is not None:+ breaks = int_breaks+ else:+ breaks = np.linspace(start, end, periods) if all(is_integer(x) for x in com.not_none(start, end, freq)): # np.linspace always produces float output breaks = maybe_downcast_numeric(breaks, dtype)pandas/tests/indexes/interval/test_interval_range.py61 + / 0 −
@@ -159,6 +159,67 @@ def test_no_invalid_float_truncation(self, start, end, freq): result = pd.interval_range(start=start, end=end, periods=4, freq=freq) tm.assert_index_equal(result, expected) + @pytest.mark.parametrize("freq", [1, 3, 1000])+ @pytest.mark.parametrize(+ "start",+ [+ 2**50 - 1,+ 2**50,+ 2**52 + 1,+ 2**53,+ 2**53 + 1,+ 2**62 + 3,+ -(2**62) - 3,+ np.iinfo(np.int64).max - 4000,+ ],+ )+ def test_large_integer_endpoints(self, start, freq):+ # GH#71288 float arithmetic dropped the last break from ~2**50 and+ # rounded the breaks above 2**53+ breaks = np.array([start + i * freq for i in range(5)], dtype=np.int64)+ expected = pd.IntervalIndex.from_breaks(breaks)+ end = int(breaks[-1])++ result = pd.interval_range(start=start, end=end, freq=freq)+ tm.assert_index_equal(result, expected)++ result = pd.interval_range(start=start, periods=4, freq=freq)+ tm.assert_index_equal(result, expected)++ result = pd.interval_range(end=end, periods=4, freq=freq)+ tm.assert_index_equal(result, expected)++ result = pd.interval_range(start=start, end=end, periods=4)+ tm.assert_index_equal(result, expected)++ def test_large_integer_early_truncation(self):+ # GH#71288+ start = 2**53 + 1+ result = pd.interval_range(start=start, end=start + 10, freq=3)+ expected = pd.IntervalIndex.from_breaks(+ np.array([start, start + 3, start + 6, start + 9], dtype=np.int64)+ )+ tm.assert_index_equal(result, expected)++ def test_large_numpy_integer_span_with_periods(self):+ # GH#71288 end - start overflows int64 for NumPy integer endpoints+ start = np.int64(-(2**62) - 1)+ end = np.int64(2**62 + 1)+ result = pd.interval_range(start=start, end=end, periods=2)+ expected = pd.IntervalIndex.from_breaks(+ np.array([start, 0, end], dtype=np.int64)+ )+ tm.assert_index_equal(result, expected)++ def test_large_unsigned_endpoints(self):+ # GH#71288+ start = np.uint64(2**63)+ result = pd.interval_range(start=start, end=start + np.uint64(4))+ expected = pd.IntervalIndex.from_breaks(+ np.array([2**63 + i for i in range(5)], dtype=np.uint64)+ )+ tm.assert_index_equal(result, expected)+ @pytest.mark.parametrize( "start, mid, end", [