ENH: astype("boolean") converts values as astype(bool) (GH#37614)

@@ -36,6 +36,7 @@ Other enhancements - :meth:`DataFrame.plot.area` and :meth:`Series.plot.area` now accept keyword arguments that only the lines or only the filled areas support, such as ``hatch``, ``edgecolor`` and ``marker``, instead of raising ``AttributeError``; ``hatch`` also accepts a list with one hatch per column (:issue:`12354`) - :meth:`DataFrame.to_csv` and other writers now accept an ``os.PathLike`` such as :class:`pathlib.Path` for the ``archive_name`` entry of a zip or tar ``compression`` dict (:issue:`31934`) - :meth:`DataFrame.to_json`, :meth:`Series.to_json` and :meth:`DataFrame.to_xml` now warn when a compression method is requested but no path is given, since the returned string is not compressed, and raise for an unrecognized ``compression`` as they do with a path (:issue:`32326`)+- :meth:`Series.astype` and :func:`array` with ``dtype="boolean"`` now convert values the same way as ``astype(bool)`` instead of raising for values other than booleans, 0 and 1 (e.g. ``"False"`` becomes ``True``), with missing values becoming ``NA`` (:issue:`37614`, :issue:`40566`) - :meth:`Series.groupby` now accepts ``as_index=False``, returning aggregation results as a :class:`DataFrame` with the group labels as columns (:issue:`36507`) - Added :meth:`Categorical.to_dense` and :meth:`Series.cat.to_dense` to get the values with the dtype of the categories, including extension dtypes (:issue:`26410`) - Added :meth:`DatetimeIndex.set_freq` and :meth:`TimedeltaIndex.set_freq`, which return a new index with the given ``freq`` instead of modifying it in place (:issue:`61094`)

@@ -5,12 +5,13 @@     TYPE_CHECKING,     ClassVar,     Self,-    cast, ) import warnings  import numpy as np +from pandas._config import is_nan_na+ from pandas._libs import (     lib,     missing as libmissing,@@ -20,16 +21,21 @@ from pandas.util._exceptions import find_stack_level  from pandas.core.dtypes.common import is_list_like-from pandas.core.dtypes.dtypes import register_extension_dtype+from pandas.core.dtypes.dtypes import (+    CategoricalDtype,+    register_extension_dtype,+) from pandas.core.dtypes.missing import isna  from pandas.core import ops from pandas.core.array_algos import masked_accumulations+from pandas.core.array_algos.take import take_nd from pandas.core.arrays import ExtensionArray from pandas.core.arrays.masked import (     BaseMaskedArray,     BaseMaskedDtype, )+from pandas.core.construction import extract_array  if TYPE_CHECKING:     import pyarrow@@ -200,45 +206,49 @@ def coerce_to_array(             mask = mask.copy()         return values, mask +    # Values are converted as by astype(bool), except that missing values+    #  (NA, None, NaT, and NaN unless future.distinguish_nan_and_na) are masked+    values = extract_array(values, extract_numpy=True)     mask_values = None     if isinstance(values, np.ndarray) and values.dtype == np.bool_:         if copy:             values = values.copy()-    elif isinstance(values, np.ndarray) and values.dtype.kind in "iufcb":-        mask_values = isna(values)-+    elif isinstance(values, np.ndarray) and values.dtype.kind in "iufcmM":+        if values.dtype.kind in "fc" and not is_nan_na():+            mask_values = np.zeros(values.shape, dtype=bool)+        else:+            mask_values = isna(values)+        values = values.astype(bool)+        values[mask_values] = False+    elif isinstance(getattr(values, "dtype", None), CategoricalDtype):+        # convert each category once; Categorical.astype(bool) can warn here+        cat_values, _ = coerce_to_array(values.categories._values)+        mask_values = values.codes == -1+        values = take_nd(cat_values, values.codes, fill_value=False)+    elif isinstance(values, ExtensionArray):+        mask_values = np.asarray(values.isna(), dtype=bool)         values_bool = np.zeros(len(values), dtype=bool)         values_bool[~mask_values] = values[~mask_values].astype(bool)--        if not np.all(-            values_bool[~mask_values].astype(values.dtype) == values[~mask_values]-        ):-            raise TypeError("Need to pass bool-like values")-         values = values_bool     else:         values_object = np.asarray(values, dtype=object)--        inferred_dtype = lib.infer_dtype(values_object, skipna=True)-        integer_like = ("floating", "integer", "mixed-integer-float")-        if inferred_dtype not in ("boolean", "empty", *integer_like):-            raise TypeError("Need to pass bool-like values")--        # mypy does not narrow the type of mask_values to npt.NDArray[np.bool_]-        # within this branch, it assumes it can also be None-        mask_values = cast("npt.NDArray[np.bool_]", isna(values_object))  # type: ignore[redundant-cast]+        if values_object.ndim != 1:+            raise TypeError("values must be a 1D list-like")+        mask_values = isna(values_object)+        if not is_nan_na():+            # NaN is a value rather than missing, so it converts to True+            idx = mask_values.nonzero()[0]+            is_nan = np.array(+                [+                    lib.is_float(x) or lib.is_complex(x) or lib.is_decimal(x)+                    for x in values_object[idx]+                ],+                dtype=bool,+            )+            mask_values[idx[is_nan]] = False         values = np.zeros(len(values), dtype=bool)         values[~mask_values] = values_object[~mask_values].astype(bool) -        # if the values were integer-like, validate it were actually 0/1's-        if (inferred_dtype in integer_like) and not (-            np.all(-                values[~mask_values].astype(float)-                == values_object[~mask_values].astype(float)-            )-        ):-            raise TypeError("Need to pass bool-like values")-     if mask is None and mask_values is None:         mask = np.zeros(values.shape, dtype=bool)     elif mask is None:

@@ -547,11 +547,9 @@ def from_dummies(             f"Dummy DataFrame contains NA value in column: '{col_isna_mask.idxmax()}'"         ) -    # index data with a list of all columns that are dummies-    try:-        data_to_decode = data.astype("boolean")-    except TypeError as err:-        raise TypeError("Passed DataFrame contains non-dummy data") from err+    if not data.isin([0, 1]).all().all():+        raise TypeError("Passed DataFrame contains non-dummy data")+    data_to_decode = data.astype("boolean")      # collect prefixes and get lists to slice data for each prefix     variables_slice = defaultdict(list)

@@ -50,6 +50,37 @@ def test_astype_to_boolean_array():     tm.assert_extension_array_equal(result, arr)  +@pytest.mark.parametrize(+    "values, dtype, expected",+    [+        ([0, 1, 2], "int64", [False, True, True]),+        ([0.0, 0.5, np.nan], "float64", [False, True, None]),+        (["", "False", None], "object", [False, True, None]),+        (["", "False", None], "string", [False, True, None]),+        (["", "False", None], pd.StringDtype(na_value=np.nan), [False, True, None]),+        ([0, 2, None], "category", [False, True, None]),+        (["", "x", None], "category", [False, True, None]),+        ([None, None], "category", [None, None]),+        ([0, 5, None], "timedelta64[ns]", [False, True, None]),+        (["1970-01-01", "2000-01-01", None], "M8[ns]", [False, True, None]),+        (["1970-01-01", "2000-01-01", None], "M8[ns, UTC]", [False, True, None]),+        (["1970-01-01", None], "period[D]", [True, None]),+        ([0, 2, None], "int64[pyarrow]", [False, True, None]),+    ],+)+def test_astype_from_other_dtypes_matches_astype_bool(values, dtype, expected):+    # GH#37614, GH#40566+    if dtype == "int64[pyarrow]":+        pytest.importorskip("pyarrow")+    ser = pd.Series(values, dtype=dtype)+    result = ser.astype("boolean")+    expected = pd.Series(expected, dtype="boolean")+    tm.assert_series_equal(result, expected)++    notna = ser.notna()+    tm.assert_series_equal(result[notna], ser[notna].astype(bool).astype("boolean"))++ def test_astype_to_integer_array():     # astype to IntegerArray     arr = pd.array([True, False, None], dtype="boolean")

@@ -1,3 +1,5 @@+from decimal import Decimal+ import numpy as np import pytest @@ -96,25 +98,65 @@ def test_to_boolean_array_missing_indicators(a, b):   @pytest.mark.parametrize(-    "values",+    "values, expected",+    [+        (["foo", "", "False", None], [True, False, True, None]),+        ([0, 1, 2, -1, None], [False, True, True, True, None]),+        ([0.0, 0.5, np.nan], [False, True, None]),+        ([0, "a", 1.5, pd.NA, pd.NaT], [False, True, True, None, None]),+        ([np.nan, {"a": 1}, {}], [None, True, False]),+        # a list is treated as object dtype, where every Timestamp is truthy+        ([pd.Timestamp(0), pd.NaT], [True, None]),+        (np.array(["foo", ""]), [True, False]),+        (np.array([0, 1, 2]), [False, True, True]),+        (np.array([0.0, 0.5, np.nan]), [False, True, None]),+        (np.array([0j, 1j, complex(np.nan, 0)]), [False, True, None]),+        (np.array([0, 5, "NaT"], dtype="m8[ns]"), [False, True, None]),+        (+            np.array(["1970-01-01", "2000-01-01", "NaT"], dtype="M8[ns]"),+            [False, True, None],+        ),+        (pd.date_range("1970-01-01", periods=2, tz="UTC"), [False, True]),+        (pd.period_range("2000", periods=2), [True, True]),+    ],+)+def test_to_boolean_array_matches_astype_bool(values, expected):+    # GH#37614 values are converted as by astype(bool), with missing values masked+    result = pd.array(values, dtype="boolean")+    expected = pd.array(expected, dtype="boolean")+    tm.assert_extension_array_equal(result, expected)+++@pytest.mark.parametrize(+    "values, expected",     [-        ["foo", "bar"],-        ["1", "2"],-        # "foo",-        [1, 2],-        [1.0, 2.0],-        pd.date_range("20130101", periods=2),-        np.array(["foo"]),-        np.array([1, 2]),-        np.array([1.0, 2.0]),-        [np.nan, {"a": 1}],+        (+            [0.0, np.nan, None, pd.NA, pd.NaT, Decimal("nan")],+            [False, True, None, None, None, True],+        ),+        (np.array([0.0, np.nan]), [False, True]),+        (np.array([0j, complex(np.nan, 0)]), [False, True]),     ], )-def test_to_boolean_array_error(values):-    # error in converting existing arrays to BooleanArray-    msg = "Need to pass bool-like value"-    with pytest.raises(TypeError, match=msg):-        pd.array(values, dtype="boolean")+def test_to_boolean_array_distinguish_nan_and_na(values, expected):+    # with NaN distinct from NA, NaN is a value and converts as bool(np.nan)+    with pd.option_context("future.distinguish_nan_and_na", True):+        result = pd.array(values, dtype="boolean")+    expected = pd.array(expected, dtype="boolean")+    tm.assert_extension_array_equal(result, expected)+++def test_float_ea_astype_boolean_distinguish_nan_and_na():+    with pd.option_context("future.distinguish_nan_and_na", True):+        arr = pd.array([0.0, np.nan, None], dtype="Float64")+        result = arr.astype("boolean")+    expected = pd.array([False, True, None], dtype="boolean")+    tm.assert_extension_array_equal(result, expected)+++def test_to_boolean_array_2d_raises():+    with pytest.raises(TypeError, match="values must be a 1D list-like"):+        pd.array([[True, False]], dtype="boolean")   def test_to_boolean_array_from_integer_array():

@@ -95,6 +95,14 @@ def test_error_contains_non_dummies():         pd.from_dummies(dummies)  +@pytest.mark.parametrize("values", [[0, 0.5], ["0", "1"]])+def test_error_contains_non_dummies_bool_castable(values):+    # values that astype("boolean") accepts but are not 0 or 1+    dummies = pd.DataFrame({"a": values, "b": [1, 0]})+    with pytest.raises(TypeError, match=r"Passed DataFrame contains non-dummy data"):+        pd.from_dummies(dummies)++ def test_error_with_prefix_multiple_separators():     dummies = pd.DataFrame(         {@@ -250,6 +258,20 @@ def test_no_prefix_string_cats_basic_mixed_bool_values():     tm.assert_frame_equal(result, expected)  +@pytest.mark.parametrize(+    "dtype", ["int64", "uint8", "float64", "Int64", "boolean", "int64[pyarrow]"]+)+def test_no_prefix_string_cats_basic_dtypes(dtype):+    if dtype == "int64[pyarrow]":+        pytest.importorskip("pyarrow")+    dummies = pd.DataFrame(+        {"a": [1, 0, 0], "b": [0, 1, 0], "c": [0, 0, 1]}, dtype=dtype+    )+    expected = pd.DataFrame({"": ["a", "b", "c"]})+    result = pd.from_dummies(dummies)+    tm.assert_frame_equal(result, expected)++ def test_no_prefix_int_cats_basic():     dummies = pd.DataFrame(         {1: [1, 0, 0, 0], 25: [0, 1, 0, 0], 2: [0, 0, 1, 0], 5: [0, 0, 0, 1]}

@@ -511,13 +511,8 @@ class TestAstypeString:         ],     )     def test_astype_string_to_extension_dtype_roundtrip(-        self, data, dtype, request, nullable_string_dtype+        self, data, dtype, nullable_string_dtype     ):-        if dtype == "boolean":-            mark = pytest.mark.xfail(-                reason="TODO StringArray.astype() with missing values #GH40566"-            )-            request.applymarker(mark)         # GH-40351         ser = pd.Series(data, dtype=dtype)