Repository: astropy/astropy

Issue:
# Deprecate use of chararray in io.fits

As raised in #3854 and earlier issues, Numpy is gradually trying to deprecate and eventually kill off the old Numarray `chararray` class.  This is fine, but `io.fits` still uses it for FITS tables.  This is especially important in supporting FITS ASCII tables, for its ability to automatically `rstrip()` individual strings returned from the array.

However, for FITS data we can just use our own class that has this capability (we don't need any of the other `chararray` methods).  It might be nice to go ahead and use some version of my [`encoded_text_array`](https://github.com/embray/PyFITS/blob/c11abd331cff7e177f0fc551188347e529cde92d/lib/pyfits/fitsrec.py#L999) class for FITS string columns (and possible string columns in other Astropy tables), and to add _optional_ rstrip support to that class (it already has automatic rstripping, but that should probably be optional).

I think for backwards compat the new class _should_ inherit `chararray` methods, but they would all raise deprecation warnings.  Similarly `isinstance(..., chararray)` checks should raise a deprecation warning (that the class will no longer be a chararray subclass after long).


Implementation patch:
diff --git a/astropy/io/fits/column.py b/astropy/io/fits/column.py
index 804adb07c95f..ff513628367f 100644
--- a/astropy/io/fits/column.py
+++ b/astropy/io/fits/column.py
@@ -14,9 +14,9 @@
 from textwrap import indent
 
 import numpy as np
-from numpy import char as chararray
 
 from astropy.utils import lazyproperty
+from astropy.utils.compat import chararray, get_chararray
 from astropy.utils.exceptions import AstropyUserWarning
 
 from .card import CARD_LENGTH, Card
@@ -696,14 +696,14 @@ def __init__(
         # input arrays can be just list or tuple, not required to be ndarray
         # does not include Object array because there is no guarantee
         # the elements in the object array are consistent.
-        if not isinstance(array, (np.ndarray, chararray.chararray, Delayed)):
+        if not isinstance(array, (np.ndarray, chararray, Delayed)):
             try:  # try to convert to a ndarray first
                 if array is not None:
                     array = np.array(array)
             except Exception:
                 try:  # then try to convert it to a strings array
                     itemsize = int(recformat[1:])
-                    array = chararray.array(array, itemsize=itemsize)
+                    array = get_chararray(array, itemsize=itemsize)
                 except ValueError:
                     # then try variable length array
                     # Note: This includes _FormatQ by inheritance
@@ -1388,7 +1388,7 @@ def _convert_to_valid_data_type(self, array):
                         fsize = dims[-1]
                     else:
                         fsize = np.dtype(format.recformat).itemsize
-                    return chararray.array(array, itemsize=fsize, copy=False)
+                    return get_chararray(array, itemsize=fsize, copy=False)
                 else:
                     return _convert_array(array, np.dtype(format.recformat))
             elif "L" in format:
@@ -2082,7 +2082,7 @@ def __new__(cls, input, dtype="S"):
             try:
                 # this handles ['abc'] and [['a','b','c']]
                 # equally, beautiful!
-                input = [chararray.array(x, itemsize=1) for x in input]
+                input = [get_chararray(x, itemsize=1) for x in input]
             except Exception:
                 raise ValueError(f"Inconsistent input data array: {input}")
 
@@ -2105,10 +2105,10 @@ def __setitem__(self, key, value):
         """
         if isinstance(value, np.ndarray) and value.dtype == self.dtype:
             pass
-        elif isinstance(value, chararray.chararray) and value.itemsize == 1:
+        elif isinstance(value, chararray) and value.itemsize == 1:
             pass
         elif self.element_dtype == "S":
-            value = chararray.array(value, itemsize=1)
+            value = get_chararray(value, itemsize=1)
         else:
             value = np.array(value, dtype=self.element_dtype)
         np.ndarray.__setitem__(self, key, value)
@@ -2262,7 +2262,7 @@ def _makep(array, descr_output, format, nrows=None):
             else:
                 rowval = [0] * data_output.max
         if format.dtype == "S":
-            data_output[idx] = chararray.array(encode_ascii(rowval), itemsize=1)
+            data_output[idx] = get_chararray(encode_ascii(rowval), itemsize=1)
         else:
             data_output[idx] = np.array(rowval, dtype=format.dtype)
 
diff --git a/astropy/io/fits/connect.py b/astropy/io/fits/connect.py
index 1439c62b6caf..6643ab1ee54a 100644
--- a/astropy/io/fits/connect.py
+++ b/astropy/io/fits/connect.py
@@ -295,7 +295,7 @@ def read_table_fits(
         coltype = col.dtype.subdtype[0].type if col.dtype.subdtype else col.dtype.type
 
         if strip_spaces and coltype is np.bytes_:
-            arr = arr.rstrip()
+            arr = np.strings.rstrip(arr)
 
         # Check if column is masked. Here, we make a guess based on the
         # presence of FITS mask values. For integer columns, this is simply
diff --git a/astropy/io/fits/fitsrec.py b/astropy/io/fits/fitsrec.py
index 6f5e0f511cc1..32b8ec436424 100644
--- a/astropy/io/fits/fitsrec.py
+++ b/astropy/io/fits/fitsrec.py
@@ -8,9 +8,9 @@
 from functools import reduce
 
 import numpy as np
-from numpy import char as chararray
 
 from astropy.utils import lazyproperty
+from astropy.utils.compat import chararray, get_chararray
 
 from .column import (
     _VLF,
@@ -831,7 +831,7 @@ def _convert_p(self, column, field, recformat):
                 dt = np.dtype(recformat.dtype + str(1))
                 arr_len = count * dt.itemsize
                 da = raw_data[offset : offset + arr_len].view(dt)
-                da = np.char.array(da.view(dtype=dt), itemsize=count)
+                da = get_chararray(da.view(dtype=dt), itemsize=count)
                 dummy[idx] = decode_ascii(da)
             else:
                 dt = np.dtype(recformat.dtype)
@@ -1204,7 +1204,7 @@ def _scale_back(self, update_heap_pointers=True):
                 if isinstance(self._coldefs, _AsciiColDefs):
                     self._scale_back_ascii(index, dummy, raw_field)
                 # binary table string column
-                elif isinstance(raw_field, chararray.chararray):
+                elif isinstance(raw_field, chararray):
                     self._scale_back_strings(index, dummy, raw_field)
                 # all other binary table columns
                 else:
@@ -1341,7 +1341,7 @@ def _scale_back_ascii(self, col_idx, input_field, output_field):
 
         # Replace exponent separator in floating point numbers
         if "D" in format:
-            output_field[:] = output_field.replace(b"E", b"D")
+            output_field[:] = np.strings.replace(output_field, b"E", b"D")
 
     def tolist(self):
         # Override .tolist to take care of special case of VLF
@@ -1361,8 +1361,8 @@ def _get_recarray_field(array, key):
     # This is currently needed for backwards-compatibility and for
     # automatic truncation of trailing whitespace
     field = np.recarray.field(array, key)
-    if field.dtype.char in ("S", "U") and not isinstance(field, chararray.chararray):
-        field = field.view(chararray.chararray)
+    if field.dtype.char in ("S", "U") and not isinstance(field, chararray):
+        field = field.view(chararray)
     return field
 
 
diff --git a/astropy/io/fits/hdu/table.py b/astropy/io/fits/hdu/table.py
index 16528f813fb7..362dc2cd8ee9 100644
--- a/astropy/io/fits/hdu/table.py
+++ b/astropy/io/fits/hdu/table.py
@@ -11,7 +11,6 @@
 from contextlib import suppress
 
 import numpy as np
-from numpy import char as chararray
 
 # This module may have many dependencies on astropy.io.fits.column, but
 # astropy.io.fits.column has fewer dependencies overall, so it's easier to
@@ -37,6 +36,7 @@
 from astropy.io.fits.header import Header, _pad_length
 from astropy.io.fits.util import _is_int, _str_to_num, path_like
 from astropy.utils import lazyproperty
+from astropy.utils.compat import chararray
 
 from .base import DELAYED, ExtensionHDU, _ValidHDU
 
@@ -1489,7 +1489,7 @@ def _binary_table_byte_swap(data):
         formats.append(field_dtype)
         offsets.append(field_offset)
 
-        if isinstance(field, chararray.chararray):
+        if isinstance(field, chararray):
             continue
 
         # only swap unswapped
@@ -1507,7 +1507,7 @@ def _binary_table_byte_swap(data):
             coldata = data.field(idx)
             for c in coldata:
                 if (
-                    not isinstance(c, chararray.chararray)
+                    not isinstance(c, chararray)
                     and c.itemsize > 1
                     and c.dtype.str[0] in swap_types
                 ):
diff --git a/astropy/utils/compat/numpycompat.py b/astropy/utils/compat/numpycompat.py
index dde37ec3701d..b4f40fb1caa9 100644
--- a/astropy/utils/compat/numpycompat.py
+++ b/astropy/utils/compat/numpycompat.py
@@ -9,7 +9,10 @@
 import numpy as np
 
 from astropy.utils import minversion
-from astropy.utils.exceptions import AstropyPendingDeprecationWarning
+from astropy.utils.exceptions import (
+    AstropyDeprecationWarning,
+    AstropyPendingDeprecationWarning,
+)
 
 __all__ = [
     "NUMPY_LT_2_1",
@@ -18,6 +21,8 @@
     "NUMPY_LT_2_4",
     "NUMPY_LT_2_4_1",
     "NUMPY_LT_2_5",
+    "chararray",
+    "get_chararray",
 ]
 
 # TODO: It might also be nice to have aliases to these named for specific
@@ -42,3 +47,43 @@ def __getattr__(attr):
         return None
 
     raise AttributeError(f"module {__name__!r} has no attribute {attr!r}.")
+
+
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+    from numpy.char import array as np_char_array
+    from numpy.char import chararray as np_chararray
+
+
+_deprecated_chararray_attributes = {
+    "capitalize", "center", "count", "decode", "encode", "endswith",
+    "expandtabs", "find", "index", "isalnum", "isalpha", "isdecimal",
+    "isdigit", "islower", "isnumeric", "isspace", "istitle", "isupper",
+    "join", "ljust", "lower", "lstrip", "replace", "rfind", "rindex",
+    "rjust", "rpartition", "rsplit", "rstrip", "split", "splitlines",
+    "startswith", "strip", "swapcase", "title", "translate", "upper",
+    "zfill"
+}  # fmt: skip
+
+
+class chararray(np_chararray):
+    """Version of np.char.chararray with deprecation warnings on special methods."""
+
+    def __getattribute__(self, name):
+        if name in _deprecated_chararray_attributes:
+            warnings.warn(
+                "chararray is deprecated, in future versions astropy will "
+                "return a normal array so the special chararray methods "
+                "(e.g., .rstrip()) will not be available. Use np.strings "
+                "functions instead.",
+                AstropyDeprecationWarning,
+            )
+        return super().__getattribute__(name)
+
+
+def get_chararray(obj, itemsize=None, copy=True, unicode=None, order=None):
+    """Get version of np.char.chararray that gives deprecation warnings on special methods."""
+    return np_char_array(
+        obj, itemsize=itemsize, copy=copy, unicode=unicode, order=order
+    ).view(chararray)


Test patch:
diff --git a/astropy/io/fits/tests/test_table.py b/astropy/io/fits/tests/test_table.py
index 3196fe9ab294..61acb9812d97 100644
--- a/astropy/io/fits/tests/test_table.py
+++ b/astropy/io/fits/tests/test_table.py
@@ -9,7 +9,6 @@
 
 import numpy as np
 import pytest
-from numpy import char as chararray
 
 try:
     import objgraph
@@ -24,7 +23,8 @@
 from astropy.io.fits.verify import VerifyError
 from astropy.table import Table
 from astropy.units import Unit, UnitsWarning, UnrecognizedUnit
-from astropy.utils.exceptions import AstropyUserWarning
+from astropy.utils.compat import get_chararray
+from astropy.utils.exceptions import AstropyDeprecationWarning, AstropyUserWarning
 
 from .conftest import FitsTestCase
 from .test_connect import TestMultipleHDU
@@ -156,7 +156,7 @@ def test_open(self, home_is_data):
         fd = fits.open(self.data("test0.fits"))
 
         # create some local arrays
-        a1 = chararray.array(["abc", "def", "xx"])
+        a1 = get_chararray(["abc", "def", "xx"])
         r1 = np.array([11.0, 12.0, 13.0], dtype=np.float32)
 
         # create a table from scratch, using a mixture of columns from existing
@@ -319,7 +319,7 @@ def test_ascii_table(self):
 
         # Test Start Column
 
-        a1 = chararray.array(["abcd", "def"])
+        a1 = get_chararray(["abcd", "def"])
         r1 = np.array([11.0, 12.0])
         c1 = fits.Column(name="abc", format="A3", start=19, array=a1)
         c2 = fits.Column(name="def", format="E", start=3, array=r1)
@@ -1970,7 +1970,7 @@ def test_string_column_padding(self):
             "p\x00\x00\x00\x00\x00\x00\x00\x00\x00"
         )
 
-        acol = fits.Column(name="MEMNAME", format="A10", array=chararray.array(a))
+        acol = fits.Column(name="MEMNAME", format="A10", array=get_chararray(a))
         ahdu = fits.BinTableHDU.from_columns([acol])
         assert ahdu.data.tobytes().decode("raw-unicode-escape") == s
         ahdu.writeto(self.temp("newtable.fits"))
@@ -2228,8 +2228,14 @@ def test_new_table_with_nd_column(self):
         with fits.open(self.temp("test.fits")) as h:
             # Need to force string arrays to byte arrays in order to compare
             # correctly on Python 3
-            assert (h[1].data["str"].encode("ascii") == arra).all()
-            assert (h[1].data["strarray"].encode("ascii") == arrb).all()
+            with pytest.warns(
+                AstropyDeprecationWarning, match="chararray is deprecated.*"
+            ):
+                assert (h[1].data["str"].encode("ascii") == arra).all()
+            with pytest.warns(
+                AstropyDeprecationWarning, match="chararray is deprecated.*"
+            ):
+                assert (h[1].data["strarray"].encode("ascii") == arrb).all()
             assert (h[1].data["intarray"] == arrc).all()
 
     def test_mismatched_tform_and_tdim(self):