Repository: astropy/astropy

Issue:
# TNULL should be ignored for floating point columns in FITS tables

I tried downloading the FITS (ascii) table from here:

http://vizier.u-strasbg.fr/viz-bin/VizieR-3?-source=J/A%2bA/587/A153/yso&-out.max=50&-out.form=HTML%20Table&-out.add=_r&-out.add=_RAJ,_DEJ&-sort=_r&-oc.form=sexa

and if I read it in with Astropy I get:

```
In [8]: hdulist[1].columns
Out[8]: 
ColDefs(
    name = '_RAJ2000'; format = 'F10.6'; unit = 'deg'; start = 2
    name = '_DEJ2000'; format = 'F10.6'; unit = 'deg'; start = 13
    name = 'VISION'; format = 'A16'; start = 24
    name = 'RAJ2000'; format = 'F10.6'; unit = 'deg'; start = 41
    name = 'DEJ2000'; format = 'F10.6'; unit = 'deg'; start = 52
    name = 'Jmag'; format = 'F6.3'; unit = 'mag'; null = 'NaN'; start = 63
    name = 'e_Jmag'; format = 'F6.3'; unit = 'mag'; null = 'NaN'; start = 70
    name = 'Hmag'; format = 'F6.3'; unit = 'mag'; null = 'NaN'; start = 77
    name = 'e_Hmag'; format = 'F6.3'; unit = 'mag'; null = 'NaN'; start = 84
    name = 'Ksmag'; format = 'F6.3'; unit = 'mag'; null = 'NaN'; start = 91
    name = 'e_Ksmag'; format = 'F6.3'; unit = 'mag'; null = 'NaN'; start = 98
    name = 'Japer'; format = 'F5.2'; unit = 'arcsec'; null = 'NaN'; start = 105
    name = 'Haper'; format = 'F5.2'; unit = 'arcsec'; null = 'NaN'; start = 111
    name = 'Ksaper'; format = 'F5.2'; unit = 'arcsec'; null = 'NaN'; start = 117
    name = 'Simbad'; format = 'A6'; start = 123
)
```

There are two issues here:

* The FITS standard doesn't allow TNULL for F columns:

https://fits.gsfc.nasa.gov/standard40/fits_standard40aa.pdf (page 23)

* Even if we decided to relax this, the null parameter should be forced to be a float or ignored (not set to a 'null' string above)

Implementation patch:
diff --git a/astropy/io/fits/fitsrec.py b/astropy/io/fits/fitsrec.py
index 32b8ec436424..29cf77b88030 100644
--- a/astropy/io/fits/fitsrec.py
+++ b/astropy/io/fits/fitsrec.py
@@ -878,11 +878,14 @@ def _convert_ascii(self, column, field):
         # array buffer.
         dummy = np.char.ljust(field, format.width)
         dummy = np.char.replace(dummy, encode_ascii("D"), encode_ascii("E"))
-        null_fill = encode_ascii(str(ASCIITNULL).rjust(format.width))
 
-        # Convert all fields equal to the TNULL value (nullval) to empty fields.
-        # TODO: These fields really should be converted to NaN or something else undefined.
-        # Currently they are converted to empty fields, which are then set to zero.
+        # Convert all fields equal to the TNULL value (nullval) to either NaN
+        # for float columns or 0 for the other fields.
+        if format.format in "DEF":
+            null_fill = "nan"
+        else:
+            null_fill = str(ASCIITNULL)
+        null_fill = encode_ascii(null_fill.rjust(format.width))
         dummy = np.where(np.char.strip(dummy) == nullval, null_fill, dummy)
 
         # always replace empty fields, see https://github.com/astropy/astropy/pull/5394


Test patch:
diff --git a/astropy/io/fits/tests/test_hdulist.py b/astropy/io/fits/tests/test_hdulist.py
index 760c29746a1a..3fa47dd11444 100644
--- a/astropy/io/fits/tests/test_hdulist.py
+++ b/astropy/io/fits/tests/test_hdulist.py
@@ -785,7 +785,7 @@ def test_fromstring(filename):
                         for n in hdul[idx].data.names:
                             c1 = hdul[idx].data[n]
                             c2 = hdul2[idx].data[n]
-                            assert (c1 == c2).all()
+                            np.testing.assert_array_equal(c1, c2)
                     elif any(dim == 0 for dim in hdul[idx].data.shape) or any(
                         dim == 0 for dim in hdul2[idx].data.shape
                     ):
diff --git a/astropy/io/fits/tests/test_table.py b/astropy/io/fits/tests/test_table.py
index 0acafd773626..ce1d99385e2a 100644
--- a/astropy/io/fits/tests/test_table.py
+++ b/astropy/io/fits/tests/test_table.py
@@ -1893,8 +1893,8 @@ def test_fits_rec_column_access(self):
         tbdata = fits.getdata(self.data("ascii.fits"))
         for col in ("a", "b"):
             data = getattr(tbdata, col)
-            assert (data == tbdata.field(col)).all()
-            assert (data == tbdata[col]).all()
+            np.testing.assert_array_equal(data, tbdata.field(col))
+            np.testing.assert_array_equal(data, tbdata[col])
 
         # with VLA column
         col1 = fits.Column(
@@ -2565,7 +2565,7 @@ def test_blank_field_zero(self):
             h.seek(0)
             h.write(nulled)
 
-        with fits.open(self.temp("ascii_null.fits"), memmap=True) as f:
+        with fits.open(self.temp("ascii_null.fits")) as f:
             assert f[1].data[2][0] == 0
 
         # Test a float column with a null value set and blank fields.
@@ -2586,10 +2586,23 @@ def test_blank_field_zero(self):
             h.seek(0)
             h.write(nulled)
 
-        with fits.open(self.temp("ascii_null2.fits"), memmap=True) as f:
-            # (Currently it should evaluate to 0.0, but if a TODO in fitsrec is
-            # completed, then it should evaluate to NaN.)
-            assert f[1].data[2][0] == 0.0 or np.isnan(f[1].data[2][0])
+        with fits.open(self.temp("ascii_null2.fits")) as f:
+            assert np.isnan(f[1].data[2][0])
+
+        # Test a float column with a NaN value
+        a = np.arange(10, dtype=float)
+        a[5] = np.nan
+        table = fits.TableHDU.from_columns(
+            [
+                fits.Column(name="a1", array=a, format="F"),
+                fits.Column(name="a2", array=a, format="F", null=np.nan),
+            ]
+        )
+        table.writeto(self.temp("ascii_null3.fits"))
+
+        with fits.open(self.temp("ascii_null3.fits")) as f:
+            assert np.isnan(f[1].data["a1"][5])
+            assert np.isnan(f[1].data["a2"][5])
 
     def test_column_array_type_mismatch(self):
         """Regression test for https://aeon.stsci.edu/ssb/trac/pyfits/ticket/218"""