Repository: RadioAstronomySoftwareGroup/pyuvdata

Issue:
Add uv objects which are metadata only
Should enable setting read_data=False when reading _list of files_.  This solution will be similar to what was worked out for select() which has a metadata_only option.

This is critically needed to load large datasets.

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d5fd40f..8bb1913e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@
 All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
+
+### Added
+- `metadata_only` property on `UVData` to automatically detect if data-like arrays are present
+- support for combining metadata only objects and reading in multiple files as metadata only
 - `utils.uvcalibrate` flag propagation bug fix
 - `UVCal.ant2ind` indexing bug fix
 - `UVCal.get_*` methods for accessing data arrays with antenna-polarization keys
@@ -9,6 +13,7 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 - Fixed a bug in select that caused bls and antenna_names/numbers to be or'ed rather than and'ed together.
+- Fixed a bug where `baseline_to_antnums` could accept a numpy array as input but not other array_like objects.
 
 ## [1.4.0] - 2019-05-23
 
diff --git a/pyuvdata/utils.py b/pyuvdata/utils.py
index 11e78dd6..55394e44 100644
--- a/pyuvdata/utils.py
+++ b/pyuvdata/utils.py
@@ -987,16 +987,25 @@ def baseline_to_antnums(baseline, Nants_telescope):
     """
     Get the antenna numbers corresponding to a given baseline number.
 
-    Args:
-        baseline: integer baseline number
-        Nant_telescope: integer number of antennas
+    Parameters
+    ----------
+    baseline : int or array_like of ints
+        baseline number
+    Nant_telescope : int
+        number of antennas
 
-    Returns:
-        tuple with the two antenna numbers corresponding to the baseline.
+    Returns
+    -------
+    int or array_like of int
+        first antenna number(s)
+    int or array_like of int
+        second antenna number(s)
     """
     if Nants_telescope > 2048:
         raise Exception('error Nants={Nants}>2048 not '
                         'supported'.format(Nants=Nants_telescope))
+
+    baseline = np.asarray(baseline, dtype=np.int64)
     if np.min(baseline) > 2**16:
         ant2 = (baseline - 2**16) % 2048 - 1
         ant1 = (baseline - 2**16 - (ant2 + 1)) / 2048 - 1
@@ -1010,16 +1019,23 @@ def antnums_to_baseline(ant1, ant2, Nants_telescope, attempt256=False):
     """
     Get the baseline number corresponding to two given antenna numbers.
 
-    Args:
-        ant1: first antenna number (integer)
-        ant2: second antenna number (integer)
-        Nant_telescope: integer number of antennas
-        attempt256: Option to try to use the older 256 standard used in
-            many uvfits files (will use 2048 standard if there are more
-            than 256 antennas). Default is False.
+    Parameters
+    ----------
+    ant1 : int or array_like of int
+        first antenna number
+    ant2 : int or array_like of int
+        second antenna number
+    Nant_telescope : int
+        number of antennas
+    attempt256 : bool
+        Option to try to use the older 256 standard used in
+        many uvfits files (will use 2048 standard if there are more
+        than 256 antennas). Default is False.
 
-    Returns:
-        integer baseline number corresponding to the two antenna numbers.
+    Returns
+    -------
+    int or array of int
+        baseline number corresponding to the two antenna numbers.
     """
     ant1, ant2 = np.int64((ant1, ant2))
     if Nants_telescope is not None and Nants_telescope > 2048:
diff --git a/pyuvdata/uvdata.py b/pyuvdata/uvdata.py
index 934c71eb..d977b48b 100644
--- a/pyuvdata/uvdata.py
+++ b/pyuvdata/uvdata.py
@@ -365,6 +365,33 @@ class UVData(UVBase):
 
         super(UVData, self).__init__()
 
+    @property
+    def _data_params(self):
+        """List of strings giving the data-like parameters"""
+        return ['data_array', 'nsample_array', 'flag_array']
+
+    @property
+    def data_like_parameters(self):
+        """An iterator of defined parameters which are data-like (not metadata-like)"""
+        for key in self._data_params:
+            if hasattr(self, key):
+                yield getattr(self, key)
+
+    @property
+    def metadata_only(self):
+        """
+        Property that determines whether this is a metadata only object.
+
+        An object is metadata only if data_array, nsample_array and flag_array
+        are all None.
+        """
+        metadata_only = all(d is None for d in self.data_like_parameters)
+
+        for param_name in self._data_params:
+            getattr(self, "_" + param_name).required = not metadata_only
+
+        return metadata_only
+
     def check(self, check_extra=True, run_check_acceptability=True):
         """
         Add some extra checks on top of checks on UVBase class.
@@ -577,15 +604,15 @@ class UVData(UVBase):
 
         Parameters
         ----------
-        baseline : int
+        baseline : int or array_like of int
             baseline number
 
         Returns
         -------
-        int
-            first antenna number
-        int
-            second antenna number
+        int or array_like of int
+            first antenna number(s)
+        int or array_like of int
+            second antenna number(s)
         """
         return uvutils.baseline_to_antnums(baseline, self.Nants_telescope)
 
@@ -595,9 +622,9 @@ class UVData(UVBase):
 
         Parameters
         ----------
-        ant1 : int
+        ant1 : int or array_like of int
             first antenna number
-        ant2 : int
+        ant2 : int or array_like of int
             second antenna number
         attempt256 : bool
             Option to try to use the older 256 standard used in many uvfits files
@@ -605,7 +632,7 @@ class UVData(UVBase):
 
         Returns
         -------
-        int
+        int or array of int
             baseline number corresponding to the two antenna numbers.
         """
         return uvutils.antnums_to_baseline(ant1, ant2, self.Nants_telescope, attempt256=attempt256)
@@ -1376,6 +1403,7 @@ class UVData(UVBase):
             this = self
         else:
             this = copy.deepcopy(self)
+
         # Check that both objects are UVData and valid
         this.check(check_extra=check_extra, run_check_acceptability=run_check_acceptability)
         if not issubclass(other.__class__, this.__class__):
@@ -1415,29 +1443,30 @@ class UVData(UVBase):
             this.freq_array[0, :], other.freq_array[0, :], return_indices=True)
         both_blts, this_blts_ind, other_blts_ind = np.intersect1d(
             this_blts, other_blts, return_indices=True)
-        if len(both_pol) > 0:
-            if len(both_freq) > 0:
-                if len(both_blts) > 0:
-                    # check that overlapping data is not valid
-                    this_all_zero = np.all(this.data_array[this_blts_ind][
-                        :, :, this_freq_ind][:, :, :, this_pol_ind] == 0)
-                    this_all_flag = np.all(this.flag_array[this_blts_ind][
-                        :, :, this_freq_ind][:, :, :, this_pol_ind])
-                    other_all_zero = np.all(other.data_array[other_blts_ind][
-                        :, :, other_freq_ind][:, :, :, other_pol_ind] == 0)
-                    other_all_flag = np.all(other.flag_array[other_blts_ind][
-                        :, :, other_freq_ind][:, :, :, other_pol_ind])
-                    if (this_all_zero and this_all_flag):
-                        # we're fine to overwrite; update history accordingly
-                        history_update_string = ' Overwrote invalid data using pyuvdata.'
-                        this.history += history_update_string
-                    elif (other_all_zero and other_all_flag):
-                        raise ValueError('To combine these data, please run the add operation again, '
-                                         'but with the object whose data is to be overwritten as the '
-                                         'first object in the add operation.')
-                    else:
-                        raise ValueError('These objects have overlapping data and'
-                                         ' cannot be combined.')
+        if not self.metadata_only and (
+                len(both_pol) > 0 and len(both_freq) > 0 and len(both_blts) > 0
+        ):
+            # check that overlapping data is not valid
+            this_all_zero = np.all(this.data_array[this_blts_ind][
+                :, :, this_freq_ind][:, :, :, this_pol_ind] == 0)
+            this_all_flag = np.all(this.flag_array[this_blts_ind][
+                :, :, this_freq_ind][:, :, :, this_pol_ind])
+            other_all_zero = np.all(other.data_array[other_blts_ind][
+                :, :, other_freq_ind][:, :, :, other_pol_ind] == 0)
+            other_all_flag = np.all(other.flag_array[other_blts_ind][
+                :, :, other_freq_ind][:, :, :, other_pol_ind])
+
+            if this_all_zero and this_all_flag:
+                # we're fine to overwrite; update history accordingly
+                history_update_string = ' Overwrote invalid data using pyuvdata.'
+                this.history += history_update_string
+            elif other_all_zero and other_all_flag:
+                raise ValueError('To combine these data, please run the add operation again, '
+                                 'but with the object whose data is to be overwritten as the '
+                                 'first object in the add operation.')
+            else:
+                raise ValueError('These objects have overlapping data and'
+                                 ' cannot be combined.')
 
         # find the blt indices in "other" but not in "this"
         temp = np.nonzero(~np.in1d(other_blts, this_blts))[0]
@@ -1511,12 +1540,13 @@ class UVData(UVBase):
         if len(bnew_inds) > 0:
             this_blts = np.concatenate((this_blts, new_blts))
             blt_order = np.argsort(this_blts)
-            zero_pad = np.zeros(
-                (len(bnew_inds), this.Nspws, this.Nfreqs, this.Npols))
-            this.data_array = np.concatenate([this.data_array, zero_pad], axis=0)
-            this.nsample_array = np.concatenate([this.nsample_array, zero_pad], axis=0)
-            this.flag_array = np.concatenate([this.flag_array,
-                                              1 - zero_pad], axis=0).astype(np.bool)
+            if not self.metadata_only:
+                zero_pad = np.zeros(
+                    (len(bnew_inds), this.Nspws, this.Nfreqs, this.Npols))
+                this.data_array = np.concatenate([this.data_array, zero_pad], axis=0)
+                this.nsample_array = np.concatenate([this.nsample_array, zero_pad], axis=0)
+                this.flag_array = np.concatenate([this.flag_array,
+                                                  1 - zero_pad], axis=0).astype(np.bool)
             this.uvw_array = np.concatenate([this.uvw_array,
                                              other.uvw_array[bnew_inds, :]], axis=0)[blt_order, :]
             this.time_array = np.concatenate([this.time_array,
@@ -1533,25 +1563,27 @@ class UVData(UVBase):
                                                   other.baseline_array[bnew_inds]])[blt_order]
 
         if len(fnew_inds) > 0:
-            zero_pad = np.zeros((this.data_array.shape[0], this.Nspws, len(fnew_inds),
-                                 this.Npols))
             this.freq_array = np.concatenate([this.freq_array,
                                               other.freq_array[:, fnew_inds]], axis=1)
             f_order = np.argsort(this.freq_array[0, :])
-            this.data_array = np.concatenate([this.data_array, zero_pad], axis=2)
-            this.nsample_array = np.concatenate([this.nsample_array, zero_pad], axis=2)
-            this.flag_array = np.concatenate([this.flag_array, 1 - zero_pad],
-                                             axis=2).astype(np.bool)
+            if not self.metadata_only:
+                zero_pad = np.zeros((this.data_array.shape[0], this.Nspws, len(fnew_inds),
+                                     this.Npols))
+                this.data_array = np.concatenate([this.data_array, zero_pad], axis=2)
+                this.nsample_array = np.concatenate([this.nsample_array, zero_pad], axis=2)
+                this.flag_array = np.concatenate([this.flag_array, 1 - zero_pad],
+                                                 axis=2).astype(np.bool)
         if len(pnew_inds) > 0:
-            zero_pad = np.zeros((this.data_array.shape[0], this.Nspws,
-                                 this.data_array.shape[2], len(pnew_inds)))
             this.polarization_array = np.concatenate([this.polarization_array,
                                                       other.polarization_array[pnew_inds]])
             p_order = np.argsort(np.abs(this.polarization_array))
-            this.data_array = np.concatenate([this.data_array, zero_pad], axis=3)
-            this.nsample_array = np.concatenate([this.nsample_array, zero_pad], axis=3)
-            this.flag_array = np.concatenate([this.flag_array, 1 - zero_pad],
-                                             axis=3).astype(np.bool)
+            if not self.metadata_only:
+                zero_pad = np.zeros((this.data_array.shape[0], this.Nspws,
+                                     this.data_array.shape[2], len(pnew_inds)))
+                this.data_array = np.concatenate([this.data_array, zero_pad], axis=3)
+                this.nsample_array = np.concatenate([this.nsample_array, zero_pad], axis=3)
+                this.flag_array = np.concatenate([this.flag_array, 1 - zero_pad],
+                                                 axis=3).astype(np.bool)
 
         # Now populate the data
         pol_t2o = np.nonzero(
@@ -1559,26 +1591,31 @@ class UVData(UVBase):
         freq_t2o = np.nonzero(
             np.in1d(this.freq_array[0, :], other.freq_array[0, :]))[0]
         blt_t2o = np.nonzero(np.in1d(this_blts, other_blts))[0]
-        this.data_array[np.ix_(blt_t2o, [0], freq_t2o,
-                               pol_t2o)] = other.data_array
-        this.nsample_array[np.ix_(
-            blt_t2o, [0], freq_t2o, pol_t2o)] = other.nsample_array
-        this.flag_array[np.ix_(blt_t2o, [0], freq_t2o,
-                               pol_t2o)] = other.flag_array
-        if len(bnew_inds) > 0:
-            this.data_array = this.data_array[blt_order, :, :, :]
-            this.nsample_array = this.nsample_array[blt_order, :, :, :]
-            this.flag_array = this.flag_array[blt_order, :, :, :]
+        if not self.metadata_only:
+            this.data_array[np.ix_(blt_t2o, [0], freq_t2o,
+                                   pol_t2o)] = other.data_array
+            this.nsample_array[np.ix_(
+                blt_t2o, [0], freq_t2o, pol_t2o)] = other.nsample_array
+            this.flag_array[np.ix_(blt_t2o, [0], freq_t2o,
+                                   pol_t2o)] = other.flag_array
+
+        if not self.metadata_only:
+            if len(bnew_inds) > 0:
+                for name, param in zip(this._data_params, this.data_like_parameters):
+                    setattr(this, name, param[blt_order, :, :, :])
+
+            if len(fnew_inds) > 0:
+                for name, param in zip(this._data_params, this.data_like_parameters):
+                    setattr(this, name, param[:, :, f_order, :])
+
+            if len(pnew_inds) > 0:
+                for name, param in zip(this._data_params, this.data_like_parameters):
+                    setattr(this, name, param[:, :, :, p_order])
+
         if len(fnew_inds) > 0:
             this.freq_array = this.freq_array[:, f_order]
-            this.data_array = this.data_array[:, :, f_order, :]
-            this.nsample_array = this.nsample_array[:, :, f_order, :]
-            this.flag_array = this.flag_array[:, :, f_order, :]
         if len(pnew_inds) > 0:
             this.polarization_array = this.polarization_array[p_order]
-            this.data_array = this.data_array[:, :, :, p_order]
-            this.nsample_array = this.nsample_array[:, :, :, p_order]
-            this.flag_array = this.flag_array[:, :, :, p_order]
 
         # Update N parameters (e.g. Npols)
         this.Ntimes = len(np.unique(this.time_array))
@@ -1740,10 +1777,10 @@ class UVData(UVBase):
             elif np.max(freq_separation) > this.channel_width:
                 warnings.warn('Combined frequencies are not contiguous. This will make '
                               'it impossible to write this data out to some file types.')
-
-            this.data_array = np.concatenate([this.data_array, other.data_array], axis=2)
-            this.nsample_array = np.concatenate([this.nsample_array, other.nsample_array], axis=2)
-            this.flag_array = np.concatenate([this.flag_array, other.flag_array], axis=2)
+            if not self.metadata_only:
+                this.data_array = np.concatenate([this.data_array, other.data_array], axis=2)
+                this.nsample_array = np.concatenate([this.nsample_array, other.nsample_array], axis=2)
+                this.flag_array = np.concatenate([this.flag_array, other.flag_array], axis=2)
         elif axis == 'polarization':
             this.polarization_array = np.concatenate([this.polarization_array,
                                                      other.polarization_array])
@@ -1754,9 +1791,10 @@ class UVData(UVBase):
                 warnings.warn('Combined polarizations are not evenly spaced. This will '
                               'make it impossible to write this data out to some file types.')
 
-            this.data_array = np.concatenate([this.data_array, other.data_array], axis=3)
-            this.nsample_array = np.concatenate([this.nsample_array, other.nsample_array], axis=3)
-            this.flag_array = np.concatenate([this.flag_array, other.flag_array], axis=3)
+            if not self.metadata_only:
+                this.data_array = np.concatenate([this.data_array, other.data_array], axis=3)
+                this.nsample_array = np.concatenate([this.nsample_array, other.nsample_array], axis=3)
+                this.flag_array = np.concatenate([this.flag_array, other.flag_array], axis=3)
         elif axis == 'blt':
             this.Nblts = this.Nblts + other.Nblts
             this.ant_1_array = np.concatenate([this.ant_1_array,
@@ -1777,9 +1815,10 @@ class UVData(UVBase):
             this.Nbls = len(np.unique(this.baseline_array))
             this.integration_time = np.concatenate([this.integration_time,
                                                    other.integration_time])
-            this.data_array = np.concatenate([this.data_array, other.data_array], axis=0)
-            this.nsample_array = np.concatenate([this.nsample_array, other.nsample_array], axis=0)
-            this.flag_array = np.concatenate([this.flag_array, other.flag_array], axis=0)
+            if not self.metadata_only:
+                this.data_array = np.concatenate([this.data_array, other.data_array], axis=0)
+                this.nsample_array = np.concatenate([this.nsample_array, other.nsample_array], axis=0)
+                this.flag_array = np.concatenate([this.flag_array, other.flag_array], axis=0)
 
         # Check final object is self-consistent
         if run_check:
@@ -2166,7 +2205,7 @@ class UVData(UVBase):
                bls=None, frequencies=None, freq_chans=None,
                times=None, polarizations=None, blt_inds=None, run_check=True,
                check_extra=True, run_check_acceptability=True, inplace=True,
-               metadata_only=False, keep_all_metadata=True):
+               metadata_only=None, keep_all_metadata=True):
         """
         Downselect data to keep on the object along various axes.
 
@@ -2238,7 +2277,9 @@ class UVData(UVBase):
             select will be done on self).
         metadata_only : bool
             Option to only do the select on the metadata. Not allowed if the
-            data_array, flag_array or nsample_array is not None.
+            data_array, flag_array or nsample_array is not None. Note this option
+            has been replaced by an automatic detection of whether the data like
+            arrays are present. The keyword will be deprecated in version 1.6.
         keep_all_metadata : bool
             Option to keep all the metadata associated with antennas, even those
             that do do not have data associated with them after the select option.
@@ -2253,12 +2294,18 @@ class UVData(UVBase):
         ------
         ValueError
             If any of the parameters are set to inappropriate values.
+
         """
-        if metadata_only is True and (self.data_array is not None
-                                      or self.flag_array is not None
-                                      or self.nsample_array is not None):
-            raise ValueError('metadata_only option cannot be used if data_array, '
-                             'flag_array or nsample_array is not None')
+        if metadata_only is not None:
+            warnings.warn('The metadata_only option has been replaced by an '
+                          'automatic detection of whether the data like arrays '
+                          'are present. The keyword will be deprecated in version 1.6.',
+                          DeprecationWarning)
+
+            if metadata_only != self.metadata_only:
+                raise ValueError('The metadata_only option can only be True if '
+                                 'data_array, flag_array or nsample_array are '
+                                 'all None and must be False otherwise.')
 
         if inplace:
             uv_object = self
@@ -2273,26 +2320,23 @@ class UVData(UVBase):
         uv_object._select_metadata(blt_inds, freq_inds, pol_inds, history_update_string,
                                    keep_all_metadata)
 
-        if metadata_only is True:
+        if self.metadata_only:
             if not inplace:
                 return uv_object
             else:
                 return
 
         if blt_inds is not None:
-            uv_object.data_array = uv_object.data_array[blt_inds, :, :, :]
-            uv_object.flag_array = uv_object.flag_array[blt_inds, :, :, :]
-            uv_object.nsample_array = uv_object.nsample_array[blt_inds, :, :, :]
+            for param_name, param in zip(self._data_params, uv_object.data_like_parameters):
+                setattr(uv_object, param_name, param[blt_inds, :, :, :])
 
         if freq_inds is not None:
-            uv_object.data_array = uv_object.data_array[:, :, freq_inds, :]
-            uv_object.flag_array = uv_object.flag_array[:, :, freq_inds, :]
-            uv_object.nsample_array = uv_object.nsample_array[:, :, freq_inds, :]
+            for param_name, param in zip(self._data_params, uv_object.data_like_parameters):
+                setattr(uv_object, param_name, param[:, :, freq_inds, :])
 
         if pol_inds is not None:
-            uv_object.data_array = uv_object.data_array[:, :, :, pol_inds]
-            uv_object.flag_array = uv_object.flag_array[:, :, :, pol_inds]
-            uv_object.nsample_array = uv_object.nsample_array[:, :, :, pol_inds]
+            for param_name, param in zip(self._data_params, uv_object.data_like_parameters):
+                setattr(uv_object, param_name, param[:, :, :, pol_inds])
 
         # check if object is uv_object-consistent
         if run_check:
@@ -2481,16 +2525,19 @@ class UVData(UVBase):
                 func = 'read_uvfits'
 
         if isinstance(filename, (list, tuple)):
-            if not read_data:
-                raise ValueError('read_data cannot be False for a list of uvfits files')
+            if not read_data and not read_metadata:
+                raise ValueError('A list of files cannot be used when just '
+                                 'reading the header (read_data and read_metadata are False)')
             if func == 'read_uvfits_data':
-                raise ValueError('A list of files cannot be used when just reading data')
+                raise ValueError('A list of files cannot be used when just '
+                                 'reading data (metadata already exists)')
 
             self.read_uvfits(filename[0], antenna_nums=antenna_nums,
                              antenna_names=antenna_names, ant_str=ant_str,
                              bls=bls, frequencies=frequencies,
                              freq_chans=freq_chans, times=times,
                              polarizations=polarizations, blt_inds=blt_inds,
+                             read_data=read_data, read_metadata=read_metadata,
                              run_check=run_check, check_extra=check_extra,
                              run_check_acceptability=run_check_acceptability,
                              keep_all_metadata=keep_all_metadata)
@@ -2502,6 +2549,7 @@ class UVData(UVBase):
                                     bls=bls, frequencies=frequencies,
                                     freq_chans=freq_chans, times=times,
                                     polarizations=polarizations, blt_inds=blt_inds,
+                                    read_data=read_data, read_metadata=read_metadata,
                                     run_check=run_check, check_extra=check_extra,
                                     run_check_acceptability=run_check_acceptability,
                                     keep_all_metadata=keep_all_metadata)
@@ -2528,13 +2576,14 @@ class UVData(UVBase):
                 self._convert_from_filetype(uvfits_obj)
                 del(uvfits_obj)
             elif func == 'read_uvfits_metadata':
-                # can only be one file, it would have errored earlier because read_data=False
+                # can only be one file, it would have errored earlier otherwise
                 uvfits_obj = self._convert_to_filetype('uvfits')
                 uvfits_obj.read_uvfits_metadata(
                     filename, run_check_acceptability=run_check_acceptability)
                 self._convert_from_filetype(uvfits_obj)
                 del(uvfits_obj)
             elif func == 'read_uvfits_data':
+                # can only be one file, it would have errored earlier otherwise
                 uvfits_obj = self._convert_to_filetype('uvfits')
                 uvfits_obj.read_uvfits_data(filename, antenna_nums=antenna_nums,
                                             antenna_names=antenna_names, ant_str=ant_str,
@@ -2777,9 +2826,6 @@ class UVData(UVBase):
         """
         from . import miriad
         if isinstance(filepath, (list, tuple)):
-            if not read_data:
-                raise ValueError('read_data cannot be False for a list of miriad files')
-
             self.read_miriad(filepath[0], correct_lat_lon=correct_lat_lon,
                              run_check=run_check, check_extra=check_extra,
                              run_check_acceptability=run_check_acceptability,
@@ -2926,9 +2972,8 @@ class UVData(UVBase):
             that do not have data associated with them after the select option.
         read_data : bool
             Read in the visibility and flag data. If set to false, only the
-            basic header info and metadata (if read_metadata is True) will be
-            read in. Setting read_data to False results in an incompletely
-            defined object (check will not pass).
+            basic header info and metadata will be read in. Setting read_data to
+            False results in an incompletely defined object (check will not pass).
         data_array_dtype : numpy dtype
             Datatype to store the output data_array as. Must be either
             np.complex64 (single-precision real and imaginary) or np.complex128 (double-
@@ -2949,8 +2994,6 @@ class UVData(UVBase):
         """
         from . import uvh5
         if isinstance(filename, (list, tuple)):
-            if not read_data and len(filename) > 1:
-                raise ValueError('read_data cannot be False for a list of uvh5 files')
 
             self.read_uvh5(filename[0], antenna_nums=antenna_nums,
                            antenna_names=antenna_names, ant_str=ant_str, bls=bls,
@@ -3345,7 +3388,7 @@ class UVData(UVBase):
         if file_type is None:
             raise ValueError('File type could not be determined.')
 
-        if (time_range is not None):
+        if time_range is not None:
             if times is not None:
                 raise ValueError(
                     'Only one of times and time_range can be provided.')
@@ -3354,7 +3397,7 @@ class UVData(UVBase):
             raise ValueError('Only one of antenna_nums and antenna_names can be provided.')
 
         if file_type == 'uvfits':
-            if (time_range is not None):
+            if time_range is not None:
                 select = True
                 warnings.warn('Warning: "time_range" keyword is set which is not '
                               'supported by read_uvfits. This select will be '
@@ -4448,7 +4491,7 @@ class UVData(UVBase):
         return uvutils.get_baseline_redundancies(baselines, baseline_vecs,
                                                  tol=tol, with_conjugates=True)
 
-    def compress_by_redundancy(self, tol=1.0, inplace=True, metadata_only=False,
+    def compress_by_redundancy(self, tol=1.0, inplace=True, metadata_only=None,
                                keep_all_metadata=True):
         """
         Downselect to only have one baseline per redundant group on the object.
@@ -4463,8 +4506,10 @@ class UVData(UVBase):
         inplace : bool
             Option to do selection on current object.
         metadata_only : bool
-            Option to only do the select on the metadata. Not allowed
-            if the data_array, flag_array or nsample_array is not None.
+            Option to only do the select on the metadata. Not allowed if the
+            data_array, flag_array or nsample_array is not None. Note this option
+            has been replaced by an automatic detection of whether the data like
+            arrays are present. The keyword will be deprecated in version 1.6.
         keep_all_metadata : bool
             Option to keep all the metadata associated with antennas,
             even those that do not remain after the select option.


Test patch:
diff --git a/pyuvdata/tests/test_miriad.py b/pyuvdata/tests/test_miriad.py
index ca22c880..64230a4e 100644
--- a/pyuvdata/tests/test_miriad.py
+++ b/pyuvdata/tests/test_miriad.py
@@ -569,7 +569,6 @@ def test_readWriteReadMiriad():
     uv_out = UVData()
     testfile = os.path.join(DATA_PATH, 'zen.2456865.60537.xy.uvcRREAA')
     write_file = os.path.join(DATA_PATH, 'test/outtest_miriad.uv')
-    write_file2 = os.path.join(DATA_PATH, 'test/outtest_miriad2.uv')
     uvtest.checkWarnings(uv_in.read, [testfile], known_warning='miriad')
     uv_in.write_miriad(write_file, clobber=True)
     uv_out.read(write_file)
@@ -628,7 +627,33 @@ def test_readWriteReadMiriad():
     assert ischan == 1
     del(uv_aipy)  # close the file so it can be used later
 
-    # check partial IO selections
+    # test load_telescope_coords w/ blank Miriad
+    uv_in = Miriad()
+    uv = aipy_extracts.UV(testfile)
+    uvtest.checkWarnings(uv_in._load_telescope_coords, [uv], known_warning='miriad')
+    assert uv_in.telescope_location_lat_lon_alt is not None
+    # test load_antpos w/ blank Miriad
+    uv_in = Miriad()
+    uv = aipy_extracts.UV(testfile)
+    uvtest.checkWarnings(uv_in._load_antpos, [uv], known_warning='miriad')
+    assert uv_in.antenna_positions is not None
+
+    # test that changing precision of integraiton_time is okay
+    # tolerance of integration_time (1e-3) is larger than floating point type conversions
+    uv_in = UVData()
+    uvtest.checkWarnings(uv_in.read, [testfile], known_warning='miriad')
+    uv_in.integration_time = uv_in.integration_time.astype(np.float32)
+    uv_in.write_miriad(write_file, clobber=True)
+    new_uv = UVData()
+    new_uv.read(write_file)
+    assert uv_in == new_uv
+
+
+def test_readWriteReadMiriad_partial():
+    testfile = os.path.join(DATA_PATH, 'zen.2456865.60537.xy.uvcRREAA')
+    write_file = os.path.join(DATA_PATH, 'test/outtest_miriad.uv')
+
+    # check partial read selections
     full = UVData()
     uvtest.checkWarnings(full.read, [testfile], known_warning='miriad')
     full.write_miriad(write_file, clobber=True)
@@ -758,9 +783,11 @@ def test_readWriteReadMiriad():
     exp_uv = full.select(blt_inds=blts_select, antenna_nums=ants_keep, inplace=False)
     assert uv_in != exp_uv
 
-    del(uv_in)
-    del(uv_out)
-    del(full)
+
+def test_readWriteReadMiriad_partial_metadata_only():
+    testfile = os.path.join(DATA_PATH, 'zen.2456865.60537.xy.uvcRREAA')
+    write_file = os.path.join(DATA_PATH, 'test/outtest_miriad.uv')
+    write_file2 = os.path.join(DATA_PATH, 'test/outtest_miriad2.uv')
 
     # try metadata only read
     uv_in = UVData()
@@ -773,40 +800,27 @@ def test_readWriteReadMiriad():
     for m in metadata:
         assert getattr(uv_in, m) is not None
 
-    # test exceptions
-    # multiple file read-in
+    # metadata only multiple file read-in
     uv_in = UVData()
     uvtest.checkWarnings(uv_in.read, [testfile], known_warning='miriad')
     new_uv = uv_in.select(freq_chans=np.arange(5), inplace=False)
     new_uv.write_miriad(write_file, clobber=True)
     new_uv = uv_in.select(freq_chans=np.arange(5) + 5, inplace=False)
     new_uv.write_miriad(write_file2, clobber=True)
-    pytest.raises(ValueError, uv_in.read, [write_file, write_file2], read_data=False)
-    # read-in when data already exists
-    uv_in = UVData()
-    uvtest.checkWarnings(uv_in.read, [testfile], known_warning='miriad')
-    pytest.raises(ValueError, uv_in.read, testfile, read_data=False)
+    uv_in.read(testfile)
+    uv_in.select(freq_chans=np.arange(10))
+    uv_in2 = UVData()
+    uv_in2.read([write_file, write_file2])
 
-    # test load_telescope_coords w/ blank Miriad
-    uv_in = Miriad()
-    uv = aipy_extracts.UV(testfile)
-    uvtest.checkWarnings(uv_in._load_telescope_coords, [uv], known_warning='miriad')
-    assert uv_in.telescope_location_lat_lon_alt is not None
-    # test load_antpos w/ blank Miriad
-    uv_in = Miriad()
-    uv = aipy_extracts.UV(testfile)
-    uvtest.checkWarnings(uv_in._load_antpos, [uv], known_warning='miriad')
-    assert uv_in.antenna_positions is not None
+    assert uv_in.history != uv_in2.history
+    uv_in2.history = uv_in.history
+    assert uv_in == uv_in2
 
-    # test that changing precision of integraiton_time is okay
-    # tolerance of integration_time (1e-3) is larger than floating point type conversions
+    # test exceptions
+    # read-in when data already exists
     uv_in = UVData()
     uvtest.checkWarnings(uv_in.read, [testfile], known_warning='miriad')
-    uv_in.integration_time = uv_in.integration_time.astype(np.float32)
-    uv_in.write_miriad(write_file, clobber=True)
-    new_uv = UVData()
-    new_uv.read(write_file)
-    assert uv_in == new_uv
+    pytest.raises(ValueError, uv_in.read, testfile, read_data=False)
 
 
 @uvtest.skipIf_no_casa
diff --git a/pyuvdata/tests/test_uvdata.py b/pyuvdata/tests/test_uvdata.py
index 77b63b9e..4ee34a7c 100644
--- a/pyuvdata/tests/test_uvdata.py
+++ b/pyuvdata/tests/test_uvdata.py
@@ -106,6 +106,21 @@ def test_parameter_iter(uvdata_props):
 
 def test_required_parameter_iter(uvdata_props):
     "Test expected required parameters."
+
+    # at first it's a metadata_only object, so need to modify required_parameters
+    required = []
+    for prop in uvdata_props.uv_object.required():
+        required.append(prop)
+    expected_required = copy.copy(uvdata_props.required_parameters)
+    expected_required.remove('_data_array')
+    expected_required.remove('_nsample_array')
+    expected_required.remove('_flag_array')
+    for a in expected_required:
+        assert a in required, 'expected attribute ' + a + ' not returned in required iterator'
+
+    uvdata_props.uv_object.data_array = 1
+    uvdata_props.uv_object.nsample_array = 1
+    uvdata_props.uv_object.flag_array = 1
     required = []
     for prop in uvdata_props.uv_object.required():
         required.append(prop)
@@ -176,6 +191,17 @@ def uvdata_data():
     return
 
 
+def test_metadata_only_property(uvdata_data):
+    uvdata_data.uv_object.data_array = None
+    assert uvdata_data.uv_object.metadata_only is False
+    pytest.raises(ValueError, uvdata_data.uv_object.check)
+    uvdata_data.uv_object.flag_array = None
+    assert uvdata_data.uv_object.metadata_only is False
+    pytest.raises(ValueError, uvdata_data.uv_object.check)
+    uvdata_data.uv_object.nsample_array = None
+    assert uvdata_data.uv_object.metadata_only is True
+
+
 def test_equality(uvdata_data):
     """Basic equality test."""
     assert uvdata_data.uv_object == uvdata_data.uv_object
@@ -297,7 +323,10 @@ def uvdata_baseline():
 def test_baseline_to_antnums(uvdata_baseline):
     """Test baseline to antnum conversion for 256 & larger conventions."""
     assert uvdata_baseline.uv_object.baseline_to_antnums(67585) == (0, 0)
-    pytest.raises(Exception, uvdata_baseline.uv_object2.baseline_to_antnums, 67585)
+    with pytest.raises(Exception) as cm:
+        uvdata_baseline.uv_object2.baseline_to_antnums(67585)
+    assert str(cm.value).startswith('error Nants={Nants}>2048'
+                                    ' not supported'.format(Nants=uvdata_baseline.uv_object2.Nants_telescope))
 
     ant_pairs = [(10, 20), (280, 310)]
     for pair in ant_pairs:
@@ -313,6 +342,18 @@ def test_baseline_to_antnums(uvdata_baseline):
         assert pair == ant_pair_out
 
 
+def test_baseline_to_antnums_vectorized(uvdata_baseline):
+    """Test vectorized antnum to baseline conversion."""
+    ant_1 = [10, 280]
+    ant_2 = [20, 310]
+    baseline_array = uvdata_baseline.uv_object.antnums_to_baseline(ant_1, ant_2)
+    assert np.array_equal(baseline_array, [88085, 641335])
+    ant_1_out, ant_2_out = uvdata_baseline.uv_object.baseline_to_antnums(baseline_array.tolist())
+    print('out:', ant_1_out, ant_2_out)
+    assert np.array_equal(ant_1, ant_1_out)
+    assert np.array_equal(ant_2, ant_2_out)
+
+
 def test_antnums_to_baselines(uvdata_baseline):
     """Test antums to baseline conversion for 256 & larger conventions."""
     assert uvdata_baseline.uv_object.antnums_to_baseline(0, 0) == 67585
@@ -612,13 +653,11 @@ def test_select_blts():
 
     # check that just doing the metadata works properly
     uv_object3 = copy.deepcopy(uv_object)
-    pytest.raises(ValueError, uv_object3.select, blt_inds=blt_inds, metadata_only=True)
     uv_object3.data_array = None
-    pytest.raises(ValueError, uv_object3.select, blt_inds=blt_inds, metadata_only=True)
     uv_object3.flag_array = None
-    pytest.raises(ValueError, uv_object3.select, blt_inds=blt_inds, metadata_only=True)
     uv_object3.nsample_array = None
-    uv_object4 = uv_object3.select(blt_inds=blt_inds, metadata_only=True, inplace=False)
+    assert uv_object3.metadata_only is True
+    uv_object4 = uv_object3.select(blt_inds=blt_inds, inplace=False)
     for param in uv_object4:
         param_name = getattr(uv_object4, param).name
         if param_name not in ['data_array', 'flag_array', 'nsample_array']:
@@ -627,12 +666,22 @@ def test_select_blts():
             assert getattr(uv_object4, param_name) is None
 
     # also check with inplace=True
-    uv_object3.select(blt_inds=blt_inds, metadata_only=True)
+    uv_object3.select(blt_inds=blt_inds)
     assert uv_object3 == uv_object4
 
+    # check for warnings & errors with the metadata_only keyword
+    uv_object3 = copy.deepcopy(uv_object)
+    with pytest.raises(ValueError) as cm:
+        uvtest.checkWarnings(uv_object3.select,
+                             func_kwargs={'blt_inds': blt_inds, 'metadata_only': True},
+                             message='The metadata_only option has been replaced',
+                             category=DeprecationWarning)
+    assert str(cm.value).startswith('The metadata_only option can only be True')
+
     # check for errors associated with out of bounds indices
     pytest.raises(ValueError, uv_object.select, blt_inds=np.arange(-10, -5))
-    pytest.raises(ValueError, uv_object.select, blt_inds=np.arange(uv_object.Nblts + 1, uv_object.Nblts + 10))
+    pytest.raises(ValueError, uv_object.select,
+                  blt_inds=np.arange(uv_object.Nblts + 1, uv_object.Nblts + 10))
 
 
 def test_select_antennas():
@@ -3572,7 +3621,19 @@ def test_compress_redundancy_metadata_only():
     uv2.data_array = None
     uv2.flag_array = None
     uv2.nsample_array = None
-    uv2.compress_by_redundancy(tol=tol, inplace=True, metadata_only=True)
+    uv2.compress_by_redundancy(tol=tol, inplace=True)
+
+    # check for deprecation warning with metadata_only keyword
+    uv1 = copy.deepcopy(uv0)
+    uv1.data_array = None
+    uv1.flag_array = None
+    uv1.nsample_array = None
+    uvtest.checkWarnings(uv1.compress_by_redundancy,
+                         func_kwargs={'tol': tol, 'inplace': True,
+                                      'metadata_only': True},
+                         category=DeprecationWarning,
+                         message='The metadata_only option has been replaced')
+    assert uv1 == uv2
 
     uv0.compress_by_redundancy(tol=tol)
     uv0.data_array = None
diff --git a/pyuvdata/tests/test_uvfits.py b/pyuvdata/tests/test_uvfits.py
index 34efe998..cd8921d1 100644
--- a/pyuvdata/tests/test_uvfits.py
+++ b/pyuvdata/tests/test_uvfits.py
@@ -35,8 +35,9 @@ def test_ReadNRAO():
                          message='Telescope EVLA is not')
     assert expected_extra_keywords.sort() == list(UV2.extra_keywords.keys()).sort()
     pytest.raises(ValueError, UV2.check)
+
     UV2.read(testfile, read_data=False)
-    pytest.raises(ValueError, UV2.check)
+    assert UV2.check()
     UV2.read(testfile)
     assert UV == UV2
     # test reading in header & metadata first, then data
@@ -44,7 +45,7 @@ def test_ReadNRAO():
     uvtest.checkWarnings(UV2.read, [testfile], {'read_data': False},
                          message='Telescope EVLA is not')
     assert expected_extra_keywords.sort() == list(UV2.extra_keywords.keys()).sort()
-    pytest.raises(ValueError, UV2.check)
+    assert UV2.check()
     UV2.read(testfile)
     assert UV == UV2
 
@@ -660,19 +661,36 @@ def test_multi_files():
     uv1.history = uv_full.history
     assert uv1 == uv_full
 
-    # check raises error if read_data and read_metadata are False
-    pytest.raises(ValueError, uv1.read, [testfile1, testfile2],
-                  read_data=False, read_metadata=False)
+    # check with metadata_only
+    uv_full = UVData()
+    uvtest.checkWarnings(uv_full.read, [uvfits_file], {'read_data': False},
+                         message='Telescope EVLA is not')
+    uv1 = UVData()
+    uv1.read([testfile1, testfile2], read_data=False)
 
-    # check raises error if read_data is False and read_metadata is True
-    pytest.raises(ValueError, uv1.read, [testfile1, testfile2],
-                  read_data=False, read_metadata=True)
+    # Check history is correct, before replacing and doing a full object check
+    assert uvutils._check_histories(uv_full.history + '  Downselected to '
+                                    'specific frequencies using pyuvdata. '
+                                    'Combined data along frequency axis '
+                                    'using pyuvdata.', uv1.history)
+
+    uv1.history = uv_full.history
+    assert uv1 == uv_full
+
+    # check raises error if read_data and read_metadata are False
+    with pytest.raises(ValueError) as cm:
+        uv1.read([testfile1, testfile2], read_data=False, read_metadata=False)
+    assert str(cm.value).startswith('A list of files cannot be used when just '
+                                    'reading the header')
 
-    # check raises error if only reading data on a list of files
+    # check raises error if only reading data on a list of files (metadata already read)
     uv1 = UVData()
     uvtest.checkWarnings(uv1.read, [uvfits_file], {'read_data': False},
                          message=['Telescope EVLA is not'])
-    pytest.raises(ValueError, uv1.read, [testfile1, testfile2])
+    with pytest.raises(ValueError) as cm:
+        uv1.read([testfile1, testfile2])
+    assert str(cm.value).startswith('A list of files cannot be used when just '
+                                    'reading data (metadata already exists)')
 
 
 @uvtest.skipIf_no_casa
diff --git a/pyuvdata/tests/test_uvh5.py b/pyuvdata/tests/test_uvh5.py
index 49484b96..742f8f5a 100644
--- a/pyuvdata/tests/test_uvh5.py
+++ b/pyuvdata/tests/test_uvh5.py
@@ -90,7 +90,6 @@ def test_ReadUVH5Errors():
     uv_in = UVData()
     fake_file = os.path.join(DATA_PATH, 'fake_file.uvh5')
     pytest.raises(IOError, uv_in.read_uvh5, fake_file)
-    pytest.raises(ValueError, uv_in.read_uvh5, ['list of', 'fake files'], read_data=False)
 
     return
 
@@ -215,6 +214,43 @@ def test_UVH5ReadMultiple_files():
     return
 
 
+@uvtest.skipIf_no_h5py
+def test_UVH5ReadMultiple_files_metadata_only():
+    """
+    Test reading multiple uvh5 files with metadata only
+    """
+    uv_full = UVData()
+    uvfits_file = os.path.join(DATA_PATH, 'day2_TDEM0003_10s_norx_1src_1spw.uvfits')
+    testfile1 = os.path.join(DATA_PATH, 'test/uv1.uvh5')
+    testfile2 = os.path.join(DATA_PATH, 'test/uv2.uvh5')
+    uvtest.checkWarnings(uv_full.read_uvfits, [uvfits_file], message='Telescope EVLA is not')
+    uv1 = copy.deepcopy(uv_full)
+    uv2 = copy.deepcopy(uv_full)
+    uv1.select(freq_chans=np.arange(0, 32))
+    uv2.select(freq_chans=np.arange(32, 64))
+    uv1.write_uvh5(testfile1, clobber=True)
+    uv2.write_uvh5(testfile2, clobber=True)
+
+    uv_full = UVData()
+    uvtest.checkWarnings(uv_full.read_uvfits, [uvfits_file], {'read_data': False},
+                         message='Telescope EVLA is not')
+    uvtest.checkWarnings(uv1.read, [[testfile1, testfile2]], {'read_data': False},
+                         nwarnings=2, message='Telescope EVLA is not')
+    # Check history is correct, before replacing and doing a full object check
+    assert uvutils._check_histories(uv_full.history + '  Downselected to '
+                                    'specific frequencies using pyuvdata. '
+                                    'Combined data along frequency axis using'
+                                    ' pyuvdata.', uv1.history)
+    uv1.history = uv_full.history
+    assert uv1 == uv_full
+
+    # clean up
+    os.remove(testfile1)
+    os.remove(testfile2)
+
+    return
+
+
 @uvtest.skipIf_no_h5py
 def test_UVH5ReadMultiple_files_axis():
     """