Repository: geocryology/globsim (language: python)

Issue:
Wind direction could be greater than 360
The scale function for wind direction may lead to a value greater than 360º if both the U and V are negtive. I prose the below scale function, details please see:

https://confluence.ecmwf.int/pages/viewpage.action?pageId=133262398 

WD = np.mod(180 + np.arctan2(U, V) * (180 / np.pi), 360)


Discussion / hints:
Hi Bin, 

thanks for pointing this out.  
I note both that our current formulation has the arguments reversed for the `atan2` function and that the link you shared says 
> The atan2 arguments vary by type of software and in some applications the arguments (u,v) are switched around. 

I would suggest wrapping the existing calculation in a `modulo 360` . What do you think?
Hi Nick,
sounds great!
Another function I previously used for other studies was

WD = (270 - (np.degrees(np.arctan2(v, u)) + 360) % 360) % 360

Bests,
BIn

Fix patch:
diff --git a/globsim/_version.py b/globsim/_version.py
index f0024e2..bdcd06f 100644
--- a/globsim/_version.py
+++ b/globsim/_version.py
@@ -1,3 +1,3 @@
-__version__ = "3.9.3"
+__version__ = "4.0.0"
 
 
diff --git a/globsim/boundingbox.py b/globsim/boundingbox.py
index dc516cf..74ef074 100644
--- a/globsim/boundingbox.py
+++ b/globsim/boundingbox.py
@@ -62,9 +62,51 @@ def stations_bbox(stations) -> BoundingBox:
 
 
 def netcdf_bbox(ncf: "Union[str, nc.Dataset]") -> BoundingBox:
+
     if isinstance(ncf, str):
         ncf = nc.Dataset(ncf)
+    if isinstance(ncf, nc.Dataset):
+        x,y = _nc_get_lat_lon(ncf)
+        xval = x[:]
+        yval = y[:]
+    
+    else:  # XARRAY
+        x,y = _xr_get_lat_lon(ncf)
+        xval = x.values
+        yval = y.values
+
+    bbx = BoundingBox(np.min(xval), np.max(xval), np.min(yval), np.max(yval))
+
+    return bbx
+
+
+def _xr_get_lat_lon(xrf):
+    # TODO: implement this
+    return xrf['longitude'], xrf['latitude']
 
+    x = xrf.filter_by_attrs(axis='X')
+    if not x:
+        x = xrf.filter_by_attrs(standard_name='longitude')
+    if not x:
+        x = xrf.filter_by_attrs(long_name='longitude')
+    if not x:
+        x = xrf['longitude'] if 'longitude' in xrf.coords else None
+    if not x:
+        raise KeyError("Could not find x-coordinate")
+
+    y = xrf.filter_by_attrs(axis='Y')
+    if not y:
+        y = xrf.filter_by_attrs(standard_name='latitude')
+    if not y:
+        y = xrf.filter_by_attrs(long_name='latitude')
+    if not y:
+        y = xrf['latitude'] if 'latitude' in xrf.coords else None
+    if not y:
+        raise KeyError("Could not find y-coordinate")
+
+    return x, y
+
+def _nc_get_lat_lon(ncf):
     x = ncf.get_variables_by_attributes(axis='X')
     if not x:
         x = ncf.get_variables_by_attributes(standard_name='longitude')
@@ -84,15 +126,8 @@ def netcdf_bbox(ncf: "Union[str, nc.Dataset]") -> BoundingBox:
         y = ncf['latitude'] if 'latitude' in ncf.variables else None
     if not y:
         raise KeyError("Could not find y-coordinate")
-
-    xval = x[:]
-    yval = y[:]
-
-    bbx = BoundingBox(np.min(xval), np.max(xval), np.min(yval), np.max(yval))
-
-    return bbx
-
-
+    
+    return x,y
 """
 
 from collections import namedtuple
diff --git a/globsim/common_utils.py b/globsim/common_utils.py
index e810939..3f2dfca 100644
--- a/globsim/common_utils.py
+++ b/globsim/common_utils.py
@@ -36,8 +36,8 @@ def variables_skip(variable_name):
     Which variable names to use? Drop the ones that are dimensions.
     """
     skip = 0
-    dims = ('time', 'number', 'level',
-            'latitude', 'longitude', 'station', 'height')
+    dims = ('time', 'number', 'level','valid_time', 'pressure_level','expver',
+            'latitude', 'longitude', 'station', 'height', 'skip')
     if variable_name in dims:
         skip = 1
     return skip
@@ -153,7 +153,6 @@ def create_globsim_directory(target_dir, name):
     mkdir(TL)
 
     # create subdirectories
-    mkdir(path.join(TL, "eraint"))
     mkdir(path.join(TL, "Grib"))
     mkdir(path.join(TL, "jra55"))
     mkdir(path.join(TL, "merra2"))
@@ -204,3 +203,4 @@ def get_begin_date(par, data_folder, match_strings):
 
     print("Found some files in directory. Beginning download on {}".format(begin_date.strftime("%Y-%m-%d")))
     return(begin_date)
+
diff --git a/globsim/download/ERAIdownload.py b/globsim/download/ERAIdownload.py
deleted file mode 100644
index 60c2111..0000000
--- a/globsim/download/ERAIdownload.py
+++ /dev/null
@@ -1,633 +0,0 @@
-import glob
-import netCDF4 as nc
-import numpy as np
-import logging
-
-from datetime import datetime, timedelta
-from ecmwfapi.api import ECMWFDataServer
-from fnmatch import filter
-from math import floor
-from os import path, listdir, remove
-
-from globsim.common_utils import str_encode
-from globsim.download.GenericDownload import GenericDownload
-from globsim.meteorology import pressure_from_elevation
-
-logger = logging.getLogger('globsim.download')
-
-try:
-    from nco import Nco
-except ImportError:
-    print("*** NCO not imported, netCDF appending not possible. ***")
-    pass
-
-
-class ERAIgeneric(object):
-    """
-    Parent class for other ERA-Interim classes.
-    """
-
-    def areaString(self, area):
-        """Converts numerical coordinates into string: North/West/South/East"""
-        res = str(round(area['north'],2)) + "/"
-        res += str(round(area['west'], 2)) + "/"
-        res += str(round(area['south'],2)) + "/"
-        res += str(round(area['east'], 2))
-        return(res)
-
-    def dateString(self, date):
-        """Converts datetime objects into string"""
-        res = (date['beg'].strftime("%Y-%m-%d") + "/to/" +
-               date['end'].strftime("%Y-%m-%d"))
-        return(res)
-
-    def getPressureLevels(self, elevation):
-        """Restrict list of ERA-interim pressure levels to be downloaded"""
-        Pmax = pressure_from_elevation(elevation['min']) + 55
-        Pmin = pressure_from_elevation(elevation['max']) - 55
-        levs = np.array([300, 350, 400, 450, 500, 550, 600, 650, 700, 750, 775,
-                         800, 825, 850, 875, 900, 925, 950, 975, 1000])
-        mask = (levs >= Pmin) * (levs <= Pmax)  # select
-        levs = '/'.join(map(str, levs[mask]))
-        return levs
-
-    def getDictionaryGen(self, area, date):
-        """
-        Makes dictionary of generic variables for a server call
-        """
-        dictionary_gen = {
-          'area'    : self.areaString(area),
-          'date'    : self.dateString(date),
-          'dataset' : "interim",
-          'stream'  : "oper",
-          'class'   : "ei",
-          'format'  : "netcdf",
-          'grid'    : "0.75/0.75"}
-        return dictionary_gen
-
-    def getDstring(self):
-        return ('_' + self.date['beg'].strftime("%Y%m%d") + "_to_" +
-                self.date['end'].strftime("%Y%m%d"))
-
-    def download(self):
-        query = self.getDictionary()
-        target = query.pop('target')
-
-        if path.isfile(target):
-            print("WARNING: File '{}' already exists and was skipped".format(target))
-        else:
-            server = ECMWFDataServer()
-            print(server.trace('=== ERA Interim: START ===='))
-            server.retrieve(self.getDictionary())
-            print(server.trace('=== ERA Interim: STOP ====='))
-
-    def TranslateCF2ERA(self, variables, dpar):
-        """
-        Translate CF Standard Names into ERA-Interim code numbers.
-        """
-        self.param = ''
-        for var in variables:
-            try:
-                self.param += dpar.get(var) + '/'
-            except TypeError:
-                pass
-        self.param = self.param.rstrip('/')  # fix last
-
-    def __str__(self):
-        string = ("List of generic variables to query ECMWF server for "
-                  "ERA-Interim data: {0}")
-        return string.format(self.getDictionary)
-
-    def netCDF_merge(self, directory):
-        """
-        To combine mutiple downloaded erai netCDF files into a large file with
-        specified chunk_size(e.g. 500),
-        -- give the full name of merged file to the output = outfile
-        -- pass all data from the first input netfile to the merged file name
-        -- loop over the files_list, append file one by one into the merge file
-        -- pass the mergae netcdf file to interpolation module to process
-        (to use nc.MFDataset by reading it)
-
-        Args:
-            ncfile_in: the full name of downloaded files (file directory + files names)
-        e.g.:
-              '/home/xquan/src/globsim/examples/erai/era_sa_*.nc'
-              '/home/xquan/src/globsim/examples/erai/era_pl_*.nc'
-              '/home/xquan/src/globsim/examples/erai/era_sf_*.nc'
-
-        Output: merged netCDF files
-        era_all_0.nc, era_all_1.nc, ...,
-
-        """
-        # set up nco operator
-        nco = Nco()
-
-        # loop over filetypes, read, report
-        file_type = ['erai_sa_*.nc', 'erai_sf_*.nc', 'erai_pl_*.nc']
-        for ft in file_type:
-            ncfile_in = path.join(directory, ft)
-
-            # get the file list
-            files_list = glob.glob(ncfile_in)
-            files_list.sort()
-            num = len(files_list)
-
-            # set up the name of merged file
-            if ncfile_in[-7:-5] == 'sa':
-                merged_file = path.join(ncfile_in[:-11],'erai_sa_all_'+ files_list[0][-23:-15] + "_" + files_list[num-1][-11:-3] +'.nc')
-            elif ncfile_in[-7:-5] == 'sf':
-                merged_file = path.join(ncfile_in[:-11],'erai_sf_all_' + files_list[0][-23:-15] + '_' + files_list[num-1][-11:-3] + '.nc')
-            elif ncfile_in[-7:-5] == 'pl':
-                merged_file = path.join(ncfile_in[:-11],'erai_pl_all_'+ files_list[0][-23:-15] + '_' + files_list[num-1][-11:-3] +'.nc')
-            else:
-                print('There is not such type of file')
-
-            # combined files into merged files
-            nco.ncrcat(input=files_list,output=merged_file, append=True)
-
-            print('The Merged File below is saved:')
-            print(merged_file)
-
-            # clear up the data
-            for fl in files_list:
-                remove(fl)
-
-    def mergeFiles(self, ncfile_in):
-        """
-        To combine mutiple downloaded erai netCDF files into a large file.
-
-        Args:
-            ncfile_in: the full name of downloaded files (file directory + files names)
-        e.g.:
-              '/home/xquan/src/globsim/examples/erai/era_sa_*.nc'
-              '/home/xquan/src/globsim/examples/erai/era_pl_*.nc'
-              '/home/xquan/src/globsim/examples/erai/era_sf_*.nc'
-
-        Output: merged netCDF files
-        erai_sa_all.nc, erai_sf_all.nc, erai_pl_all.nc
-
-        """
-
-        # read in one type of mutiple netcdf files
-        ncf_in = nc.MFDataset(ncfile_in, 'r', aggdim='time')
-
-        # is it a file with pressure levels?
-        pl = 'level' in ncf_in.dimensions.keys()
-
-        # get spatial dimensions
-        lat  = ncf_in.variables['latitude'][:]
-        lon  = ncf_in.variables['longitude'][:]
-        if pl: # only for pressure level files
-            lev  = ncf_in.variables['level'][:]
-            nlev = len(lev)
-
-        # get time and convert to datetime object
-        nctime = ncf_in.variables['time'][:]
-
-        #set up the name of merged file
-        if ncfile_in[-7:-5] == 'sa':
-            ncfile_out = path.join(ncfile_in[:-11],'erai_sa_all' + '.nc')
-        elif ncfile_in[-7:-5] == 'sf':
-            ncfile_out = path.join(ncfile_in[:-11],'erai_sf_all' + '.nc')
-        elif ncfile_in[-7:-5] == 'pl':
-            ncfile_out = path.join(ncfile_in[:-11],'erai_pl_all' + '.nc')
-        else:
-            print('There is not such type of file'    )
-
-        # get variables
-        varlist = [str_encode(x) for x in ncf_in.variables.keys()]
-        varlist.remove('time')
-        varlist.remove('latitude')
-        varlist.remove('longitude')
-        if pl: #only for pressure level files
-            varlist.remove('level')
-
-        #Build the netCDF file
-        rootgrp = nc.Dataset(ncfile_out, 'w', format='NETCDF4_CLASSIC')
-        rootgrp.Conventions = 'CF-1.6'
-        rootgrp.source      = 'ERA_Interim, merged downloaded original files'
-        rootgrp.featureType = "timeSeries"
-
-        # dimensions
-        latitude = rootgrp.createDimension('latitude', len(lat))
-        longitude = rootgrp.createDimension('longitude', len(lon))
-        time    = rootgrp.createDimension('time', None)
-
-        # base variables
-        time           = rootgrp.createVariable('time', 'i4',('time'))
-        time.long_name = 'time'
-        time.units     = 'hours since 1900-01-01 00:00:0.0'
-        time.calendar  = 'gregorian'
-        latitude            = rootgrp.createVariable('latitude', 'f4',('latitude'))
-        latitude.long_name  = 'latitude'
-        latitude.units      = 'degrees_north'
-        longitude           = rootgrp.createVariable('longitude','f4',('longitude'))
-        longitude.long_name = 'longitude'
-        longitude.units     = 'degrees_east'
-
-        # assign station characteristics
-        latitude[:]  = lat[:]
-        longitude[:] = lon[:]
-        time[:]    = nctime[:]
-
-        # extra treatment for pressure level files
-        try:
-            lev = ncf_in.variables['level'][:]
-            print("== 3D: file has pressure levels")
-            level = rootgrp.createDimension('level', len(lev))
-            level           = rootgrp.createVariable('level','i4',('level'))
-            level.long_name = 'pressure_level'
-            level.units     = 'hPa'
-            level[:] = lev
-        except:
-            print("== 2D: file without pressure levels")
-            lev = []
-
-        # create and assign variables based on input file
-        for n, var in enumerate(varlist):
-            print("VAR: ", var)
-            # extra treatment for pressure level files
-            if len(lev):
-                tmp = rootgrp.createVariable(var,'f4',('time', 'level', 'latitude', 'longitude'))
-            else:
-                tmp = rootgrp.createVariable(var,'f4',('time', 'latitude', 'longitude'))
-            tmp.long_name = ncf_in.variables[var].long_name.encode('UTF8') # for erai
-            tmp.units     = ncf_in.variables[var].units.encode('UTF8')
-
-            # assign values
-            if pl: # only for pressure level files
-                tmp[:] = ncf_in.variables[var][:,:,:,:]
-            else:
-                tmp[:] = ncf_in.variables[var][:,:,:]
-
-
-        #close the file
-        rootgrp.close()
-        ncf_in.close()
-
-        #get the file list
-        files_list = glob.glob(ncfile_in)
-        files_list.sort()
-
-        #clear up the data
-        for fl in files_list:
-            remove(fl)
-
-
-class ERAIpl(ERAIgeneric):
-    """Returns an object for ERA-Interim data that has methods for querying the
-    ECMWF server.
-
-    Args:
-        date: A dictionary specifying the time period desired with a begin
-              and an end date given as a datetime.datetime object.
-
-        area: A dictionary delimiting the area to be queried with the latitudes
-              north and south, and the longitudes west and east [decimal deg].
-
-        elevation: A dictionary specifying the min/max elevation of the area of
-                   interest. This is used to determine the pressure levels
-                   needed. Unit: [m].
-
-        variables:  List of variable(s) to download based on CF Standard Names.
-
-        directory: Directory to hold output files
-
-    Example:
-        from datetime import datetime
-        date  = {'beg' : datetime(1994, 1, 1),
-                 'end' : datetime(2013, 1, 1)}
-        area  = {'north' :  40.0,
-                 'south' :  15.0,
-                 'west'  :  60.0,
-                 'east'  : 105.0}
-        elevation = {'min' :    0,
-                     'max' : 8850}
-        variables  = ['air_temperature', 'relative_humidity']
-        directory = '/Users/stgruber/Desktop'
-        ERAIpl = ERAIpl(date, area, elevation, variables, directory)
-        ERAIpl.download()
-    """
-    def __init__(self, date, area, elevation, variables, directory):
-        self.date       = date
-        self.area       = area
-        self.elevation  = elevation
-        self.directory  = directory
-        outfile = 'erai_pl' + self.getDstring() + '.nc'
-        self.file_ncdf  = path.join(self.directory, outfile)
-
-        # dictionary to translate CF Standard Names into ERA-Interim
-        # pressure level variable names.
-        dpar = {'air_temperature'   : '130.128',           # [K]
-                'relative_humidity' : '157.128',           # [%]
-                'wind_speed'        : '131.128/132.128'}   # [m s-1]
-
-        # translate variables into those present in ERA pl data
-        self.TranslateCF2ERA(variables, dpar)
-        self.param += '/129.128' # geopotential always needed [m2 s-2]
-
-    def getDictionary(self):
-        self.dictionary = {
-           'levtype'  : "pl",
-           'levellist': self.getPressureLevels(self.elevation),
-           'time'     : "00/06/12/18",
-           'step'     : "0",
-           'type'     : "an",
-           'param'    : self.param,
-           'target'   : self.file_ncdf
-           }
-        self.dictionary.update(self.getDictionaryGen(self.area, self.date))
-        return self.dictionary
-
-    def __str__(self):
-        string = ("List of variables to query ECMWF server for "
-                  "ERA-Interim data: {0}")
-        return string.format(self.getDictionary)
-
-
-class ERAIsa(ERAIgeneric):
-    """
-    Returns an object for ERA-Interim data that has methods for querying the
-    ECMWF server for surface analysis variables (airt2,dewp2, ozone, vapor,
-    wind10).
-
-    Args:
-
-        target:    File name of the netcdf file to be created.
-
-        variable:  List of variable(s) to download based on CF Standard Names
-
-    Example:
-        from datetime import datetime
-        date  = {'beg' : datetime(1994, 1, 1),
-                 'end' : datetime(2013, 1, 1)}
-        area  = {'north' :  40.0,
-                 'south' :  15.0,
-                 'west'  :  60.0,
-                 'east'  : 105.0}
-        variables  = ['air_temperature', 'relative_humidity']
-        directory = '/Users/stgruber/Desktop'
-        ERAIsa = ERAIsa(date, area, variables, directory)
-        ERAIsa.download()
-    """
-    def __init__(self, date, area, variables, directory):
-        self.date       = date
-        self.area       = area
-        self.directory  = directory
-        outfile = 'erai_sa' + self.getDstring() + '.nc'
-        self.file_ncdf  = path.join(self.directory, outfile)
-
-        # dictionary to translate CF Standard Names into ERA-Interim
-        # pressure level variable names.
-        dpar = {'air_temperature'   : '167.128',  # [K] 2m values
-                'relative_humidity' : '168.128',  # [K] 2m values
-                'downwelling_shortwave_flux_in_air_assuming_clear_sky' :
-                    '206.128/137.128',  # [kg m-2] Total column ozone
-                                        # [kg m-2] Total column W vapor
-                'wind_speed' : '165.128/166.128'}   # [m s-1] 10m values
-
-        # translate variables into those present in ERA pl data
-        self.TranslateCF2ERA(variables, dpar)
-        print(self.param)
-
-
-    def getDictionary(self):
-        self.dictionary = {
-           'levtype'  : "sfc",
-           'time'     : "00/06/12/18",
-           'step'     : "0",
-           'type'     : "an",
-           'param'    : self.param,
-           'target'   : self.file_ncdf
-           }
-        self.dictionary.update(self.getDictionaryGen(self.area, self.date))
-        return self.dictionary
-
-    def __str__(self):
-        string = ("Class for ERA-Interim surface analysis data: {0}")
-        return string.format(self.getDictionary)
-
-
-class ERAIsf(ERAIgeneric):
-    """
-    Returns an object for ERA-Interim data that has methods for querying the
-    ECMWF server for surface forecast variables (prec, swin, lwin).
-
-    Args:
-        target:    File name of the netcdf file to be created.
-
-        variables:  List of variable(s) to download that can include one, several
-                   , or all of these: ['airt', 'rh', 'geop', 'wind'].
-
-        ERAgen:    ERAIgeneric() object with generic information on the area and
-                   time span to be
-
-    Example:
-        from datetime import datetime
-        date  = {'beg' : datetime(1994, 1, 1),
-                 'end' : datetime(1994, 1, 1)}
-        area  = {'north' :  40.0,
-                 'south' :  40.0,
-                 'west'  :  60.0,
-                 'east'  :  60.0}
-        variables  = ['prec','swin','lwin']
-        directory = '/Users/stgruber/Desktop'
-        ERAIsf = ERAIsf(date, area, variables, directory)
-        ERAIsf.download()
-    """
-    def __init__(self, date, area, variables, directory):
-        self.date       = date
-        self.area       = area
-        self.directory  = directory
-        outfile = 'erai_sf' + self.getDstring() + '.nc'
-        self.file_ncdf  = path.join(self.directory, outfile)
-
-        # dictionary to translate CF Standard Names into ERA-Interim
-        # pressure level variable names.
-        # [m] total precipitation
-        # [J m-2] short-wave downward
-        # [J m-2] long-wave downward
-        dpar = {'precipitation_amount'              : '228.128',
-                'downwelling_shortwave_flux_in_air' : '169.128',
-                'downwelling_longwave_flux_in_air'  : '175.128'}
-
-        # translate variables into those present in ERA pl data
-        self.TranslateCF2ERA(variables, dpar)
-        print(self.param)
-
-
-    def getDictionary(self):
-        self.dictionary = {
-           'levtype'  : "sfc",
-           'time'     : "00/12",
-           'step'     : "3/6/9/12",
-           'type'     : "fc",
-           'param'    : self.param,
-           'target'   : self.file_ncdf
-           }
-        self.dictionary.update(self.getDictionaryGen(self.area, self.date))
-        return self.dictionary
-
-    def __str__(self):
-        string = ("List of variables to query ECMWF server for "
-                  "ERA-Interim air tenperature data: {0}")
-        return string.format(self.getDictionary)
-
-class ERAIto(ERAIgeneric):
-    """
-    Returns an object for downloading and handling ERA-Interim
-    topography (invariant).
-
-    Args:
-
-    Example:
-        area  = {'north' :  40.0,
-                 'south' :  45.0,
-                 'west'  :  60.0,
-                 'east'  :  65.0}
-        directory = '/Users/stgruber/Desktop'
-        ERAIto = ERAIto(area, directory)
-        ERAIto.download()
-    """
-    def __init__(self, area, directory):
-        self.area       = area
-        self.date       = {'beg' : datetime(1979, 1, 1),
-                           'end' : datetime(1979, 1, 1)}
-        self.directory  = directory
-        self.file_ncdf  = path.join(self.directory,'erai_to.nc')
-
-    def getDictionary(self):
-        self.dictionary = {
-           'levtype'  : "sfc",
-           'time'     : "12",
-           'step'     : "0",
-           'type'     : "an",
-           'param'    : "129.128/172.128", # geopotential and land-sea mask
-           'target'   : self.file_ncdf
-           }
-        self.dictionary.update(self.getDictionaryGen(self.area, self.date))
-        return self.dictionary
-
-    def __str__(self):
-        string = ("List of variables to query ECMWF server for "
-                  "ERA-Interim air tenperature data: {0}")
-        return string.format(self.getDictionary)
-
-
-class ERAIdownload(GenericDownload):
-    """
-    Class for ERA-Interim data that has methods for querying
-    the ECMWF server, returning all variables usually needed.
-
-    Args:
-        pfile: Full path to a Globsim Download Parameter file.
-
-    Example:
-        ERAd = ERAdownload(pfile)
-        ERAd.retrieve()
-    """
-
-    def __init__(self, pfile):
-        super().__init__(pfile)
-        par = self.par
-        self._set_input_directory("erai")
-
-        # time bounds
-        self.date  = {'beg': datetime.strptime(par['beg'], '%Y/%m/%d'),
-                      'end': datetime.strptime(par['end'], '%Y/%m/%d')}
-
-        # chunk size for downloading and storing data [days]
-        self.chunk_size = par['chunk_size']
-
-    def retrieve(self):
-        """
-        Retrieve all required ERA-Interim data from MARS server.
-        """
-        # prepare time loop
-        date_i = {}
-        slices = floor(float((self.date['end'] - self.date['beg']).days)/
-                       self.chunk_size)+1
-        # topography
-        top = ERAIto(self.area, self.directory)
-        top.download()
-
-        for ind in range (0, int(slices)):
-            #prepare time slices
-            date_i['beg'] = self.date['beg'] + timedelta(days =
-                            self.chunk_size * ind)
-            date_i['end'] = self.date['beg'] + timedelta(days =
-                            self.chunk_size * (ind+1) - 1)
-            if ind == (slices-1):
-                date_i['end'] = self.date['end']
-
-            #actual functions
-            pl = ERAIpl(date_i, self.area, self.elevation,
-                       self.variables, self.directory)
-            sa = ERAIsa(date_i, self.area, self.variables, self.directory)
-            sf = ERAIsf(date_i, self.area, self.variables, self.directory)
-
-            #download from ECMWF server convert to netCDF
-            ERAli = [pl, sa, sf]
-            for era in ERAli:
-                era.download()
-
-
-        # report inventory
-        self.inventory()
-
-
-    def inventory(self):
-        """
-        Report on data avaialbe in directory: time slice, variables, area
-        """
-        print("\n\n\n")
-        print("=== INVENTORY FOR GLOBSIM ERA-INTERIM DATA === \n")
-        print("Download parameter file: \n" + self.pfile + "\n")
-        # loop over filetypes, read, report
-        file_type = ['erai_pl_*.nc', 'erai_sa_*.nc', 'erai_sf_*.nc', 'erai_t*.nc']
-        for ft in file_type:
-            infile = path.join(self.directory, ft)
-            nf = len(filter(listdir(self.directory), ft))
-            print(str(nf) + " FILE(S): " + infile)
-
-            if nf > 0:
-                # open dataset
-                ncf = nc.MFDataset(infile, 'r')
-
-                # list variables
-                keylist = [str_encode(x) for x in ncf.variables.keys()]
-
-                print("    VARIABLES:")
-                print("        " + str(len(keylist)) +
-                      " variables, inclusing dimensions")
-                for key in keylist:
-                    print("        " + ncf.variables[key].long_name)
-
-                # time slice
-                time = ncf.variables['time']
-                tmin = nc.num2date(min(time[:]), time.units,
-                                   calendar=time.calendar).strftime('%Y/%m/%d')
-                tmax = nc.num2date(max(time[:]), time.units,
-                                   calendar=time.calendar).strftime('%Y/%m/%d')
-                print("    TIME SLICE")
-                print("        " + str(len(time[:])) + " time steps")
-                print("        " + tmin + " to " + tmax)
-
-                # area
-                lon = ncf.variables['longitude']
-                lat = ncf.variables['latitude']
-                nlat = str(len(lat))
-                nlon = str(len(lon))
-                ncel = str(len(lat) * len(lon))
-                print("    BOUNDING BOX / AREA")
-                print("        " + ncel + " cells, " + nlon +
-                      " W-E and " + nlat + " S-N")
-                print("        N: " + str(max(lat)))
-                print("        S: " + str(min(lat)))
-                print("        W: " + str(min(lon)))
-                print("        E: " + str(max(lon)))
-
-                ncf.close()
-
-    def __str__(self):
-        return "Object for ERA-Interim data download and conversion"
diff --git a/globsim/download/GenericDownload.py b/globsim/download/GenericDownload.py
index 258fa36..02e4325 100644
--- a/globsim/download/GenericDownload.py
+++ b/globsim/download/GenericDownload.py
@@ -14,7 +14,8 @@ class GenericDownload(ABC):
     """
     Generic functionality for download classes
     """
-
+    retry_delay_min = 0.5
+    
     def __init__(self, pfile):
         # read parameter file
         self.pfile = pfile
diff --git a/globsim/download/JRA3QJMAdownload.py b/globsim/download/JRA3QJMAdownload.py
new file mode 100644
index 0000000..e21d115
--- /dev/null
+++ b/globsim/download/JRA3QJMAdownload.py
@@ -0,0 +1,536 @@
+import logging
+import netCDF4 as nc
+import numpy as np
+import pandas as pd
+import pygrib
+import xarray as xr
+
+from abc import ABC, abstractmethod
+from datetime import datetime, timedelta
+from os import rename
+from pathlib import Path
+from typing import Optional, Union
+from urllib.error import URLError
+
+from globsim.download.GenericDownload import GenericDownload
+from globsim.download.JRA3Q_helpers import GribSubsetter, download_daily_gribs, download_constant
+from globsim.download.JRAdownload import getDate
+from globsim.download.JRA3Q_dl import GetAccessor
+from globsim.download.jra_dict_formatters import getPressureLevels
+
+
+logger = logging.getLogger('globsim.download')
+
+CHUNK_LAT = 4
+CHUNK_LON = 4
+
+
+class J3QD(GenericDownload):
+
+    _tunits = "hours since 1947-01-01"
+    _tcal = "standard"
+    REANALYSIS = 'jra3q'
+
+    def __init__(self, pfile):
+        super().__init__(pfile)
+        self.retry_delay_min = 1
+        par = self.par
+        self._set_input_directory("jra3q")
+        levels = getPressureLevels(GribSubsetter.DEFAULT_LEV_HPA, self.elevation['min'], self.elevation['max'])
+        self._subsetter = GribSubsetter(round((float(par['bbW']) - 1.25) / 1.25) * 1.25,
+                                        round(float(par['bbE']) / 1.25) * 1.25,
+                                        round((float(par['bbS']) - 1.25) / 1.25) * 1.25,
+                                        round(float(par['bbN']) / 1.25) * 1.25,
+                                        levels=levels)
+
+        # time bounds
+        self.date  = getDate(par)
+
+        # Connect
+        self.credential = Path(par['credentials_directory'], ".netrc")
+        self.connect()
+
+        # chunk size for downloading and storing data [days]
+        logger.info("Chunk size ignored for JRA3Q download. Using monthly")
+        # self.chunk_size = par['chunk_size'] * 2000
+
+        # variables
+        logger.info(f"Variable list ignored for {self.REANALYSIS}")
+
+        if par.get("download_only") is not None:
+            self.download_only = par.get("download_only")
+        else:
+            self.download_only = False
+
+    def file_in_progress(self, filename):
+        return str(Path(filename).with_suffix(".inp"))
+    
+    def complete(self, filename):
+        f = self.file_in_progress(filename)
+        rename(f, filename)
+
+    def connect(self):
+        try:
+            self.access = GetAccessor(netrc_file=self.credential)
+        except ValueError:
+            logger.error("Could not find credentials. Aborting")
+            raise ValueError(f"Could not find credentials file {self.credential}")
+        except URLError:
+            logger.error("Could not connect to JRA server. This aint gonna work.")
+
+    @staticmethod
+    def date2num(dt:datetime) -> float:
+        t = nc.date2num(dt, J3QD._tunits, J3QD._tcal)
+        t = np.round(t, 5)  # catch any floating-point errors
+        return t
+
+    def nc_filename(self, year, month, kind="sa") -> str:
+        return f"{self.REANALYSIS}_{kind}_{year}{month}01_to_{year}{month}31.nc"
+
+    def retrieve(self):
+        days = pd.date_range(self.date['beg'], self.date['end'] + timedelta(days=1))  # add extra day but end when we get to it
+
+        # Constant surface data
+        to_grib = [download_constant(self.access, self.directory)]
+        t = days[0]
+        to_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_to_{t.strftime(r"%Y%m%d")}_to_{t.strftime(r"%Y%m%d")}.nc')
+        gribs_to_netcdf(*to_grib,
+                        netcdf_file=to_ncfile,
+                        tstart=self.date2num(t),
+                        tstop=self.date2num(days[1]),
+                        kind="to",
+                        subsetter=self._subsetter)
+
+        # Temporal data
+        running_sa, running_sf, running_pl, running_dates = [], [], [], []
+
+        for date in days:
+            end_of_days = (date == days[-1])
+            
+            # Aggregation subroutine
+            if (not self.download_only) and ((date.day == 1 and len(running_dates) > 1) or end_of_days):
+                t1 = running_dates[0]
+                t2 = running_dates[-1]
+                tstart = J3QD.date2num(t1)
+                tstop = J3QD.date2num(date)  # t2 + timedelta(days=1)
+
+                sa_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_sa_{t1.strftime(r"%Y%m%d")}_to_{t2.strftime(r"%Y%m%d")}.nc')
+                sf_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_sf_{t1.strftime(r"%Y%m%d")}_to_{t2.strftime(r"%Y%m%d")}.nc')
+                pl_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_pl_{t1.strftime(r"%Y%m%d")}_to_{t2.strftime(r"%Y%m%d")}.nc')
+
+                # aggregate gribs then delete them
+                logger.info(f"Aggregating gribs to netcdf: {t1} to {t2}")
+
+                if not sa_ncfile.is_file():
+                    gribs_to_netcdf(*running_sa, netcdf_file=self.file_in_progress(sa_ncfile),
+                                    tstart=tstart, tstop=tstop, kind="sa", subsetter=self._subsetter, engine="pygrib")
+                    self.complete(sa_ncfile)
+                
+                if not sf_ncfile.is_file():
+                    gribs_to_netcdf(*running_sf, netcdf_file=self.file_in_progress(sf_ncfile), 
+                                    tstart=tstart, tstop=tstop, kind="sf", subsetter=self._subsetter, engine="pygrib")
+                    self.complete(sf_ncfile)
+                
+                if not pl_ncfile.is_file():
+                    gribs_to_netcdf(*running_pl, netcdf_file=self.file_in_progress(pl_ncfile),
+                                    tstart=tstart, tstop=tstop, kind="pl", subsetter=self._subsetter, engine="xarray")
+                    self.complete(pl_ncfile)
+                
+                #  delete grib files
+                logger.info("Not deleting grib files")
+                #for grib in running_sa + running_sf + running_pl:
+                    #Path(grib).unlink()
+
+                # clear running
+                running_sa, running_sf, running_pl, running_dates = [], [], [], []
+
+            if end_of_days:
+                break
+
+            # Download
+            logger.info(f"Downloading gribs for {date}")
+            sa, sf, pl = download_daily_gribs(self.access, self.directory, date.year, date.month, date.day)
+            running_sa += sa
+            running_sf += sf
+            running_pl += pl
+            running_dates.append(date)
+
+
+class VariableError(ValueError):
+    pass
+
+
+class ConversionHandler(ABC):
+    """
+    Handles the translation of grib files into a netcdf file
+    """
+    _include = []
+    _dims = ()
+
+    def __init__(self,
+                 tstart: float,
+                 tstop: float,
+                 subsetter:Optional[GribSubsetter] = None):
+
+        self._tstart = tstart
+        self._tstop = tstop
+        self.subsetter = subsetter if subsetter else GribSubsetter(0, 360, -90, 90)
+
+    @property
+    def tstep(self):
+        raise NotImplementedError("implement in child class")
+
+    @property
+    def times(self):
+        t = np.arange(self._tstart, self._tstop, self.tstep)
+        return t
+
+    @abstractmethod
+    def empty_file(self, *args, **kwargs):
+        raise NotImplementedError("implement in child class")
+
+    def create_variable(self, shortName, ncd, ncvar_dict):
+        var = ncd.createVariable(shortName, "f8", self._dims, chunksizes=self.chunk_sizes(ncd))
+        for attr in ncvar_dict:
+            var.setncattr(attr, ncvar_dict[attr])
+
+    def chunk_sizes(self, ncd):
+        return None
+    
+    def valid_record(self, record):
+        is_valid = record.shortName in self._include
+        return is_valid
+    
+    def get_valid_vars(self, dataset) -> "list[str]":
+        return [v for v in dataset.variables if v in self._include.keys()]
+
+    def xread_grib(self, gribfile) -> "xr.Dataset":
+        d = xr.open_dataset(gribfile, engine='cfgrib')
+        return d
+
+    def xgrib_to_nc(self, ncfile, gribfile):
+        logger.debug(f"de-gribbing {Path(gribfile).name} with xarray")
+        dataset = self.xread_grib(gribfile)
+        for v in self.get_valid_vars(dataset):
+            var = dataset[v]
+            self.xvar_to_nc(var, ncfile)
+    
+    def xvar_to_nc(self, var, ncfile):
+        if self.subsetter is not None:
+            vals, lats, lons = self.subsetter.xsubset(var)
+        else:
+            vals, lats, lons = var.values, var.latitude.values, var.longitude.values
+
+        # check if ncfile exists
+        if not Path(ncfile).exists():
+            logger.warning(f"missing file {ncfile}. Creating it")
+            self.empty_file(ncfile, lats, lons, self.times)
+
+        with nc.Dataset(ncfile, 'a') as ncd:
+            # check if variable is in ncfile
+            # if not, add it
+            ncvar_dict = {"units" : var.units,
+                          "long_name" : var.name}
+
+            if var.name not in ncd.variables:
+                self.create_variable(var.name, ncd, ncvar_dict)
+
+            # check if timestep is in ncfile
+            timestep = J3QD.date2num(pd.Timestamp(var.valid_time.values).to_pydatetime())
+            try:
+                t_i = np.where(self.times == timestep)[0][0]
+            except IndexError:
+                t_i = self.handle_missing_timestep(var.valid_time.values, ncfile)
+
+            # write data
+            if t_i is not None:
+                self.xwrite_record(ncd=ncd,
+                                   varname=var.name,
+                                   t_i=t_i,
+                                   vals=vals,
+                                   var=var)
+
+    def grib_to_nc(self, ncfile, record):
+        varname = record.shortName
+
+        if not self.valid_record(record):
+            raise VariableError(f"{varname}")
+
+        if self.subsetter is not None:
+            vals, lats, lons = self.subsetter.subset(record)
+        else:
+            vals, lats, lons = record.data()
+
+        # check if ncfile exists
+        if not Path(ncfile).exists():
+            logger.warning(f"missing file {ncfile}. Creating it")
+            lats = lats[:,0]
+            lons = lons[0,:]
+
+            self.empty_file(ncfile, lats, lons, self.times)  # if not, create it
+
+        with nc.Dataset(ncfile, 'a') as ncd:
+            # check if variable is in ncfile
+            # if not, add it
+            ncvar_dict = {"units" : record.units,
+                          "long_name" : record.name,
+                          "comment": record.__repr__()}
+
+            if varname not in ncd.variables:
+                self.create_variable(record.shortName, ncd, ncvar_dict)
+
+            # check if timestep is in ncfile
+            timestep = J3QD.date2num(record.validDate)
+            try:
+                t_i = np.where(self.times == timestep)[0][0]
+            except IndexError:
+                t_i = self.handle_missing_timestep(record, ncfile)
+            
+            # write temporary array
+            # self.write_mem(ncd, t_i, vals, record)
+
+            # write data
+            if t_i is not None:
+                self.write_record(ncd=ncd,
+                                  varname=varname,
+                                  t_i=t_i,
+                                  vals=vals,
+                                  record=record)
+
+    def handle_missing_timestep(self, date, ncfile):
+        print(f"err: timestamp {date} not in ncfile {ncfile}")
+        return None
+
+    def write_record(self, ncd, varname, t_i, vals, record):
+        ncd[varname][t_i, :, :] = vals
+
+    def xwrite_record(self, ncd, varname, t_i, vals, var):
+        ncd[varname][t_i, :, :] = vals
+
+    def valid_messages(self):
+        m = [d["messagenumber"] for d in list(self._include.values()) if "messagenumber" in d]
+        return m
+
+
+class ToConverter(ConversionHandler):
+    _include = {"z": None}
+    _dims = ("time", "latitude", "longitude",)
+
+    @property
+    def tstep(self):
+        return 24
+
+    def empty_file(self, filename:str, lats, lons, times):
+        empty_surface_file(filename, lats, lons, times)
+
+    def write_record(self, ncd, varname, t_i, vals, record):
+        ncd[varname][0, :, :] = vals
+
+    def xwrite_record(self, ncd, varname, t_i, vals, var):
+        ncd[varname][0, :, :] = vals
+
+    def handle_missing_timestep(self, date, ncfile):
+        return 0
+
+
+class SaConverter(ConversionHandler):
+    _include = {"sp": None,
+                "2t": None,
+                "2sh": None,
+                "2r": None,
+                "10u": None,
+                "10v": None}
+    _dims = ("time", "latitude", "longitude",)
+
+    @property
+    def tstep(self):
+        # 6-hourly, calculated on the fly in case we change time units in the future
+        t1 = J3QD.date2num(datetime(1980, 3, 1, 0))
+        t2 = J3QD.date2num(datetime(1980, 3, 1, 6))
+        return t2 - t1
+
+    def chunk_sizes(self, ncd):
+        return (ncd['time'].shape[0], CHUNK_LAT, CHUNK_LON,)
+    
+    def xread_grib(self, gribfile) -> xr.Dataset:
+        d = xr.open_dataset(gribfile, engine='cfgrib', backend_kwargs={"filter_by_keys":{'typeOfLevel': 'surface'}})
+        return d
+
+    def empty_file(self, filename:str, lats, lons, times):
+        empty_surface_file(filename, lats, lons, times)
+
+
+class FcstConverter(ConversionHandler):
+    _include = {"tprate": {"messagenumber": 6},
+                "sp": {"messagenumber": 12},
+                'dswrf': {"messagenumber": 13},
+                "dlwrf": {"messagenumber": 17}
+                }
+    _dims = ("time", "latitude", "longitude",)
+   
+    @property
+    def tstep(self):
+        # 6-hourly, calculated on the fly in case we change time units in the future
+        t1 = J3QD.date2num(datetime(1980, 3, 1, 0))
+        t2 = J3QD.date2num(datetime(1980, 3, 1, 1))
+        return t2 - t1
+
+    def valid_record(self, record):
+        return record.messagenumber in self.valid_messages()
+        
+    def empty_file(self, filename:str, lats, lons, times):
+        empty_surface_file(filename, lats, lons, times)
+
+    def xread_grib(self, gribfile) -> xr.Dataset:
+        d = xr.open_dataset(gribfile, engine='cfgrib', backend_kwargs={"filter_by_keys":{'typeOfLevel': 'surface'}})
+        return d
+
+    def chunk_sizes(self, ncd):
+        return (ncd['time'].shape[0], CHUNK_LAT, CHUNK_LON,)
+
+
+class PlConverter(ConversionHandler):
+    _dims = ("time", "level", "latitude", "longitude",)
+    _include = {"t":None,  # temperature
+                "u":None,  # u-wind
+                "v":None,  # v-wind
+                "q":None,  # specific humidity
+                "r":None,   # relative humidity
+                "gh":None    # geopotential
+                }
+
+    def __init__(self, tstart, tstop, subsetter):
+        super().__init__(tstart, tstop, subsetter)
+        # self.mem_array = np.empty(len(self.times), len(self.subsetter.levels), len(self.subsetter.lats), len(self.subsetter.lons))
+
+    def valid_record(self, record):
+        good_variable = record.shortName in self._include
+        lv_hPa = record.level if record.pressureUnits == "hPa" else record.level / 10
+        good_level = lv_hPa in self.subsetter.levels
+
+        return (good_variable and good_level)
+
+    def xread_grib(self, gribfile) -> xr.Dataset:
+        d = xr.open_dataset(gribfile, engine='cfgrib', backend_kwargs={"filter_by_keys":{'typeOfLevel': 'isobaricInhPa'}})
+        return d
+    
+    @property
+    def tstep(self):
+        # 6-hourly, calculated on the fly in case we change time units in the future
+        t1 = J3QD.date2num(datetime(1980, 3, 1, 0))
+        t2 = J3QD.date2num(datetime(1980, 3, 1, 6))
+        return t2 - t1
+
+    def empty_file(self, filename:str, lats, lons, times):
+        levels = self.subsetter.levels
+        empty_pressure_levels_file(filename, lats, lons, times, levels)
+
+    def write_record(self, ncd, varname, t_i, vals, record):
+        lev = record.level if record.pressureUnits == "hPa" else record.level / 10
+
+        lev_i = np.where(ncd['level'][:] == lev)[0][0]
+
+        ncd[varname][t_i, lev_i, :, :] = vals
+
+    def xwrite_record(self, ncd, varname, t_i, vals, var):
+        ncd[varname][t_i, :, :, :] = vals[::-1, :, :]  # put levels in backwards to match JRA55
+    
+    def chunk_sizes(self, ncd):
+        return (ncd['time'].shape[0], ncd['level'].shape[0], CHUNK_LAT, CHUNK_LON,)
+
+
+def empty_constant_file(filename: str, lats, lons):
+    with nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC') as ncd:
+        _ = ncd.createDimension('latitude', len(lats))
+        _ = ncd.createDimension('longitude', len(lons))
+
+        var_lat = ncd.createVariable('latitude', 'f8', ('latitude',))
+        var_lat.units = "degrees_north"
+        var_lon = ncd.createVariable('longitude', 'f8', ('longitude',))
+        var_lon.units = "degrees_east"
+
+        var_lat[:] = lats
+        var_lon[:] = lons
+        
+
+def empty_surface_file(filename:str, lats, lons, times):
+    with nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC') as ncd:
+        _ = ncd.createDimension('time', len(times))
+        _ = ncd.createDimension('latitude', len(lats))
+        _ = ncd.createDimension('longitude', len(lons))
+
+        var_time = ncd.createVariable('time', 'f8', ('time',))
+        var_time.units = J3QD._tunits
+        var_time.calendar = J3QD._tcal
+        var_lat = ncd.createVariable('latitude', 'f8', ('latitude',))
+        var_lat.units = "degrees_north"
+        var_lon = ncd.createVariable('longitude', 'f8', ('longitude',))
+        var_lon.units = "degrees_east"
+
+        var_time[:] = times
+        var_lat[:] = lats
+        var_lon[:] = lons
+
+
+def empty_pressure_levels_file(filename:str, lats, lons, times, levs):
+    with nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC') as ncd:
+        _ = ncd.createDimension('level', len(levs))
+        _ = ncd.createDimension('time', len(times))
+        _ = ncd.createDimension('latitude', len(lats))
+        _ = ncd.createDimension('longitude', len(lons))
+
+        var_time = ncd.createVariable('time', 'f8', ('time',))
+        var_time.units = J3QD._tunits
+        var_time.calendar = J3QD._tcal
+        var_lat = ncd.createVariable('latitude', 'f8', ('latitude',))
+        var_lat.units = "degrees_north"
+        var_lon = ncd.createVariable('longitude', 'f8', ('longitude',))
+        var_lon.units = "degrees_east"
+        var_lev = ncd.createVariable('level', 'f8', ('level',))
+        var_lev.units = "hPa"
+
+        var_time[:] = times
+        var_lat[:] = lats
+        var_lon[:] = lons
+        var_lev[:] = levs
+
+
+def gribs_to_netcdf(*grib_files:"Union[str,Path]", netcdf_file:"Union[str,Path]", tstart:float, tstop:float, kind:str, subsetter=None, engine="xarray"):
+    logger.info(f"creating {netcdf_file}")
+    notified = []
+    Converter = get_converter(kind)
+    converter = Converter(tstart, tstop, subsetter)
+
+    for gribfile in grib_files:
+        if engine == "xarray":
+            converter.xgrib_to_nc(netcdf_file, gribfile)
+
+        elif engine == "pygrib":
+            logger.debug(f"de-gribbing {Path(gribfile).name} with pygrib")
+
+            with pygrib.open(str(gribfile)) as f:  # pygrib doesn't like Path objects
+                for record in f:
+                    if not converter.valid_record(record):
+                        continue
+                    try:
+                        converter.grib_to_nc(netcdf_file, record)
+                    except VariableError as e:
+                        if e.args[0] not in notified:
+                            logger.warning(f"warning: skip variable {e}")
+                            notified.append(e.args[0])
+
+        else:
+            raise ValueError("engine must be 'pygrib' or 'xarray'")
+
+
+def get_converter(kind:str) -> "type[ConversionHandler]":
+    if kind == "sf":
+        return FcstConverter
+    elif kind == "sa":
+        return SaConverter
+    elif kind == "pl":
+        return PlConverter
+    elif kind == "to":
+        return ToConverter
+    else:
+        raise ValueError(f"unknown kind {kind}")
diff --git a/globsim/download/JRA3Qdownload.py b/globsim/download/JRA3Qdownload.py
index e190b93..a98fa33 100644
--- a/globsim/download/JRA3Qdownload.py
+++ b/globsim/download/JRA3Qdownload.py
@@ -1,536 +1,27 @@
 import logging
-import netCDF4 as nc
-import numpy as np
-import pandas as pd
-import pygrib
-import xarray as xr
-
-from abc import ABC, abstractmethod
-from datetime import datetime, timedelta
-from os import rename
-from pathlib import Path
-from typing import Optional, Union
-from urllib.error import URLError
-
-from globsim.download.GenericDownload import GenericDownload
-from globsim.download.JRA3Q_helpers import GribSubsetter, download_daily_gribs, download_constant
-from globsim.download.JRAdownload import getDate
-from globsim.download.JRA3Q_dl import GetAccessor
-from globsim.download.JRAdownload import getPressureLevels
 
+from globsim.download.JRAdownload import JRAdownload
+from globsim.download.RDA import Rdams 
+from globsim.download.jra_dict_formatters import J3QDictFormatter, J3QGDictFormatter
+from globsim.download.JraDownloadHandler import J3QDownloadHandler
 
 logger = logging.getLogger('globsim.download')
 
-CHUNK_LAT = 4
-CHUNK_LON = 4
-
-
-class J3QD(GenericDownload):
-
-    _tunits = "hours since 1947-01-01"
-    _tcal = "standard"
-    REANALYSIS = 'jra3q'
-
-    def __init__(self, pfile):
-        super().__init__(pfile)
-        self.retry_delay_min = 1
-        par = self.par
-        self._set_input_directory("jra3q")
-        levels = getPressureLevels(GribSubsetter.DEFAULT_LEV_HPA, self.elevation['min'], self.elevation['max'])
-        self._subsetter = GribSubsetter(round((float(par['bbW']) - 1.25) / 1.25) * 1.25,
-                                        round(float(par['bbE']) / 1.25) * 1.25,
-                                        round((float(par['bbS']) - 1.25) / 1.25) * 1.25,
-                                        round(float(par['bbN']) / 1.25) * 1.25,
-                                        levels=levels)
-
-        # time bounds
-        self.date  = getDate(par)
-
-        # Connect
-        self.credential = Path(par['credentials_directory'], ".netrc")
-        self.connect()
-
-        # chunk size for downloading and storing data [days]
-        logger.info("Chunk size ignored for JRA3Q download. Using monthly")
-        # self.chunk_size = par['chunk_size'] * 2000
-
-        # variables
-        logger.info(f"Variable list ignored for {self.REANALYSIS}")
-
-        if par.get("download_only") is not None:
-            self.download_only = par.get("download_only")
-        else:
-            self.download_only = False
-
-    def file_in_progress(self, filename):
-        return str(Path(filename).with_suffix(".inp"))
-    
-    def complete(self, filename):
-        f = self.file_in_progress(filename)
-        rename(f, filename)
-
-    def connect(self):
-        try:
-            self.access = GetAccessor(netrc_file=self.credential)
-        except ValueError:
-            logger.error("Could not find credentials. Aborting")
-            raise ValueError(f"Could not find credentials file {self.credential}")
-        except URLError:
-            logger.error("Could not connect to JRA server. This aint gonna work.")
-
-    @staticmethod
-    def date2num(dt:datetime) -> float:
-        t = nc.date2num(dt, J3QD._tunits, J3QD._tcal)
-        t = np.round(t, 5)  # catch any floating-point errors
-        return t
-
-    def nc_filename(self, year, month, kind="sa") -> str:
-        return f"{self.REANALYSIS}_{kind}_{year}{month}01_to_{year}{month}31.nc"
-
-    def retrieve(self):
-        days = pd.date_range(self.date['beg'], self.date['end'] + timedelta(days=1))  # add extra day but end when we get to it
-
-        # Constant surface data
-        to_grib = [download_constant(self.access, self.directory)]
-        t = days[0]
-        to_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_to_{t.strftime(r"%Y%m%d")}_to_{t.strftime(r"%Y%m%d")}.nc')
-        gribs_to_netcdf(*to_grib,
-                        netcdf_file=to_ncfile,
-                        tstart=self.date2num(t),
-                        tstop=self.date2num(days[1]),
-                        kind="to",
-                        subsetter=self._subsetter)
-
-        # Temporal data
-        running_sa, running_sf, running_pl, running_dates = [], [], [], []
-
-        for date in days:
-            end_of_days = (date == days[-1])
-            
-            # Aggregation subroutine
-            if (not self.download_only) and ((date.day == 1 and len(running_dates) > 1) or end_of_days):
-                t1 = running_dates[0]
-                t2 = running_dates[-1]
-                tstart = J3QD.date2num(t1)
-                tstop = J3QD.date2num(date)  # t2 + timedelta(days=1)
-
-                sa_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_sa_{t1.strftime(r"%Y%m%d")}_to_{t2.strftime(r"%Y%m%d")}.nc')
-                sf_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_sf_{t1.strftime(r"%Y%m%d")}_to_{t2.strftime(r"%Y%m%d")}.nc')
-                pl_ncfile = Path(self.output_directory, f'{self.REANALYSIS}_pl_{t1.strftime(r"%Y%m%d")}_to_{t2.strftime(r"%Y%m%d")}.nc')
-
-                # aggregate gribs then delete them
-                logger.info(f"Aggregating gribs to netcdf: {t1} to {t2}")
-
-                if not sa_ncfile.is_file():
-                    gribs_to_netcdf(*running_sa, netcdf_file=self.file_in_progress(sa_ncfile),
-                                    tstart=tstart, tstop=tstop, kind="sa", subsetter=self._subsetter, engine="pygrib")
-                    self.complete(sa_ncfile)
-                
-                if not sf_ncfile.is_file():
-                    gribs_to_netcdf(*running_sf, netcdf_file=self.file_in_progress(sf_ncfile), 
-                                    tstart=tstart, tstop=tstop, kind="sf", subsetter=self._subsetter, engine="pygrib")
-                    self.complete(sf_ncfile)
-                
-                if not pl_ncfile.is_file():
-                    gribs_to_netcdf(*running_pl, netcdf_file=self.file_in_progress(pl_ncfile),
-                                    tstart=tstart, tstop=tstop, kind="pl", subsetter=self._subsetter, engine="xarray")
-                    self.complete(pl_ncfile)
-                
-                #  delete grib files
-                logger.info("Not deleting grib files")
-                #for grib in running_sa + running_sf + running_pl:
-                    #Path(grib).unlink()
-
-                # clear running
-                running_sa, running_sf, running_pl, running_dates = [], [], [], []
-
-            if end_of_days:
-                break
-
-            # Download
-            logger.info(f"Downloading gribs for {date}")
-            sa, sf, pl = download_daily_gribs(self.access, self.directory, date.year, date.month, date.day)
-            running_sa += sa
-            running_sf += sf
-            running_pl += pl
-            running_dates.append(date)
-
 
-class VariableError(ValueError):
-    pass
+class J3QD(JRAdownload):
 
-
-class ConversionHandler(ABC):
-    """
-    Handles the translation of grib files into a netcdf file
-    """
-    _include = []
-    _dims = ()
-
-    def __init__(self,
-                 tstart: float,
-                 tstop: float,
-                 subsetter:Optional[GribSubsetter] = None):
-
-        self._tstart = tstart
-        self._tstop = tstop
-        self.subsetter = subsetter if subsetter else GribSubsetter(0, 360, -90, 90)
-
-    @property
-    def tstep(self):
-        raise NotImplementedError("implement in child class")
-
-    @property
-    def times(self):
-        t = np.arange(self._tstart, self._tstop, self.tstep)
-        return t
-
-    @abstractmethod
-    def empty_file(self, *args, **kwargs):
-        raise NotImplementedError("implement in child class")
-
-    def create_variable(self, shortName, ncd, ncvar_dict):
-        var = ncd.createVariable(shortName, "f8", self._dims, chunksizes=self.chunk_sizes(ncd))
-        for attr in ncvar_dict:
-            var.setncattr(attr, ncvar_dict[attr])
-
-    def chunk_sizes(self, ncd):
-        return None
-    
-    def valid_record(self, record):
-        is_valid = record.shortName in self._include
-        return is_valid
+    JRA_VERSION = 'jra3q'
+    API = Rdams
+    DICT_FORMATTER = J3QDictFormatter
+    FILE_HANDLER = J3QDownloadHandler
+    dsID = 'd640000'
+    timeName = 'time'
     
-    def get_valid_vars(self, dataset) -> "list[str]":
-        return [v for v in dataset.variables if v in self._include.keys()]
-
-    def xread_grib(self, gribfile) -> "xr.Dataset":
-        d = xr.open_dataset(gribfile, engine='cfgrib')
-        return d
-
-    def xgrib_to_nc(self, ncfile, gribfile):
-        logger.debug(f"de-gribbing {Path(gribfile).name} with xarray")
-        dataset = self.xread_grib(gribfile)
-        for v in self.get_valid_vars(dataset):
-            var = dataset[v]
-            self.xvar_to_nc(var, ncfile)
-    
-    def xvar_to_nc(self, var, ncfile):
-        if self.subsetter is not None:
-            vals, lats, lons = self.subsetter.xsubset(var)
-        else:
-            vals, lats, lons = var.values, var.latitude.values, var.longitude.values
-
-        # check if ncfile exists
-        if not Path(ncfile).exists():
-            logger.warning(f"missing file {ncfile}. Creating it")
-            self.empty_file(ncfile, lats, lons, self.times)
-
-        with nc.Dataset(ncfile, 'a') as ncd:
-            # check if variable is in ncfile
-            # if not, add it
-            ncvar_dict = {"units" : var.units,
-                          "long_name" : var.name}
-
-            if var.name not in ncd.variables:
-                self.create_variable(var.name, ncd, ncvar_dict)
-
-            # check if timestep is in ncfile
-            timestep = J3QD.date2num(pd.Timestamp(var.valid_time.values).to_pydatetime())
-            try:
-                t_i = np.where(self.times == timestep)[0][0]
-            except IndexError:
-                t_i = self.handle_missing_timestep(var.valid_time.values, ncfile)
-
-            # write data
-            if t_i is not None:
-                self.xwrite_record(ncd=ncd,
-                                   varname=var.name,
-                                   t_i=t_i,
-                                   vals=vals,
-                                   var=var)
-
-    def grib_to_nc(self, ncfile, record):
-        varname = record.shortName
-
-        if not self.valid_record(record):
-            raise VariableError(f"{varname}")
-
-        if self.subsetter is not None:
-            vals, lats, lons = self.subsetter.subset(record)
-        else:
-            vals, lats, lons = record.data()
-
-        # check if ncfile exists
-        if not Path(ncfile).exists():
-            logger.warning(f"missing file {ncfile}. Creating it")
-            lats = lats[:,0]
-            lons = lons[0,:]
-
-            self.empty_file(ncfile, lats, lons, self.times)  # if not, create it
-
-        with nc.Dataset(ncfile, 'a') as ncd:
-            # check if variable is in ncfile
-            # if not, add it
-            ncvar_dict = {"units" : record.units,
-                          "long_name" : record.name,
-                          "comment": record.__repr__()}
-
-            if varname not in ncd.variables:
-                self.create_variable(record.shortName, ncd, ncvar_dict)
-
-            # check if timestep is in ncfile
-            timestep = J3QD.date2num(record.validDate)
-            try:
-                t_i = np.where(self.times == timestep)[0][0]
-            except IndexError:
-                t_i = self.handle_missing_timestep(record, ncfile)
-            
-            # write temporary array
-            # self.write_mem(ncd, t_i, vals, record)
-
-            # write data
-            if t_i is not None:
-                self.write_record(ncd=ncd,
-                                  varname=varname,
-                                  t_i=t_i,
-                                  vals=vals,
-                                  record=record)
-
-    def handle_missing_timestep(self, date, ncfile):
-        print(f"err: timestamp {date} not in ncfile {ncfile}")
-        return None
-
-    def write_record(self, ncd, varname, t_i, vals, record):
-        ncd[varname][t_i, :, :] = vals
-
-    def xwrite_record(self, ncd, varname, t_i, vals, var):
-        ncd[varname][t_i, :, :] = vals
-
-    def valid_messages(self):
-        m = [d["messagenumber"] for d in list(self._include.values()) if "messagenumber" in d]
-        return m
-
-
-class ToConverter(ConversionHandler):
-    _include = {"z": None}
-    _dims = ("time", "latitude", "longitude",)
-
-    @property
-    def tstep(self):
-        return 24
-
-    def empty_file(self, filename:str, lats, lons, times):
-        empty_surface_file(filename, lats, lons, times)
-
-    def write_record(self, ncd, varname, t_i, vals, record):
-        ncd[varname][0, :, :] = vals
-
-    def xwrite_record(self, ncd, varname, t_i, vals, var):
-        ncd[varname][0, :, :] = vals
-
-    def handle_missing_timestep(self, date, ncfile):
-        return 0
-
-
-class SaConverter(ConversionHandler):
-    _include = {"sp": None,
-                "2t": None,
-                "2sh": None,
-                "2r": None,
-                "10u": None,
-                "10v": None}
-    _dims = ("time", "latitude", "longitude",)
-
-    @property
-    def tstep(self):
-        # 6-hourly, calculated on the fly in case we change time units in the future
-        t1 = J3QD.date2num(datetime(1980, 3, 1, 0))
-        t2 = J3QD.date2num(datetime(1980, 3, 1, 6))
-        return t2 - t1
-
-    def chunk_sizes(self, ncd):
-        return (ncd['time'].shape[0], CHUNK_LAT, CHUNK_LON,)
-    
-    def xread_grib(self, gribfile) -> xr.Dataset:
-        d = xr.open_dataset(gribfile, engine='cfgrib', backend_kwargs={"filter_by_keys":{'typeOfLevel': 'surface'}})
-        return d
-
-    def empty_file(self, filename:str, lats, lons, times):
-        empty_surface_file(filename, lats, lons, times)
-
-
-class FcstConverter(ConversionHandler):
-    _include = {"tprate": {"messagenumber": 6},
-                "sp": {"messagenumber": 12},
-                'dswrf': {"messagenumber": 13},
-                "dlwrf": {"messagenumber": 17}
-                }
-    _dims = ("time", "latitude", "longitude",)
-   
-    @property
-    def tstep(self):
-        # 6-hourly, calculated on the fly in case we change time units in the future
-        t1 = J3QD.date2num(datetime(1980, 3, 1, 0))
-        t2 = J3QD.date2num(datetime(1980, 3, 1, 1))
-        return t2 - t1
-
-    def valid_record(self, record):
-        return record.messagenumber in self.valid_messages()
-        
-    def empty_file(self, filename:str, lats, lons, times):
-        empty_surface_file(filename, lats, lons, times)
-
-    def xread_grib(self, gribfile) -> xr.Dataset:
-        d = xr.open_dataset(gribfile, engine='cfgrib', backend_kwargs={"filter_by_keys":{'typeOfLevel': 'surface'}})
-        return d
-
-    def chunk_sizes(self, ncd):
-        return (ncd['time'].shape[0], CHUNK_LAT, CHUNK_LON,)
-
-
-class PlConverter(ConversionHandler):
-    _dims = ("time", "level", "latitude", "longitude",)
-    _include = {"t":None,  # temperature
-                "u":None,  # u-wind
-                "v":None,  # v-wind
-                "q":None,  # specific humidity
-                "r":None,   # relative humidity
-                "gh":None    # geopotential
-                }
-
-    def __init__(self, tstart, tstop, subsetter):
-        super().__init__(tstart, tstop, subsetter)
-        # self.mem_array = np.empty(len(self.times), len(self.subsetter.levels), len(self.subsetter.lats), len(self.subsetter.lons))
-
-    def valid_record(self, record):
-        good_variable = record.shortName in self._include
-        lv_hPa = record.level if record.pressureUnits == "hPa" else record.level / 10
-        good_level = lv_hPa in self.subsetter.levels
-
-        return (good_variable and good_level)
-
-    def xread_grib(self, gribfile) -> xr.Dataset:
-        d = xr.open_dataset(gribfile, engine='cfgrib', backend_kwargs={"filter_by_keys":{'typeOfLevel': 'isobaricInhPa'}})
-        return d
-    
-    @property
-    def tstep(self):
-        # 6-hourly, calculated on the fly in case we change time units in the future
-        t1 = J3QD.date2num(datetime(1980, 3, 1, 0))
-        t2 = J3QD.date2num(datetime(1980, 3, 1, 6))
-        return t2 - t1
-
-    def empty_file(self, filename:str, lats, lons, times):
-        levels = self.subsetter.levels
-        empty_pressure_levels_file(filename, lats, lons, times, levels)
-
-    def write_record(self, ncd, varname, t_i, vals, record):
-        lev = record.level if record.pressureUnits == "hPa" else record.level / 10
-
-        lev_i = np.where(ncd['level'][:] == lev)[0][0]
-
-        ncd[varname][t_i, lev_i, :, :] = vals
-
-    def xwrite_record(self, ncd, varname, t_i, vals, var):
-        ncd[varname][t_i, :, :, :] = vals[::-1, :, :]  # put levels in backwards to match JRA55
-    
-    def chunk_sizes(self, ncd):
-        return (ncd['time'].shape[0], ncd['level'].shape[0], CHUNK_LAT, CHUNK_LON,)
-
-
-def empty_constant_file(filename: str, lats, lons):
-    with nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC') as ncd:
-        _ = ncd.createDimension('latitude', len(lats))
-        _ = ncd.createDimension('longitude', len(lons))
-
-        var_lat = ncd.createVariable('latitude', 'f8', ('latitude',))
-        var_lat.units = "degrees_north"
-        var_lon = ncd.createVariable('longitude', 'f8', ('longitude',))
-        var_lon.units = "degrees_east"
-
-        var_lat[:] = lats
-        var_lon[:] = lons
-        
-
-def empty_surface_file(filename:str, lats, lons, times):
-    with nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC') as ncd:
-        _ = ncd.createDimension('time', len(times))
-        _ = ncd.createDimension('latitude', len(lats))
-        _ = ncd.createDimension('longitude', len(lons))
-
-        var_time = ncd.createVariable('time', 'f8', ('time',))
-        var_time.units = J3QD._tunits
-        var_time.calendar = J3QD._tcal
-        var_lat = ncd.createVariable('latitude', 'f8', ('latitude',))
-        var_lat.units = "degrees_north"
-        var_lon = ncd.createVariable('longitude', 'f8', ('longitude',))
-        var_lon.units = "degrees_east"
-
-        var_time[:] = times
-        var_lat[:] = lats
-        var_lon[:] = lons
-
-
-def empty_pressure_levels_file(filename:str, lats, lons, times, levs):
-    with nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC') as ncd:
-        _ = ncd.createDimension('level', len(levs))
-        _ = ncd.createDimension('time', len(times))
-        _ = ncd.createDimension('latitude', len(lats))
-        _ = ncd.createDimension('longitude', len(lons))
-
-        var_time = ncd.createVariable('time', 'f8', ('time',))
-        var_time.units = J3QD._tunits
-        var_time.calendar = J3QD._tcal
-        var_lat = ncd.createVariable('latitude', 'f8', ('latitude',))
-        var_lat.units = "degrees_north"
-        var_lon = ncd.createVariable('longitude', 'f8', ('longitude',))
-        var_lon.units = "degrees_east"
-        var_lev = ncd.createVariable('level', 'f8', ('level',))
-        var_lev.units = "hPa"
-
-        var_time[:] = times
-        var_lat[:] = lats
-        var_lon[:] = lons
-        var_lev[:] = levs
-
-
-def gribs_to_netcdf(*grib_files:"Union[str,Path]", netcdf_file:"Union[str,Path]", tstart:float, tstop:float, kind:str, subsetter=None, engine="xarray"):
-    logger.info(f"creating {netcdf_file}")
-    notified = []
-    Converter = get_converter(kind)
-    converter = Converter(tstart, tstop, subsetter)
-
-    for gribfile in grib_files:
-        if engine == "xarray":
-            converter.xgrib_to_nc(netcdf_file, gribfile)
-
-        elif engine == "pygrib":
-            logger.debug(f"de-gribbing {Path(gribfile).name} with pygrib")
-
-            with pygrib.open(str(gribfile)) as f:  # pygrib doesn't like Path objects
-                for record in f:
-                    if not converter.valid_record(record):
-                        continue
-                    try:
-                        converter.grib_to_nc(netcdf_file, record)
-                    except VariableError as e:
-                        if e.args[0] not in notified:
-                            logger.warning(f"warning: skip variable {e}")
-                            notified.append(e.args[0])
-
-        else:
-            raise ValueError("engine must be 'pygrib' or 'xarray'")
-
-
-def get_converter(kind:str) -> "type[ConversionHandler]":
-    if kind == "sf":
-        return FcstConverter
-    elif kind == "sa":
-        return SaConverter
-    elif kind == "pl":
-        return PlConverter
-    elif kind == "to":
-        return ToConverter
-    else:
-        raise ValueError(f"unknown kind {kind}")
+class J3QgD(J3QD):
+
+    JRA_VERSION = 'jra3qg'
+    API = Rdams
+    DICT_FORMATTER = J3QGDictFormatter
+    FILE_HANDLER = J3QDownloadHandler
+    dsID = 'd640000'
+    timeName = 'time'
diff --git a/globsim/download/JRAdownload.py b/globsim/download/JRAdownload.py
index bda8ac2..ad50732 100644
--- a/globsim/download/JRAdownload.py
+++ b/globsim/download/JRAdownload.py
@@ -1,543 +1,42 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-import glob
-import http.cookiejar
-import json
-import netCDF4 as nc
-import numpy as np
-import re
-import sys
-import tarfile
 import time
-import urllib.error
-import urllib.request
 import logging
 
 from datetime import datetime, timedelta
 from math import floor
-from os import path, remove, makedirs
+from os import path, remove
+from typing import Optional
+from pathlib import Path
+from requests import exceptions as req_exceptions
 
-from globsim.common_utils import variables_skip
 from globsim.download.GenericDownload import GenericDownload
-from globsim.meteorology import pressure_from_elevation
-
+from globsim.download.RDA import Rdams 
+from globsim.download.jra_dict_formatters import J55DictFormatter
+from globsim.download.JraDownloadHandler import J55DownloadHandler
 logger = logging.getLogger('globsim.download')
 
 def get_userinfo():
     return None, None
 
 
-class RDA(object):
-
-    def __init__(self, username, password):
-        '''Return an object for RDA data sets, to submit subset requests
-        on select gridded data sets, and to check on the processing status of
-        any subset requests. Details could be found:
-            https://www2.cisl.ucar.edu/data-portals/research-data-archive/command-line-subset-requests-%E2%80%93-rdams'''
-
-        self.base = 'https://rda.ucar.edu/apps/'
-        self.loginurl = 'https://rda.ucar.edu/cgi-bin/login'
-        self.cookie_file = 'auth.rda_ucar_edu'
-        self.username = username
-        self.password = password
-
-    def makeOpener(self, theurl):
-        '''make the opener based on username and password'''
-
-        passman = urllib.request.HTTPPasswordMgrWithDefaultRealm()
-        passman.add_password(None, theurl, self.username, self.password)
-        authhandler = urllib.request.HTTPBasicAuthHandler(passman)
-        opener = urllib.request.build_opener(authhandler)
-        urllib.request.install_opener(opener)
-
-        return opener
-
-    def tarExtract(self, tarf, directory):
-        '''extract the tar file'''
-
-        filename, file_extension = path.splitext(tarf)
-
-        if file_extension == '.tar':
-            tar = tarfile.open(tarf)
-            tar.extractall(path=directory)
-            tar.close()
-            remove(tarf)
-
-    def urlOpen(self, theurl):
-        '''open the url'''
-
-        opener = self.makeOpener(theurl)
-        request = urllib.request.Request(theurl)
-
-        try:
-            url = opener.open(request)
-        except urllib.error.HTTPError as e:
-            if e.code == 401:
-                logger.error('RDA username and password invalid. Please try again\n')
-                opener = self.makeOpener(theurl)
-                try:
-                    url = opener.open(request)
-                except urllib.error.HTTPError as e:
-                    if e.code == 401:
-                        logger.error(
-                            'RDA username and password invalid, or you are'
-                            'not authorized to access this dataset. Please verify your login information at http://rda.ucar.edu')
-                        sys.exit()
-            else:
-                logger.error(f"Received response: '{e}' when opening url {theurl}")
-                sys.exit()
-        return url
-
-    def add_http_cookie(self, url, authstring):
-        '''add_http_cookie(url,authstring): Get and add authentication cookie
-        to http file download handler'''
-
-        cj = http.cookiejar.MozillaCookieJar(self.cookie_file)
-        openrf = urllib.request.build_opener(
-            urllib.request.HTTPCookieProcessor(cj))
-        frequest = urllib.request.Request(url, authstring)
-        cj.add_cookie_header(frequest)
-        _ = openrf.open(frequest)
-        openerf = urllib.request.build_opener(
-            urllib.request.HTTPCookieProcessor(cj))
-        urllib.request.install_opener(openerf)
-
-    def getHelp(self):
-        '''get the help information'''
-
-        theurl = self.base + 'help'
-        url = self.urlOpen(theurl)
-
-        logger.debug(url.read().decode())
-
-    def getSummary(self, dsID=None):
-        '''get the summary of given dataset'''
-
-        logger.info('Getting summary information. Please wait as this may take a while.')
-
-        theurl = theurl = self.base + 'summary/' + dsID
-        url = self.urlOpen(theurl)
-
-        logger.info(url.read().decode())
-
-    def getMeta(self, dsID):
-        '''get the metadata of gieven dataset'''
-
-        theurl = self.base + 'metadata/' + dsID
-        url = self.urlOpen(theurl)
-
-        logger.info(url.read().decode())
-
-    def getParaSummary(self, dsID):
-        '''submit a subset request control file.
-           Subset request control files are built from the parameters dumped
-           out by the '-get_metadata <dsnnn.n>' option.'''
-
-        logger.info('Getting parameter summary. Please wait as this may take a while.')
-
-        theurl = self.base + 'paramsummary/' + dsID
-        url = self.urlOpen(theurl)
-
-        logger.info(url.read().decode())
-
-    def getStatus(self):
-        '''get the status of requests'''
-
-        theurl = self.base + 'request'
-        url = self.urlOpen(theurl)
-
-        logger.info(url.read().decode())
-
-    def submit(self, controlparms):
-        '''submit JRA55 dataset request based on predescribed control
-        parameters'''
-
-        theurl = self.base + 'request'
-
-        jsondata = '{'
-        for k in list(controlparms.keys()):
-            jsondata += '"' + k + '"' + ":" + '"' + controlparms[k] + '",'
-        jsondata = jsondata[:-1]
-        jsondata += '}'
-        logger.info('Submitting request')
-
-        if len(jsondata) > 1:
-            request = urllib.request.Request(
-                theurl, jsondata.encode(), {'Content-type':'application/json'})
-        else:
-            request = urllib.request.Request(theurl)
-
-        opener = self.makeOpener(theurl)
-        try:
-            url = opener.open(request)
-        except urllib.error.HTTPError as e:
-            if e.code == 401:
-                logger.error('RDA username and password invalid.  Please try again\n')
-                (username, password) = get_userinfo()
-                opener = self.makeOpener(theurl)
-                try:
-                    url = opener.open(request)
-                except urllib.error.HTTPError as e:
-                    if e.code == 401:
-                        logger.error('RDA username and password invalid, or you are not authorized to access this dataset.')
-                        logger.error('Please verify your login information at http://rda.ucar.edu.')
-                        sys.exit()
-
-        logger.info(url.read().decode())
-
-    def downloadSinglefile(self, remfile, outfile):
-        '''download_file(remfile,outfile) : download a file from a remote
-         server (remfile) to a local location (outfile)'''
-
-        _ = urllib.request.Request(remfile)
-        fresponse = urllib.request.urlopen(remfile)
-        with open(outfile, 'wb') as handle:
-            handle.write(fresponse.read())
-
-    def downloadFile(self, filelist, directory):
-        '''download_files(filelist,directory): Download multiple files from the
-        rda server and save them to a local directory'''
-
-        backslash = '/'
-        filecount = 0
-        percentcomplete = 0
-        localsize = ''
-        length = 0
-        length = len(filelist)
-        if not path.exists(directory):
-            makedirs(directory)
-        for key, value in filelist.items():
-            downloadpath, localfile = key.rsplit("/", 1)
-            outpath = directory + backslash + localfile
-            percentcomplete = (float(filecount) / float(length))
-            self.update_progress(percentcomplete, directory)
-            if path.isfile(outpath):
-                localsize = path.getsize(outpath)
-                if(str(localsize) != value):
-                    self.downloadSinglefile(key, outpath)
-            elif(not path.isfile(outpath)):
-                self.downloadSinglefile(key, outpath)
-            self.tarExtract(outpath, directory)
-
-            filecount = filecount + 1
-            percentcomplete = (float(filecount) / float(length))
-        self.update_progress(percentcomplete, directory)
-
-    def update_progress(self, progress, outdir):
-
-        barLength = 20  # Modify this to change the length of the progress bar
-        status = ""
-        if isinstance(progress, int):
-            progress = float(progress)
-        if not isinstance(progress, float):
-            progress = 0
-            status = "error: progress var must be float\r\n\n"
-        if progress < 0:
-            progress = 0
-            status = "Halt...\r\n\n"
-        if progress >= 1:
-            progress = 1
-            status = "Done...\r\n\n"
-        block = int(round(barLength * progress))
-        text = "\r ====== Downloading Request ======\n "
-        sys.stdout.write(text)
-        sys.stdout.flush()
-
-    def getDSindex(self):
-        '''get the index of submitted dataset index'''
-
-        theurl = self.base + 'request/'
-        url = self.urlOpen(theurl)
-
-        authdata = 'email=' + self.username + '&password=' + self.password + '&action=login'
-        authdata = authdata.encode()
-
-        responses = url.read().decode()
-        responses = responses.split('\n')
-        responses = [item for item in responses if item.startswith("RequestIndex")]
-
-        status = [item.split('- ')[1] for item in responses]
-        status = np.where(np.asarray(status) == 'Online')[0]
-        dsIndex = [item.split(':  ')[1] for item in responses]
-        dsIndex = [item.split(', ')[0] for item in dsIndex]
-        dsIndex = np.asarray(dsIndex)[status]
-
-        return dsIndex
-
-    def download(self, directory, ds):
-        '''download all the data completed from the NCAR server'''
-
-        if not isinstance(ds, str):
-            ds = str(ds)
-
-        theurl = self.base + 'request/' + ds + '/filelist'
-        url = self.urlOpen(theurl)
-
-        authdata = 'email=' + self.username + '&password=' + self.password + '&action=login'
-        authdata = authdata.encode()
-
-        jsonfilelist = url.read().decode()
-
-        if not jsonfilelist[0] != "{":
-
-            filelist = json.loads(jsonfilelist)
-
-            # get cookie required to download data files
-            self.add_http_cookie(self.loginurl, authdata)
-
-            logger.info("Starting Download.")
-
-            self.downloadFile(filelist, directory)
-
-    def purge(self, dsIndex):
-        '''delete dataset from NCAR server based on given dsIndex'''
-
-        for ds in dsIndex:
-            if not isinstance(ds, str):
-                ds = str(ds)
-            theurl = self.base + 'request/' + ds
-
-            opener = self.makeOpener(theurl)
-            request = urllib.request.Request(theurl)
-            request.get_method = lambda: 'DELETE'
-
-            try:
-                url = opener.open(request)
-            except urllib.error.HTTPError as e:
-                if e.code == 401:
-                    logger.error('RDA username and password invalid. Please try again')
-                    opener = self.makeOpener(theurl)
-                    try:
-                        url = opener.open(request)
-                    except urllib.error.HTTPError as e:
-                        if e.code == 401:
-                            logger.error('RDA username and password invalid, or you are not authorized to access this dataset.')
-                            logger.error('Please verify your login information at http://rda.ucar.edu.')
-                            sys.exit()
-
-            logger.info(url.read().decode())
-
-
-class JRApl(object):
-    VALID_LEVELS = [100, 125, 150, 175, 200, 225, 250, 300, 350, 400, 450, 500, 550,
-                    600, 650, 700, 750, 775, 800, 825, 850, 875, 900,
-                    925, 950, 975, 1000]
-    
-    def __init__(self, date, area, elevation, variables, rda):
-        '''Returns an object for JRA55 data that has methods for querying the
-        NCAR server for pressure level variables (prec, swin, lwin). '''
-
-        self.date = date
-        self.area = area
-        self.elevation = elevation
-
-        dpar = {'air_temperature'   : ['Temperature'],
-                'relative_humidity' : ['Relative humidity'],
-                'wind_speed'        : ['u-component of wind',
-                                       'v-component of wind']}
-
-        self.param = self.getParam(dpar, variables)
-
-    def getParam(self, dpar, variables):
-
-        varlist = []
-        for var in variables:
-            varlist.append(dpar.get(var))
-
-        varlist = [item for item in varlist if item is not None]
-        varlist = [item for sublist in varlist for item in sublist]
-
-        if len(varlist) > 0:
-            varlist += ['Geopotential height', 'level', 'pressure']
-
-        return varlist
-
-    def makeDate(self):
-        '''convert data format to NCAR RDA request'''
-
-        beg = self.date['beg'].strftime('%Y%m%d%H%M')
-        end = self.date['end'].strftime('%Y%m%d%H%M')
-        dateRange = beg + '/to/' + end
-
-        return dateRange
-
-    def getDictionary(self):
-        levels = getPressureLevels(self.VALID_LEVELS, self.elevation['min'], self.elevation['max'])
-        elevation = [str(ele) for ele in levels]
-        elevation = '/'.join(elevation)
-
-        self.dictionary = {
-                'dataset': 'ds628.0',
-                'date': self.makeDate(),
-                'param': '/'.join(self.param),
-                'level': 'Isobaric surface:' + elevation,
-                'oformat': 'netCDF',
-                'nlat': str(self.area['north']),
-                'slat': str(self.area['south']),
-                'wlon': str(self.area['west']),
-                'elon': str(self.area['east']),
-                'product': 'Analysis',
-                'compression': 'NN',
-                'gridproj': 'latLon',
-                'griddef': '288:145:90N:0E:90S:1.25W:1.25:1.25'}
-
-        return self.dictionary
-
-
-class JRAsa(object):
-
-    def __init__(self, date, area, variables, rda):
-        '''Returns an object for JRA55 data that has methods for querying the
-        NCAR server for surface forecast variables (prec, swin, lwin). '''
-
-        self.date = date
-        self.area = area
-
-        dpar = {'air_temperature'  : ['Temperature'],
-                'relative_humidity': ['Relative humidity'],
-                'specific_humidity': ['Specific humidity'],
-                'wind_speed'       : ['u-component of wind',
-                                      'v-component of wind']}
-
-        self.param = self.getParam(dpar, variables)
-
-    def getParam(self, dpar, variables):
-
-        varlist = []
-        for var in variables:
-            varlist.append(dpar.get(var))
-
-        varlist = [item for item in varlist if item is not None]
-        varlist = [item for sublist in varlist for item in sublist]
-
-        return varlist
-
-    def makeDate(self):
-        '''convert data format to NCAR RDA request'''
-
-        beg = self.date['beg'].strftime('%Y%m%d%H%M')
-        end = self.date['end'].strftime('%Y%m%d%H%M')
-        dateRange = beg + '/to/' + end
-
-        return dateRange
-
-    def getDictionary(self):
-        self.dictionary = {
-                'dataset': 'ds628.0',
-                'date': self.makeDate(),
-                'param': '/'.join(self.param),
-                'level': 'Specified height above ground:2/10',
-                'oformat': 'netCDF',
-                'nlat': str(self.area['north']),
-                'slat': str(self.area['south']),
-                'wlon': str(self.area['west']),
-                'elon': str(self.area['east']),
-                'product': 'Analysis',
-                'compression': 'NN',
-                'gridproj': 'latLon',
-                'griddef': '288:145:90N:0E:90S:1.25W:1.25:1.25'}
-
-        return self.dictionary
-
-
-class JRAto:
-
-    def __init__(self, area, rda):
-        self.area=area
-        self.param=["Geopotential"]
-    
-    def getDictionary(self):
-        self.dictionary = {
-                'dataset': 'ds628.0',
-                'date': "198101010000/to/198101010000",
-                'param': 'Geopotential',
-                'level': 'Ground or water surface:0',
-                'oformat': 'netCDF',
-                'nlat': str(self.area['north']),
-                'slat': str(self.area['south']),
-                'wlon': str(self.area['west']),
-                'elon': str(self.area['east']),
-                'product': 'Analysis',
-                'compression': 'NN',
-                'gridproj': 'latLon',
-                'griddef': '288:145:90N:0E:90S:1.25W:1.25:1.25'}
-
-        return self.dictionary
-
-
-class JRAsf(object):
-
-    def __init__(self, date, area, variables, rda):
-        '''Returns an object for JRA55 data that has methods for querying the
-        NCAR server for surface forecast variables (prec, swin, lwin). '''
-
-        self.date = date
-        self.area = area
-
-        dpar = {'precipitation_amount':
-                    ['Total precipitation'],
-                'downwelling_shortwave_flux_in_air':
-                    ['Downward solar radiation flux'],
-                'downwelling_longwave_flux_in_air':
-                    ['Downward longwave radiation flux'],
-                'downwelling_shortwave_flux_in_air_assuming_clear_sky':
-                    ['Clear sky downward solar radiation flux'],
-                'downwelling_longwave_flux_in_air_assuming_clear_sky':
-                    ['Clear sky downward longwave radiation flux']}
-
-        self.param = self.getParam(dpar, variables)
-
-    def getParam(self, dpar, variables):
-
-        varlist = []
-        for var in variables:
-            varlist.append(dpar.get(var))
-
-        varlist = [item for item in varlist if item is not None]
-        varlist = [item for sublist in varlist for item in sublist]
-        varlist.append('Pressure')
-
-        return varlist
-
-    def makeDate(self):
-        '''convert data format to NCAR RDA request'''
-
-        beg = self.date['beg'].strftime('%Y%m%d%H%M')
-        end = self.date['end'].strftime('%Y%m%d%H%M')
-        dateRange = beg + '/to/' + end
-
-        return dateRange
-
-    def getDictionary(self):
-        self.dictionary = {
-                'dataset': 'ds628.0',
-                'date': self.makeDate(),
-                'param': '/'.join(self.param),
-                'level': 'Ground or water surface:0',
-                'oformat': 'netCDF',
-                'nlat': str(self.area['north']),
-                'slat': str(self.area['south']),
-                'wlon': str(self.area['west']),
-                'elon': str(self.area['east']),
-                'product': '3-hour Average (initial+0 to initial+3)',
-                'compression': 'NN',
-                'gridproj': 'latLon',
-                'griddef': '288:145:90N:0E:90S:1.25W:1.25:1.25'}
-
-        return self.dictionary
-
-
 class JRAdownload(GenericDownload):
     '''Return an objet to download JRA55 dataset client based on RDA'''
-
-    # TODO: add credential
+    JRA_VERSION = 'jra55'
+    API = Rdams
+    DICT_FORMATTER = J55DictFormatter
+    FILE_HANDLER = J55DownloadHandler
+    dsID = 'ds628.0'
+    timeName = 'initial_time0_hours'
+    levName = 'lv_ISBL1'
+    
     def __init__(self, pfile):
         super().__init__(pfile)
-        self.retry_delay_min = 1
+        self.retry_delay_min = 0.2
         par = self.par
-        self._set_input_directory("jra55")
-
-        self.dsID = 'ds628.0'
+        self._set_input_directory(self.JRA_VERSION)
+        
+        self.api = self.get_accessor()
 
         self.__varCheck(par)
 
@@ -552,264 +51,223 @@ def __init__(self, pfile):
 
         # chunk size for downloading and storing data [days]
         self.chunk_size = par['chunk_size'] * 2000
-
-        self.ncfVar  = {
-                'initial_time0_hours':   'time',
-                'initial_time0':         'time',
-                'initial_time0_encoded': 'time',
-                'lv_ISBL1':              'level',
-                'g0_lat_1':              'latitude',
-                'g0_lon_2':              'longitude',
-                'g0_lat_2':              'latitude',
-                'g0_lon_3':              'longitude',
-                'GP_GDS0_SFC':           'Geopotential',
-                'HGT_GDS0_ISBL':         'Geopotential height',
-                'RH_GDS0_ISBL':          'Relative humidity',
-                'VGRD_GDS0_ISBL':        'v-component of wind',
-                'UGRD_GDS0_ISBL':        'u-component of wind',
-                'TMP_GDS0_ISBL':         'Temperature',
-                'TMP_GDS0_HTGL':         'Temperature',
-                'VGRD_GDS0_HTGL':        'v-component of wind',
-                'UGRD_GDS0_HTGL':        'u-component of wind',
-                'RH_GDS0_HTGL':          'Relative humidity',
-                'SPFH_GDS0_HTGL':        'Specific humidity',
-                'PRES_GDS0_SFC_ave3h':   'Pressure',
-                'TPRAT_GDS0_SFC_ave3h':  'Total precipitation',
-                'CSDSF_GDS0_SFC_ave3h':  'Clear sky downward solar radiation flux',
-                'CSDLF_GDS0_SFC_ave3h':  'Clear sky downward longwave radiation flux',
-                'DSWRF_GDS0_SFC_ave3h':  'Downward solar radiation flux',
-                'DLWRF_GDS0_SFC_ave3h':  'Downward longwave radiation flux'}
-
+        
+    def get_accessor(self):
+        cred = path.join(self.par['credentials_directory'], "rdams_token.txt")
+        return self.API(cred)
+    
     def __varCheck(self, par):
         '''convert one variable to a list'''
 
         if not isinstance(par['variables'], (list,)):
             par['variables'] = [par['variables']]
 
-    def getDataLev(self, dsi):
-        '''get data level of the download data set'''
-
-        flist = glob.glob(path.join(self.directory, '*' + dsi + '*'))
-        lev = path.basename(flist[0]).split('_')[1].split('.')[0]
-        if lev == 'p125':
-            dataLev = 'pl'
-        elif lev == 'surf125':
-            dataLev = 'sa'
-        elif lev == 'phy2m125':
-            dataLev = 'sf'
-        elif lev == 'gp':
-            dataLev = 'to'
-        else:
-            raise KeyError(f"File name unknown for data {dsi}")
-
-        return dataLev
-
-    def getVars(self, dsi):
-        '''get all download variable names'''
-
-        flist = glob.glob(path.join(self.directory, '*' + dsi + '*'))
-        varlist = []
-        for f in flist:
-            fname = path.basename(f)
-            try:
-                var = fname.split('_')[2]
-                varlist.append(var.split('.')[0])
-            except IndexError:  # one variable (i.e. 'gp' geopotential)
-                var = re.search(r"\.\d{3}_([a-zA-Z]*)\.", fname).group(1)
-                varlist.append(var)
-
-        variables = np.unique(varlist)
-
-        return variables
-
-    def getOutFile(self, ncf, dataLev):
-
-        times = nc.num2date(ncf[self.timeName][:],
-                            units=ncf[self.timeName].units,
-                            calendar='standard')
-        begStr = np.min(times).strftime('%Y%m%d')
-        endStr = np.max(times).strftime('%Y%m%d')
-
-        fileName = ['jra55', dataLev, begStr, 'to', endStr]
-        fileName = '_'.join(fileName) + '.nc'
-        fileName = path.join(self.directory, fileName)
-
-        return fileName
-
-    def getDimName(self, dataLev):
-        '''get the dimension [time, level, latitude, longitude] name in the
-        original JRA55 ncf'''
-
-        self.timeName = 'initial_time0_hours'
-        self.levName = 'lv_ISBL1'
-
-        if dataLev == 'pl':
-            self.lonName = 'g0_lon_3'
-            self.latName = 'g0_lat_2'
-        elif dataLev in ['sa', 'sf', 'to']:
-            self.lonName = 'g0_lon_2'
-            self.latName = 'g0_lat_1'
-
-    def makeNCF(self, dsi):
-
-        variables = self.getVars(dsi)
-        dataLev = self.getDataLev(dsi)
-        self.getDimName(dataLev)
-
-        varf = np.sort(glob.glob(path.join(self.directory,
-                                           f'*{variables[0]}*')))
-        ncf = nc.MFDataset(varf.tolist(), aggdim='initial_time0_hours')
-
-        if dataLev == 'pl':
-            Levs = ncf['lv_ISBL1'][:].data
-
-        Times = ncf[self.timeName][:]
-        Lats  = ncf[self.latName][:].data
-        Lons  = ncf[self.lonName][:].data
-
-        file_new = self.getOutFile(ncf, dataLev)
-
-        # initialize new data file and create group
-        ncn = nc.Dataset(file_new, 'w', format='NETCDF4_CLASSIC')
-
-        # make dimensions
-        if dataLev == 'pl':
-            Levs = ncf[self.levName][:].data
-            ncn.createDimension('level', len(Levs))
-            levels     = ncn.createVariable('level', 'i4',('level',))
-            levels.long_name  = 'pressure level'
-            levels.units      = 'mbar'
-            levels[:] = Levs
-        ncn.createDimension('time', len(Times))
-        ncn.createDimension('latitude', len(Lats))
-        ncn.createDimension('longitude', len(Lons))
-
-        # make dimension variables
-        times      = ncn.createVariable('time', 'd',('time',))
-        latitudes  = ncn.createVariable('latitude', 'f8', ('latitude',))
-        longitudes = ncn.createVariable('longitude', 'f8', ('longitude',))
-
-        times.standard_name = 'time'
-        times.units     = ncf[self.timeName].units
-        times.calendar  = 'standard'
-        latitudes.standard_name  = ncf[self.latName].long_name
-        latitudes.units      = ncf[self.latName].units
-        longitudes.standard_name = ncf[self.lonName].long_name
-        longitudes.units     = ncf[self.lonName].units
-
-        ncf.close()
-
-        # assign dimensions
-        times[:] = Times
-        longitudes[:] = Lons
-        latitudes[:] = Lats
-
-        for vari in variables:
-            flist = np.sort(glob.glob(path.join(self.directory, f'*{vari}*')))
-            ncf = nc.MFDataset(flist.tolist(), aggdim=self.timeName)
-            for n, var in enumerate(ncf.variables.keys()):
-                if variables_skip(self.ncfVar[var]):
-                    continue
-                logger.info(f"Creating variable: {var}")
-                if dataLev == 'pl':
-                    vari = ncn.createVariable(self.ncfVar[var], 'f4',
-                                              ('time', 'level',
-                                               'latitude', 'longitude',))
-                    vari[:,:,:,:] = ncf[var][:,:,:,:]
-                else:
-                    vari = ncn.createVariable(self.ncfVar[var],'f4',
-                                              ('time',
-                                               'latitude', 'longitude'))
-                    vari[:,:,:] = ncf[var][:,:,:]
-
-                vari.long_name = ncf[var].long_name
-                vari.units     = ncf[var].units
-
-            ncf.close()
-            for f in flist:
-                remove(f)
-
-        ncn.close()
-
-    def requestClear(self, rda):
+    def requestClear(self):
         '''clear online datasets before downloading'''
 
         logger.info('Clear Online Datasets From NCAR Server')
-        dsIndex = rda.getDSindex()
+        dsIndex = self.api.get_status()['data']
         if len(dsIndex) > 1:
-            rda.purge(dsIndex)
-        logger.info('Online Dateset Cleared')
-
-    def requestSubmit(self, rda):
+            for req in dsIndex:
+                i = req['request_index']
+                self.api.purge_request(str(i))
+                logger.info(f'Cleared request {i}')
+
+    def requestSubmit(self) -> dict:
+        """submit request to the server
+        
+        Returns
+        -------
+        dict
+            dictionary of submitted requests and their file type (e.g. 'to', 'sa', 'sf', 'pl')
+            for example, {'1234':'to', '1235':'sa', '1236':'sf', '1237':'pl'}
+        """
 
         logger.info('Submit Request')
 
         slices = floor(float((self.date['end'] - self.date['beg']).days) /
                        self.chunk_size) + 1
-
-        dsN = 0
-        for ind in range(0, int(slices)):
+        submitted_requests = dict()
+        
+        for chunk_index in range(0, int(slices)):
             date_i = {}
             # prepare time slices
             date_i['beg'] = (self.date['beg']
-                             + timedelta(days=self.chunk_size * ind))
+                             + timedelta(days=self.chunk_size * chunk_index))
             date_i['end'] = (self.date['beg']
-                             + timedelta(days=self.chunk_size * (ind + 1) - 1))
-            if ind == (slices - 1):
+                             + timedelta(days=self.chunk_size * (chunk_index + 1) - 1))
+            if chunk_index == (slices - 1):
                 date_i['end'] = self.date['end']
+            
+            Formatter = self.DICT_FORMATTER
+            self.formatter = Formatter(date=date_i,
+                                       area=self.area,
+                                       elevation=self.elevation,
+                                       variables=self.variables)
+            
+            pl = self.formatter.get_pl_dict(self.variables) 
+            sa = self.formatter.get_sa_dict(self.variables) 
+            sf = self.formatter.get_sf_dict(self.variables) 
+            to = self.formatter.get_to_dict()
+            
+            for (request_dict, filetype) in zip([to, sa, sf, pl], ['to','sa','sf','pl']):
+                if len(request_dict['param']) == 0:
+                    continue
+                
+                if (chunk_index > 0) and (filetype == 'to'):
+                    continue  # only request constant data once
+                
+                # check if file already exists
+                output_file = Path(self.directory,
+                                   f"{self.JRA_VERSION}_{filetype}_{request_dict['date'][:8]}_to_{request_dict['date'][-12:-4]}.nc")
+                if output_file.exists():
+                    logger.info(f"File {output_file.name} already exists. Skipping")
+                    continue
+
+                summary = self.api.get_param_summary(self.dsID)['data']['data']
+                request_dict['param'] = self.formatter.param_descriptions_to_name(request_dict['param'], summary)
+                logger.debug("Requesting data for: " + str(request_dict))
+                try:
+                    req = self.api.submit_json(request_dict)
+                except req_exceptions.JSONDecodeError as e:
+                    logger.error(f"Error in request {request_dict}")
+                    logger.error(e)
+                    continue
 
-            pl = JRApl(date_i, self.area, self.elevation,
-                       self.variables, rda)
-            sa = JRAsa(date_i, self.area, self.variables, rda)
-            sf = JRAsf(date_i, self.area, self.variables, rda)
-            to = JRAto(self.area, rda)
-
-            # get download data level
-            JRAli = []
-            for jrai in [to, sa, sf, pl]:
-                if len(jrai.param) > 0:
-                    JRAli.append(jrai)
-
-            for jrai in JRAli:
-                rda.submit(jrai.getDictionary())
-                dsN += 1
+                if req['http_response'] != 200:
+                    logger.error(f"Error in request {request_dict}")
+                else:
+                    logger.info(f"Request {req['data']['request_id']} submitted")
+                
+                # append request ID
+                rid = str(req['data']['request_id'])
+                try:
+                    submitted_requests[rid] = filetype
+                except KeyError:
+                    logger.error(f"Error in request {request_dict}")
+                    logger.error(f"Error messages: {req.get('error_messages')}")
+                    continue
+        
         logger.info('Request submitted')
 
-        return dsN
+        return submitted_requests
+
+    def update_status(self, submitted_requests:Optional[dict], failed=None, done=None):
+        done = done if done is not None else []
+        failed = failed if failed is not None else []
+        to_get = []
+        logger.info('Geting Active Requets from NCAR Server')
+        active_requests = self.api.get_status()['data']  # type: dict
+        
+        if submitted_requests is None:
+            submitted_requests = {str(r['request_index']):None for r in active_requests}
+
+        for rid in submitted_requests.keys():  # Omit requests that 
+            if (rid in done) or (rid in failed):
+                continue
+            if rid not in [str(r['request_index']) for r in active_requests]:
+                logger.error(f"Request {rid} not found in active requests")
+                failed.append(rid)
+
+        for r in active_requests:  # Omit requests that are already done or failed
+            ix = r['request_index']
+            if (ix in done) or (ix in failed):
+                continue
+            elif str(r['request_index']) in submitted_requests.keys():
+                to_get.append(r)
+        
+        return to_get, failed, done
+
+    def requestDownload(self, submitted_requests:Optional[dict]=None):
+        '''download all the requests'''
+        failed=None
+        done=None
+        to_get = [None]
+
+        while len(to_get) > 0:
+            to_get, failed, done = self.update_status(submitted_requests, failed=failed, done=done)
+
+            remaining = {r['request_id']: r['status'] for r in to_get}
+            logger.info(f"remaining requests :{remaining}")
+            logger.debug(f"done: {done} ")
+            logger.debug(f"failed: {failed} ")
+            
+            for ds in to_get:
+                if (ds['request_index'] in done) or(ds['request_index'] in failed):
+                    continue
 
-    def requestDownload(self, rda, dsN):
-        '''download all the request'''
+                if ds['status'] == 'Error':
+                    logger.error(f"Error in request {ds['request_index']}")
+                    failed.append(ds['request_index'])
+                    continue
+                    
+                elif ds['status'] == 'Queued for Processing':
+                    logger.info(f"Request {ds['request_index']} Queued for Processing")
+                    continue
+                
+                elif ds['status'] == 'Processing':
+                    logger.info(f"Request {ds['request_index']} Processing")
+                    continue
 
-        doneI = []
-        while len(doneI) < dsN:
+                elif ds['status'] == 'Set for Purge':
+                    logger.info(f"Request {ds['request_index']} Set for Purge")
+                    done.append(ds['request_index'])
+                    continue
 
-            logger.info('Geting Available Dataset')
-            dsIndex = rda.getDSindex()
-            dsIndex = [item for item in dsIndex if item not in doneI]
-            if len(dsIndex) > 0:
-                for ds in dsIndex:
-                    rda.download(self.directory, ds)
-                    self.makeNCF(ds)
-                doneI += dsIndex
+                elif ds['status'] == 'Completed':
+                    logger.info(f"Request {ds['request_index']} Complete")
+                    res = self.api.download(ds['request_index'], path.join(self.directory, self.JRA_VERSION))
+                    
+                    # untar / extract
+                    Handler = self.FILE_HANDLER
+                    handler = Handler()
+                    if submitted_requests is not None:
+                        filetype = submitted_requests[str(ds['request_index'])]
+                    else:
+                        filetype = None
+                    handler.make_globsim_dataset(self.directory, str(ds['request_index']), filetype=filetype)
+                    done.append(ds['request_index'])
 
-            if self.retry_delay_min < 10:
-                self.retry_delay_min *= 1.6
+                else:
+                    logger.error(f"Unknown status {ds['status']} for request {ds['request_index']}")
+                    failed.append(ds['request_index'])
+                    continue
+                
+            if (len(to_get) == 1):
+                last_ri = to_get[0]['request_index']
+                if (last_ri in done) or (last_ri in failed):
+                    break
 
+            if self.retry_delay_min < 10:
+                self.retry_delay_min *= 1.2
+            
+            logger.debug(f"Waiting {self.retry_delay_min:0.2f} minutes before next check")
             time.sleep(60 * self.retry_delay_min)  # check available data every 10 mins
 
+        logger.info("All requests processed")
+        if done is not None:
+            for rid in done:
+                logger.info(f"{rid}: Done")
+        if failed is not None:
+            for rid in failed:
+                logger.error(f"{rid}: Failed ()")
+            
         logger.info('''Download Completed''')
 
     def retrieve(self):
         '''submit and download all the dataset'''
 
-        logger.info('''Starting JRA-55''')
+        logger.info(f'''Starting {self.JRA_VERSION} download''')
 
-        rda = RDA(self.username, self.password)  # initialize RDA server
-        self.requestClear(rda)  # clear online request
-        dsN = self.requestSubmit(rda)  # submit request
-        self.requestDownload(rda, dsN)  # download dataset
+        active_requests = self.api.get_status()['data']
+        if len(active_requests) > 5:
+            logger.error(f"Found {len(active_requests)} active requests (>5). Will clear them before proceeding")
+            self.requestClear()  # don't clear online requests
+            time.sleep(10)  # allow time for purge to register
+        submitted_requests = self.requestSubmit()  # submit request
+        self.requestDownload(submitted_requests)  # download dataset
 
-        logger.info('''JRA-55 Complete''')
+        logger.info(f'''{self.JRA_VERSION} Complete''')
 
 
 def getDate(par):
@@ -822,26 +280,4 @@ def getDate(par):
     return dateRange
 
 
-def getPressureLevels(levels: list, min_elev: float, max_elev: float) -> "list[float]":
-    # flip max and min because 1000 is the bottom and 0 is the top
-    elevationMax = pressure_from_elevation(min_elev)
-    elevationMin = pressure_from_elevation(max_elev)
-
-    minNum = min(levels, key=lambda x:abs(x - elevationMin))
-    maxNum = min(levels, key=lambda x:abs(x - elevationMax))
-
-    if (minNum > elevationMin and levels.index(minNum) > 0):
-        elevationMinRange = levels.index(minNum) - 1
-    else:
-        elevationMinRange = levels.index(minNum)
-
-    if (maxNum < elevationMin and levels.index(maxNum) < 36):
-        elevationMaxRange = levels.index(maxNum) - 1
-    else:
-        elevationMaxRange = levels.index(maxNum)
-
-    elevation = []
-    for e in range(elevationMinRange, elevationMaxRange + 1):
-        elevation.append(levels[e])
 
-    return elevation
diff --git a/globsim/download/JraDownloadHandler.py b/globsim/download/JraDownloadHandler.py
new file mode 100644
index 0000000..8669aae
--- /dev/null
+++ b/globsim/download/JraDownloadHandler.py
@@ -0,0 +1,317 @@
+import logging
+import netCDF4 as nc
+import tarfile
+import xarray as xr
+
+from pathlib import Path
+from typing import Optional
+
+from globsim.common_utils import variables_skip
+from globsim.download.jra_dict_formatters import lookup_param
+
+logger = logging.getLogger(__name__)
+
+
+class NcarDownloadHandler:
+    REANALYSIS = ''
+
+    def __init__(self):
+        pass
+
+    def format_output_filename(self, beg, end, filetype):
+        return f"{self.REANALYSIS}_{filetype}_{beg.strftime(r'%Y%m%d')}_to_{end.strftime(r'%Y%m%d')}.nc"
+
+
+class J3QDownloadHandler(NcarDownloadHandler):
+    REANALYSIS = 'jra3q'
+    DIMS = {'time': 'time', 
+            'latitude': 'lat',
+            'longitude': 'lon',
+            'level': 'pressure_level'}
+    DATASET_ID = 'd640000'
+    
+    def __init__(self):
+        super().__init__()
+    
+    def extract_dimensions(self, nc_template_files):
+        '''extract dimensions from the template files'''
+
+        dataset = xr.open_mfdataset(nc_template_files, decode_times=False)
+        Times = dataset[self.DIMS['time']]
+        Lats = dataset[self.DIMS['latitude']]
+        Lons = dataset[self.DIMS['longitude']]
+
+        if self.DIMS['level'] in dataset.dims: 
+            Levs = dataset[self.DIMS['level']]
+        else:
+            Levs = None
+        
+        dataset.close()
+
+        return Times, Lats, Lons, Levs
+
+    def variable_to_globsim_name(self, param):
+        varname = param.split('-')[0]
+        d = {'rh2m': 'Relative humidity',
+            'tmp2m': 'Temperature',
+            'ugrd10m': 'u-component of wind',
+            'vgrd10m': 'v-component of wind',
+            'lat': 'latitude',
+            'lon': 'longitude',
+            'time': 'time',
+            'gp' : 'Geopotential',
+            'hgt': 'Geopotential height',
+            'rh': 'Relative humidity',
+            'tmp': 'Temperature',
+            'ugrd': 'u-component of wind',
+            'vgrd': 'v-component of wind',
+            'dswrf1have': 'Downward solar radiation flux',
+            'dswrfcs1have': 'Clear sky downward solar radiation flux',
+            'dlwrf1have': 'Downward longwave radiation flux',
+            'dlwrfcs1have': 'Clear sky downward longwave radiation flux',
+            'tprate1have': 'Total precipitation',
+            'pres': 'Pressure',
+            }
+        return d.get(varname)
+
+    def get_dim_names(self, *args, **kwargs):
+        return self.DIMS
+    
+    def get_nc_files(self, directory, variable, dsi) -> list[Path]:
+        def filter_func(f):
+            return dsi in f.name and variable in f.name
+        
+        valid_files = filter(filter_func, Path(directory).glob('*nc'))
+
+        return list(valid_files)
+    
+    def make_globsim_dataset(self, directory:str, request_id:str, filetype:Optional[str]=None):
+        '''make a globsim dataset from JRA-3Q dataset'''
+
+        # extract the downloaded tar files
+        # get all variables associated with the dataset
+        # return the extracted tar files   
+        logger.debug(f"Creating {filetype if filetype is not None else ''} dataset from requeset {request_id}")
+        
+        extract_downloaded_tar_files(directory, request_id)
+        
+        variables = get_downloaded_variable_names(directory, request_id)
+        
+        if len(variables) == 0:
+            raise ValueError(f"No variables found in dataset {request_id}")
+        
+        if filetype is None:
+            filetype = determine_output_file_type(directory, request_id)
+        
+        dims = self.get_dim_names(directory, request_id)
+        
+        nc_template_files = self.get_nc_files(directory, variables[0], request_id)
+        
+        Times, Lats, Lons, Levs = self.extract_dimensions(nc_template_files)
+        
+        beg, end = nc.num2date(Times.values[[0,-1]], units=Times.units, calendar=Times.calendar)
+        file_new = self.format_output_filename(beg, end, filetype)
+
+        ncn = new_jra_download_file(str(Path(directory,file_new)),
+                                     Times, Lats, Lons, Levs)
+        
+        for varname in variables:
+            if variables_skip(varname):
+                logger.debug(f"Skipping variable: {varname}")
+
+            flist = self.get_nc_files(directory, varname, request_id)
+            ncf = xr.open_mfdataset(flist, decode_times=False)
+            
+            output_name = self.variable_to_globsim_name(varname)
+
+            if output_name is None:
+                logger.warning(f"Variable {varname} not found in lookup table")
+                continue
+
+            logger.debug(f"Creating variable: {varname}")
+            
+            if filetype == 'pl':
+                vari = ncn.createVariable(output_name, 'f4',
+                                            ('time', 'level',
+                                            'latitude', 'longitude',))
+                vari[:,:,:,:] = ncf[varname][:,:,:,:]
+            else:
+                vari = ncn.createVariable(output_name,'f4',
+                                            ('time',
+                                            'latitude', 'longitude'))
+                vari[:,:,:] = ncf[varname][:,:,:]
+            
+            vari.long_name = ncf[varname].long_name
+            vari.units     = ncf[varname].units
+            vari.orig_name = varname
+
+            ncf.close()
+            for f in flist:
+                f.unlink()
+
+        ncn.close()
+
+
+class J55DownloadHandler(J3QDownloadHandler):
+    REANALYSIS = 'jra55'
+    DATASET_ID = 'ds628.0'
+
+    def __init__(self):
+        super().__init__()
+    
+    def variable_to_globsim_name(self, param):
+        varname = param.split('-')[0]
+        return  {
+                'initial_time0_hours':   'time',
+                'initial_time0':         'time',
+                'initial_time0_encoded': 'time',
+                'lv_ISBL1':              'level',
+                'g0_lat_1':              'latitude',
+                'g0_lon_2':              'longitude',
+                'g0_lat_2':              'latitude',
+                'g0_lon_3':              'longitude',
+                'GP_GDS0_SFC':           'Geopotential',
+                'HGT_GDS0_ISBL':         'Geopotential height',
+                'RH_GDS0_ISBL':          'Relative humidity',
+                'VGRD_GDS0_ISBL':        'v-component of wind',
+                'UGRD_GDS0_ISBL':        'u-component of wind',
+                'TMP_GDS0_ISBL':         'Temperature',
+                'TMP_GDS0_HTGL':         'Temperature',
+                'VGRD_GDS0_HTGL':        'v-component of wind',
+                'UGRD_GDS0_HTGL':        'u-component of wind',
+                'RH_GDS0_HTGL':          'Relative humidity',
+                'SPFH_GDS0_HTGL':        'Specific humidity',
+                'PRES_GDS0_SFC_ave3h':   'Pressure',
+                'TPRAT_GDS0_SFC_ave3h':  'Total precipitation',
+                'CSDSF_GDS0_SFC_ave3h':  'Clear sky downward solar radiation flux',
+                'CSDLF_GDS0_SFC_ave3h':  'Clear sky downward longwave radiation flux',
+                'DSWRF_GDS0_SFC_ave3h':  'Downward solar radiation flux',
+                'DLWRF_GDS0_SFC_ave3h':  'Downward longwave radiation flux',
+                'DSWRF_GDS0_NTAT_ave3h':  'skip',}[varname]
+    
+    def get_dim_names(self, filetype, *args, **kwargs) -> dict:
+    
+        if filetype == 'pl':
+            lon = 'g0_lon_3'
+            lat = 'g0_lat_2'
+        elif filetype in ['sa', 'sf', 'to']:
+            lon = 'g0_lon_2'
+            lat = 'g0_lat_1'
+        
+        return {'time': 'initial_time0_hours', 
+                'latitude': lat,
+                'longitude': lon,
+                'level': 'lv_ISBL'}
+
+
+def determine_output_file_type(directory:str, request_id:str) -> str:
+    '''determine the output file type for data request
+    
+    returns one of 'pl' 'sa' 'sf' or 'to'
+    '''
+   # TODO: [NB] make this more robust- its super janky
+    flist = list(Path(directory).glob(f'*{request_id}*nc'))
+    product = flist[0].name.split('_')[1]
+
+    if 'gp' in product:
+        file_type = 'to'
+    elif 'p125' in product:
+        file_type = 'pl'
+    elif 'surf' in product:
+        file_type = 'sa'
+    elif 'phy2m125' in product:
+        file_type = 'sf'
+        
+    else:
+        raise KeyError(f"File type unknown for data request {request_id}")
+
+    return file_type
+
+
+class J3QgDownloadHandler(J3QDownloadHandler):
+    REANALYSIS = 'jra3q'
+
+
+def new_jra_download_file(filename:str,
+                          Times:xr.DataArray,
+                          Lats:xr.DataArray,
+                          Lons:xr.DataArray,
+                          Levs:Optional[xr.DataArray]=None) -> nc.Dataset:
+    output_file = nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC')
+
+    logger.debug(f"Creating new empty output file: {Path(filename).name}")
+    
+    if Levs is not None:
+        output_file.createDimension('level', len(Levs))
+        levels     = output_file.createVariable('level', 'i4',('level',))
+        levels.long_name  = 'pressure level'
+        levels.units      = 'mbar'
+        levels[:] = Levs.values
+
+    dim_time = output_file.createDimension('time', len(Times))
+    dim_lat = output_file.createDimension('latitude', len(Lats))
+    dim_lon = output_file.createDimension('longitude', len(Lons))
+    
+    # make dimension variables
+    var_times      = output_file.createVariable('time', 'd',('time',))
+    var_latitude  = output_file.createVariable('latitude', 'f8', ('latitude',))
+    var_longitude = output_file.createVariable('longitude', 'f8', ('longitude',))
+
+    var_times[:] = Times.values
+    var_latitude[:] = Lats.values
+    var_longitude[:] = Lons.values
+
+    var_times.standard_name = 'time'
+    var_times.units     = Times.units
+    var_times.calendar  = 'standard'
+    var_latitude.standard_name  = Lats.long_name
+    var_latitude.units      = Lats.units
+    var_longitude.standard_name = Lons.long_name
+    var_longitude.units     = Lons.units
+
+    return output_file
+   
+
+def extract_downloaded_tar_files(directory:str, request_id:str, remove_when_completed:bool=True) -> list[Path]:
+    '''find downloaded tar files for a given dataset id and extract them'''
+
+    tarf = list(Path(directory).glob(f"*{request_id}*.tar"))
+    
+    for f in tarf:
+        if Path(f).suffix == '.tar':
+            tar = tarfile.open(f)
+            tar.extractall(path=directory, filter='fully_trusted')
+            tar.close()
+            
+            if remove_when_completed:
+                f.unlink()
+    
+    return tarf
+
+
+def get_downloaded_variable_names(directory:str, request_id:str) -> list[str]:
+    '''get all variables associated with a given dataset id'''
+
+    flist = list(Path(directory).glob(f'*{request_id}*'))
+    varlist = []
+    for f in flist:
+        fname = f.name
+        for part in fname.split("."):
+            if len(part.split("-")) == 4:
+                # var = part.split("-")[0]
+                var = part
+                varlist.append(var)
+    unique_variables = []
+    
+    for v in varlist:
+        if v not in unique_variables:
+            unique_variables.append(v)
+
+    return unique_variables
+
+if __name__ == "__main__":
+    import sys
+    dir = sys.argv[1]
+    ds = str(sys.argv[2])
+    j = J3QDownloadHandler()
+    j.make_globsim_dataset(dir, ds)
\ No newline at end of file
diff --git a/globsim/download/MERRAdownload.py b/globsim/download/MERRAdownload.py
index 08c4d99..ef5b18a 100644
--- a/globsim/download/MERRAdownload.py
+++ b/globsim/download/MERRAdownload.py
@@ -13,7 +13,7 @@
 from netCDF4           import Dataset
 from os                import path
 from pathlib           import Path
-from pydap.client      import open_url, open_dods
+from pydap.client      import open_url, open_dods_url
 from pydap.cas.urs     import setup_session
 from scipy.interpolate import interp1d
 from typing            import Optional
@@ -688,6 +688,9 @@ def retrieve(self):
 
         # build chunks
         all_dates = pd.date_range(self.date['beg'], self.date['end'])
+        if len(all_dates) < chunk_size:
+            raise ValueError("The date range is too small for the chunk size. Please increase the date range or decrease the chunk size.")
+        
         chunks = [{'beg': x, 'end': y} for (x,y) in zip(all_dates[0::chunk_size], all_dates[chunk_size - 1::chunk_size])]
 
         if not chunks[-1]['end'] == self.date['end']:  # if chunks not an even multiple
@@ -941,7 +944,7 @@ def subset_dataset(self, url, metadata=False):
         dods_url = self.create_dods_url(url)
 
         logger.debug(f"Downloading subset from {dods_url}")
-        ds = open_dods(dods_url, session=self.session, metadata=metadata)
+        ds = open_dods_url(dods_url, session=self.session, metadata=metadata)
 
         return ds
 
diff --git a/globsim/download/RDA.py b/globsim/download/RDA.py
new file mode 100644
index 0000000..fca5c6f
--- /dev/null
+++ b/globsim/download/RDA.py
@@ -0,0 +1,599 @@
+#!/usr/bin/env python
+"""List dataset metadata, subset data subset requests,
+check on request status.
+
+Usage:
+```
+rdams-client.py -get_summary <dsnnn.n>
+rdams-client.py -get_metadata <dsnnn.n> <-f>
+rdams-client.py -get_param_summary <dsnnn.n> <-f>
+rdams-client.py -submit [control_file_name]
+rdams-client.py -get_status <RequestIndex> <-proc_status>
+rdams-client.py -download [RequestIndex]
+rdams-client.py -globus_download [RequestIndex]
+rdams-client.py -get_control_file_template <dsnnn.n>
+rdams-client.py -help
+```
+"""
+__version__ = '3.0.0'
+__author__ = 'Doug Schuster (schuster@ucar.edu), Riley Conroy (rpconroy@ucar.edu)'
+
+import sys
+import os
+import requests
+import json
+import argparse
+
+from pathlib import Path
+
+
+class Rdams(object):
+    BASE_URL = 'https://rda.ucar.edu/api/'
+    ENVAUTHFILE = 'GLOBSIM_RDA_AUTH_FILE'
+
+    def __init__(self, auth_file):
+        if auth_file is None:
+            auth_file = self.look_for_auth_file()
+
+        self.DEFAULT_AUTH_FILE = auth_file
+        
+    def look_for_auth_file(self):
+        env_auth = os.environ.get(self.ENVAUTHFILE, None)
+        if env_auth is not None:
+            return env_auth
+
+        default_auth = Path("~", 'rdams_token.txt').expanduser()
+        if default_auth.exists():
+            return str(default_auth)
+        
+        else:
+            print(f"Could not find an authentication for RDAMS file. Please set the environment variable {self.ENVAUTHFILE} or create a file at {default_auth}")
+            print("See 'https://rda.ucar.edu/accounts/profile/' to get a token.")
+            sys.exit(1)
+
+    def query(self, args=None):
+        """Perform a query based on command line like arguments.
+
+        Args:
+            args (list): argument list of querying commands.
+
+        Returns:
+            (dict): Output of json decoded API query.
+
+        Example:
+            ```
+            >>> query(['-get_status', '123456'])
+
+            >>> query(['-get_metadata', 'ds083.2'])
+            ```
+        """
+        parser = self.get_parser()
+        if args is None or len(args) == 0:
+            parser.parse_args(['-h'])
+        args = parser.parse_args(args)
+        args_dict = args.__dict__
+        func,params = self.get_selected_function(args_dict)
+        if args_dict['outdir'] and func==self.download:
+            out_dir = args_dict['outdir']
+            return func(params, out_dir)
+        result = func(params)
+        if not args.noprint:
+            print(json.dumps(result, indent=3))
+        return result
+
+    def add_ds_str(self, ds_num):
+        """Adds 'ds' to ds_num if needed.
+        Throws error if ds number isn't valid.
+        """
+        ds_num = ds_num.strip()
+        if ds_num[0:2] != 'ds':
+            ds_num = 'ds' + ds_num
+        if len(ds_num) != 7:
+            print("'" + ds_num + "' is not valid.")
+            sys.exit()
+        return ds_num
+
+    def get_userinfo(self):
+        """Get token from command line."""
+        print('Please visit https://rda.ucar.edu/accounts/profile/ to access token.')
+        token = input("Paste that token here: ")
+        self.write_token_file(token)
+        return token
+
+    def write_token_file(self, token, token_file):
+        """Write token to a file."""
+        with open(token_file, "w") as fo:
+            fo.write(token)
+
+    def read_token_file(self, token_file):
+        """Read user information from token file.
+
+        Args:
+            token_file (str): location of token file.
+
+        Returns:
+            (str): token
+        """
+        with open(token_file, 'r') as f:
+            token = f.read()
+        return token.strip()
+
+    def read_control_file(self, control_file):
+        """Reads control file, and return python dict.
+
+        Args:
+            control_file (str): Location of control file to parse.
+                    Or control file string.
+
+        Returns:
+            (dict) python dict representing control file.
+        """
+        control_params = {}
+        if os.path.exists(control_file):
+            myfile = open(control_file, 'r')
+        else:
+            myfile = control_file.split('\n')
+
+        for line in myfile:
+            line = line.strip()
+            if line.startswith('#') or line == "":
+                continue
+            li = line.rstrip()
+            (key, value) = li.split('=', 2)
+            control_params[key] = value
+
+        # Handle empty params
+        if 'param' in control_params and control_params['param'].strip() == '':
+            all_params = self.get_all_params(control_params['dataset'])
+            control_params['param'] = '/'.join(all_params)
+
+        try:
+            myfile.close()
+        except:
+            pass
+        return control_params
+
+    def get_parser(self, ):
+        """Creates and returns parser object.
+
+        Returns:
+            (argparse.ArgumentParser): Parser object from which to parse arguments.
+        """
+        description = "Queries NCAR RDA REST API."
+        parser = argparse.ArgumentParser(prog='rdams', description=description)
+        parser.add_argument('-noprint', '-np',
+                action='store_true',
+                required=False,
+                help="Do not print result of queries.")
+        parser.add_argument('-outdir', '-od',
+                nargs='?',
+                required=False,
+                help="Change the output directory of downloaded files")
+        group = parser.add_mutually_exclusive_group(required=True)
+        group.add_argument('-get_summary', '-gsum',
+                type=str,
+                metavar='<dsid>',
+                required=False,
+                help="Get a summary of the given dataset.")
+        group.add_argument('-get_metadata', '-gm',
+                type=str,
+                metavar='<dsid>',
+                required=False,
+                help="Get metadata for a given dataset.")
+        group.add_argument('-get_param_summary', '-gpm',
+                type=str,
+                metavar='<dsid>',
+                required=False,
+                help="Get only parameters for a given dataset.")
+        group.add_argument('-submit', '-s',
+                type=str,
+                metavar='<control file>',
+                required=False,
+                help="Submit a request using a control file.")
+        group.add_argument('-get_status', '-gs',
+                type=str,
+                nargs='?',
+                const='ALL',
+                metavar='<Request Index>',
+                required=False,
+                help="Get a summary of the given dataset.")
+        group.add_argument('-download', '-d',
+                type=str,
+                required=False,
+                metavar='<Request Index>',
+                help="Download data given a request id.")
+        group.add_argument('-get_filelist', '-gf',
+                type=str,
+                required=False,
+                metavar='<Request Index>',
+                help="Query the filelist for a completed request.")
+        group.add_argument('-globus_download', '-gd',
+                type=str,
+                required=False,
+                metavar='<Request Index>',
+                help="Start a globus transfer for a give request index.")
+        group.add_argument('-get_control_file_template', '-gt',
+                type=str,
+                metavar='<dsid>',
+                required=False,
+                help="Get a template control file used for subsetting.")
+        group.add_argument('-purge', # Sorry no -p
+                type=str,
+                metavar='<Request Index>',
+                required=False,
+                help="Purge a request.")
+        return parser
+
+    def check_status(self, ret):
+        """Checks that status of return object.
+
+        Exits if a 401 status code.
+
+        Args:
+            ret (response.Response): Response of a request.
+            token_file (str) : password file. Will remove if auth incorrect
+
+        Returns:
+            None
+        """
+        if ret.status_code == 401: # Not Authorized
+            print(ret.content)
+            exit(1)
+
+    def check_file_status(self, filepath, filesize):
+        """Prints file download status as percent of file complete.
+
+        Args:
+            filepath (str): File being downloaded.
+            filesize (int): Expected total size of file in bytes.
+
+        Returns:
+            None
+        """
+        sys.stdout.write('\r')
+        sys.stdout.flush()
+        size = int(os.stat(filepath).st_size)
+        percent_complete = (size/filesize)*100
+        sys.stdout.write('%.3f %s' % (percent_complete, '% Completed'))
+        sys.stdout.flush()
+
+    def download_files(self, filelist, out_dir='./', cookie_file=None):
+        """Download files in a list.
+
+        Args:
+            filelist (list): List of web files to download.
+            out_dir (str): directory to put downloaded files
+
+        Returns:
+            None
+        """
+        for _file in filelist:
+            file_base = os.path.basename(_file)
+            out_file = out_dir + file_base
+            print('Downloading',file_base)
+            header = requests.head(_file, allow_redirects=True, stream=True)
+            filesize = int(header.headers['Content-Length'])
+            req = requests.get(_file, allow_redirects=True, stream=True)
+            with open(out_file, 'wb') as outfile:
+                chunk_size=1048576
+                for chunk in req.iter_content(chunk_size=chunk_size):
+                    outfile.write(chunk)
+                    if chunk_size < filesize:
+                        self.check_file_status(out_file, filesize)
+            self.check_file_status(out_file, filesize)
+            print()
+
+    def encode_url(self, url, token):
+        return url + '?token=' + token
+
+    def get_authentication(self):
+        """Attempts to get authentication.
+
+        Args:
+            token_file (str): location of password file.
+
+        Returns:
+            (tuple): token
+        """
+        token_file=self.DEFAULT_AUTH_FILE
+        if os.path.isfile(token_file) and os.path.getsize(token_file) > 0:
+            return self.read_token_file(token_file)
+        else:
+            return self.get_userinfo()
+
+
+    def get_summary(self, ds):
+        """Returns summary of dataset.
+
+        Args:
+            ds (str): Datset id. e.g. 'ds083.2'
+
+        Returns:
+            dict: JSON decoded result of the query.
+        """
+        url = self.BASE_URL + 'summary/'
+        url += ds
+
+        token = self.get_authentication()
+        ret = requests.get(self.encode_url(url,token))
+
+        self.check_status(ret)
+        return ret.json()
+
+    def get_metadata(self, ds):
+        """Return metadata of dataset.
+
+        Args:
+            ds (str): Datset id. e.g. 'ds083.2'
+
+        Returns:
+            dict: JSON decoded result of the query.
+        """
+        url = self.BASE_URL + 'metadata/'
+        url += ds
+
+        token = self.get_authentication()
+        ret = requests.get(self.encode_url(url,token))
+
+        self.check_status(ret)
+        return ret.json()
+
+    def get_all_params(self, ds):
+        """Return set of parameters for a dataset.
+
+        Args:
+            ds (str): Datset id. e.g. 'ds083.2'
+
+        Returns:
+            set: All unique params in dataset.
+        """
+        res = self.get_param_summary(ds)
+        res_data = res['data']['data']
+        param_names = set()
+        for param in res_data:
+            param_names.add(param['param'])
+        return param_names
+
+
+    def get_param_summary(self, ds):
+        """Return summary of parameters for a dataset.
+
+        Args:
+            ds (str): Datset id. e.g. 'ds083.2'
+
+        Returns:
+            dict: JSON decoded result of the query.
+        """
+        url = self.BASE_URL + 'paramsummary/'
+        url += ds
+
+        token = self.get_authentication()
+        ret = requests.get(self.encode_url(url,token))
+
+        self.check_status(ret)
+        return ret.json()
+
+
+    def submit_json(self, json_file):
+        """Submit a RDA subset or format conversion request using json file or dict.
+
+        Args:
+            json_file (str): json control file to submit.
+                    OR
+                    Python dict to submit.
+
+        Returns:
+            dict: JSON decoded result of the query.
+        """
+        if type(json_file) is str:
+            assert os.path.isfile(json_file)
+            with open(json_file) as fh:
+                control_dict = json.load(fh)
+        else:
+            assert type(json_file) is dict
+            control_dict = json_file
+
+        url = self.BASE_URL + 'submit/'
+
+        token = self.get_authentication()
+        ret = requests.post(self.encode_url(url,token), json=control_dict)
+
+        self.check_status(ret)
+        
+        return ret.json()
+
+    def submit(self, control_file_name):
+        """Submit a RDA subset or format conversion request.
+        Calls submit json after reading control_file
+
+        Args:
+            control_file_name (str): control file to submit.
+
+        Returns:
+            dict: JSON decoded result of the query.
+        """
+        _dict = self.read_control_file(control_file_name)
+        return self.submit_json(_dict)
+
+
+    def get_status(self, request_idx=None):
+        """Get status of request.
+        If request_ix not provided, get all open requests
+
+        Args:
+            request_idx (str, Optional): Request Index, typcally a 6-digit integer.
+
+        Returns:
+            dict: JSON decoded result of the query.
+        """
+        if request_idx is None:
+            request_idx = 'ALL'
+        url = self.BASE_URL + 'status/'
+        url += str(request_idx)
+
+
+        token = self.get_authentication()
+        ret = requests.get(self.encode_url(url,token))
+
+        self.check_status(ret)
+        return ret.json()
+
+    def get_filelist(self, request_idx):
+        """Gets filelist for request
+
+        Args:
+            request_idx (str): Request Index, typically a 6-digit integer.
+
+        Returns:
+            dict: JSON decoded result of the query.
+        """
+        url = self.BASE_URL + 'get_req_files/'
+        url += str(request_idx)
+
+        token = self.get_authentication()
+        ret = requests.get(self.encode_url(url,token))
+
+        self.check_status(ret)
+
+        return ret.json()
+
+
+    def download(self, request_idx, out_dir='./'):
+        """Download files given request Index
+
+        Args:
+            request_idx (str): Request Index, typically a 6-digit integer.
+
+        Returns:
+            None
+        """
+        ret = self.get_filelist(request_idx)
+        if len(ret['data']) == 0:
+            return ret
+
+        filelist = ret['data']['web_files']
+
+        token = self.get_authentication()
+
+        web_files = list(map(lambda x: x['web_path'], filelist))
+
+        # Only download unique files.
+        self.download_files(set(web_files), out_dir)
+        return ret
+
+    def globus_download(self, request_idx):
+        """Begin a globus transfer.
+
+        Args:
+            request_ix (str): Request Index, typically a 6-digit integer.
+
+        Returns:
+            dict: JSON decoded result of the query.
+        """
+        url = self.BASE_URL + 'request/'
+        url += request_idx
+        url += '-globus_download'
+
+        token = self.get_authentication()
+        ret = requests.get(self.encode_url(url,token))
+
+        self.check_status(ret)
+        return ret.json()
+
+    def get_control_file_template(self, ds):
+        """Write a control file for use in subset requests.
+
+        Args:
+            ds (str): datset id. e.g. 'ds083.2'
+
+        Returns:
+            dict: JSON decoded result of the query.
+        """
+        url = self.BASE_URL + 'control_file_template/'
+        url += ds
+
+        token = self.get_authentication()
+        ret = requests.get(self.encode_url(url,token))
+
+        self.check_status(ret)
+        return ret.json()
+
+    def write_control_file_template(self, ds, write_location='./'):
+        """Write a control file for use in subset requests.
+
+        Args:
+            ds (str): datset id. e.g. 'ds083.2'
+            write_location (str, Optional): Directory in which to write.
+                    Defaults to working directory
+
+        Returns:
+            dict: JSON decoded result of the query.
+        """
+        _json = self.get_control_file_template(ds)
+        control_str = _json['data']['template']
+
+        template_filename = write_location + self.add_ds_str(ds) + '_control.ctl'
+        if os.path.exists(template_filename):
+            print(template_filename + " already exists.\nExiting")
+            exit(1)
+        with open(template_filename, 'w') as fh:
+            fh.write(control_str)
+
+        return _json
+
+    def purge_request(self, request_idx):
+        """Write a control file for use in subset requests.
+
+        Args:
+            ds (str): datset id. e.g. 'ds083.2'
+            write_location (str, Optional): Directory in which to write.
+                    Defaults to working directory
+
+        Returns:
+            None
+        """
+        url = self.BASE_URL + 'purge/'
+        url += request_idx
+
+        token = self.get_authentication()
+        ret = requests.delete(self.encode_url(url,token))
+
+        self.check_status(ret)
+        return ret.json()
+
+    def get_selected_function(self, args_dict):
+        """Returns correct function based on options.
+        Args:
+            options (dict) : Command with options.
+
+        Returns:
+            (function): function that the options specified
+        """
+        # Maps an argument to function call
+        action_map = {
+                'get_summary' : self.get_summary,
+                'get_metadata' : self.get_metadata,
+                'get_param_summary' : self.get_param_summary,
+                'submit' : self.submit,
+                'get_status' : self.get_status,
+                'download' : self.download,
+                'get_filelist' : self.get_filelist,
+                'globus_download' : self.globus_download,
+                'get_control_file_template' : self.write_control_file_template,
+                'purge' : self.purge_request
+                }
+        for opt,value in args_dict.items():
+            if opt in action_map and value is not None:
+                return (action_map[opt], value)
+
+class RdamsWrapper(Rdams):
+    
+    def __init__(self, auth_file):
+        super().__init__(auth_file)
+
+    # wrappers for the functions
+    def purge(self, ds):
+        self.purge_request(ds)
+
+    def download(self, directory, ds):
+        super().download(ds, directory)
diff --git a/globsim/download/__init__.py b/globsim/download/__init__.py
index 0462c0f..27c83b5 100644
--- a/globsim/download/__init__.py
+++ b/globsim/download/__init__.py
@@ -1,14 +1,12 @@
 from .ERA5download import ERA5download
 from .ERA5Edownload import ERA5Edownload
-from .ERAIdownload import ERAIdownload
 from .JRAdownload import JRAdownload
 from .MERRAdownload import MERRAdownload
-from .JRA3Qdownload import J3QD
+from .JRA3Qdownload import J3QD, J3QgD
 from .era5_monthly import download_threadded, ERA5MonthlyDownload
 
 __all__ = ["ERA5download",
            "ERA5Edownload",
-           "ERAIdownload",
            "JRAdownload",
            "MERRAdownload",
            "download_threadded",
diff --git a/globsim/download/era5_monthly.py b/globsim/download/era5_monthly.py
index af51e10..25021a6 100644
--- a/globsim/download/era5_monthly.py
+++ b/globsim/download/era5_monthly.py
@@ -39,10 +39,6 @@ def __init__(self, pfile, ensemble=False):
             self._set_input_directory("era5")
             self.product_type = 'reanalysis'
             self.topo_file = 'era5_to.nc'
-
-    def retrieve(self):
-        requests = self.list_requests()
-        self.download_threadded(requests, 12)
         
     def list_requests(self):
         variables = self.par.get('variables')
@@ -62,7 +58,9 @@ def list_requests(self):
                                          day="01",
                                          time="00:00",
                                          area=area,
-                                         format='netcdf')
+                                         format='netcdf',
+                                         data_format= 'netcdf',
+                                         download_format = 'unarchived')
 
         request_to = Era5Request('reanalysis-era5-single-levels', self.directory, to_param)
         request_to.set_output_file(self.topo_file)
@@ -80,7 +78,9 @@ def list_requests(self):
                                        time=Era5RequestParameters.all_times(),
                                        area=area,
                                        pressure_level=self.levels,
-                                       format='netcdf')
+                                       format='netcdf',
+                                       data_format= 'netcdf',
+                                       download_format = 'unarchived')
             rpl = Era5Request('reanalysis-era5-pressure-levels', self.directory, pl)
             requests_pl.append(rpl)
 
@@ -91,7 +91,9 @@ def list_requests(self):
                                        day=day,
                                        time=Era5RequestParameters.all_times(),
                                        area=area,
-                                       format='netcdf')
+                                       format='netcdf',
+                                       data_format= 'netcdf',
+                                       download_format = 'unarchived')
 
             rsl = Era5Request('reanalysis-era5-single-levels', self.directory, sl)
             requests_sl.append(rsl)
@@ -99,61 +101,136 @@ def list_requests(self):
         return [request_to] + requests_pl + requests_sl
 
     def rename_files(self):
-        rename_pl(self.directory)
-        rename_sl(self.directory)
+        rename_pl_dir(self.directory)
+        rename_sl_dir(self.directory)
 
     def download_threadded(self, cds_requests, workers=6):
         logger.info(f"Starting ERA5 multi-threaded download with {workers} workers")
         incomplete_requests = list()
         for request in cds_requests:
-            if request.exists():
-                logger.warning(f"Found {request.output_file}. Will not re-download.")
-            elif request.is_downloaded():
-                logger.warning(f"Found {request.renamed_files}. Will not re-download.")
+            if request.is_downloaded():
+                logger.warning(f"Found {[file.name for file in request.renamed_files]}. Will not re-download.")
+            elif request.exists():
+                logger.warning(f"Found {request.output_file.name}. Will not re-download.")
             else:
                 incomplete_requests.append(request)
 
-        download_threadded(incomplete_requests, workers)
+        failed_downloads = download_threadded(incomplete_requests, workers)
         logger.info(f"Finished ERA5 multi-threaded download with {workers} workers")
-        rename_pl(self.directory)
-        rename_sl(self.directory)
+        
+        logger.info(f"Renaming files in {self.directory}")
+        rename_pl_dir(self.directory)
+        rename_sl_dir(self.directory)
+
+        failfile_path = Path(self.directory, "failed_downloads.txt")
+        
+        if len(failed_downloads) > 0:
+            logger.warning(f"Failed to download {len(failed_downloads)} files. See {failfile_path} for more information")
+        
+            with open(failfile_path, 'w') as f:
+                for failed in failed_downloads:
+                    f.write(f"{failed.output_file}\n")
+        
+        else:
+            logger.info("All files downloaded successfully")
+            failfile_path.unlink()
 
     def retrieve(self, workers=6):
         requests = self.list_requests()
-        self.download_threadded(requests, workers)
+        failed_downloads = self.download_threadded(requests, workers)
+        return failed_downloads
 
 
 def download_threadded(cds_requests, workers=6):
     def download_request(request):
-        if not request.exists():
+        if not request.is_downloaded():
             request.download()
-        else:
-            print("skipping request")
+        
+        if not request.exists():
+            logger.error(f"Failed to download {request.output_file}")
+            return request
 
+    failed_downloads = []
+    
     with ThreadPoolExecutor(max_workers=workers) as executor:
-        executor.map(download_request, cds_requests)
+        for result in executor.map(download_request, cds_requests):
+            if result is not None:
+                failed_downloads.append(result)
         executor.shutdown()
+    
+    return failed_downloads
 
 
-def rename_pl(dir):
+def rename_pl_dir(dir):
     pl_pattern = re.compile(r"era5_re_repl_(\d{8}_to_\d{8}).nc")
     files = [str(f) for f in Path(dir).iterdir() if pl_pattern.search(str(f))]
-    new_files = [pl_pattern.sub(r"era5_pl_\1.nc", f) for f in files]
-    
-    for old, new in zip(files, new_files):
-        rename(old, new)
+    for f in files:
+        rename_pl_file(f)
+
+
+def rename_pl_file(f, overwrite=True):
+    pl_pattern = re.compile(r"era5_re_repl_(\d{8}_to_\d{8}).nc")
+    new_file = pl_pattern.sub(r"era5_pl_\1.nc", f)
+    if not overwrite and Path(new_file).exists():
+        print(f"Skipping {new_file}")
+        return
+    logger.debug(f"Renaming {Path(f).name}")
+    print(new_file)
+    rename(f, new_file)
 
 
-def rename_sl(dir):
+def rename_sl_dir(dir):
     orig = re.compile(r"era5_re_resl_(\d{8}_to_\d{8}).nc")
     files = [str(f) for f in Path(dir).iterdir()]
     matched_files = [f for f in files if orig.search(f)]
     for f in matched_files:
-        sf = orig.sub(r'era5_sf_\1.nc', f)
-        sa = orig.sub(r'era5_sa_\1.nc', f)
-        cmd1 = f"nccopy -V time,latitude,longitude,ssrd,strd,tp {f} {sf}"
-        cmd2 = f"nccopy -V time,latitude,longitude,d2m,t2m,tco3,tcwv,u10,v10 {f} {sa}"
-        logger.debug(cmd1)
-        subprocess.Popen(cmd1.split(" "))
-        logger.debug(cmd2)
-        subprocess.Popen(cmd2.split(" "))
+        split_sl(f)
+
+
+def split_sl(f, overwrite=False, time_var='valid_time'):
+    orig = re.compile(r"era5_re_resl_(\d{8}_to_\d{8}).nc")
+    sf = orig.sub(r'era5_sf_\1.nc', f)
+    sa = orig.sub(r'era5_sa_\1.nc', f)
+    if not overwrite and Path(sf).exists():
+        print(f"Skipping {sf}")
+        return
+    if not overwrite and Path(sa).exists():
+        print(f"Skipping {sa}")
+        return
+    logger.debug(f"Splitting {Path(f).name}")
+    cmd1 = f"nccopy -V {time_var},latitude,longitude,ssrd,strd,tp {f} {sf}"
+    cmd2 = f"nccopy -V {time_var},latitude,longitude,d2m,t2m,tco3,tcwv,u10,v10 {f} {sa}"
+    logger.debug(cmd1)
+    p1 = subprocess.Popen(cmd1.split(" "))
+    p1.wait()
+    logger.debug(cmd2)
+    subprocess.Popen(cmd2.split(" "))
+    if Path(sf).exists() and Path(sa).exists():
+        logger.debug(f"Removing {f}")
+        # Path(f).unlink()
+
+
+if __name__ == "__main__":
+    import argparse
+    logger.setLevel(logging.DEBUG)
+    parser = argparse.ArgumentParser(description="Download ERA5 data")
+    
+    # add subparser for splitting
+    subparsers = parser.add_subparsers(dest='command')
+
+    # add parser for download
+    split = subparsers.add_parser('split', help="Download ERA5 data")
+    split.add_argument('-s', '--sl', type=str, default=None, help="Single-level file")
+    split.add_argument('-S', '--sl-dir', type=str, dest="sldir", default=None, help="Single-level file")
+    split.add_argument('-p', '--pl', type=str, default=None, help="Pressure-level file")
+    split.add_argument('-o', '--overwrite', action='store_true', help="Overwrite existing files")
+
+    args = parser.parse_args()
+    
+    if args.command == 'split':
+        if args.sl:
+            split_sl(args.sl, overwrite=args.overwrite)
+        if args.pl:
+            rename_pl_file(args.pl, overwrite=args.overwrite)
+        if args.sldir:
+            rename_sl_dir(args.sldir)
diff --git a/globsim/download/era_helpers.py b/globsim/download/era_helpers.py
index 85ce186..c1b894d 100644
--- a/globsim/download/era_helpers.py
+++ b/globsim/download/era_helpers.py
@@ -1,4 +1,5 @@
 import cdsapi
+import re
 
 import numpy as np
 
@@ -15,10 +16,10 @@
 class Era5RequestParameters(MutableMapping):
     """ Request dictionary """
     VALID_KEYS = ['product_type','format','year',
-                  'month','day','time','area','variable',
+                  'month','day','time','area','variable','data_format','download_format',
                   'pressure_level']
 
-    REQUIRED_KEYS = ['product_type','format','year',
+    REQUIRED_KEYS = ['product_type','data_format','year',
                      'month','day','time','variable']
 
     """A dictionary that applies an arbitrary key-altering
@@ -41,7 +42,7 @@ def __setitem__(self, key, value):
         if key in self.VALID_KEYS:
             self.store[key] = value
         else:
-            raise KeyError(f"{value} is not a valid ERA5 request parameter")
+            raise KeyError(f"{key}:{value} is not a valid ERA5 request parameter")
 
     def __delitem__(self, key):
         del self.store[key]
@@ -123,6 +124,12 @@ def __init__(self, dataset: str, directory: str, request_params: Era5RequestPara
         self.dataset = dataset
         self._output_file = None
 
+    def __repr__(self):
+        return f"ERA5 Request: {self.dataset} {self.params}"
+
+    def __str__(self):
+        return f"{self.dataset} request for {len(self.params)} variables at {self.params['area']}"
+    
     def download(self, target=None):
         ''' download '''
         server = cdsapi.Client()
@@ -208,6 +215,23 @@ def output_file(self) -> Path:
             file = Path(self.directory, f"era5_{era_type}_{dataset}_{time}.nc")
 
         return file
+    
+    @property
+    def globsim_outputs(self) -> "list[Path]":
+        if self.dataset == "reanalysis-era5-single-levels": 
+            orig = re.compile(r"era5_re_resl_(\d{8}_to_\d{8}).nc")
+            sf = orig.sub(r'era5_sf_\1.nc', self.output_file.name)
+            sa = orig.sub(r'era5_sa_\1.nc', self.output_file.name)
+            
+            files = [self.output_file.with_name(n) for n in [sa, sf]]
+
+        elif self.dataset == "reanalysis-era5-pressure-levels":
+            pl_pattern = re.compile(r"era5_re_repl_(\d{8}_to_\d{8}).nc")
+            pl = pl_pattern.sub(r"era5_pl_\1.nc", self.output_file.name)
+            
+            files = [self.output_file.with_name(pl)]
+
+        return files
 
 
 def era5_pressure_levels(elev_min: float, elev_max: float) -> "list[str]":
diff --git a/globsim/download/jra_dict_formatters.py b/globsim/download/jra_dict_formatters.py
new file mode 100644
index 0000000..a949d7e
--- /dev/null
+++ b/globsim/download/jra_dict_formatters.py
@@ -0,0 +1,354 @@
+import re
+
+from typing import Optional
+
+from globsim.meteorology import pressure_from_elevation
+from globsim.download.RDA import Rdams
+
+
+def getPressureLevels(levels: list, min_elev: float, max_elev: float) -> "list[float]":
+    # flip max and min because 1000 is the bottom and 0 is the top
+    elevationMax = pressure_from_elevation(min_elev)
+    elevationMin = pressure_from_elevation(max_elev)
+
+    minNum = min(levels, key=lambda x:abs(x - elevationMin))
+    maxNum = min(levels, key=lambda x:abs(x - elevationMax))
+
+    if (minNum > elevationMin and levels.index(minNum) > 0):
+        elevationMinRange = levels.index(minNum) - 1
+    else:
+        elevationMinRange = levels.index(minNum)
+
+    if (maxNum < elevationMin and levels.index(maxNum) < 36):
+        elevationMaxRange = levels.index(maxNum) - 1
+    else:
+        elevationMaxRange = levels.index(maxNum)
+
+    elevation = []
+    for e in range(elevationMinRange, elevationMaxRange + 1):
+        elevation.append(levels[e])
+
+    return elevation
+
+
+class JRAformatter:
+
+    VALID_LEVELS = [100, 125, 150, 175, 200, 225, 250, 300, 350, 400, 450, 500, 550,
+                    600, 650, 700, 750, 775, 800, 825, 850, 875, 900,
+                    925, 950, 975, 1000]
+    
+    def __init__(self, date, area, elevation={'min': 0, 'max': 2000}, variables=[]):
+        '''Returns an object for JRA55 data that has methods for querying the
+        NCAR server for pressure level variables (prec, swin, lwin). '''
+
+        self.date = date
+        self.area = area
+        self.elevation = elevation
+        self.variables = variables
+
+    def makeDate(self):
+        '''convert data format to NCAR RDA request'''
+
+        beg = self.date['beg'].strftime('%Y%m%d%H%M')
+        end = self.date['end'].strftime('%Y%m%d%H%M')
+        dateRange = beg + '/to/' + end
+
+        return dateRange
+
+    def getParam(self, dpar, variables, add=None):
+
+        varlist = []
+        for var in variables:
+            varlist.append(dpar.get(var))
+        
+        varlist = [item for item in varlist if item is not None]
+        varlist = [item for sublist in varlist for item in sublist]
+
+        if add is not None:
+            for var in add:
+                varlist.append(var)
+
+        return varlist
+
+    @staticmethod
+    def param_descriptions_to_name(params: str, param_dict: dict) -> str:
+        if isinstance(params, list):
+            param_list = params
+        elif isinstance(params, str):
+            param_list = params.split('/')
+        
+        param_codes = {v['param_description']:v['param'] for v in param_dict}
+        
+        shortpar = "/".join([i for i in param_list])
+        
+        return shortpar
+    
+
+class J55DictFormatter(JRAformatter):
+    
+    DATASET = 'ds628.0'
+
+    pl_dict = {'air_temperature'   : ['Temperature'],
+                'relative_humidity' : ['Relative humidity'],
+                'wind_speed'        : ['u-component of wind',
+                                       'v-component of wind']}
+    
+    sf_dict = {'precipitation_amount':
+            ['Total precipitation'],
+        'downwelling_shortwave_flux_in_air':
+            ['Downward solar radiation flux'],
+        'downwelling_longwave_flux_in_air':
+            ['Downward longwave radiation flux'],
+        'downwelling_shortwave_flux_in_air_assuming_clear_sky':
+            ['Clear sky downward solar radiation flux'],
+        'downwelling_longwave_flux_in_air_assuming_clear_sky':
+            ['Clear sky downward longwave radiation flux']}
+
+    sa_dict = {'air_temperature'  : ['Temperature'],
+                'relative_humidity': ['Relative humidity'],
+                'specific_humidity': ['Specific humidity'],
+                'wind_speed'       : ['u-component of wind',
+                                      'v-component of wind']}
+
+    def get_dict_template(self):
+        dictionary = {
+                'dataset': self.DATASET,
+                'date': self.makeDate(),
+                'param': None,
+                'level': None,
+                'oformat': 'netCDF',
+                'nlat': str(self.area['north']),
+                'slat': str(self.area['south']),
+                'wlon': str(self.area['west']),
+                'elon': str(self.area['east']),
+                'product': 'Analysis',
+                'compression': 'NN',
+                'gridproj': 'latLon',
+                'griddef': '288:145:90N:0E:90S:1.25W:1.25:1.25'}
+        
+        return dictionary
+
+    def get_pl_dict(self, variables):
+        levels = getPressureLevels(self.VALID_LEVELS, self.elevation['min'], self.elevation['max'])
+        elevation = [str(ele) for ele in levels]
+        temp = self.get_dict_template()
+        param = self.getParam(self.pl_dict, variables, add=['Geopotential height', 'Pressure'])
+        temp['param'] = '/'.join(param)
+        temp['level'] = 'Isobaric surface:' + '/'.join(elevation)
+        return temp
+
+    def get_sf_dict(self, variables):
+        temp = self.get_dict_template()
+        param = self.getParam(self.sf_dict, variables, add=['Pressure'])
+        temp['param'] = '/'.join(param)
+        temp['level'] = 'Ground or water surface:0'
+        temp['product'] = '3-hour Average (initial+0 to initial+3)'
+
+        return temp
+    
+    def get_sa_dict(self, variables):
+        temp = self.get_dict_template()
+        param = self.getParam(self.sa_dict, variables)
+        temp['param'] = '/'.join(param)
+        temp['level'] = 'Specified height above ground:2/10'
+
+        return temp
+
+    def get_to_dict(self):
+        temp = self.get_dict_template()
+        temp['date'] = "194709010000/to/194709010000"
+        temp['param'] = 'Geopotential'
+        temp['level'] = 'Ground or water surface:0'
+        
+        return temp
+    
+
+class J3QDictFormatter(JRAformatter):
+    DATASET = 'd640000'
+    REANALYSIS = 'jra3q'
+
+    _pl_dict = {'air_temperature'   : ['tmp-pres-an-{grid}'],
+                'relative_humidity' : ['rh-pres-an-{grid}'],
+                'specific_humidity': ['spfh-pres-an-{grid}'],
+                'wind_speed'        : ['ugrd-pres-an-{grid}',
+                                       'vgrd-pres-an-{grid}']}
+
+    _sa_dict = {'air_temperature'  : ['tmp2m-hgt-an-{grid}'],
+                'relative_humidity': ['rh2m-hgt-an-{grid}'],
+                'specific_humidity': ['spfh2m-hgt-an-{grid}'],
+                'wind_speed'       : ['ugrd10m-hgt-an-{grid}',
+                                      'vgrd10m-hgt-an-{grid}']}
+
+    _to_dict = {'geopotential': ['gp-sfc-cn-{grid}']}
+
+    _sf_dict = {'precipitation_amount':
+            ['tprate1have-sfc-fc-{grid}'],
+        'downwelling_shortwave_flux_in_air':
+            ['dswrf1have-sfc-fc-{grid}'],
+        'downwelling_longwave_flux_in_air':
+            ['dlwrf1have-sfc-fc-{grid}'],
+        'downwelling_shortwave_flux_in_air_assuming_clear_sky':
+            ['dswrfcs1have-sfc-fc-{grid}'],
+        'downwelling_longwave_flux_in_air_assuming_clear_sky':
+            ['dlwrfcs1have-sfc-fc-{grid}']}
+    
+    def __init__(self, grid='ll125', metadata=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.set_grid(grid)
+        rda = Rdams(metadata)
+        self.metadata = rda.get_metadata(self.DATASET)['data']['data']
+        
+    def lookup_param(self, param, returns='param_description'):
+        return lookup_param(self.metadata, param, returns)
+    
+    @property
+    def pl_dict(self):
+        return {k:[i.format(grid=self.grid) for i in v] 
+                for k,v in self._pl_dict.items()}
+    
+    @property
+    def sa_dict(self):
+        return {k:[i.format(grid=self.grid) for i in v] 
+                for k,v in self._sa_dict.items()}
+
+    @property
+    def sf_dict(self):
+        return {k:[i.format(grid=self.grid) for i in v] 
+                for k,v in self._sf_dict.items()}
+    
+    @property
+    def to_dict(self):
+        return {k:[i.format(grid=self.grid) for i in v] 
+                for k,v in self._to_dict.items()}
+
+    def get_pl_dict(self, variables):
+        levels = getPressureLevels(self.VALID_LEVELS, self.elevation['min'], self.elevation['max'])
+        elevation = [str(ele) for ele in levels]
+        temp = self.get_dict_template()
+        param = self.getParam(self.pl_dict, variables, add=[f'hgt-pres-an-{self.grid}'])
+        temp['param'] = '/'.join(param)
+        temp['level'] = 'pressure (isobaric) level:' + '/'.join(elevation)
+
+        return temp
+
+    def get_sf_dict(self, variables):
+        temp = self.get_dict_template()
+        param = self.getParam(self.sf_dict, variables, add=[f'pres-sfc-fcst-{self.grid}'])
+        temp['param'] = '/'.join(param)
+        temp['level'] = 'Surface:0'
+        
+
+        return temp
+    
+    def get_sa_dict(self, variables):
+        temp = self.get_dict_template()
+        param = self.getParam(self.sa_dict, variables)
+        temp['param'] = '/'.join(param)
+        temp['level'] = 'Surface:0'
+
+        return temp
+
+    def get_to_dict(self, *args, **kwargs):
+        temp = self.get_dict_template()
+        param = self.getParam(self.to_dict, ['geopotential'])
+        metadata = find_param(self.metadata, level_description_pattern=None, 
+                              name_pattern=None, 
+                              param_pattern=f'.*-sfc-cn-{self.grid}',
+                              names_only=False, drop_levels=False)
+
+        temp['date'] = f"{metadata[0]['start_date']}/to/{metadata[0]['end_date']}"
+        temp['param'] = '/'.join(param)
+        temp['level'] = 'Surface:0'
+        
+        return temp
+    
+    def set_grid(self, grid):
+        if grid not in ['ll125', 'gauss']:
+            raise ValueError(f'Invalid grid ({grid}). Must be "ll125" or "gauss"')
+        self.griddef = {'ll125': '288:145:90N:0E:90S:358.75E:1.25:1.25',
+                        'gauss': '960:480:89.713N:0E:89.713S:359.625E:0.375:0.375'}[grid]
+        self.grid = grid
+
+    def get_dict_template(self):
+        dictionary = {
+                'dataset': self.DATASET,
+                'date': self.makeDate(),
+                'param': None,
+                'level': None,
+                'oformat': 'netCDF',
+                'nlat': str(self.area['north']),
+                'slat': str(self.area['south']),
+                'wlon': str(self.area['west']),
+                'elon': str(self.area['east']),
+                'product': 'Analysis',
+                'compression': 'NN',
+                'gridproj': 'latLon',
+                'griddef': self.griddef}
+        
+        return dictionary
+    
+    @staticmethod
+    def param_descriptions_to_name(params: str, param_dict: dict) -> str:
+        if isinstance(params, list):
+            param_list = params
+        elif isinstance(params, str):
+            param_list = params.split('/')
+        
+        shortpar = "/".join(param_list)
+        
+        return shortpar
+
+
+def find_param(md:dict, name_pattern:Optional[str]='wind', 
+               level_description_pattern:Optional[str]='isobaric',
+               param_pattern:Optional[str]='ll125', 
+               names_only:bool=False,
+               drop_levels:bool=True) -> list:
+    param = md.copy()
+    if name_pattern is not None:
+        param = filter(lambda x: re.search(name_pattern, x['param_description']), md)
+
+    if level_description_pattern is not None:
+        param = filter(lambda x: re.search(level_description_pattern, x['levels'][0]['level_description']), param)
+    
+    if param_pattern is not None:
+        param = filter(lambda x: re.search(param_pattern, x['param']), param)
+    
+    if names_only:
+        return set([p['param'] for p in param])
+    else:
+        if drop_levels:
+            param = [{k:v for k,v in p.items() if k != 'levels'} for p in param]
+        
+        return list(param)
+    
+
+def lookup_param(param, metadata=None, dsid=None, returns='param_description'):
+    if metadata is None:
+        if dsid is not None:
+            rda = Rdams(None)
+            metadata = rda.get_metadata(dsid)['data']['data']
+        else:
+            raise ValueError('metadata or dsid must be provided')
+    
+    return list(filter(lambda x: x['param'] == param, metadata))[0][returns]
+
+
+class J3QGDictFormatter(J3QDictFormatter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.set_grid('gauss')
+
+
+if __name__ =="__main__":
+    
+    rda = Rdams(None)
+    md = rda.get_metadata('d640000')['data']['data']
+    a = find_param(md, 
+                level_description_pattern=None, 
+                name_pattern=None, 
+                param_pattern=f'.*-sfc-cn',
+                names_only=False, drop_levels=False)
+    print(a)
+    # [print(p['param_description'], p['param']) for p in q]
diff --git a/globsim/globsim_cli.py b/globsim/globsim_cli.py
index 87054ce..eaa0ffb 100644
--- a/globsim/globsim_cli.py
+++ b/globsim/globsim_cli.py
@@ -61,7 +61,7 @@ def main():
 
         parser.add_argument("-d",
                             default=None, nargs="*",required=True, type=str.upper,
-                            choices=["ERAI", "ERA5", "ERA5ENS", "MERRA", "JRA", "JRA3Q"],
+                            choices=["ERA5", "ERA5ENS", "MERRA", "JRA", "JRA3Q", "JRA3QG"],
                             dest='d', help="What data sources should run?")
 
     for parser in [download, interpolate, scale, convert]:
@@ -76,6 +76,7 @@ def main():
     interpolate.add_argument('--skip-sa', dest='skip_sa', action='store_true', help='skip creation of "sa" file')
     interpolate.add_argument('--skip-sf', dest='skip_sf', action='store_true', help='skip creation of "sf" file')
     interpolate.add_argument('--skip-pl', dest='skip_pl', action='store_true', help='skip creation of "pl" file')
+    interpolate.add_argument('--resume', dest='resume', action='store_true', help='pick up interpolation where it left off')
     interpolate.add_argument('--skip-checks', dest='skip_checks', action='store_true', help='skip data integrity checks')
     
     download.set_defaults(func=globsim_download.main)
diff --git a/globsim/globsim_download.py b/globsim/globsim_download.py
index 8ca0536..e107f0a 100755
--- a/globsim/globsim_download.py
+++ b/globsim/globsim_download.py
@@ -28,24 +28,24 @@
 
 def main(args):
     pfile = args.f
-    ERAI    = True if args.d is None or "ERAI"  in args.d else False
     ERA5    = True if args.d is None or "ERA5"  in args.d else False
     ERA5ENS = True if args.d is None or "ERA5ENS"  in args.d else False
     JRA     = True if args.d is None or "JRA"   in args.d else False
     MERRA   = True if args.d is None or "MERRA" in args.d else False
     JRA3Q   = True if args.d is None or "JRA3Q" in args.d else False
+    JRA3QG   = True if args.d is None or "JRA3QG" in args.d else False
 
     r_max = args.retry
     i = 0
 
     if r_max <= 1:
-        GlobsimDownload(pfile, ERAI=ERAI, ERA5=ERA5, ERA5ENS=ERA5ENS, 
-                        JRA=JRA, MERRA=MERRA, JRA3Q=JRA3Q, multithread=args.multi)
+        GlobsimDownload(pfile, ERA5=ERA5, ERA5ENS=ERA5ENS, 
+                        JRA=JRA, MERRA=MERRA, JRA3Q=JRA3Q, JRA3QG=JRA3QG, multithread=args.multi)
     else:
         while i < r_max:
             try:
-                GlobsimDownload(pfile, ERAI=ERAI, ERA5=ERA5, ERA5ENS=ERA5ENS, 
-                                JRA=JRA, MERRA=MERRA, JRA3Q=JRA3Q, multithread=args.multi)
+                GlobsimDownload(pfile, ERA5=ERA5, ERA5ENS=ERA5ENS, 
+                                JRA=JRA, MERRA=MERRA, JRA3Q=JRA3Q, JRA3QG=JRA3QG, multithread=args.multi)
             except Exception as e:
                 print(e)
             time.sleep(360)
@@ -57,7 +57,7 @@ def main(args):
     parser = argparse.ArgumentParser(description="Get MET data",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('-f',             default=None, type=str, help="file path to download parameter file")
-    parser.add_argument('-d',             default=None, type=str, nargs="*", help="What data sources should run? ERAI, ERA5, MERRA, JRA")
+    parser.add_argument('-d',             default=None, type=str, nargs="*", help="What data sources should run? ERA5, MERRA, JRA")
     parser.add_argument('-r', '--retry',  default=1,    type=int, help="Number of times to re-launch download if it crashes.")
     parser.add_argument('-m', '--multi', action='store_true',    help="Download all data sources simultaneously")
 
diff --git a/globsim/globsim_interpolate.py b/globsim/globsim_interpolate.py
index a1df70c..489b232 100755
--- a/globsim/globsim_interpolate.py
+++ b/globsim/globsim_interpolate.py
@@ -28,25 +28,25 @@
 def main(args):
     pfile = args.f
 
-    ERAI     = bool(args.d is None or "ERAI"  in args.d)
     ERA5     =  bool(args.d is None or "ERA5"  in args.d)
     ERA5ENS =  bool(args.d is None or "ERA5ENS"  in args.d)
     JRA     =  bool(args.d is None or "JRA"   in args.d)
     JRA3Q   = bool(args.d is None or "JRA3Q"   in args.d)
+    JRA3QG   = bool(args.d is None or "JRA3QG"   in args.d)
     MERRA     = bool(args.d is None or "MERRA" in args.d)
 
-    if sum([ERAI, ERA5, ERA5ENS, JRA, MERRA, JRA3Q]) > 0:
+    if sum([ERA5, ERA5ENS, JRA, MERRA, JRA3Q, JRA3QG]) > 0:
 
         GlobsimInterpolateStation(pfile,
-                                  ERAI=ERAI,
                                   ERA5=ERA5,
                                   ERA5ENS=ERA5ENS,
                                   JRA=JRA,
                                   MERRA=MERRA,
                                   JRA3Q=JRA3Q,
+                                  JRA3QG=JRA3QG,
                                   **vars(args))
 
-    else: print("Failed! Reanalysis source should be ERAI, ERA5, MERRA, JRA, JRA3Q, please check")
+    else: print("Failed! Reanalysis source should be  ERA5, MERRA, JRA, JRA3Q, please check")
 
 
 # ===interpolate the variables from multiple re-analysis data to individual stations===
@@ -56,7 +56,7 @@ def main(args):
     parser.add_argument('-f',    default=None, type=str, 
                         help="file path to download parameter file")
     parser.add_argument('-d',    default=None, nargs="*", type=str, 
-                        help="What data sources should run? ERAI, ERA5, MERRA, JRA, JRA3Q")
+                        help="What data sources should run?  ERA5, MERRA, JRA, JRA3Q, JRA3QG")
 
     args = parser.parse_args()
 
diff --git a/globsim/globsim_main.py b/globsim/globsim_main.py
index 28c75b8..1937ca8 100755
--- a/globsim/globsim_main.py
+++ b/globsim/globsim_main.py
@@ -47,20 +47,15 @@
 
 
 def GlobsimDownload(pfile, multithread=True, 
-                    ERAI=True, ERA5=True, 
-                    ERA5ENS=True, MERRA=True, JRA=True, JRA3Q=True):
+                    ERA5=True, 
+                    ERA5ENS=True, MERRA=True, JRA=True, JRA3Q=True, JRA3QG=True):
     """
     Download data from multiple reanalyses. Each reanalysis is run as one 
     separate thread if 'multithread = True'. If 'multithread = False', each
     download is run sequentially. This is easier for interpreting the output.
     """
     # make list of objects to execute
-    objects = [] 
-    
-    # === ERA-Interim ===
-    if ERAI:
-        ERAIdownl = download.ERAIdownload(pfile)
-        objects.append(ERAIdownl)
+    objects = []
     
     # === ERA5 ===
     if ERA5:
@@ -87,6 +82,10 @@ def GlobsimDownload(pfile, multithread=True,
         JRA3Qdownl = download.J3QD(pfile)
         objects.append(JRA3Qdownl)
 
+    if JRA3QG:
+        JRA3QGdownl = download.J3QgD(pfile)
+        objects.append(JRA3QGdownl)
+
     # serial of parallel execution
     if multithread:
         # Make the Pool of workers and run as lambda functions
@@ -102,8 +101,8 @@ def GlobsimDownload(pfile, multithread=True,
         print('Serial download finished')
     
 
-def GlobsimInterpolateStation(ifile, ERAI=True, ERA5=True, ERA5ENS=True,
-                              MERRA=True, JRA=True, JRA3Q=True, **kwargs):
+def GlobsimInterpolateStation(ifile, ERA5=True, ERA5ENS=True,
+                              MERRA=True, JRA=True, JRA3Q=True,JRA3QG=True, **kwargs):
     """
     Interpolate re-analysis data to individual stations (points: lat, lon, ele).
     The temporal granularity and variables of each re-analysis are preserved.
@@ -111,11 +110,6 @@ def GlobsimInterpolateStation(ifile, ERAI=True, ERA5=True, ERA5ENS=True,
     differing processors as memory may be a limiting factor.
     """
     
-    # === ERA-Interim ===
-    if ERAI:
-        ERAIinterp = interpolate.ERAIinterpolate(ifile, **kwargs)
-        ERAIinterp.process()
-    
     # === ERA5 ===
     if ERA5:
         ERA5interp = interpolate.ERA5interpolate(ifile, **kwargs)
@@ -139,18 +133,18 @@ def GlobsimInterpolateStation(ifile, ERAI=True, ERA5=True, ERA5ENS=True,
     if JRA3Q:
         JRA3Qinterp = interpolate.J3QI(ifile)
         JRA3Qinterp.process()
+
+    if JRA3QG:
+        JRA3QGinterp = interpolate.J3QgI(ifile)
+        JRA3QGinterp.process()
             
-def GlobsimScale(sfile, ERAI=True, ERA5=True, ERA5ENS=True, MERRA=True, JRA=True, JRA3Q=True):
+def GlobsimScale(sfile, ERA5=True, ERA5ENS=True, MERRA=True, JRA=True, JRA3Q=True,JRA3QG=True):
     """
     Use re-analysis data that has been interpolated to station locations to 
     derive fluxes scaled / converted / adjusted to drive point-scale 
     near-surface models. The resulting data has coherent variables for all 
     reanalyses, optionally coherent temporal granularity, and is saved as netCDF.
     """
-    # === ERA-Interim ===
-    if ERAI:
-        ERAIsc = scale.ERAIscale(sfile)
-        ERAIsc.process()
     
     # === ERA5 ===
     if ERA5:
@@ -176,4 +170,8 @@ def GlobsimScale(sfile, ERAI=True, ERA5=True, ERA5ENS=True, MERRA=True, JRA=True
     if JRA3Q:
         JRA3Qsc = scale.J3QS(sfile)
         JRA3Qsc.process()
+
+    if JRA3QG:
+        JRA3QGsc = scale.J3QgS(sfile)
+        JRA3QGsc.process()
                   
diff --git a/globsim/globsim_scale.py b/globsim/globsim_scale.py
index e80ea04..8758e53 100755
--- a/globsim/globsim_scale.py
+++ b/globsim/globsim_scale.py
@@ -28,20 +28,20 @@
 
 def main(args):
         sfile = args.f
-        
-        ERAI    = True if args.d is None or "ERAI"     in args.d else False
+
         ERA5    = True if args.d is None or "ERA5"     in args.d else False
         ERA5ENS = True if args.d is None or "ERA5ENS"  in args.d else False
         JRA     = True if args.d is None or "JRA"      in args.d else False
         JRA3Q   = True if args.d is None or "JRA3Q"    in args.d else False
+        JRA3QG   = True if args.d is None or "JRA3QG"    in args.d else False
         MERRA   = True if args.d is None or "MERRA"    in args.d else False
         
-        if sum([ERAI, ERA5, ERA5ENS, JRA, MERRA, JRA3Q]) > 0:
-            GlobsimScale(sfile, ERAI=ERAI, ERA5=ERA5, ERA5ENS=ERA5ENS, 
-                        JRA=JRA, MERRA=MERRA, JRA3Q=JRA3Q)
+        if sum([ ERA5, ERA5ENS, JRA, MERRA, JRA3Q, JRA3QG]) > 0:
+            GlobsimScale(sfile, ERA5=ERA5, ERA5ENS=ERA5ENS, 
+                        JRA=JRA, MERRA=MERRA, JRA3Q=JRA3Q, JRA3QG=JRA3QG)
 
         
-        else: print("Failed! Reanalysis source should be ERAI, ERA5, MERRA, JRA, JRA3Q please check")
+        else: print("Failed! Reanalysis source should be ERA5, MERRA, JRA, JRA3Q please check")
 
 #===scale the variables from mutiple re-analysis data at stations===
 if __name__ == "__main__":
@@ -50,7 +50,7 @@ def main(args):
     parser.add_argument('-f', default=None, type=str, 
                         help="file path to download parameter file")
     parser.add_argument('-d', default=None, nargs="*", type=str, 
-                        help="What data sources should run? ERAI, ERA5, ERA5ENS, MERRA, JRA, JRA3Q")
+                        help="What data sources should run?  ERA5, ERA5ENS, MERRA, JRA, JRA3Q, JRA3QG")
     
     
     args = parser.parse_args()
diff --git a/globsim/interpolate/ERA5interpolate.py b/globsim/interpolate/ERA5interpolate.py
index 9ad72ac..013908d 100644
--- a/globsim/interpolate/ERA5interpolate.py
+++ b/globsim/interpolate/ERA5interpolate.py
@@ -1,10 +1,13 @@
 import netCDF4 as nc
+import gc
+import xarray as xr
+import logging
 import numpy as np
 import urllib3
-import logging
 
 from datetime import datetime
 from os import path
+from pathlib import Path
 
 from globsim.common_utils import variables_skip, str_encode
 from globsim.interpolate.GenericInterpolate import GenericInterpolate
@@ -38,40 +41,54 @@ def __init__(self, ifile, era5type='reanalysis', **kwargs):
         # convert longitude to ERA notation if using negative numbers
         self.stations['longitude_dd'] = self.stations['longitude_dd'] % 360
 
-        self.mf_to = nc.MFDataset(self.getInFile('to'), 'r', aggdim='time')
+        self.mf_to = xr.open_mfdataset(self.get_input_file_glob('to'), decode_times=False)
         
         # Check dataset integrity
         if not self.skip_checks:
-            with nc.MFDataset(self.getInFile('sa'), 'r', aggdim='time') as sa:
+            with xr.open_mfdataset(self.get_input_file_paths('sa'), decode_times=False) as sa:
                 logger.info("Check data integrity (sa)")
-                self.ensure_datset_integrity(sa['time'], 1)
+                self.ensure_datset_integrity(sa[self.vn_time], 3600)
             
-            with nc.MFDataset(self.getInFile('sf'), 'r', aggdim='time') as sf:
+            with xr.open_mfdataset(self.get_input_file_paths('sf'), decode_times=False) as sf:
                 logger.info("Check data integrity (sf)")
-                self.ensure_datset_integrity(sf['time'], 1)
+                self.ensure_datset_integrity(sf[self.vn_time], 3600)
 
-            with nc.MFDataset(self.getInFile('pl'), 'r', aggdim='time') as pl:
+            with xr.open_mfdataset(self.get_input_file_paths('pl'), decode_times=False) as pl:
                 logger.info("Check data integrity (pl)")
-                self.ensure_datset_integrity(pl['time'], 1)
+                self.ensure_datset_integrity(pl[self.vn_time], 3600)
 
             logger.info("Data integrity ok")
 
-    def getInFile(self, levStr):
+    @property
+    def vn_time(self):
+        return 'valid_time'
+    
+    @property
+    def vn_level(self):
+        return 'pressure_level'
+    
+    def get_input_file_glob(self, levStr):
+        """ Return the input file(s) for the interpolation. """
         # edited naming conventions for simplicity and to avoid errors
         if self.ens:
             typeStr = '_ens'
         else:
             typeStr = ''
 
-        nome = 'era5{}_{}_*.nc'
+        f_glob = 'era5{}_{}_*.nc'
 
         if levStr == 'to':
             infile = path.join(self.input_dir, 'era5{}_to.nc'.format(typeStr))
         else:
-            infile = path.join(self.input_dir, nome.format(typeStr, levStr))
+            infile = path.join(self.input_dir, f_glob.format(typeStr, levStr))
 
         return infile
 
+    def get_input_file_paths(self, levStr):
+        glob = self.get_input_file_glob(levStr)
+        infiles = self.prefilter_mf_paths(glob)
+        return infiles
+    
     def getOutFile(self, levStr):
         if self.ens:
             nome = 'era5_ens_{}_'.format(levStr) + self.list_name + '.nc'
@@ -114,122 +131,150 @@ def ERA2station(self, ncf_in, ncfile_out, points,
         self.validate_stations_extent(ncf_in)
 
         # is it a file with pressure levels?
-        pl = 'level' in ncf_in.dimensions.keys()
-        ens = 'number' in ncf_in.dimensions.keys()
+        pl = self.vn_level in ncf_in.sizes.keys()
+        ens = 'number' in ncf_in.sizes.keys()
 
         # reduce chunk size for pressure-level interpolation
         cs = self.cs
         if pl:
-            cs = cs // len(ncf_in.variables['level'][:])  # get actual number of levels
+            cs = cs // len(ncf_in.variables[self.vn_level][:])  # get actual number of levels
 
         # build the output of empty netCDF file
-        rootgrp = new_interpolated_netcdf(ncfile_out, self.stations, ncf_in,
-                                          time_units='hours since 1900-01-01 00:00:0.0')
-        if self.ens:
-            rootgrp.source = 'ERA5 10-member ensemble, interpolated bilinearly to stations'
-        else:
-            rootgrp.source = 'ERA5, interpolated bilinearly to stations'
+        level_var = self.vn_level if pl else None
+        
+        if self.resume:
+            self.require_file_can_be_resumed(ncfile_out)
+
+        if not (self.resume and Path(ncfile_out).exists()):
+            rootgrp = new_interpolated_netcdf(ncfile_out, self.stations, ncf_in,
+                                            time_units=ncf_in[self.vn_time].units,  # 'hours since 1900-01-01 00:00:0.0'
+                                            calendar=ncf_in[self.vn_time].calendar,
+                                            level_var=level_var)
+            
+            rootgrp.globsim_interpolate_start = self.par['beg']
+            rootgrp.globsim_interpolate_end = self.par['end']
+            rootgrp.globsim_chunk_size = self.cs
+            rootgrp.globsim_interpolate_success = 0
+            rootgrp.globsim_last_chunk_written = -1
 
-        rootgrp.close()  # TODO: there's no need to close this file and then re-open it a moment later
+            if self.ens:
+                rootgrp.source = 'ERA5 10-member ensemble, interpolated bilinearly to stations'
+            else:
+                rootgrp.source = 'ERA5, interpolated bilinearly to stations'
+
+            rootgrp.close()  # close the file to write the header (needed for appending?)
 
         # open the output netCDF file, set it to be appendable ('a')
-        ncf_out = nc.Dataset(ncfile_out, 'a')
-
-        # get time and convert to datetime object
-        nctime = ncf_in.variables['time'][:]
-        # "hours since 1900-01-01 00:00:0.0"
-        t_unit = ncf_in.variables['time'].units
-        try:
-            t_cal = ncf_in.variables['time'].calendar
-        except AttributeError:  # attribute doesn't exist
-            t_cal = u"gregorian"  # standard
-        time = nc.num2date(nctime, units=t_unit, calendar=t_cal)
-
-        # detect invariant files (topography etc.)
-        invariant = True if len(time) == 1 else False
-
-        # restrict to date/time range if given
-        if date is None:
-            tmask = np.array([True])
-        else:
-            tmask = (time < date['end']) * (time >= date['beg'])
-
-        # get time vector for output
-        time_in = nctime[tmask]
-
-        # write time
-        ncf_out.variables['time'][:] = time_in
-
-        # ensure that chunk sizes cover entire period even if
-        # len(time_in) is not an integer multiple of cs
-        niter = len(time_in) // cs
-        niter += ((len(time_in) % cs) > 0)
-
-        # Create source grid
-        sgrid = self.create_source_grid(ncf_in)
-        subset_grid, lon_slice, lat_slice = self.create_subset_source_grid(sgrid, self.stations_bbox)
-
-        # loop over chunks
-        for n in range(niter):
-            # indices (relative to index of the output file)
-            beg = n * cs
-            # restrict last chunk to lenght of tmask plus one (to get last time)
-            end = min(n * cs + cs, len(time_in)) - 1
-
-            # time to make tmask for chunk
-            beg_time = nc.num2date(time_in[beg], units=t_unit, calendar=t_cal)
-            if invariant:
-                # allow topography to work in same code, len(nctime) = 1
-                end_time = nc.num2date(nctime[0], units=t_unit, calendar=t_cal)
-                # end = 1
+        with nc.Dataset(ncfile_out, 'a') as ncf_out:
+
+            # get time and convert to datetime object
+            nctime = ncf_in.variables[self.vn_time][:]
+            
+            t_unit = ncf_in.variables[self.vn_time].attrs['units']
+            try:
+                t_cal = ncf_in.variables[self.vn_time].attrs['calendar']
+            except AttributeError:  # attribute doesn't exist
+                t_cal = u"gregorian"  # standard
+            time = nc.num2date(nctime, units=t_unit, calendar=t_cal)
+
+            # detect invariant files (topography etc.)
+            invariant = True if len(time) == 1 else False
+
+            # restrict to date/time range if given
+            if date is None:
+                tmask = np.array([True])
             else:
-                end_time = nc.num2date(time_in[end], units=t_unit, calendar=t_cal)
-
-            # '<= end_time', would damage appending
-            tmask_chunk = (time <= end_time) * (time >= beg_time)
-            if invariant:
-                # allow topography to work in same code
-                tmask_chunk = np.array([True])
-
-            # get the interpolated variables
-            dfield, variables = self.interp2D(ncf_in,
-                                              self.stations, tmask_chunk,
-                                              subset_grid, lon_slice, lat_slice,
-                                              variables=None, date=None)
-            # append time
-            # ncf_out.variables['time'][:] = np.append(ncf_out.variables['time'][:],
-                                                     # time_in[beg:end+1])
-
-            # append variables
-            for i, var in enumerate(variables):
-                if variables_skip(var):
+                tmask = (time < date['end']) * (time >= date['beg'])
+
+            # get time vector for output
+            time_in = nctime[tmask]
+
+            # write time
+            ncf_out.variables['time'][:] = time_in
+
+            # ensure that chunk sizes cover entire period even if
+            # len(time_in) is not an integer multiple of cs
+            niter = len(time_in) // cs
+            niter += ((len(time_in) % cs) > 0)
+
+            # Create source grid
+            sgrid = self.create_source_grid(ncf_in)
+            subset_grid, lon_slice, lat_slice = self.create_subset_source_grid(sgrid, self.stations_bbox)
+
+            # loop over chunks
+            for n in range(niter):
+                if self.resume and n <= ncf_out.globsim_last_chunk_written:
+                    if n == ncf_out.globsim_last_chunk_written:
+                        logger.info(f"Resuming interpolation at chunk {n+1}")
                     continue
 
-                if ens:
-                    num = ncf_in.variables['number'][:]
-                    for ni in num:
-                        if pl:
-                            # dfield [station, variables, time, levels, number]
-                            vi = dfield[ni].data[:,i,:,:].transpose((1,2,0))
-                            ncf_out.variables[var][beg:end+1,ni,:,:] = vi
-                        else:
-                            vi = dfield[ni].data[:,i,:].transpose((1,0))
-                            ncf_out.variables[var][beg:end+1,ni,:] = vi
+                self.require_safe_mem_usage()
+
+                # indices (relative to index of the output file)
+                beg = n * cs
+                # restrict last chunk to length of tmask plus one (to get last time)
+                end = min(n * cs + cs, len(time_in)) - 1
+
+                # time to make tmask for chunk
+                beg_time = nc.num2date(time_in[beg], units=t_unit, calendar=t_cal)
+                if invariant:
+                    # allow topography to work in same code, len(nctime) = 1
+                    end_time = nc.num2date(nctime[0], units=t_unit, calendar=t_cal)
+                    # end = 1
                 else:
-                    if pl:
-                        vi = dfield.data[:,i,:,:].transpose((1,2,0))
-                        ncf_out.variables[var][beg:end+1,:,:] = vi
+                    end_time = nc.num2date(time_in[end], units=t_unit, calendar=t_cal)
+
+                # '<= end_time', would damage appending
+                tmask_chunk = (time <= end_time) * (time >= beg_time)
+                if invariant:
+                    # allow topography to work in same code
+                    tmask_chunk = np.array([True])
+
+                # get the interpolated variables
+                dfield, variables = self.interp2D(ncf_in,
+                                                self.stations, tmask_chunk,
+                                                subset_grid, lon_slice, lat_slice,
+                                                variables=None, date=None)
+                
+                # append variables
+                for i, var in enumerate(variables):
+                    if variables_skip(var):
+                        continue
+
+                    if ens:
+                        num = ncf_in.variables['number'][:]
+                        for ni in num:
+                            if pl:
+                                # dfield [station, variables, time, levels, number]
+                                vi = dfield[ni].data[:,i,:,:].transpose((1,2,0))
+                                ncf_out.variables[var][beg:end+1,ni,:,:] = vi
+                            else:
+                                vi = dfield[ni].data[:,i,:].transpose((1,0))
+                                ncf_out.variables[var][beg:end+1,ni,:] = vi
                     else:
-                        vi = dfield.data[:,i,:].transpose((1,0))
-                        ncf_out.variables[var][beg:end+1,:] = vi
+                        if pl:
+                            vi = dfield.data[:,i,:,:].transpose((1,2,0))
+                            ncf_out.variables[var][beg:end+1,:,:] = vi
+                        else:
+                            vi = dfield.data[:,i,:].transpose((1,0))
+                            ncf_out.variables[var][beg:end+1,:] = vi
+
+                # delete objects and free memory
+                del dfield, vi, tmask_chunk
+                gc.collect()
+
+                ncf_out.globsim_last_chunk_written = n
 
-        # close the file
-        ncf_out.close()
+            # Write success flag
+            ncf_out.globsim_interpolate_success = 1
+        ncf_in.close()
 
     def levels2elevation(self, ncfile_in, ncfile_out):
         """
-        ncf : nc.Dataset
-            netcdf Dataset or MFDataset
+        ncfile_in : nc.Dataset
+            2-d Interpolated ERA5 file on pressure-levels
+        ncfile_out : str
+            output file name
         Linear 1D interpolation of pressure level data available for individual
         stations to station elevation. Where and when stations are below the
         lowest pressure level, they are assigned the value of the lowest
@@ -237,17 +282,18 @@ def levels2elevation(self, ncfile_in, ncfile_out):
 
         """
         # open file
-        # TODO: check the aggdim does not work
-        ncf = nc.MFDataset(ncfile_in, 'r', aggdim='time')
+        ncf = xr.open_mfdataset(ncfile_in, decode_times=False)
         height = ncf.variables['height'][:]
-        nt = len(ncf.variables['time'][:])
+
+        nt = len(ncf.variables['time'][:])  # these are reading from a globsim file, not an ERA5 file
         nl = len(ncf.variables['level'][:])
 
         # list variables
-        varlist = [str_encode(x) for x in ncf.variables.keys()]
+        varlist = [key for key in ncf.variables.keys()]
         for V in ['time', 'station', 'latitude', 'longitude',
-                  'level','height','z']:
-            varlist.remove(V)
+                  'level','height','z','expver']:
+            if V in varlist:
+                varlist.remove(V)
         if self.ens:
             varlist.remove('number')
 
@@ -258,7 +304,9 @@ def levels2elevation(self, ncfile_in, ncfile_out):
         # stations are integer numbers
         # create a file (Dataset object, also the root group).
         rootgrp = netcdf_base(ncfile_out, len(height), nt,
-                              'hours since 1900-01-01 00:00:0.0', ncf)
+                              time_units=ncf['time'].units,
+                              nc_in=ncf,
+                              calendar=ncf['time'].calendar)
         if self.ens:
             rootgrp.source = 'ERA5 10-member ensemble, interpolated (bi)linearly to stations'
         else:
@@ -284,8 +332,9 @@ def levels2elevation(self, ncfile_in, ncfile_out):
                                              'f4',('time','number','station'))
             else:
                 tmp = rootgrp.createVariable(var,'f4',('time', 'station'))
-            tmp.long_name = str_encode(ncf.variables[var].long_name)
-            tmp.units     = str_encode(ncf.variables[var].units)
+
+            tmp.long_name = ncf[var].long_name
+            tmp.units     = ncf[var].units
 
         # add air pressure as new variable
         var = 'air_pressure'
@@ -303,23 +352,23 @@ def levels2elevation(self, ncfile_in, ncfile_out):
             if self.ens:
                 num = ncf.variables['number'][:]
                 for ni in num:
-                    elevation = ncf.variables['z'][:,ni,:,n] / const.G
+                    elevation = ncf.variables['z'][:,ni,:,n].values / const.G
                     elev_diff, va, vb = ele_interpolate(elevation, h, nl)
                     wa, wb = calculate_weights(elev_diff, va, vb)
                     for v, var in enumerate(varlist):
                         if var == 'air_pressure':
                             # pressure [Pa] variable from levels, shape: (time, level)
-                            data = np.repeat([ncf.variables['level'][:]],
+                            data = np.repeat([ncf.variables['level'][:].values],
                                              len(time),axis=0).ravel()
                         else:
                             # read data from netCDF
-                            data = ncf.variables[var][:,ni,:,n].ravel()
+                            data = ncf.variables[var][:,ni,:,n].values.ravel()
 
                         ipol = data[va] * wa + data[vb] * wb   # interpolated value
                         rootgrp.variables[var][:,ni,n] = ipol  # assign to file
             else:
                 # convert geopotential [mbar] to height [m], shape: (time, level)
-                elevation = ncf.variables['z'][:,:,n] / const.G
+                elevation = ncf.variables['z'][:,:,n].values / const.G
                 elev_diff, va, vb = ele_interpolate(elevation, h, nl)
                 wa, wb = calculate_weights(elev_diff, va, vb)
 
@@ -327,11 +376,11 @@ def levels2elevation(self, ncfile_in, ncfile_out):
                 for v, var in enumerate(varlist):
                     if var == 'air_pressure':
                         # pressure [Pa] variable from levels, shape: (time, level)
-                        data = np.repeat([ncf.variables['level'][:]],
+                        data = np.repeat([ncf.variables['level'][:].values],
                                          len(time),axis=0).ravel()
                     else:
                         # read data from netCDF
-                        data = ncf.variables[var][:,:,n].ravel()
+                        data = ncf.variables[var][:,:,n].values.ravel()
 
                     ipol = data[va] * wa + data[vb] * wb   # interpolated value
                     rootgrp.variables[var][:,n] = ipol  # assign to file
@@ -358,10 +407,13 @@ def _process_sa(self):
                                         # [kg m-2] Total column W vapor
                 'wind_speed': ['u10', 'v10']}   # [m s-1] 10m values
         varlist = self.TranslateCF2short(dpar)
-
-        with nc.MFDataset(self.getInFile('sa'), 'r', aggdim='time') as sa:
-            self.ERA2station(sa, self.getOutFile('sa'),
-                             self.stations, varlist, date=self.date)
+        
+        if self.resume and self.completed_successfully(self.getOutFile('sa')):
+            logger.info("Skipping surface analysis interpolation")
+        else:
+            with xr.open_mfdataset(self.get_input_file_paths('sa'), decode_times=False) as sa:
+                self.ERA2station(sa, self.getOutFile('sa'),
+                                 self.stations, varlist, date=self.date)
     
     def _process_pl(self):
         # === 2D Interpolation for Pressure Level Data ===
@@ -371,7 +423,7 @@ def _process_pl(self):
                 'relative_humidity' : ['r'],           # [%]
                 'wind_speed'        : ['u', 'v']}      # [m s-1]
         varlist = self.TranslateCF2short(dpar).append('z')
-        with nc.MFDataset(self.getInFile('pl'), 'r', aggdim='time') as pl:
+        with xr.open_mfdataset(self.get_input_file_paths('pl'), decode_times=False) as pl:
             self.ERA2station(pl, self.getOutFile('pl'),
                              self.stations, varlist, date=self.date)
 
@@ -394,6 +446,13 @@ def _process_sf(self):
                 'downwelling_shortwave_flux_in_air' : ['ssrd'],
                 'downwelling_longwave_flux_in_air'  : ['strd']}
         varlist = self.TranslateCF2short(dpar)
-        with nc.MFDataset(self.getInFile('sf'), 'r', aggdim='time') as sf:
-            self.ERA2station(sf, self.getOutFile('sf'),
-                            self.stations, varlist, date=self.date)
+        if self.resume and self.completed_successfully(self.getOutFile('sa')):
+            logger.info("Skipping surface analysis interpolation")
+        else:
+            with xr.open_mfdataset(self.get_input_file_paths('sf'), decode_times=False) as sf:
+                self.ERA2station(sf, self.getOutFile('sf'),
+                                self.stations, varlist, date=self.date)
+
+
+# vv = nc.Dataset(self.getOutFile('pl'))
+# xx = xr.open_dataset(self.getOutFile('sa'))
\ No newline at end of file
diff --git a/globsim/interpolate/ERAIinterpolate.py b/globsim/interpolate/ERAIinterpolate.py
deleted file mode 100644
index 7543f32..0000000
--- a/globsim/interpolate/ERAIinterpolate.py
+++ /dev/null
@@ -1,327 +0,0 @@
-import re
-import netCDF4 as nc
-import numpy as np
-import logging
-
-from datetime import datetime
-from os import path
-
-from globsim.common_utils import variables_skip, str_encode
-from globsim.interpolate.GenericInterpolate import GenericInterpolate
-from globsim.nc_elements import netcdf_base, new_interpolated_netcdf
-from globsim.interp import calculate_weights, ele_interpolate
-import globsim.constants as const
-
-logger = logging.getLogger('globsim.interpolate')
-
-try:
-    import ESMF
-
-    # Check ESMF version.  7.0.1 behaves differently than 7.1.0r
-    ESMFv = int(re.sub("[^0-9]", "", ESMF.__version__))
-    ESMFnew = ESMFv > 701
-
-except ModuleNotFoundError:
-        print("*** ESMF not imported, trying esmpy. ***")
-        try:
-            import esmpy as ESMF
-        except ImportError:
-            print('Could not import ESMF or esmpy')
-            pass
-
-
-class ERAIinterpolate(GenericInterpolate):
-    """
-    Collection of methods to interpolate ERA-Interim netCDF files to station
-    coordinates. All variables retain theit original units and time stepping.
-    """
-
-    def __init__(self, ifile, **kwargs):
-        super().__init__(ifile, **kwargs)
-        par = self.par
-        self.input_dir = path.join(par['project_directory'], 'erai')
-
-        #convert longitude to ERA notation if using negative numbers
-        self.stations['longitude_dd'] = self.stations['longitude_dd'] % 360
-
-    def ERA2station(self, ncfile_in, ncfile_out, points,
-                    variables = None, date = None):
-
-        """
-        Biliner interpolation from fields on regular grid (latitude, longitude)
-        to individual point stations (latitude, longitude). This works for
-        surface and for pressure level files (all Era_Interim files). The type
-        of variable and file structure are determined from the input.
-
-        This function creates an empty of netCDF file to hold the interpolated
-        results, by calling ERAIgeneric().netCDF_empty. Then, data is
-        interpolated in temporal chunks and appended. The temporal chunking can
-        be set in the interpolation parameter file.
-
-        Args:
-        ncfile_in: Full path to an Era-Interim derived netCDF file. This can
-                   contain wildcards to point to multiple files if temporal
-                  chunking was used.
-
-        ncfile_out: Full path to the output netCDF file to write.
-
-        points: A dataframe of locations. See method StationListRead in
-                common_utils.py for more details.
-
-        variables:  List of variable(s) to interpolate such as
-                    ['r', 't', 'u','v', 't2m', 'u10', 'v10', 'ssrd', 'strd', 'tp'].
-                    Defaults to using all variables available.
-
-        date: Directory to specify begin and end time for the derived time
-                series. Defaluts to using all times available in ncfile_in.
-
-        cs: chunk size, i.e. how many time steps to interpolate at once. This
-            helps to manage overall memory usage (small cs is slower but less
-            memory intense).
-        """
-
-        # read in one type of mutiple netcdf files
-        ncf_in = nc.MFDataset(ncfile_in, 'r', aggdim ='time')
-
-        # is it a file with pressure levels?
-        pl = 'level' in ncf_in.dimensions.keys()
-
-        # build the output of empty netCDF file
-        rootgrp = new_interpolated_netcdf(ncfile_out, self.stations, ncf_in,
-                                          time_units= 'hours since 1900-01-01 00:00:0.0')
-        rootgrp.source = 'ERA_Interim, interpolated bilinearly to stations'
-        rootgrp.close()
-
-        # open the output netCDF file, set it to be appendable ('a')
-        ncf_out = nc.Dataset(ncfile_out, 'a')
-
-        # get time and convert to datetime object
-        nctime = ncf_in.variables['time'][:]
-        #"hours since 1900-01-01 00:00:0.0"
-        t_unit = ncf_in.variables['time'].units
-        try :
-            t_cal = ncf_in.variables['time'].calendar
-        except AttributeError :  # attribute doesn't exist
-            t_cal = u"gregorian" # standard
-        time = nc.num2date(nctime, units = t_unit, calendar = t_cal)
-
-        # detect invariant files (topography etc.)
-        invariant = True if len(time) == 1 else False
-
-        # restrict to date/time range if given
-        if date is None:
-            tmask = time < datetime(3000, 1, 1)
-        else:
-            tmask = (time < date['end']) * (time >= date['beg'])
-
-        # get time vector for output
-        time_in = nctime[tmask]
-
-        # ensure that chunk sizes cover entire period even if
-        # len(time_in) is not an integer multiple of cs
-        niter  = len(time_in) // self.cs
-        niter += ((len(time_in) % self.cs) > 0)
-
-        # loop over chunks
-        for n in range(niter):
-            # indices (relative to index of the output file)
-            beg = n * self.cs
-            # restrict last chunk to lenght of tmask plus one (to get last time)
-            end = min(n*self.cs + self.cs, len(time_in))-1
-
-            # time to make tmask for chunk
-            beg_time = nc.num2date(time_in[beg], units=t_unit, calendar=t_cal)
-            if invariant:
-                # allow topography to work in same code, len(nctime) = 1
-                end_time = nc.num2date(nctime[0], units=t_unit, calendar=t_cal)
-                #end = 1
-            else:
-                end_time = nc.num2date(time_in[end], units=t_unit, calendar=t_cal)
-
-            #'<= end_time', would damage appending
-            tmask_chunk = (time <= end_time) * (time >= beg_time)
-            if invariant:
-                # allow topography to work in same code
-                tmask_chunk = [True]
-
-            # get the interpolated variables
-            dfield, variables = self.interp2D(ncfile_in, ncf_in,
-                                                     self.stations, tmask_chunk,
-                                                     variables=None, date=None)
-
-            # append time
-            ncf_out.variables['time'][:] = np.append(ncf_out.variables['time'][:],
-                                                     time_in[beg:end+1])
-
-            #append variables
-            for i, var in enumerate(variables):
-                if variables_skip(var):
-                    continue
-
-                if pl:
-                    if ESMFnew:
-                        # dfield has dimensions (station, variables, time, pressure levels)
-                        ncf_out.variables[var][beg:end+1,:,:] = dfield.data[:,i,:,:].transpose((1,2,0))
-                    else:
-                        # dimension: time, level, station (pressure level files)
-                        ncf_out.variables[var][beg:end+1,:,:] = dfield.data[i,:,:,:]
-                else:
-                    if ESMFnew:
-                        # dfield has dimensions (station, variables, time)
-                        ncf_out.variables[var][beg:end+1,:] = dfield.data[:,i,:].transpose((1,0))
-                    else:
-                        # dfield has dimensions time, station (2D files)
-                        ncf_out.variables[var][beg:end+1,:] = dfield.data[i,:,:]
-
-        #close the file
-        ncf_in.close()
-        ncf_out.close()
-
-
-    def levels2elevation(self, ncfile_in, ncfile_out):
-        """
-        Linear 1D interpolation of pressure level data available for individual
-        stations to station elevation. Where and when stations are below the
-        lowest pressure level, they are assigned the value of the lowest
-        pressure level.
-
-        """
-        # open file
-
-        ncf = nc.MFDataset(ncfile_in, 'r', aggdim='time')
-        height = ncf.variables['height'][:]
-        nt = len(ncf.variables['time'][:])
-        nl = len(ncf.variables['level'][:])
-
-        # list variables
-        varlist = [str_encode(x) for x in ncf.variables.keys()]
-        for V in ['time', 'station', 'latitude', 'longitude', 'level','height','z']:
-            varlist.remove(V)
-
-        # === open and prepare output netCDF file ==============================
-        # dimensions: station, time
-        # variables: latitude(station), longitude(station), elevation(station)
-        #            others: ...(time, station)
-        # stations are integer numbers
-        # create a file (Dataset object, also the root group).
-        rootgrp = netcdf_base(ncfile_out=ncfile_out, n_stations=len(height), n_time=nt, time_units='hours since 1900-01-01 00:00:0.0')
-        rootgrp.source = 'ERA-Interim, interpolated (bi)linearly to stations'
-
-        time = rootgrp['time']
-        station = rootgrp['station']
-        latitude = rootgrp['latitude']
-        longitude = rootgrp['longitude']
-        height = rootgrp['height']
-
-        # assign base variables
-        time[:]      = ncf.variables['time'][:]
-        station[:]   = ncf.variables['station'][:]
-        latitude[:]  = ncf.variables['latitude'][:]
-        longitude[:] = ncf.variables['longitude'][:]
-        height[:]    = ncf.variables['height'][:]
-
-        # create and assign variables from input file
-        for var in varlist:
-            tmp   = rootgrp.createVariable(var,'f4',('time', 'station'))
-            tmp.long_name = str_encode(ncf.variables[var].long_name)
-            tmp.units     = str_encode(ncf.variables[var].units)
-
-        # add air pressure as new variable
-        var = 'air_pressure'
-        varlist.append(var)
-        tmp   = rootgrp.createVariable(var,'f4',('time', 'station'))
-        tmp.long_name = var.encode('UTF8')
-        tmp.units     = 'hPa'.encode('UTF8')
-        # end file prepation ===================================================
-
-        # loop over stations
-        for n, h in enumerate(height):
-            # convert geopotential [mbar] to height [m], shape: (time, level)
-            elevation = ncf.variables['z'][:,:,n] / const.G
-            # TODO: check if height of stations in data range
-
-            elev_diff, va, vb = ele_interpolate(elevation, h, nl)
-            wa, wb = calculate_weights(elev_diff, va, vb)
-
-            #loop over variables and apply interpolation weights
-            for v, var in enumerate(varlist):
-                if var == 'air_pressure':
-                    # pressure [Pa] variable from levels, shape: (time, level)
-                    data = np.repeat([ncf.variables['level'][:]],
-                                      len(time),axis=0).ravel()
-                else:
-                    #read data from netCDF
-                    data = ncf.variables[var][:,:,n].ravel()
-                ipol = data[va]*wa + data[vb]*wb   # interpolated value
-                rootgrp.variables[var][:,n] = ipol # assign to file
-
-        rootgrp.close()
-        ncf.close()
-        # closed file ==========================================================
-
-
-    def process(self):
-        """
-        Interpolate point time series from downloaded data. Provides access to
-        the more generically ERA-like interpolation functions.
-        """
-
-        # 2D Interpolation for Invariant Data
-        # dictionary to translate CF Standard Names into ERA-Interim
-        # pressure level variable keys.
-        dummy_date  = {'beg' : datetime(1979, 1, 1, 12, 0),
-                       'end' : datetime(1979, 1, 1, 12, 0)}
-        self.ERA2station(path.join(self.input_dir,'erai_to.nc'),
-                         path.join(self.output_dir,'erai_to_' +
-                                   self.list_name + '.nc'), self.stations,
-                                   ['z', 'lsm'], date = dummy_date)
-
-        # === 2D Interpolation for Surface Analysis Data ===
-        # dictionary to translate CF Standard Names into ERA-Interim
-        # pressure level variable keys.
-        dpar = {'air_temperature'   : ['t2m'],  # [K] 2m values
-                'relative_humidity' : ['d2m'],  # [K] 2m values
-                'downwelling_shortwave_flux_in_air_assuming_clear_sky' :
-                    ['tco3', 'tcwv'],   # [kg m-2] Total column ozone
-                                        # [kg m-2] Total column W vapor
-                'wind_speed' : ['u10', 'v10']}   # [m s-1] 10m values
-        varlist = self.TranslateCF2short(dpar)
-        self.ERA2station(path.join(self.input_dir,'erai_sa_*.nc'),
-                         path.join(self.output_dir,'erai_sa_' +
-                                   self.list_name + '.nc'), self.stations,
-                                   varlist, date = self.date)
-
-        # 2D Interpolation for Surface Forecast Data    'tp', 'strd', 'ssrd'
-        # dictionary to translate CF Standard Names into ERA-Interim
-        # pressure level variable keys.
-        # [m] total precipitation
-        # [J m-2] short-wave downward
-        # [J m-2] long-wave downward
-        dpar = {'precipitation_amount'              : ['tp'],
-                'downwelling_shortwave_flux_in_air' : ['ssrd'],
-                'downwelling_longwave_flux_in_air'  : ['strd']}
-        varlist = self.TranslateCF2short(dpar)
-        self.ERA2station(path.join(self.input_dir,'erai_sf_*.nc'),
-                         path.join(self.output_dir,'erai_sf_' +
-                                   self.list_name + '.nc'), self.stations,
-                                   varlist, date = self.date)
-
-
-        # === 2D Interpolation for Pressure Level Data ===
-        # dictionary to translate CF Standard Names into ERA-Interim
-        # pressure level variable keys.
-        dpar = {'air_temperature'   : ['t'],           # [K]
-                'relative_humidity' : ['r'],           # [%]
-                'wind_speed'        : ['u', 'v']}      # [m s-1]
-        varlist = self.TranslateCF2short(dpar).append('z')
-        self.ERA2station(path.join(self.input_dir,'erai_pl_*.nc'),
-                         path.join(self.output_dir,'erai_pl_' +
-                                   self.list_name + '.nc'), self.stations,
-                                   varlist, date = self.date)
-
-        # 1D Interpolation for Pressure Level Data
-        self.levels2elevation(path.join(self.output_dir,'erai_pl_' +
-                                        self.list_name + '.nc'),
-                              path.join(self.output_dir,'erai_pl_' +
-                                        self.list_name + '_surface.nc'))
-
diff --git a/globsim/interpolate/GenericInterpolate.py b/globsim/interpolate/GenericInterpolate.py
index 52ae16a..cd9d302 100644
--- a/globsim/interpolate/GenericInterpolate.py
+++ b/globsim/interpolate/GenericInterpolate.py
@@ -1,9 +1,13 @@
+import esmpy
+import gc
 import numpy as np
 import netCDF4 as nc
 import re
 import tomlkit
 import warnings
 import logging
+import sys
+import psutil
 
 from datetime import datetime, timedelta
 from os import path, makedirs, listdir
@@ -19,22 +23,11 @@
 
 logger = logging.getLogger('globsim.interpolate')
 
-try:
-    import ESMF
-
-    # Check ESMF version.  7.0.1 behaves differently than 7.1.0r
-    ESMFv = int(re.sub("[^0-9]", "", ESMF.__version__))
-    ESMFnew = ESMFv > 701
-
-except ModuleNotFoundError:
-        print("*** ESMF not imported, trying esmpy. ***")
-        try:
-            import esmpy as ESMF
-        except ImportError:
-            print('Could not import ESMF or esmpy')
-            pass
+import esmpy as ESMF
 
 class GenericInterpolate:
+    REANALYSIS = ''
+    SAFE_MEM_LIMIT_PERCENT = 90
 
     def __init__(self, ifile: str, **kwargs):
         # read parameter file
@@ -46,6 +39,7 @@ def __init__(self, ifile: str, **kwargs):
         self.variables = par.get('variables')
         self.skip_checks = kwargs.get('skip_checks', bool(par.get("skip_checks", False)))
         self.list_name = path.basename(path.normpath(par.get('station_list'))).split(path.extsep)[0]
+        
 
         # read station points
         self.stations_csv = self.find_stations_csv(par)
@@ -63,10 +57,19 @@ def __init__(self, ifile: str, **kwargs):
         self._array = np.array([])  # recycled numpy array
         self._plarray = np.array([])
 
-        self.__skip_sa = kwargs.get('skip_sa', False)
-        self.__skip_sf = kwargs.get('skip_sf', False)
-        self.__skip_pl = kwargs.get('skip_pl', False)
+        self._skip_sa = kwargs.get('skip_sa', False)
+        self._skip_sf = kwargs.get('skip_sf', False)
+        self._skip_pl = kwargs.get('skip_pl', False)
+        self.resume = kwargs.get('resume', False)
+        
+    @property
+    def vn_time(self):
+        return 'time'
 
+    @property
+    def vn_level(self):
+        return 'level'
+    
     def find_stations_csv(self, par):
         if Path(par.get('station_list')).is_file():
             return Path(par.get('station_list'))
@@ -94,20 +97,26 @@ def validate_stations_extent(self, ncdf):
         except KeyError:
             logger.error("Could not verify whether stations are within downloaded netcdf")
 
+    def getOutFile(self, kind):
+        """
+        Get the output file name for the given kind of interpolation.
+        """
+        return path.join(self.output_dir, f'{self.REANALYSIS}_{kind}_{self.list_name}.nc')
+    
     def process(self):
         self._preprocess()
 
-        if not self.__skip_sa:
+        if not self._skip_sa:
             self._process_sa()
         else:
             logger.info("skipping interpolation of _sa file")
 
-        if not self.__skip_sf:
+        if not self._skip_sf:
             self._process_sf()
         else:
             logger.info("skipping interpolation of _sf file")
 
-        if not self.__skip_pl:
+        if not self._skip_pl:
             self._process_pl()
         else:
             logger.info("skipping interpolation of _pl file")
@@ -143,7 +152,7 @@ def TranslateCF2short(self, dpar):
     def interp2D(self,  ncf_in, points, tmask_chunk: "np.ndarray",
                  sgrid: "ESMF.Grid",
                  lon_subset:slice, lat_subset:slice,
-                 variables=None, date=None):
+                 variables=None, date=None) -> "tuple[ESMF.Field, list[str]]":
         """
         Bilinear interpolation from fields on regular grid (latitude, longitude)
         to individual point stations (latitude, longitude). This works for
@@ -178,17 +187,17 @@ def interp2D(self,  ncf_in, points, tmask_chunk: "np.ndarray",
             ERA2station('era_sa.nc', 'era_sa_inter.nc', stations,
                         variables=variables, date=date)
         """
-        tbeg = nc.num2date(ncf_in['time'][np.where(tmask_chunk)[0][0]], ncf_in['time'].units, 'standard')
-        tend = nc.num2date(ncf_in['time'][np.where(tmask_chunk)[0][-1]], ncf_in['time'].units, 'standard')
+        tbeg = nc.num2date(ncf_in[self.vn_time][np.where(tmask_chunk)[0][0]], ncf_in[self.vn_time].units, ncf_in[self.vn_time].calendar)
+        tend = nc.num2date(ncf_in[self.vn_time][np.where(tmask_chunk)[0][-1]],  ncf_in[self.vn_time].units, ncf_in[self.vn_time].calendar)
         logger.debug(f"2d interpolation for period {tbeg} to {tend}")
 
         # is it a file with pressure levels?
-        pl = 'level' in ncf_in.dimensions.keys()
-        ens = 'number' in ncf_in.dimensions.keys()
+        pl = self.vn_level in ncf_in.sizes.keys()
+        ens = 'number' in ncf_in.sizes.keys()
 
         # get spatial dimensions
         if pl:  # only for pressure level files
-            nlev = len(ncf_in.variables['level'][:])
+            nlev = len(ncf_in.variables[self.vn_level][:])
         else:
             nlev = 1
         
@@ -255,6 +264,10 @@ def interp2D(self,  ncf_in, points, tmask_chunk: "np.ndarray",
 
             dfield = self.regrid(sfield, dfield)
         
+        # clean up and GC
+        del sfield, locstream, ncf_in
+        gc.collect()
+
         logger.debug("Created destination field")
 
         return dfield, variables
@@ -289,9 +302,25 @@ def create_source_grid(self, ncf_in: "nc.MFDataset") -> "ESMF.Grid":
         # flist = np.sort(fnmatch_filter(listdir(self.input_dir),
         #                               path.basename(ncfile_in)))
         # ncsingle = path.join(self.input_dir, flist[0])
-        ncsingle = ncf_in._files[0]
-        sgrid = ESMF.Grid(filename=ncsingle, filetype=ESMF.FileFormat.GRIDSPEC)
+        ncsingle = None
+        for v in ncf_in.variables:
+            if ncsingle is not None:
+                break
+            try:
+                ncsingle = ncf_in[v].encoding["source"]
+            except Exception: 
+                pass
+            #ncf_in._files[0]
+
+        #sgrid = ESMF.Grid(filename=ncsingle, filetype=ESMF.FileFormat.GRIDSPEC)
         
+        # sg = ESMF.Grid(max_index = np.array([205,45]), num_peri_dims=1, periodic_dim=0,staggerloc=ESMF.StaggerLoc.CENTER)
+
+        template = nc.Dataset(ncsingle)
+        lat = template.variables['latitude'][:]
+        lon = template.variables['longitude'][:]
+
+        sgrid = grid_create_from_coordinates_periodic(lon, lat)
         return sgrid
 
     def create_subset_source_grid(self, sgrid: "ESMF.Grid", bbox: BoundingBox) -> "tuple[ESMF.Grid, slice, slice]":
@@ -382,27 +411,28 @@ def nc_data_subset_to_source_field(self, variables, sfield: "ESMF.Field", ncf_in
             if pl:
                 logger.debug(f"Reading {v} data from source.")  # NB: writing is almost instantaneous
                 if self._plarray.shape[0] == dtime:
-                    self._plarray[:] = var[time_slice, :, lat_slice, lon_slice]
+                    self._plarray[:] = var[time_slice, :, lat_slice, lon_slice].values
                 else:
-                    self._plarray = var[time_slice, :, lat_slice, lon_slice]
+                    self._plarray = var[time_slice, :, lat_slice, lon_slice].values
                 # logger.debug(f"Chunksize {(self._plarray.size * self._plarray.itemsize * 1e-6)} Megabytes")
                 sfield.data[:, :, n, :, :] = self._plarray.transpose((3,2,0,1))
 
             else:
                 logger.debug(f"Reading {v} data from source")
                 if self._array.shape == (dtime, dlat, dlon):
-                    self._array[:] = var[time_slice, lat_slice, lon_slice]
+                    self._array[:] = var[time_slice, lat_slice, lon_slice].values
                 else:
-                    self._array = var[time_slice, lat_slice, lon_slice]
+                    self._array = var[time_slice, lat_slice, lon_slice].values
+                
                 # logger.debug(f"Chunksize {(self._plarray.size * self._plarray.itemsize * 1e-6)} Megabytes")
+                
                 sfield.data[:, :, n, :] = self._array.transpose((2,1,0))
 
         filltime = (datetime.now() - t0).total_seconds()
         logger.info(f"Finished reading source data for chunk ({filltime} seconds)")
 
-    @staticmethod
-    def remove_select_variables(varlist: list, pl: bool, ens: bool = False):
-        varlist.remove('time')
+    def remove_select_variables(self, varlist: list, pl: bool, ens: bool = False):
+        varlist.remove(self.vn_time)
         try:
             varlist.remove('latitude')
         except ValueError as e:
@@ -416,9 +446,11 @@ def remove_select_variables(varlist: list, pl: bool, ens: bool = False):
             print('continue')
             pass
         if pl:  # only for pressure level files
-            varlist.remove('level')
-        if ens:
+            varlist.remove(self.vn_level)
+        if 'number' in varlist:
             varlist.remove('number')
+        if 'expver' in varlist:
+            varlist.remove('expver')
 
     @check
     def ensure_datset_integrity(self, time: "nc.Variable", interval: float):
@@ -426,6 +458,7 @@ def ensure_datset_integrity(self, time: "nc.Variable", interval: float):
         # Check coverage
         interpolate_start = self.date['beg']
         interpolate_end = self.date['end'] - timedelta(days=1)  # take off last day added in __init__
+
         data_start = nc.num2date(time[0], time.units, time.calendar)
         data_end = nc.num2date(time[-1], time.units, time.calendar)
 
@@ -437,6 +470,7 @@ def ensure_datset_integrity(self, time: "nc.Variable", interval: float):
 
         # Check gaps
         critical = False
+
         for gap_i, gap_start, gap_end in zip(*check_time_integrity(time, interval)):
             message = f"Data gap found at index {gap_i}. Missing data from {gap_start} to {gap_end}."
             
@@ -449,6 +483,96 @@ def ensure_datset_integrity(self, time: "nc.Variable", interval: float):
         if critical:
             raise ValueError("Data gaps found within interpolation bounds.")
 
+    def prefilter_mf_paths(self, pattern):
+        return prefilter_mf_paths(pattern, self.date['beg'], self.date['end'])
+    
+    def completed_successfully(self, file:str) -> bool:
+        """ Check if interpolation was successful """
+        if not Path(file).is_file():
+            return False
+        with nc.Dataset(file) as ncfile:
+            if 'globsim_interpolate_success' in ncfile.ncattrs():
+                return bool(int(ncfile.getncattr('globsim_interpolate_success')))
+            else:
+                return False
+            
+    def require_safe_mem_usage(self):
+        """ Check if memory usage is safe. Kill globsim if not """
+        mem = psutil.virtual_memory()
+        logger.info(f"Memory usage: {mem.used / 1024**3:.2f} GB ({mem.percent}%)")
+        safe = mem.percent < self.SAFE_MEM_LIMIT_PERCENT
+        if not safe:
+            logger.critical(f"Memory use exceeds safe limit of {self.SAFE_MEM_LIMIT_PERCENT}%. Exiting safely")
+            sys.exit(1)
+
+    def file_can_be_resumed(self, file:str, fail_on_missing=False):
+        ''' check if file can be resumed '''
+        errmsgs = []
+    
+        if not (Path(file).is_file() or fail_on_missing):
+            return True
+        
+        with nc.Dataset(file) as ncfile:
+            if not 'globsim_last_chunk_written' in ncfile.ncattrs():
+                errmsgs.append("No 'globsim_last_chunk_written' attribute found in file.")
+            if not 'globsim_chunk_size' in ncfile.ncattrs():
+                errmsgs.append("No 'globsim_chunk_size' attribute found in file.")
+            else:
+                if ncfile.getncattr('globsim_chunk_size') != self.cs:
+                    errmsgs.append(f"Chunk size in file ({ncfile.getncattr('globsim_chunk_size')}) does not match configuration ({self.cs}).")
+            if not 'globsim_interpolate_start' in ncfile.ncattrs():
+                errmsgs.append("No 'globsim_interpolate_start' attribute found in file.")
+            else:
+                if ncfile.getncattr('globsim_interpolate_start') != self.par['beg']:
+                    errmsgs.append(f"Interpolation start in file ({ncfile.getncattr('globsim_interpolate_start')}) does not match configuration ({self.date['beg']}).")
+            if not 'globsim_interpolate_end' in ncfile.ncattrs():
+                errmsgs.append("No globsim_interpolate_end attribute found in file.")
+            else:
+                if ncfile.getncattr('globsim_interpolate_end') != self.par['end']:
+                    errmsgs.append(f"Interpolation end in file ({ncfile.getncattr('globsim_interpolate_end')}) does not match configuration ({self.date['end']}).")
+        
+        if errmsgs:
+            logger.error(f"Cannot resume interpolation from file {file}:")
+            for msg in errmsgs:
+                logger.error(msg)
+            return False
+        else:
+            return True
+    
+    def require_file_can_be_resumed(self, file:str, fail_on_missing:bool = False):
+
+        if not self.file_can_be_resumed(file, fail_on_missing=fail_on_missing):
+            logger.critical(f"Cannot resume interpolation from file {file}. Exiting safely.")
+            sys.exit(1)
+        
+
+def prefilter_mf_paths(pattern:str, beg: "datetime", end: "datetime") -> list:
+    """ Filter out files that do not fall within date range """ 
+    pattern = Path(pattern)
+    files = list(Path(pattern.parent).glob(pattern.name))
+    dates = [re.search(r'(\d{8})_to_(\d{8})', str(f)).groups() for f in files]
+    filtered_files = []
+    for file, (b, e) in zip(files, dates):
+        filebeg = datetime.strptime(b, '%Y%m%d')
+
+        try:
+            fileend = datetime.strptime(e, '%Y%m%d')
+        except ValueError:  # end on 31st
+            try: 
+                fileend = datetime.strptime(e[:-2] + "30", '%Y%m%d')
+            except ValueError:
+                try:
+                    fileend = datetime.strptime(e[:-2] + "29", '%Y%m%d')
+                except ValueError:
+                    fileend = datetime.strptime(e[:-2] + "28", '%Y%m%d')
+
+
+        if fileend < beg or filebeg > end:
+            pass
+        else:
+            filtered_files.append(file)
+
+    return sorted(filtered_files)
 
 def create_stations_bbox(stations) -> BoundingBox:
     # get max/min of points lat/lon from self.stations
@@ -496,3 +620,62 @@ def create_field(sgrid: "ESMF.Grid", variables: list, nt: int, nlev:int = 1) ->
         raise Exception(msg).with_traceback(e.__traceback__)
 
     return field
+
+
+def grid_create_from_coordinates_periodic(longitudes, latitudes, lon_corners=False, lat_corners=False, corners=False, domask=False):
+    """
+    Create a 2 dimensional periodic Grid using the 'longitudes' and 'latitudes'.
+    :param longitudes: longitude coordinate values at cell centers
+    :param latitudes: latitude coordinate values at cell centers
+    :param lon_corners: longitude coordinate values at cell corners
+    :param lat_corners: latitude coordinate values at cell corners
+    :param corners: boolean to determine whether or not to add corner coordinates to this grid
+    :param domask: boolean to determine whether to set an arbitrary mask or not
+    :return: grid
+    """
+    [lon, lat] = [0, 1]
+
+    # create a grid given the number of grid cells in each dimension the center stagger location is allocated
+    max_index = np.array([len(longitudes), len(latitudes)])
+    grid = esmpy.Grid(max_index, num_peri_dims=1, staggerloc=[esmpy.StaggerLoc.CENTER])
+
+    # set the grid coordinates using numpy arrays, parallel case is handled using grid bounds
+    gridXCenter = grid.get_coords(lon)
+    lon_par = longitudes[grid.lower_bounds[esmpy.StaggerLoc.CENTER][lon]:grid.upper_bounds[esmpy.StaggerLoc.CENTER][lon]]
+    gridXCenter[...] = lon_par.reshape((lon_par.size, 1))
+
+    gridYCenter = grid.get_coords(lat)
+    lat_par = latitudes[grid.lower_bounds[esmpy.StaggerLoc.CENTER][lat]:grid.upper_bounds[esmpy.StaggerLoc.CENTER][lat]]
+    gridYCenter[...] = lat_par.reshape((1, lat_par.size))
+
+    # create grid corners in a slightly different manner to account for the bounds format common in CF-like files
+    if corners:
+        grid.add_coords([esmpy.StaggerLoc.CORNER])
+        lbx = grid.lower_bounds[esmpy.StaggerLoc.CORNER][lon]
+        ubx = grid.upper_bounds[esmpy.StaggerLoc.CORNER][lon]
+        lby = grid.lower_bounds[esmpy.StaggerLoc.CORNER][lat]
+        uby = grid.upper_bounds[esmpy.StaggerLoc.CORNER][lat]
+
+        gridXCorner = grid.get_coords(lon, staggerloc=esmpy.StaggerLoc.CORNER)
+        for i0 in range(ubx - lbx - 1):
+            gridXCorner[i0, :] = lon_corners[i0+lbx, 0]
+        gridXCorner[i0 + 1, :] = lon_corners[i0+lbx, 1]
+
+        gridYCorner = grid.get_coords(lat, staggerloc=esmpy.StaggerLoc.CORNER)
+        for i1 in range(uby - lby - 1):
+            gridYCorner[:, i1] = lat_corners[i1+lby, 0]
+        gridYCorner[:, i1 + 1] = lat_corners[i1+lby, 1]
+
+    # add an arbitrary mask
+    if domask:
+        mask = grid.add_item(esmpy.GridItem.MASK)
+        mask[:] = 1
+        mask[np.where((1.75 <= gridXCenter.any() < 2.25) &
+                      (1.75 <= gridYCenter.any() < 2.25))] = 0
+
+    return grid
+
+    
+
+
+
diff --git a/globsim/interpolate/JRA3Qinterpolate.py b/globsim/interpolate/JRA3Qinterpolate.py
index 08337f0..ae4b7fb 100644
--- a/globsim/interpolate/JRA3Qinterpolate.py
+++ b/globsim/interpolate/JRA3Qinterpolate.py
@@ -13,7 +13,12 @@ class J3QI(JRAinterpolate):
     PL_INTERVAL = 6
     SF_INTERVAL = 1
 
-    T_UNITS = J3QD._tunits
+    T_UNITS = "hours since 1900-01-01 00:00:00"
+
+
+class J3QgI(J3QI):
+
+    REANALYSIS = "jra3qg"
+
 
-    GEOPOTENTIAL = "gh"
 
diff --git a/globsim/interpolate/JRAinterpolate.py b/globsim/interpolate/JRAinterpolate.py
index 81db7b4..cfe7947 100644
--- a/globsim/interpolate/JRAinterpolate.py
+++ b/globsim/interpolate/JRAinterpolate.py
@@ -3,8 +3,11 @@
 import netCDF4 as nc
 import numpy as np
 import logging
+import gc
+import xarray as xr
 from datetime import datetime
 from os import path
+from pathlib import Path
 
 from globsim.common_utils import str_encode, variables_skip
 from globsim.interpolate.GenericInterpolate import GenericInterpolate
@@ -63,10 +66,12 @@ def __init__(self, ifile, **kwargs):
         self.cs *= 200
 
         # Load MF Datasets
-        self.mf_to = nc.MFDataset(path.join(self.input_dir, f'{self.REANALYSIS}_to_*.nc'), 'r', aggdim="time")
-        self.mf_sa = nc.MFDataset(path.join(self.input_dir, f'{self.REANALYSIS}_sa_*.nc'), 'r', aggdim="time")
-        self.mf_sf = nc.MFDataset(path.join(self.input_dir, f'{self.REANALYSIS}_sf_*.nc'), 'r', aggdim="time")
-        self.mf_pl = nc.MFDataset(path.join(self.input_dir, f'{self.REANALYSIS}_pl_*.nc'), 'r', aggdim="time")
+        p = path.join(self.input_dir, f'{self.REANALYSIS}' + '{}')
+
+        self.mf_to = xr.open_mfdataset(p.format('_to_*.nc'), decode_times=False)
+        self.mf_sa = xr.open_mfdataset(self.prefilter_mf_paths(p.format('_sa_*.nc')), decode_times=False)
+        self.mf_sf = xr.open_mfdataset(self.prefilter_mf_paths(p.format('_sf_*.nc')), decode_times=False)
+        self.mf_pl = xr.open_mfdataset(self.prefilter_mf_paths(p.format('_pl_*.nc')), decode_times=False)
         
         # Check dataset integrity
         logger.info("Check data integrity (sa)")
@@ -119,98 +124,119 @@ def JRA2station(self, ncf_in: "nc.MFDataset", ncfile_out, points,
         self.validate_stations_extent(ncf_in)
 
         # is it a file with pressure levels?
-        pl = 'level' in ncf_in.dimensions.keys()
+        pl = 'level' in ncf_in.sizes.keys()
 
         # build the output of empty netCDF file
-        rootgrp = new_interpolated_netcdf(ncfile_out, self.stations, ncf_in,
-                                          time_units=self.T_UNITS)
-        rootgrp.source = f'{self.REANALYSIS}, interpolated bilinearly to stations'
-        rootgrp.close()
+        level_var = 'level' if pl else None
+
+        if self.resume:
+            self.require_file_can_be_resumed(ncfile_out)
+        
+        if not (self.resume and Path(ncfile_out).exists()):
+            rootgrp = new_interpolated_netcdf(ncfile_out, self.stations, ncf_in,
+                                                time_units=self.T_UNITS,
+                                                level_var=level_var)
+            rootgrp.source = f'{self.REANALYSIS}, interpolated bilinearly to stations'
+            rootgrp.globsim_interpolate_start = self.par['beg']
+            rootgrp.globsim_interpolate_end = self.par['end']
+            rootgrp.globsim_chunk_size = self.cs
+            rootgrp.globsim_interpolate_success = 0
+            rootgrp.globsim_last_chunk_written = -1
+            rootgrp.close()
 
         # open the output netCDF file, set it to be appendable ('a')
-        ncf_out = nc.Dataset(ncfile_out, 'a')
+        with nc.Dataset(ncfile_out, 'a') as ncf_out:
+            # get time and convert to datetime object
+            nctime = ncf_in['time'][:]
+            # "hours since 1900-01-01 00:00:0.0"
+            t_unit = ncf_in['time'].units
+            try:
+                t_cal = ncf_in['time'].calendar
+            except AttributeError:  # attribute doesn't exist
+                t_cal = u"gregorian"  # standard
+            time = nc.num2date(nctime, units=t_unit, calendar=t_cal)
+
+            # detect invariant files (topography etc.)
+            invariant = True if len(np.unique(time)) == 1 else False
+
+            # restrict to date/time range if given
+            if date is None:
+                tmask = time < datetime(3000, 1, 1)
+            else:
+                tmask = (time < date['end']) * (time >= date['beg'])
+
+            # get time vector for output
+            time_in = nctime[tmask]
+
+            # write time
+            ncf_out.variables['time'][:] = time_in
+
+            # ensure that chunk sizes cover entire period even if
+            # len(time_in) is not an integer multiple of cs
+            niter = len(time_in) // self.cs
+            niter += ((len(time_in) % self.cs) > 0)
+
+            # Create source grid
+            sgrid = self.create_source_grid(ncf_in)
+            subset_grid, lon_slice, lat_slice = self.create_subset_source_grid(sgrid, self.stations_bbox)
+            
+            # loop over chunks
+            for n in range(niter):
+                if self.resume and n <= ncf_out.globsim_last_chunk_written:
+                    if n == ncf_out.globsim_last_chunk_written:
+                        logger.info(f"Resuming interpolation at chunk {n+1}")
+                    continue
 
-        # get time and convert to datetime object
-        nctime = ncf_in.variables['time'][:]
-        # "hours since 1900-01-01 00:00:0.0"
-        t_unit = ncf_in.variables['time'].units
-        try:
-            t_cal = ncf_in.variables['time'].calendar
-        except AttributeError:  # attribute doesn't exist
-            t_cal = u"gregorian"  # standard
-        time = nc.num2date(nctime, units=t_unit, calendar=t_cal)
+                self.require_safe_mem_usage()
 
-        # detect invariant files (topography etc.)
-        invariant = True if len(np.unique(time)) == 1 else False
+                # indices
+                beg = n * self.cs
+                # restrict last chunk to lenght of tmask plus one (to get last time)
+                if invariant:
+                    end = beg
+                else:
+                    end = min(n * self.cs + self.cs, len(time_in)) - 1
+
+                # time to make tmask for chunk
+                beg_time = nc.num2date(time_in[beg], units=t_unit, calendar=t_cal)
+                if invariant:
+                    # allow topography to work in same code, len(nctime) = 1
+                    end_time = nc.num2date(nctime[0], units=t_unit, calendar=t_cal)
+                    # end = 1
+                else:
+                    end_time = nc.num2date(time_in[end], units=t_unit, calendar=t_cal)
 
-        # restrict to date/time range if given
-        if date is None:
-            tmask = time < datetime(3000, 1, 1)
-        else:
-            tmask = (time < date['end']) * (time >= date['beg'])
+                # '<= end_time', would damage appending
+                tmask_chunk = (time <= end_time) * (time >= beg_time)
+                if invariant:
+                    # allow topography to work in same code
+                    tmask_chunk = np.array([True])
 
-        # get time indices
-        time_in = nctime[tmask]
+                # get the interpolated variables
+                dfield, variables = self.interp2D(ncf_in,
+                                                self.stations, tmask_chunk,
+                                                subset_grid, lon_slice, lat_slice,
+                                                variables=None, date=None)
 
-        # ensure that chunk sizes cover entire period even if
-        # len(time_in) is not an integer multiple of cs
-        niter = len(time_in) // self.cs
-        niter += ((len(time_in) % self.cs) > 0)
+                # append variables
+                for i, var in enumerate(variables):
+                    if variables_skip(var):
+                        continue
 
-        # Create source grid
-        sgrid = self.create_source_grid(ncf_in)
-        subset_grid, lon_slice, lat_slice = self.create_subset_source_grid(sgrid, self.stations_bbox)
-        
-        # loop over chunks
-        for n in range(niter):
-            # indices
-            beg = n * self.cs
-            # restrict last chunk to lenght of tmask plus one (to get last time)
-            if invariant:
-                end = beg
-            else:
-                end = min(n * self.cs + self.cs, len(time_in)) - 1
-
-            # time to make tmask for chunk
-            beg_time = nc.num2date(time_in[beg], units=t_unit, calendar=t_cal)
-            if invariant:
-                # allow topography to work in same code, len(nctime) = 1
-                end_time = nc.num2date(nctime[0], units=t_unit, calendar=t_cal)
-                # end = 1
-            else:
-                end_time = nc.num2date(time_in[end], units=t_unit, calendar=t_cal)
-
-            # '<= end_time', would damage appending
-            tmask_chunk = (time <= end_time) * (time >= beg_time)
-            if invariant:
-                # allow topography to work in same code
-                tmask_chunk = np.array([True])
-
-            # get the interpolated variables
-            dfield, variables = self.interp2D(ncf_in,
-                                              self.stations, tmask_chunk,
-                                              subset_grid, lon_slice, lat_slice,
-                                              variables=None, date=None)
-
-            # append time
-            ncf_out.variables['time'][:] = np.append(ncf_out.variables['time'][:],
-                                                     time_in[beg:end + 1])
-
-            # append variables
-            for i, var in enumerate(variables):
-                if variables_skip(var):
-                    continue
+                    if pl:
+                        lev = ncf_in.variables['level'][:]
 
-                if pl:
-                    lev = ncf_in.variables['level'][:]
+                        ncf_out.variables[var][beg:end + 1,:,:] = dfield.data[:,i,:,:].transpose((1,2,0))
+                    else:
+                        ncf_out.variables[var][beg:end + 1,:] = dfield.data[:,i,:].transpose((1,0))
 
-                    ncf_out.variables[var][beg:end + 1,:,:] = dfield.data[:,i,:,:].transpose((1,2,0))
-                else:
-                    ncf_out.variables[var][beg:end + 1,:] = dfield.data[:,i,:].transpose((1,0))
+                ncf_out.globsim_last_chunk_written = n
+                del dfield, tmask_chunk
+                gc.collect()
 
+            ncf_out.globsim_interpolate_success = 1
         # close the file
         ncf_in.close()
-        ncf_out.close()
 
     def levels2elevation(self, ncfile_in, ncfile_out):
         """
@@ -228,7 +254,7 @@ def levels2elevation(self, ncfile_in, ncfile_out):
         nl = len(ncf.variables['level'][:])
 
         # list variables
-        varlist = [str_encode(x) for x in ncf.variables.keys()]
+        varlist = [x for x in ncf.variables.keys()]
         for V in ['time', 'station', 'latitude', 'longitude', 'level', 'height']:
             varlist.remove(V)
 
@@ -258,17 +284,17 @@ def levels2elevation(self, ncfile_in, ncfile_out):
 
         # create and assign variables from input file
         for var in varlist:
-            vname = str_encode(ncf.variables[var].long_name)
-            tmp   = rootgrp.createVariable(vname,'f4',('time', 'station'))
-            tmp.long_name = str_encode(ncf.variables[var].long_name)
-            tmp.units     = str_encode(ncf.variables[var].units)
+            vname = var  # ncf.variables[var].long_name
+            tmp   = rootgrp.createVariable(vname, 'f4',('time', 'station'))
+            tmp.long_name = ncf.variables[var].long_name
+            tmp.units     = ncf.variables[var].units
 
         # add air pressure as new variable
         var = 'air_pressure'
         varlist.append(var)
         tmp   = rootgrp.createVariable(var,'f4',('time', 'station'))
-        tmp.long_name = str_encode(var)
-        tmp.units     = str_encode('hPa')
+        tmp.long_name = "Air pressure"
+        tmp.units     = 'hPa'
         # end file preparation ================================================
 
         # loop over stations
@@ -299,11 +325,7 @@ def levels2elevation(self, ncfile_in, ncfile_out):
         ncf.close()
         # closed file =========================================================
 
-    def process(self):
-        """
-        Interpolate point time series from downloaded data. Provides access to
-        the more generically JRA-like interpolation functions.
-        """
+    def _preprocess(self):
         try:
             self.JRA2station(self.mf_to,
                              path.join(self.output_dir,f'{self.REANALYSIS}_to_{self.list_name}.nc'),
@@ -312,32 +334,44 @@ def process(self):
             logger.error("Could not find invariant ('*_to') geopotential files for JRA. These were not downloaded in earlier versions of globsim. You may need to download them."
                          "  . Some scaling kernels may not work. Future versions of globsim may be less accepting of missing files.")
 
+    def _process_sa(self):
         # === 2D Interpolation for Surface  Data ===
         # dictionary to translate CF Standard Names into JRA55
         # pressure level variable keys.
         varlist = self.TranslateCF2short(self.dpar_sa)
-        self.JRA2station(self.mf_sa,
-                         path.join(self.output_dir,f'{self.REANALYSIS}_sa_' + self.list_name + '.nc'),
-                         self.stations,
-                         varlist, date=self.date)
-
+        if self.resume and self.completed_successfully(self.getOutFile('sa')):
+            logger.info("Skipping surface analysis interpolation")
+        else:
+            self.JRA2station(self.mf_sa,
+                            self.getOutFile('sa'),
+                            self.stations,
+                            varlist, date=self.date)
+    
+    def _process_sf(self):
         # 2D Interpolation for Radiation Data
         # dictionary to translate CF Standard Names into JRA55
         # pressure level variable keys.
         varlist = self.TranslateCF2short(self.dpar_sf)
-        self.JRA2station(self.mf_sf,
-                         path.join(self.output_dir,f'{self.REANALYSIS}_sf_' + self.list_name + '.nc'),
-                         self.stations,
-                         varlist,
-                         date=self.date)
-        
+        if self.resume and self.completed_successfully(self.getOutFile('sf')):
+            logger.info("Skipping surface analysis interpolation")
+        else:
+            self.JRA2station(self.mf_sf,
+                            self.getOutFile('sf'),
+                            self.stations,
+                            varlist,
+                            date=self.date)
+    
+    def _process_pl(self):
         varlist = self.TranslateCF2short(self.dpar_pl).append('geopotential_height')
-        self.JRA2station(self.mf_pl,
-                         path.join(self.output_dir,f'{self.REANALYSIS}_pl_' + self.list_name + '.nc'),
-                         self.stations,
-                         varlist,
-                         date=self.date)
+        if self.resume and self.completed_successfully(self.getOutFile('pl')):
+            logger.info("Skipping surface analysis interpolation")
+        else:
+            self.JRA2station(self.mf_pl,
+                            self.getOutFile('pl'),
+                            self.stations,
+                            varlist,
+                            date=self.date)
 
         # 1D Interpolation for Pressure Level Data
-        self.levels2elevation(path.join(self.output_dir,f'{self.REANALYSIS}_pl_' + self.list_name + '.nc'),
+        self.levels2elevation(self.getOutFile('pl'),
                               path.join(self.output_dir,f'{self.REANALYSIS}_pl_' + self.list_name + '_surface.nc'))
diff --git a/globsim/interpolate/MERRAinterpolate.py b/globsim/interpolate/MERRAinterpolate.py
index aaa780c..5671c08 100644
--- a/globsim/interpolate/MERRAinterpolate.py
+++ b/globsim/interpolate/MERRAinterpolate.py
@@ -1,10 +1,13 @@
 import netCDF4 as nc
 import numpy as np
+import xarray as xr
 import sys
 import logging
+import gc
 
 from datetime          import datetime
 from os                import path, makedirs
+from pathlib import Path
 
 from globsim.common_utils import str_encode, variables_skip
 from globsim.interpolate.GenericInterpolate import GenericInterpolate
@@ -24,7 +27,8 @@ class MERRAinterpolate(GenericInterpolate):
 
     Referenced from era_interim.py (Dr.Stephan Gruber): Class ERAinterpolate()
     """
-
+    REANALYSIS = 'merra2'
+    
     def __init__(self, ifile, **kwargs):
         super().__init__(ifile, **kwargs)
         par = self.par
@@ -32,19 +36,20 @@ def __init__(self, ifile, **kwargs):
         self.input_dir = path.join(par['project_directory'],'merra2')
 
         # Load MF Datasets
-        self.mf_sc = nc.MFDataset(path.join(self.input_dir,'merra_sc.nc'), 'r', aggdim="time")
-        self.mf_sa = nc.MFDataset(path.join(self.input_dir,'merra_sa_*.nc'), 'r', aggdim="time")
-        self.mf_sf = nc.MFDataset(path.join(self.input_dir,'merra_sf_*.nc'), 'r', aggdim="time")
-        self.mf_pl = nc.MFDataset(path.join(self.input_dir,'merra_pl_*.nc'), 'r', aggdim="time")
+        self.mf_sc = xr.open_mfdataset(path.join(self.input_dir,'merra_sc.nc'), decode_times=False)
+        self.mf_sa = xr.open_mfdataset(self.prefilter_mf_paths(path.join(self.input_dir,'merra_sa_*.nc')), decode_times=False)
+        self.mf_sf = xr.open_mfdataset(self.prefilter_mf_paths(path.join(self.input_dir,'merra_sf_*.nc')), decode_times=False)
+        self.mf_pl = xr.open_mfdataset(self.prefilter_mf_paths(path.join(self.input_dir,'merra_pl_*.nc')), decode_times=False)
 
         # Check dataset integrity
-        logger.info("Check data integrity (sa)")
-        self.ensure_datset_integrity(self.mf_sa['time'], 1)
-        logger.info("Check data integrity (sf)")
-        self.ensure_datset_integrity(self.mf_sf['time'], 1)
-        logger.info("Check data integrity (pl)")
-        self.ensure_datset_integrity(self.mf_pl['time'], 6)
-        logger.info("Data integrity ok")
+        if not self.skip_checks:
+            logger.info("Check data integrity (sa)")
+            self.ensure_datset_integrity(self.mf_sa['time'], 1)
+            logger.info("Check data integrity (sf)")
+            self.ensure_datset_integrity(self.mf_sf['time'], 1)
+            logger.info("Check data integrity (pl)")
+            self.ensure_datset_integrity(self.mf_pl['time'], 6)
+            logger.info("Data integrity ok")
 
     def netCDF_empty(self, ncfile_out, stations, nc_in):
         # TODO: change date type from f4 to f8 for lat and lon
@@ -86,7 +91,7 @@ def netCDF_empty(self, ncfile_out, stations, nc_in):
             lev = []
 
         # remove extra variables
-        varlist_merra = [str_encode(x) for x in nc_in.variables.keys()]
+        varlist_merra = [x for x in nc_in.variables.keys()]
 
         # create and assign variables based on input file
         for n, var in enumerate(varlist_merra):
@@ -98,12 +103,12 @@ def netCDF_empty(self, ncfile_out, stations, nc_in):
                 tmp = rootgrp.createVariable(var,'f4', ('time', 'level', 'station'))
             else:
                 tmp = rootgrp.createVariable(var,'f4', ('time', 'station'))
-            tmp.long_name = str_encode(nc_in.variables[var].long_name)  # for merra2
-            tmp.units     = str_encode(nc_in.variables[var].units)
+            
+            tmp.long_name = nc_in[var].long_name  # for merra2
+            tmp.units     = nc_in[var].units
 
-        # close the file
-        rootgrp.close()
         logger.debug(f"Created empty netcdf file {ncfile_out}")
+        return rootgrp
 
     def MERRA2station(self, ncf_in, ncfile_out, points,
                       variables=None, date=None):
@@ -136,102 +141,125 @@ def MERRA2station(self, ncf_in, ncfile_out, points,
         self.validate_stations_extent(ncf_in)
 
         # is it a file with pressure levels?
-        pl = 'level' in ncf_in.dimensions.keys()
+        pl = 'level' in ncf_in.sizes.keys()
 
         # build the output of empty netCDF file
-        self.netCDF_empty(ncfile_out, self.stations, ncf_in)
+        if self.resume:
+            self.require_file_can_be_resumed(ncfile_out)
+            
+        if not (self.resume and Path(ncfile_out).exists()):
+            rootgrp = self.netCDF_empty(ncfile_out, self.stations, ncf_in)
+            rootgrp.globsim_interpolate_start = self.par['beg']
+            rootgrp.globsim_interpolate_end = self.par['end']
+            rootgrp.globsim_chunk_size = self.cs
+            rootgrp.globsim_interpolate_success = 0
+            rootgrp.globsim_last_chunk_written = -1
+            rootgrp.close()
 
         # open the output netCDF file, set it to be appendable ('a')
-        ncf_out = nc.Dataset(ncfile_out, 'a')
-
-        # get time and convert to datetime object
-        nctime = ncf_in.variables['time'][:]
-        # "hours since 1980-01-01 00:00:00"
-        t_unit = "hours since 1980-01-01 00:00:00"  # ncf_in.variables['time'].units
-        try :
-            t_cal = ncf_in.variables['time'].calendar
-        except AttributeError :  # Attribute doesn't exist
-            t_cal = u"gregorian"  # or standard
-        # TODO: rm time = [nc.num2date(timei, units=t_unit, calendar=t_cal) for timei in nctime]
-        # TODO: rm time = np.asarray(time)
-
-        # detect invariant files (topography etc.)
-        invariant = True if len(np.unique(nctime)) <= 2 else False
-
-        # restrict to date/time range if given
-        if date is None:
-            tmask = nctime < nc.date2num(datetime(3000, 1, 1), units=t_unit, calendar=t_cal)
-        else:
-            beg_num = nc.date2num(date['beg'], units=t_unit, calendar=t_cal)
-            end_num = nc.date2num(date['end'], units=t_unit, calendar=t_cal)
-            tmask = (nctime < end_num) * (nctime >= beg_num)
-
-        if not any(tmask):
-            sys.exit('''\n ERROR: No downloaded data exist within date range specified by interpolation control file.
-                     Download new data or change 'beg' / 'end' in interpolation control file''')
-
-        # get time indices
-        time_in = nctime[tmask]
-
-        # ensure that chunk sizes cover entire period even if
-        # len(time_in) is not an integer multiple of cs
-        niter  = len(time_in) // self.cs
-        niter += ((len(time_in) % self.cs) > 0)
-
-        # Create source grid
-        sgrid = self.create_source_grid(ncf_in)
-        subset_grid, lon_slice, lat_slice = self.create_subset_source_grid(sgrid, self.stations_bbox)
-
-        # loop in chunk size cs
-        for n in range(niter):
-            # indices
-            beg = n * self.cs
-            # restrict last chunk to lenght of tmask plus one (to get last time)
-            if invariant:
-                end = beg
+        with nc.Dataset(ncfile_out, 'a') as ncf_out:
+            # get time and convert to datetime object
+            nctime = ncf_in.variables['time'][:]
+            # "hours since 1980-01-01 00:00:00"
+            t_unit = "hours since 1980-01-01 00:00:00"  # ncf_in.variables['time'].units
+            try :
+                t_cal = ncf_in.variables['time'].calendar
+            except AttributeError :  # Attribute doesn't exist
+                t_cal = u"gregorian"  # or standard
+            # TODO: rm time = [nc.num2date(timei, units=t_unit, calendar=t_cal) for timei in nctime]
+            # TODO: rm time = np.asarray(time)
+
+            # detect invariant files (topography etc.)
+            invariant = True if len(np.unique(nctime)) <= 2 else False
+
+            # restrict to date/time range if given
+            if date is None:
+                tmask = nctime < nc.date2num(datetime(3000, 1, 1), units=t_unit, calendar=t_cal)
             else:
-                end = min(n * self.cs + self.cs, len(time_in)) - 1
-
-            # make tmask for chunk
-            beg_time = time_in[beg]
-            if invariant:
-                # allow topography to work in same code, len(nctime) = 1
-                # TODO: rm end_time = nc.num2date(nctime[0], units=t_unit, calendar=t_cal)
-                end_time = nctime[0]
-                # end = 1
-            else:
-                # TODO: rm end_time = nc.num2date(time_in[end], units=t_unit, calendar=t_cal)
-                end_time = time_in[end]
-
-            # !! CAN'T HAVE '<= end_time', would damage appeding
-            tmask_chunk = (nctime <= end_time) * (nctime >= beg_time)
-            if invariant:
-                # allow topography to work in same code
-                tmask_chunk = np.array([True])
-
-            # get the interpolated variables
-            dfield, variables = self.interp2D(ncf_in,
-                                              self.stations, tmask_chunk,
-                                              subset_grid, lon_slice, lat_slice,
-                                              variables=None, date=None)
-
-            # append time
-            ncf_out.variables['time'][:] = np.append(ncf_out.variables['time'][:],
-                                                     time_in[beg:end + 1])
-            # append variables
-            for i, var in enumerate(variables):
-                if variables_skip(var):
+                beg_num = nc.date2num(date['beg'], units=t_unit, calendar=t_cal)
+                end_num = nc.date2num(date['end'], units=t_unit, calendar=t_cal)
+                tmask = (nctime < end_num) * (nctime >= beg_num)
+
+            if not any(tmask):
+                sys.exit('''\n ERROR: No downloaded data exist within date range specified by interpolation control file.
+                        Download new data or change 'beg' / 'end' in interpolation control file''')
+
+            # get time indices
+            time_in = nctime[tmask]
+
+            # write time
+            ncf_out.variables['time'][:] = time_in
+
+            # ensure that chunk sizes cover entire period even if
+            # len(time_in) is not an integer multiple of cs
+            niter  = len(time_in) // self.cs
+            niter += ((len(time_in) % self.cs) > 0)
+
+            # Create source grid
+            sgrid = self.create_source_grid(ncf_in)
+            subset_grid, lon_slice, lat_slice = self.create_subset_source_grid(sgrid, self.stations_bbox)
+
+            # loop in chunk size cs
+            for n in range(niter):
+                if self.resume and n <= ncf_out.globsim_last_chunk_written:
+                    if n == ncf_out.globsim_last_chunk_written:
+                        logger.info(f"Resuming interpolation at chunk {n+1}")
                     continue
-
-                # extra treatment for pressure level files
-                if pl:
-                    # lev = ncf_in.variables['level'][:]
-                    ncf_out.variables[var][beg:end + 1, :, :] = dfield.data[:, i, :, :].transpose(1,2,0)
+                
+                self.require_safe_mem_usage()
+
+                # indices
+                beg = n * self.cs
+                # restrict last chunk to lenght of tmask plus one (to get last time)
+                if invariant:
+                    end = beg
                 else:
-                    ncf_out.variables[var][beg:end + 1, :] = dfield.data[:, i, :].transpose(1,0)
-
+                    end = min(n * self.cs + self.cs, len(time_in)) - 1
+
+                # make tmask for chunk
+                beg_time = time_in[beg]
+                if invariant:
+                    # allow topography to work in same code, len(nctime) = 1
+                    # TODO: rm end_time = nc.num2date(nctime[0], units=t_unit, calendar=t_cal)
+                    end_time = nctime[0]
+                    # end = 1
+                else:
+                    # TODO: rm end_time = nc.num2date(time_in[end], units=t_unit, calendar=t_cal)
+                    end_time = time_in[end]
+
+                # !! CAN'T HAVE '<= end_time', would damage appeding
+                tmask_chunk = (nctime <= end_time) * (nctime >= beg_time)
+                if invariant:
+                    # allow topography to work in same code
+                    tmask_chunk = np.array([True])
+
+                # get the interpolated variables
+                dfield, variables = self.interp2D(ncf_in,
+                                                self.stations, tmask_chunk,
+                                                subset_grid, lon_slice, lat_slice,
+                                                variables=None, date=None)
+
+                # append variables
+                for i, var in enumerate(variables):
+                    if variables_skip(var):
+                        continue
+
+                    # extra treatment for pressure level files
+                    if pl:
+                        # lev = ncf_in.variables['level'][:]
+                        ncf_out.variables[var][beg:end + 1, :, :] = dfield.data[:, i, :, :].transpose(1,2,0)
+                    else:
+                        ncf_out.variables[var][beg:end + 1, :] = dfield.data[:, i, :].transpose(1,0)
+                # delete objects and free memory
+                del dfield, tmask_chunk
+                gc.collect()
+                ncf_out.globsim_last_chunk_written = n
+
+            # Write success flag
+            ncf_out.globsim_interpolate_success = 1
+        
         ncf_in.close()
-        ncf_out.close()
+
 
     def levels2elevation(self, ncfile_in, ncfile_out):
         """
@@ -243,7 +271,7 @@ def levels2elevation(self, ncfile_in, ncfile_out):
         """
         # open file
 
-        ncf = nc.MFDataset(ncfile_in, 'r', aggdim='time')
+        ncf = nc.Dataset(ncfile_in, 'r', aggdim='time')
         height = ncf.variables['height'][:]
         nt = len(ncf.variables['time'][:])
         nl = len(ncf.variables['level'][:])
@@ -326,26 +354,16 @@ def levels2elevation(self, ncfile_in, ncfile_out):
         rootgrp.close()
         ncf.close()
         # closed file ==========================================================
-
-    def process(self):
-        """
-        Interpolate point time series from downloaded data. Provides access to
-        the more generically MERRA-like interpolation functions.
-        """
-
-        # 2D Interpolation for Constant Model Parameters
-        # dictionary to translate CF Standard Names into MERRA
-        # pressure level variable keys.
-        # dummy_date = {'beg' : datetime(1992, 1, 2, 3, 0),
-        #              'end' : datetime(1992, 1, 2, 4, 0)}
-
+    
+    def _preprocess(self):
         if not path.isdir(self.output_dir):
             makedirs(self.output_dir)
-
+        
         self.MERRA2station(self.mf_sc,
                            path.join(self.output_dir,'merra2_sc_' + self.list_name + '.nc'),
                            self.stations, ['PHIS','FRLAND'], date=None)
-
+    
+    def _process_sa(self):
         # === 2D Interpolation for Surface Analysis Data ===
         # dictionary to translate CF Standard Names into MERRA2
         # pressure level variable keys.
@@ -353,10 +371,14 @@ def process(self):
                 'wind_speed' : ['U2M', 'V2M', 'U10M','V10M'],   # [m s-1] 2m & 10m values
                 'relative_humidity' : ['QV2M']}  # 2m value
         varlist = self.TranslateCF2short(dpar)
-        self.MERRA2station(self.mf_sa,
-                           path.join(self.output_dir,'merra2_sa_' + self.list_name + '.nc'),
-                           self.stations, varlist, date=self.date)
-
+        if self.resume and self.completed_successfully(self.getOutFile('sa')):
+            logger.info("Skipping surface analysis interpolation")
+        else:
+            self.MERRA2station(self.mf_sa,
+                            self.getOutFile('sa'),
+                            self.stations, varlist, date=self.date)
+    
+    def _process_sf(self):
         # 2D Interpolation for Single-level Radiation Diagnostics Data 'SWGDN',
         # 'LWGDN', 'SWGDNCLR'. 'LWGDNCLR'
         # dictionary to translate CF Standard Names into MERRA2
@@ -368,10 +390,14 @@ def process(self):
                 'downwelling_shortwave_flux_in_air_assuming_clear_sky': ['SWGDNCLR'],  # [W/m2] short-wave downward assuming clear sky
                 'downwelling_longwave_flux_in_air_assuming_clear_sky': ['LWGDNCLR']}  # [W/m2] long-wave downward assuming clear sky
         varlist = self.TranslateCF2short(dpar)
-        self.MERRA2station(self.mf_sf,
-                           path.join(self.output_dir,'merra2_sf_' + self.list_name + '.nc'),
-                           self.stations, varlist, date=self.date)
-
+        if self.resume and self.completed_successfully(self.getOutFile('sf')):
+            logger.info("Skipping surface forecast interpolation")
+        else:
+            self.MERRA2station(self.mf_sf,
+                            self.getOutFile('sf'),
+                            self.stations, varlist, date=self.date)
+            
+    def _process_pl(self):
         # NEED ADD 'H' in it!
         # === 2D Interpolation for Pressure-Level, Analyzed Meteorological DATA ===
         # dictionary to translate CF Standard Names into MERRA2
@@ -380,10 +406,17 @@ def process(self):
                 'wind_speed'        : ['U', 'V'],      # [m s-1]
                 'relative_humidity' : ['RH']}          # [1]
         varlist = self.TranslateCF2short(dpar).append('H')
-        self.MERRA2station(self.mf_pl,
-                           path.join(self.output_dir,'merra2_pl_' + self.list_name + '.nc'),
-                           self.stations, varlist, date=self.date)
+        if self.resume and self.completed_successfully(self.getOutFile('pl')):
+            logger.info("Skipping pressure level interpolation")
+        else:
+            self.MERRA2station(self.mf_pl,
+                            self.getOutFile('pl'),
+                            self.stations, varlist, date=self.date)
 
         # 1D Interpolation for Pressure Level Analyzed Meteorological Data
         self.levels2elevation(path.join(self.output_dir,'merra2_pl_' + self.list_name + '.nc'),
                               path.join(self.output_dir,'merra2_pl_' + self.list_name + '_surface.nc'))
+
+        
+
+        
diff --git a/globsim/interpolate/__init__.py b/globsim/interpolate/__init__.py
index 8e3bf2a..303df13 100644
--- a/globsim/interpolate/__init__.py
+++ b/globsim/interpolate/__init__.py
@@ -1,5 +1,11 @@
 from .ERA5interpolate import ERA5interpolate
-from .ERAIinterpolate import ERAIinterpolate
 from .JRAinterpolate import JRAinterpolate
-from .JRA3Qinterpolate import J3QI
-from .MERRAinterpolate import MERRAinterpolate
\ No newline at end of file
+from .JRA3Qinterpolate import J3QI, J3QgI
+from .MERRAinterpolate import MERRAinterpolate
+
+
+__all__ = ['ERA5interpolate', 
+           'JRAinterpolate', 
+           'J3QI', 
+           'J3QgI',
+           'MERRAinterpolate']
\ No newline at end of file
diff --git a/globsim/interpolate/create_grid_helper.py b/globsim/interpolate/create_grid_helper.py
index 0a97a85..3746de7 100644
--- a/globsim/interpolate/create_grid_helper.py
+++ b/globsim/interpolate/create_grid_helper.py
@@ -1,20 +1,7 @@
 import numpy as np
 import re
 
-try:
-    import ESMF
-
-    # Check ESMF version.  7.0.1 behaves differently than 7.1.0r
-    ESMFv = int(re.sub("[^0-9]", "", ESMF.__version__))
-    ESMFnew = ESMFv > 701
-
-except ModuleNotFoundError:
-        print("*** ESMF not imported, trying esmpy. ***")
-        try:
-            import esmpy as ESMF
-        except ImportError:
-            print('Could not import ESMF or esmpy')
-            pass
+import esmpy as ESMF
 
 from globsim.boundingbox import BoundingBox
 
diff --git a/globsim/nc_elements.py b/globsim/nc_elements.py
index b73262a..04130bd 100755
--- a/globsim/nc_elements.py
+++ b/globsim/nc_elements.py
@@ -5,6 +5,8 @@
 import netCDF4 as nc
 import numpy as np
 from os import path
+from typing import Optional
+import xarray as xr
 
 from globsim.common_utils import variables_skip, str_encode
 from globsim import __version__ as globsim_version
@@ -142,7 +144,10 @@ def new_scaled_netcdf(ncfile_out, nc_interpol, times_out,
     return rootgrp
 
 
-def new_interpolated_netcdf(ncfile_out, stations, nc_in, time_units):
+def new_interpolated_netcdf(ncfile_out:str, stations, 
+                            nc_in:xr.Dataset, time_units:str, 
+                            calendar:Optional[str]=None,
+                            level_var:Optional[list]=None) -> nc.Dataset:
     """
     Creates an empty station file to hold interpolated reults. The number of
     stations is defined by the variable stations, variables are determined by
@@ -150,12 +155,13 @@ def new_interpolated_netcdf(ncfile_out, stations, nc_in, time_units):
 
     ncfile_out: full name of the file to be created
     stations:   station list read with common_utils.StationListRead()
+    nc_in: 
     variables:  variables read from netCDF handle
     lev:        list of pressure levels, empty is [] (default)
     """
     logger.info(f"Creating new file {ncfile_out} from ")
 
-    rootgrp = netcdf_base(ncfile_out, len(stations), None, time_units, nc_in)
+    rootgrp = netcdf_base(ncfile_out, len(stations), None, time_units, nc_in, calendar)
 
     station = rootgrp['station']
     latitude = rootgrp['latitude']
@@ -169,15 +175,15 @@ def new_interpolated_netcdf(ncfile_out, stations, nc_in, time_units):
     height[:]    = list(stations['elevation_m'])
 
     # extra treatment for pressure level files
-    try:
-        lev = nc_in.variables['level'][:]
+    if level_var is not None:
+        lev = nc_in.variables[level_var][:]
         logger.info(f"Source dataset is 3D (has pressure levels)")
         level           = rootgrp.createDimension('level', len(lev))
         level           = rootgrp.createVariable('level', 'i4', ('level'))
         level.long_name = 'pressure_level'
         level.units     = 'hPa'
         level[:] = lev
-    except Exception:
+    else:
         logger.info(f"Source dataset is 2D (without pressure levels)")
         lev = []
 
@@ -208,17 +214,24 @@ def new_interpolated_netcdf(ncfile_out, stations, nc_in, time_units):
 
         # copy attributes
         input_var = nc_in.variables[var]
-        for key in input_var.ncattrs():
-            if key in ['_FillValue']:
-                continue
-            tmp.setncattr(key, getattr(input_var, key))
-        
-        logger.info(f"Created new empty variable: {str_encode(var)} [{tmp.units}]")
+
+        if isinstance(input_var, xr.Variable):  # XARRAY
+            for key, val in input_var.attrs.items():
+                if key in ['_FillValue']:
+                    continue
+                tmp.setncattr(key, val)
+        else:
+            for key in input_var.ncattrs():  # NETCDF4
+                if key in ['_FillValue']:
+                    continue
+                tmp.setncattr(key, getattr(input_var, key))
+        units = tmp.units if hasattr(tmp, 'units') else '??'
+        logger.info(f"Created new empty variable: {str_encode(var)} [{units}]")
     
     return rootgrp
 
 
-def netcdf_base(ncfile_out, n_stations, n_time, time_units, nc_in=None):
+def netcdf_base(ncfile_out, n_stations, n_time, time_units, nc_in=None, calendar=None):
     # Build the netCDF file
     rootgrp = nc_new_file(ncfile_out)
 
@@ -227,8 +240,10 @@ def netcdf_base(ncfile_out, n_stations, n_time, time_units, nc_in=None):
     _ = rootgrp.createDimension('time', None)
 
     # base variables
+    if calendar is None:
+        calendar = 'gregorian'
     _ = ncvar_add_time(rootgrp, units=time_units,
-                       calendar='gregorian', dimensions=('time'))
+                       calendar=calendar, dimensions=('time'))
     _ = ncvar_add_station(rootgrp)
     _ = ncvar_add_latitude(rootgrp)
     _ = ncvar_add_longitude(rootgrp)
diff --git a/globsim/scale/ERA5scale.py b/globsim/scale/ERA5scale.py
index 427265b..97f470d 100755
--- a/globsim/scale/ERA5scale.py
+++ b/globsim/scale/ERA5scale.py
@@ -152,7 +152,7 @@ def PRESS_Pa_pl(self):
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             # scale from hPa to Pa
             self.rg.variables[vn][:, n] = series_interpolate(self.times_out_nc,
-                                                             time_in * 3600,
+                                                             time_in,
                                                              values[:, n]) * 100
 
     def AIRT_C_pl(self):
@@ -167,11 +167,11 @@ def AIRT_C_pl(self):
         var.standard_name = 'air_temperature'
 
         # interpolate station by station
-        time_in = self.nc_pl_sur.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_pl_sur)
         values  = self.getValues(self.nc_pl_sur, 't')  # self.nc_pl_sur.variables['t'][:]
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             self.rg.variables[vn][:, n] = series_interpolate(self.times_out_nc,
-                                                             time_in * 3600, values[:, n] - 273.15)
+                                                             time_in, values[:, n] - 273.15)
 
     def AIRT_C_sur(self):
         """
@@ -185,11 +185,11 @@ def AIRT_C_sur(self):
         var.standard_name = 'air_temperature' 
 
         # interpolate station by station
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_sa)
         values  = self.getValues(self.nc_sa, 't2m')  # self.nc_sa.variables['t2m'][:]
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             self.rg.variables[vn][:, n] = series_interpolate(self.times_out_nc,
-                                                             time_in * 3600,
+                                                             time_in,
                                                              values[:, n] - 273.15)
 
     def AIRT_redcapp(self):
@@ -216,11 +216,11 @@ def AIRT_redcapp(self):
                                       elevation=elevation,
                                       h_sur=h_sur)  
 
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_sa)
         values  = Delta_T_c
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             var[:, n] = series_interpolate(self.times_out_nc,
-                                                             time_in * 3600,
+                                                             time_in,
                                                              values[:, n])
 
     def PREC_mm_sur(self):
@@ -235,17 +235,16 @@ def PREC_mm_sur(self):
         var.standard_name = 'precipitation_flux'
 
         # interpolation scale factor
-        time_in = self.nc_sf.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_sf)
 
         # total prec [mm] in 1 second
         values  = self.getValues(self.nc_sf, 'tp')  # self.nc_sf.variables['tp'][:]*1000/self.interval_in
         # https://confluence.ecmwf.int/pages/viewpage.action?pageId=197702790
         # We assume interval_in is in hours (1 for regular, 3 for ensemble)
         values  = values * 1000 / (self.interval_in)  # [m] -> [mm s-1]
-
         # interpolate station by station
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            f = interp1d(time_in * 3600, values[:, n], kind='linear')
+            f = interp1d(time_in, values[:, n], kind='linear')
             self.rg.variables[vn][:, n] = f(self.times_out_nc) * self.scf
 
     def RH_per_sur(self):
@@ -255,11 +254,11 @@ def RH_per_sur(self):
         """
         # temporary variable,  interpolate station by station
         dewp = np.zeros((self.nt, self.nstation), dtype=np.float32)
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_sa)
         values  = self.getValues(self.nc_sa, 'd2m')  # self.nc_sa.variables['d2m'][:]
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             dewp[:, n] = series_interpolate(self.times_out_nc,
-                                            time_in * 3600, values[:, n] - 273.15)
+                                            time_in, values[:, n] - 273.15)
 
         # add variable to ncdf file
         vn = 'RH_sur'  # variable name
@@ -283,7 +282,7 @@ def RH_per_pl(self):
 
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             rh[:, n] = series_interpolate(self.times_out_nc,
-                                            time_in * 3600, values[:, n])
+                                            time_in, values[:, n])
 
         # add variable to ncdf file
         vn = 'RH_pl'  # variable name
@@ -301,19 +300,19 @@ def WIND_sur(self):
         """
         # temporary variable, interpolate station by station
         U = np.zeros((self.nt, self.nstation), dtype=np.float32)
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_sa)
         values  = self.getValues(self.nc_sa, 'u10')  # self.nc_sa.variables['u10'][:]
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             U[:, n] = series_interpolate(self.times_out_nc,
-                                         time_in * 3600, values[:, n])
+                                         time_in, values[:, n])
 
         # temporary variable, interpolate station by station
         V = np.zeros((self.nt, self.nstation), dtype=np.float32)
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_sa)
         values  = self.getValues(self.nc_sa, 'v10')  # self.nc_sa.variables['v10'][:]
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             V[:, n] = series_interpolate(self.times_out_nc,
-                                         time_in * 3600, values[:, n])
+                                         time_in, values[:, n])
 
 
         # wind speed, add variable to ncdf file, convert
@@ -338,6 +337,7 @@ def WIND_sur(self):
         # convert to "clockwise from north" from "anti-clockwise from x-axis" : 90 - angle
         # convert "from direction" : + 180
         WD = 90 - (np.arctan2(V, U) * (180 / np.pi)) + 180
+        WD = np.mod(WD, 360)
         self.rg.variables[vn_dir][:, :] = WD
 
     def SW_Wm2_sur(self):
@@ -354,11 +354,11 @@ def SW_Wm2_sur(self):
         var.standard_name = 'surface_downwelling_shortwave_flux'
         
         # interpolate station by station
-        time_in = self.nc_sf.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_sf)
         values  = self.getValues(self.nc_sf, 'ssrd')  # self.nc_sf.variables['ssrd'][:]/3600/self.interval_in#[w/m2/s]
         values  = values / (self.interval_in)  # [J m-2] -> [w m-2]
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            f = interp1d(time_in * 3600, values[:, n], kind='linear')
+            f = interp1d(time_in, values[:, n], kind='linear')
             self.rg.variables[vn][:, n] = f(self.times_out_nc)
 
     def SW_Wm2_topo(self):
@@ -381,8 +381,8 @@ def SW_Wm2_topo(self):
         var.standard_name = 'surface_direct_downwelling_shortwave_flux_in_air'
 
         # interpolate station by station
-        nc_time = self.nc_sf.variables['time']
-        py_time = nc.num2date(nc_time[:], nc_time.units, nc_time.calendar, only_use_cftime_datetimes=False)
+        nc_time = self.input_times_in_output_units(self.nc_sf)
+        py_time = nc.num2date(nc_time[:], self.scaled_t_units, self.t_cal, only_use_cftime_datetimes=False)
         py_time = np.array([pytz.utc.localize(t) for t in py_time])
         lat = self.getValues(self.nc_pl_sur, 'latitude')
         lon = self.getValues(self.nc_pl_sur, 'longitude')
@@ -394,9 +394,6 @@ def SW_Wm2_topo(self):
         slope = self.get_slope()
         aspect = self.get_aspect()
 
-
-        interpolation_time = nc_time[:].astype(np.int64)
-        
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             zenith = solar_zenith(lat=lat[n], lon=lon[n], time=py_time)
     
@@ -419,10 +416,10 @@ def SW_Wm2_topo(self):
                 sensible_values_mask = np.where(cos_i_grid < 0.001, 0, 1) * np.where(corrected_direct > 1366, 0, 1)
                 corrected_direct *= sensible_values_mask
 
-            f = interp1d(interpolation_time * 3600, corrected_direct, kind='linear')
+            f = interp1d(nc_time, corrected_direct, kind='linear')
             self.rg.variables[vn_dir][:, n] = f(self.times_out_nc)
 
-            f = interp1d(interpolation_time * 3600, diffuse, kind='linear')
+            f = interp1d(nc_time, diffuse, kind='linear')
             self.rg.variables[vn_diff][:, n] = f(self.times_out_nc)
 
     def LW_Wm2_sur(self):
@@ -439,11 +436,11 @@ def LW_Wm2_sur(self):
         var.units     = 'W m-2'
 
         # interpolate station by station
-        time_in = self.nc_sf.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_sf)
         values  = self.getValues(self.nc_sf, 'strd')  # self.nc_sf.variables['strd'][:]/3600/self.interval_in #[w m-2 s-1]
         values  = values / (self.interval_in)  # [J m-2] -> [w m-2]
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            f = interp1d(time_in * 3600, values[:, n], kind='linear')
+            f = interp1d(time_in, values[:, n], kind='linear')
             self.rg.variables[vn][:, n] = f(self.times_out_nc)
 
     def LW_Wm2_topo(self):
@@ -456,7 +453,7 @@ def LW_Wm2_topo(self):
         var.units     = 'W m-2'
 
         # interpolate station by station
-        time_in = self.nc_sf.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_sf)
         t_sub = self.getValues(self.nc_pl_sur, 't')  # [K]
         rh_sub = self.getValues(self.nc_pl_sur, 'r')  # [%]
         t_grid = self.getValues(self.nc_sa, 't2m')  # [K]
@@ -471,7 +468,7 @@ def LW_Wm2_topo(self):
 
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             data = lw_sub[:, n] * svf[n]
-            f = interp1d(time_in * 3600, data, kind='linear')
+            f = interp1d(time_in, data, kind='linear')
             self.rg.variables[vn][:, n] = f(self.times_out_nc) 
 
     def SH_kgkg_sur(self):
@@ -482,11 +479,11 @@ def SH_kgkg_sur(self):
 
         # temporary variable,  interpolate station by station
         dewp = np.zeros((self.nt, self.nstation), dtype=np.float32)
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
+        time_in = self.input_times_in_output_units(self.nc_sa)
         values  = self.getValues(self.nc_sa, 'd2m')  # self.nc_sa.variables['d2m'][:]
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             dewp[:, n] = series_interpolate(self.times_out_nc,
-                                            time_in * 3600, values[:, n] - 273.15)
+                                            time_in, values[:, n] - 273.15)
 
         # compute
         SH = spec_hum_kgkg(dewp[:, :],
@@ -499,3 +496,13 @@ def SH_kgkg_sur(self):
         var.units     = '1'
         var.standard_name = 'specific_humidity'
         self.rg.variables[vn][:, :] = SH
+
+
+    def input_times_in_output_units(self, ncf):
+        """
+        Convert time in input data to time in Globsim.
+        """
+        raw = ncf['time'][:].astype(np.int64)
+        time = nc.num2date(raw, units=ncf['time'].units, calendar=ncf['time'].calendar)
+        converted = nc.date2num(time, units=self.scaled_t_units, calendar=self.t_cal)
+        return converted.astype(np.int64)
\ No newline at end of file
diff --git a/globsim/scale/ERAIscale.py b/globsim/scale/ERAIscale.py
deleted file mode 100755
index 50ad1d5..0000000
--- a/globsim/scale/ERAIscale.py
+++ /dev/null
@@ -1,392 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Methods for downloading ERA-Interim data from the ECMWF server for limited
-# areas and limited times.
-#
-#
-# (C) Copyright Stephan Gruber (2013–2017)
-#
-#     This program is free software: you can redistribute it and/or modify
-#     it under the terms of the GNU General Public License as published by
-#     the Free Software Foundation, either version 3 of the License, or
-#     (at your option) any later version.
-#
-#     This program is distributed in the hope that it will be useful,
-#     but WITHOUT ANY WARRANTY; without even the implied warranty of
-#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#     GNU General Public License for more details.
-#
-#     You should have received a copy of the GNU General Public License
-#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-#  OVERALL WORKFLOW
-#
-#  See: https://github.com/geocryology/globsim/wiki/Globsim
-#       and the examples directory on github
-#
-# ECMWF and netCDF information:
-# https://software.ecmwf.int/wiki/display/WEBAPI/Python+ERA-interim+examples
-#
-# For variable codes and units, see:
-#     http://www.ecmwf.int/publications/manuals/d/gribapi/param/
-#
-# Check ECMWF job status: http://apps.ecmwf.int/webmars/joblist/
-#
-#  CF-Convention: this is how you check netcdf file conformity:
-#  http://pumatest.nerc.ac.uk/cgi-bin/cf-checker-dev.pl
-#
-# ===============================================================================
-import netCDF4 as nc
-import numpy as np
-import logging
-
-from math import atan2, pi
-from os import path
-from scipy.interpolate import interp1d
-
-from globsim.common_utils import series_interpolate, cummulative2total
-from globsim.meteorology import spec_hum_kgkg, LW_downward
-from globsim.nc_elements import new_scaled_netcdf
-from globsim.scale.GenericScale import GenericScale
-
-logger = logging.getLogger('globsim.scale')
-
-
-class ERAIscale(GenericScale):
-    NAME = "ERA-I"
-
-    """
-    Class for ERA-Interim data that has methods for scaling station data to
-    better resemble near-surface fluxes.
-
-    Processing kernels have names in UPPER CASE.
-
-    Args:
-        sfile: Full path to a Globsim Scaling Parameter file.
-
-    Example:
-        ERAd = ERAIscale(sfile)
-        ERAd.process()
-    """
-
-    def __init__(self, sfile):
-        super().__init__(sfile)
-        par = self.par
-
-        # input file handles
-        self.nc_pl_sur = nc.Dataset(path.join(self.intpdir, 'erai_pl_' +
-                                          self.list_name + '_surface.nc'), 'r')
-        self.nc_sa = nc.Dataset(path.join(self.intpdir,'erai_sa_' +
-                                          self.list_name + '.nc'), 'r')
-        self.nc_sf = nc.Dataset(path.join(self.intpdir, 'erai_sf_' +
-                                          self.list_name + '.nc'), 'r')
-        self.nc_to = nc.Dataset(path.join(self.intpdir, 'erai_to_' +
-                                          self.list_name + '.nc'), 'r')
-        self.nstation = len(self.nc_to.variables['station'][:])
-
-        # check if output file exists and remove if overwrite parameter is set
-        self.output_file = self.getOutNCF(par, 'erai')
-
-        # time vector for output data
-        # get time and convert to datetime object
-        self.set_time_scale(self.nc_sf.variables['time'], par['time_step'])
-        self.scaled_t_units = 'seconds since 1900-01-01 00:00:00'
-        
-        self.times_out_nc = self.build_datetime_array(start_time=self.min_time,
-                                                      timestep_in_hours=self.time_step,
-                                                      num_times=self.nt,
-                                                      output_units=self.scaled_t_units,
-                                                      output_calendar=self.t_cal)
-
-    def process(self):
-        """
-        Run all relevant processes and save data. Each kernel processes one
-        variable and adds it to the netCDF file.
-        """
-        if path.isfile(self.output_file):
-            print("Warning, output file already exists. This may cause problems")
-        self.rg = new_scaled_netcdf(self.output_file, self.nc_pl_sur,
-                                 self.times_out_nc,
-                                 t_unit = self.scaled_t_units,
-                                 station_names = self.stations['station_name'])
-
-        # iterate through kernels and start process
-        for kernel_name in self.kernels:
-            if hasattr(self, kernel_name):
-                print(kernel_name)
-                getattr(self, kernel_name)()
-
-        # close netCDF files
-        self.rg.close()
-        self.nc_pl_sur.close()
-        self.nc_sf.close()
-        self.nc_sa.close()
-        self.nc_to.close()
-
-    def PRESS_Pa_pl(self):
-        """
-        Surface air pressure from pressure levels.
-        """
-        # add variable to ncdf file
-        vn = 'PRESS_pl' # variable name
-        var           = self.rg.createVariable(vn,'f4',('time','station'))
-        var.long_name = 'air_pressure {} pressure levels only'.format(self.NAME)
-        var.units     = 'Pa'.encode('UTF8')
-        var.standard_name = 'surface_air_pressure'
-
-        # interpolate station by station
-        time_in = self.nc_pl_sur.variables['time'][:].astype(np.int64)
-        values  = self.nc_pl_sur.variables['air_pressure'][:]
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            #scale from hPa to Pa
-            self.rg.variables[vn][:, n] = series_interpolate(self.times_out_nc,
-                             time_in * 3600, values[:, n]) * 100
-
-    def AIRT_C_pl(self):
-        """
-        Air temperature derived from pressure levels, exclusively.
-        """
-        # add variable to ncdf file
-        vn = 'AIRT_pl' # variable name
-        var           = self.rg.createVariable(vn,'f4',('time','station'))
-        var.long_name = 'air_temperature {} pressure levels only'.format(self.NAME)
-        var.units     = self.nc_pl_sur.variables['t'].units.encode('UTF8')
-
-        # interpolate station by station
-        time_in = self.nc_pl_sur.variables['time'][:].astype(np.int64)
-        values  = self.nc_pl_sur.variables['t'][:]
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            self.rg.variables[vn][:, n] = series_interpolate(self.times_out_nc,
-                                        time_in*3600, values[:, n]-273.15)
-
-    def AIRT_C_sur(self):
-        """
-        Air temperature derived from surface data, exclusively.
-        """
-        # add variable to ncdf file
-        vn = 'AIRT_sur' # variable name
-        var           = self.rg.createVariable(vn,'f4',('time', 'station'))
-        var.long_name = '2_metre_temperature {} surface only'.format(self.NAME)
-        var.units     = self.nc_sa.variables['t2m'].units.encode('UTF8')
-
-        # interpolate station by station
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
-        values  = self.nc_sa.variables['t2m'][:]
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            self.rg.variables[vn][:, n] = series_interpolate(self.times_out_nc,
-                                                    time_in*3600,
-                                                    values[:, n]-273.15)
-
-    def AIRT_redcapp(self):
-        """
-        Air temperature derived from surface data and pressure level data as
-        shown by the method REDCAPP.
-        """
-        print("AIRT_redcapp")
-
-    def PREC_mm_sur(self):
-        """
-        Precipitation sum in mm for the time step given.
-        """
-        # add variable to ncdf file
-        vn = 'PREC_sur' # variable name
-        var           = self.rg.createVariable(vn,'f4',('time', 'station'))
-        var.long_name = 'Total precipitation {} surface only'.format(self.NAME)
-        var.units     = 'kg m-2 s-1'
-        var.standard_name = 'precipitation_amount'
-
-        # interpolate station by station
-        time_in = self.nc_sf.variables['time'][:].astype(np.int64)
-        values  = self.nc_sf.variables['tp'][:]*1000 #[mm]
-
-        nctime = self.nc_sf['time'][:]
-        self.t_unit = self.nc_sf['time'].units #"hours since 1900-01-01 00:00:0.0"
-        self.t_cal  = self.nc_sf['time'].calendar
-        time = nc.num2date(nctime, units = self.t_unit, calendar = self.t_cal)
-
-        interval_in = (time[1]-time[0]).seconds
-
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            f = interp1d(time_in*3600,
-                         cummulative2total(values[:, n], time)/interval_in,
-                         kind = 'linear')
-            self.rg.variables[vn][:, n] = f(self.times_out_nc) * self.time_step * self.scf
-
-    def RH_per_sur(self):
-        """
-        Relative humdity derived from surface data, exclusively. Clipped to
-        range [0.001, 0.999]. Kernel AIRT_ERAI_C_sur must be run before.
-        """
-        # temporary variable,  interpolate station by station
-        dewp = np.zeros((self.nt, self.nstation), dtype=np.float32)
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
-        values  = self.nc_sa.variables['d2m'][:]
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            dewp[:, n] = series_interpolate(self.times_out_nc,
-                                            time_in*3600, values[:, n]-273.15)
-
-        # add variable to ncdf file
-        vn = 'RH_sur' # variable name
-        var           = self.rg.createVariable(vn,'f4',('time', 'station'))
-        var.long_name = 'Relative humidity {} surface only'.format(self.NAME)
-        var.units     = 'percent'
-        var.standard_name = 'relative_humidity'
-
-        # simple: https://doi.org/10.1175/BAMS-86-2-225
-        RH = 100 - 5 * (self.rg.variables['AIRT_sur'][:, :]-dewp[:, :])
-        self.rg.variables[vn][:, :] = RH.clip(min=0.1, max=99.9)/100
-
-
-    def WIND_sur(self):
-        """
-        Wind speed and direction temperature derived from surface data,
-        exclusively.
-        """
-        # temporary variable, interpolate station by station
-        U = np.zeros((self.nt, self.nstation), dtype=np.float32)
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
-        values  = self.nc_sa.variables['u10'][:]
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            U[:, n] = series_interpolate(self.times_out_nc,
-                                         time_in*3600, values[:, n])
-
-        # temporary variable, interpolate station by station
-        V = np.zeros((self.nt, self.nstation), dtype=np.float32)
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
-        values  = self.nc_sa.variables['v10'][:]
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            V[:, n] = series_interpolate(self.times_out_nc,
-                                         time_in*3600, values[:, n])
-
-        # wind speed, add variable to ncdf file, convert
-        vna = 'WSPD_sur' # variable name
-        var           = self.rg.createVariable(vna,'f4',('time', 'station'))
-        var.long_name = '10 wind speed {} surface only'.format(self.NAME)
-        var.units     = 'm s-1'
-        var.standard_name = 'wind_speed'
-
-        # wind direction, add variable to ncdf file, convert, relative to North
-        vnb = 'WDIR_sur' # variable name
-        var           = self.rg.createVariable(vnb,'f4',('time', 'station'))
-        var.long_name = '10 wind direction {} surface only'.format(self.NAME)
-        var.units     = 'degree'
-        var.standard_name = 'wind_from_direction'
-
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            WS = np.sqrt(np.power(V,2) + np.power(U,2))
-            WD = [atan2(V[i, n], U[i, n])*(180/pi) +
-                  180 for i in np.arange(V.shape[0])]
-            self.rg.variables[vna][:, n] = WS
-            self.rg.variables[vnb][:,n] = WD
-
-    def SW_Wm2_sur(self):
-        """
-        Short-wave downwelling radiation derived from surface data, exclusively.
-        This kernel only interpolates in time.
-        """
-
-        # add variable to ncdf file
-        vn = 'SW_sur' # variable name
-        var           = self.rg.createVariable(vn,'f4',('time', 'station'))
-        var.long_name = 'Surface solar radiation downwards {} surface only'.format(self.NAME)
-        var.units     = self.nc_sf.variables['ssrd'].units.encode('UTF8')
-        var.standard_name = 'surface_downwelling_shortwave_flux'
-
-        # interpolation scale factor
-        time_in = self.nc_sf.variables['time'][:].astype(np.int64)
-        values  = self.nc_sf.variables['ssrd'][:]/3600 #w/m2
-
-        nctime = self.nc_sf['time'][:]
-        self.t_unit = self.nc_sf['time'].units #"hours since 1900-01-01 00:00:0.0"
-        self.t_cal  = self.nc_sf['time'].calendar
-        time = nc.num2date(nctime, units = self.t_unit, calendar = self.t_cal)
-
-        interval_in = (time[1]-time[0]).seconds
-
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            f = interp1d(time_in*3600,
-                         cummulative2total(values[:, n], time)/interval_in,
-                         kind = 'linear')
-            self.rg.variables[vn][:, n] = f(self.times_out_nc) * self.time_step
-
-
-    def LW_Wm2_sur(self):
-        """
-        Long-wave downwelling radiation derived from surface data, exclusively.
-        This kernel only interpolates in time.
-        """
-
-        # add variable to ncdf file
-        vn = 'LW_sur' # variable name
-        var           = self.rg.createVariable(vn,'f4',('time', 'station'))
-        var.long_name = 'Surface thermal radiation downwards {} surface only'.format(self.NAME)
-        var.units     = 'W m-2'
-        var.standard_name = 'surface_downwelling_longwave_flux'
-
-        # interpolate station by station
-        time_in = self.nc_sf.variables['time'][:].astype(np.int64)
-        values  = self.nc_sf.variables['strd'][:]/3600 #[w m-2]
-
-        nctime = self.nc_sf['time'][:]
-        self.t_unit = self.nc_sf['time'].units #"hours since 1900-01-01 00:00:0.0"
-        self.t_cal  = self.nc_sf['time'].calendar
-        time = nc.num2date(nctime, units = self.t_unit, calendar = self.t_cal)
-
-        # interpolation scale factor
-        interval_in = (time[1]-time[0]).seconds
-
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            f = interp1d(time_in*3600,
-                         cummulative2total(values[:, n], time)/interval_in,
-                         kind='linear')
-            self.rg.variables[vn][:, n] = f(self.times_out_nc)*self.time_step
-
-    def SH_kgkg_sur(self):
-        '''
-        Specific humidity [kg/kg]
-        https://crudata.uea.ac.uk/cru/pubs/thesis/2007-willett/2INTRO.pdf
-        '''
-        # add variable to ncdf file
-        vn = 'SH_sur' # variable name
-        var           = self.rg.createVariable(vn,'f4',('time', 'station'))
-        var.long_name = 'Specific humidity {} surface only'.format(self.NAME)
-        var.units     = '1'
-        var.standard_name = 'specific_humidity'
-
-        # temporary variable, interpolate station by station
-        dewp = np.zeros((self.nt, self.nstation), dtype=np.float32)
-        time_in = self.nc_sa.variables['time'][:].astype(np.int64)
-        values  = self.nc_sa.variables['d2m'][:]
-        for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-            dewp[:, n] = series_interpolate(self.times_out_nc,
-                                            time_in*3600, values[:, n]-273.15)
-
-        # compute
-        SH = spec_hum_kgkg(dewp[:, :],
-                           self.rg.variables['PRESS_pl'][:, :])
-
-
-        self.rg.variables[vn][:, :] = SH
-
-    def LW_Wm2_topo(self):
-        """
-        Long-wave radiation downwards [W/m2]
-        https://www.geosci-model-dev.net/7/387/2014/gmd-7-387-2014.pdf
-        """
-        # get sky view, and interpolate in time
-        N = np.asarray(list(self.stations['sky_view'][:]))
-
-        # add variable to ncdf file
-        vn = 'LW_topo' # variable name
-        var           = self.rg.createVariable(vn,'f4',('time', 'station'))
-        var.long_name = 'Incoming long-wave radiation {} surface only'.format(self.NAME)
-        var.units     = 'W m-2'
-        var.standard_name = 'surface_downwelling_longwave_flux'
-
-        # compute
-        for i in range(0, len(self.rg.variables['RH_sur'][:])):
-            for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-                LW = LW_downward(self.rg.variables['RH_sur'][i, n],
-                     self.rg.variables['AIRT_sur'][i, n]+273.15, N[n])
-                self.rg.variables[vn][i, n] = LW
diff --git a/globsim/scale/GenericScale.py b/globsim/scale/GenericScale.py
index e49d4d1..427b5ee 100755
--- a/globsim/scale/GenericScale.py
+++ b/globsim/scale/GenericScale.py
@@ -168,7 +168,7 @@ def set_time_scale(self, time_variable, time_step):
         t1 = nc.num2date(nctime[1], units=self.t_unit, calendar=self.t_cal)
         t0 = nc.num2date(nctime[0], units=self.t_unit, calendar=self.t_cal)
         self.interval_in = (t1 - t0).seconds
-
+        
         # number of time steps
         self.nt = floor((self.max_time - self.min_time).total_seconds() / (3600 * time_step)) + 1
         logger.debug(f"Output time array has {self.nt} elements between "
diff --git a/globsim/scale/JRA3Qscale.py b/globsim/scale/JRA3Qscale.py
index ec52027..f13fdf4 100644
--- a/globsim/scale/JRA3Qscale.py
+++ b/globsim/scale/JRA3Qscale.py
@@ -1,9 +1,9 @@
 from globsim.scale.JRAscale import JRAscale
 
 
-class J3QS(JRAscale):
-    NAME = "JRA-3Q"
-    REANALYSIS = "jra3q"
+class J3QSjma(JRAscale):
+    NAME = "JRA-3Q-JMA"
+    REANALYSIS = "jra3q-jma"
 
     # translator for JRA-55 names
     J55T = {"sa":{"Temperature": "2t",
@@ -15,15 +15,24 @@ class J3QS(JRAscale):
                   'Downward solar radiation flux': "dswrf",
                   'Downward longwave radiation flux': "dswrf",},
             "pl":{"Temperature": "t",
-                  }}
+                  },
+            "pl_sur":{},
+            "to":{}}
 
     SCALING = {"sa": {},
                "sf": {},
-               "pl": {}}
-    
-    def get_name(self, file:str, jra55name:str) -> str:
-        n = self.J55T[file].get(jra55name)
-        if n is None:
-            return jra55name
-        else:
-            return n
+               "pl": {},
+               "pl_sur": {},
+               "to": {}}
+
+
+class J3QS(JRAscale):
+    NAME = "JRA-3Q"
+    REANALYSIS = "jra3q"
+
+
+
+class J3QgS(JRAscale):
+    NAME = "JRA-3QG"
+    REANALYSIS = "jra3qg"
+
diff --git a/globsim/scale/JRAscale.py b/globsim/scale/JRAscale.py
index 4051efa..4a269ef 100755
--- a/globsim/scale/JRAscale.py
+++ b/globsim/scale/JRAscale.py
@@ -169,7 +169,7 @@ def PRESS_Pa_pl(self):
         var.standard_name = 'surface_air_pressure'
 
         # interpolate station by station
-        time_in = self.get_values("pl_sur","time").astype(np.int64)
+        time_in = self.get_values("pl_sur", "time").astype(np.int64)
         values  = self.get_values("pl_sur", "air_pressure")
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             # scale from hPa to Pa
@@ -207,7 +207,7 @@ def AIRT_C_sur(self):
         var.standard_name = 'air_temperature'
 
         # interpolate station by station
-        time_in = self.get_values("sa","time")
+        time_in = self.get_values("sa", "time")
         values  = self.get_values("sa", "Temperature")
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             self.rg.variables[vn][:, n] = np.interp(self.times_out_nc,
@@ -330,13 +330,9 @@ def WIND_sur(self):
         V = self.rg.variables['10 metre V wind component'][:]
         U = self.rg.variables['10 metre U wind component'][:]
         
-        
         WS = np.sqrt(np.power(V, 2) + np.power(U, 2))
         WD = 90 - (np.arctan2(V, U) * (180 / np.pi)) + 180
-
-        #for n, s in enumerate(self.rg.variables['station'][:].tolist()):
-         #   WD = [atan2(V[i, n], U[i, n]) * (180 / pi) +
-          #        180 for i in np.arange(V.shape[0])]
+        WD = np.mod(WD, 360)
 
         self.rg.variables['WSPD_sur'][:] = WS
         self.rg.variables['WDIR_sur'][:] = WD
diff --git a/globsim/scale/MERRAscale.py b/globsim/scale/MERRAscale.py
index c370d1d..2b74ce1 100755
--- a/globsim/scale/MERRAscale.py
+++ b/globsim/scale/MERRAscale.py
@@ -391,7 +391,7 @@ def WIND_sur(self):
         for n, s in enumerate(self.rg.variables['station'][:].tolist()):
             WS = np.sqrt(np.power(V[:, n], 2) + np.power(U[:, n], 2))
             WD = [atan2(V[i, n], U[i, n]) * (180 / pi) + 180 for i in np.arange(V.shape[0])]
-
+            WD = np.mod(WD, 360)
             self.rg.variables['WSPD_sur'][:, n] = WS
             self.rg.variables['WDIR_sur'][:,n] = WD
 
diff --git a/globsim/scale/__init__.py b/globsim/scale/__init__.py
index cbc517f..3c6da41 100644
--- a/globsim/scale/__init__.py
+++ b/globsim/scale/__init__.py
@@ -1,12 +1,12 @@
 from .ERA5scale import ERA5scale
 from .ERA5Escale import ERA5Escale
-from .ERAIscale import ERAIscale
 from .JRAscale import JRAscale
-from .JRA3Qscale import J3QS
+from .JRA3Qscale import J3QS, J3QgS
 from .MERRAscale import MERRAscale
 
 __all__ = ["ERA5scale",
            "ERA5Escale",
-           "ERAIscale",
            "JRAscale",
-           "MERRAscale"]
+           "MERRAscale",
+           "J3QS",
+           "J3QgS"]
diff --git a/setup.py b/setup.py
index 9cd8f2d..7f243fc 100644
--- a/setup.py
+++ b/setup.py
@@ -38,14 +38,16 @@
         ]},
 
     install_requires=['lxml',
-                      'numpy',
+                      'numpy==1.26.4',
                       'pandas',
                       'netCDF4',
                       'scipy',
-                      'pydap==3.2.2',  # TODO: resolve the change to open_dods_url and make sure it works
+                      'dask',
+                      'xarray',
+                      'pydap>=3.4.1',  # TODO: resolve the change to open_dods_url and make sure it works
                       'tomlkit',
-                      'ecmwf-api-client',
                       'cdsapi',
+                      'psutil',
                       'pysolar',
                       'pytz']
 )


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 7884df1..a73d202 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.11,3.12]
 
     steps:
     - uses: actions/checkout@v2
@@ -34,6 +34,7 @@ jobs:
         sudo apt-get install libhdf5-serial-dev netcdf-bin libnetcdf-dev
         python -m pip install --upgrade pip
         pip install flake8
+        python -m pip install setuptools
         python -m pip install -r requirements.txt
         python -m pip install pygrib 
         python -m pip install xarray