#!/usr/bin/env python
u"""
reanalysis_geopotential_heights.py
Written by Tyler Sutterley (10/2021)
Reads temperature and specific humidity data to calculate geopotential height
    and pressure difference fields at half levels from reanalysis

INPUTS:
    Reanalysis model to run
    ERA-Interim: http://apps.ecmwf.int/datasets/data/interim-full-moda
    ERA5: http://apps.ecmwf.int/data-catalogues/era5/?class=ea
    MERRA-2: https://gmao.gsfc.nasa.gov/reanalysis/MERRA-2/

COMMAND LINE OPTIONS:
    -D X, --directory X: Working data directory
    -Y X, --year X: years to run
    -V, --verbose: Output information for each output file
    -M X, --mode X: Permission mode of directories and files

PYTHON DEPENDENCIES:
    numpy: Scientific Computing Tools For Python
        https://numpy.org
        https://numpy.org/doc/stable/user/numpy-for-matlab-users.html
    netCDF4: Python interface to the netCDF C library
        https://unidata.github.io/netcdf4-python/netCDF4/index.html

PROGRAM DEPENDENCIES:
    utilities.py: download and management utilities for files

UPDATE HISTORY:
    Updated 10/2021: using python logging for handling verbose output
    Updated 07/2021: can use input files to define command line arguments
        added check for ERA5 expver dimension (denotes mix of ERA5 and ERA5T)
    Updated 05/2021: define int/float precision to prevent deprecation warning
    Updated 03/2021: automatically update years to run based on current time
    Updated 01/2021: read from netCDF4 file in slices to reduce memory load
    Updated 12/2020: using argparse to set command line options
    Updated 01/2020: outputs variables as 32-bit floats instead of 64-bit floats
        clear variables and iterate years to reduce required memory
        iterate over time variable to calculate heights using incomplete files
    Updated 08/2019: adjust time scale variable for MERRA-2
    Updated 07/2018: added parameters for ERA5
    Written 03/2018
"""
from __future__ import print_function

import os
import re
import time
import logging
import netCDF4
import argparse
import numpy as np
import gravity_toolkit.utilities as utilities

#-- PURPOSE: reads temperature and specific humidity data to calculate
#-- geopotential height fields at half levels from reanalysis
def reanalysis_geopotential_heights(base_dir, MODEL, YEAR=None,
    VERBOSE=False, MODE=0o775):

    #-- create logger for verbosity level
    loglevel = logging.INFO if VERBOSE else logging.CRITICAL
    logging.basicConfig(level=loglevel)

    #-- directory setup
    ddir = os.path.join(base_dir,MODEL)
    #-- set model specific parameters
    if (MODEL == 'ERA-Interim'):
        #-- invariant parameters file
        input_invariant_file = 'ERA-Interim-Invariant-Parameters.nc'
        #-- coordinate parameters file
        input_coordinate_file = 'ERA-Interim_coordvars.nc'
        #-- surface pressure file format
        input_pressure_file = 'ERA-Interim-Monthly-SP-{0:4d}.nc'
        #-- regular expression pattern for finding files
        regex_pattern = r'ERA\-Interim\-Monthly\-Levels\-({0})\.nc$'
        #-- output file format
        output_file_format = 'ERA-Interim-GPH-Levels-{0:4d}.nc'
        SURFNAME = 'z'
        ZNAME = 'z'
        VARNAME = 'sp'
        TNAME = 't'
        QNAME = 'q'
        DIFFNAME = 'dp'
        LONNAME = 'longitude'
        LATNAME = 'latitude'
        TIMENAME = 'time'
        LEVELNAME = 'lvl'
        ANAME,BNAME = ('a_model_alt','b_model_alt')
        AINTERFACE,BINTERFACE = ('a_interface','b_interface')
        #-- hours since 1900-01-01 00:00:0.0
        TIME_LONGNAME = 'Time'
        UNITS = 'm**2 s**-2'
        GRAVITY = 1.0
    elif (MODEL == 'ERA5'):
        #-- invariant parameters file
        input_invariant_file = 'ERA5-Invariant-Parameters.nc'
        #-- coordinate parameters file
        input_coordinate_file = 'ERA5_coordvars.nc'
        #-- surface pressure file format
        input_pressure_file = 'ERA5-Monthly-SP-{0:4d}.nc'
        #-- regular expression pattern for finding files
        regex_pattern = r'ERA5\-Monthly\-Levels\-({0})\.nc$'
        #-- output file format
        output_file_format = 'ERA5-GPH-Levels-{0:4d}.nc'
        SURFNAME = 'z'
        ZNAME = 'z'
        VARNAME = 'sp'
        TNAME = 't'
        QNAME = 'q'
        DIFFNAME = 'dp'
        LONNAME = 'longitude'
        LATNAME = 'latitude'
        TIMENAME = 'time'
        LEVELNAME = 'lvl'
        ANAME,BNAME = ('a_half','b_half')
        AINTERFACE,BINTERFACE = ('a_interface','b_interface')
        #-- hours since 1900-01-01 00:00:0.0
        TIME_LONGNAME = 'Time'
        UNITS = 'm**2 s**-2'
        GRAVITY = 1.0
    elif (MODEL == 'MERRA-2'):
        #-- invariant parameters file
        input_invariant_file = 'MERRA2_101.const_2d_asm_Nx.00000000.nc4'
        #-- coordinate parameters file
        input_coordinate_file = 'MERRA2_101.Coords_Nx.00000000.nc'
        #-- regular expression pattern for finding files
        regex_pattern = r'MERRA2_(\d+).instM_3d_ana_Nv.({0})(\d{{2}}).SUB.nc$'
        #-- output file format
        output_file_format='MERRA2_{0:0.0f}.GPH_levels.{1:4.0f}{2:02.0f}.SUB.nc'
        SURFNAME = 'PHIS'
        ZNAME = 'PHIS'
        VARNAME = 'PS'
        TNAME = 'T'
        QNAME = 'QV'
        DIFFNAME = 'dP'
        LONNAME = 'lon'
        LATNAME = 'lat'
        TIMENAME = 'time'
        LEVELNAME = 'lev'
        ANAME,BNAME = ('a_half','b_half')
        AINTERFACE,BINTERFACE = ('a_interface','b_interface')
        #-- minutes since start of file
        TIME_LONGNAME = 'Time'
        UNITS = 'm+2 s-2'
        GRAVITY = 1.0

    #-- read model orography for dimensions
    geopotential,lon,lat=ncdf_invariant(os.path.join(ddir,input_invariant_file),
        LONNAME,LATNAME,SURFNAME)
    #-- read parameters for calculating pressures at levels
    lev,A,B,AI,BI=ncdf_coordinates(os.path.join(ddir,input_coordinate_file),
        LEVELNAME,ANAME,BNAME,AINTERFACE,BINTERFACE)
    #-- Gas constant for dry air
    R_dry = 287.06

    #-- read each reanalysis pressure field for each year
    regex_years = r'\d{4}' if (YEAR is None) else '|'.join(map(str,YEAR))
    rx = re.compile(regex_pattern.format(regex_years), re.VERBOSE)
    input_files = [fi for fi in os.listdir(ddir) if rx.match(fi)]
    #-- for each reanalysis file
    for fi in sorted(input_files):
        #-- read input temperature and specific humidity data
        fid1 = netCDF4.Dataset(os.path.join(ddir,fi),'r')
        #-- extract shape from temperature variable
        ntime,nlevels,nlat,nlon = fid1.variables[TNAME].shape
        #-- invalid value
        fill_value = fid1.variables[TNAME]._FillValue
        #-- save output variables into a python dictionary.
        dinput = {}
        dinput[ZNAME] = np.zeros((ntime,nlevels,nlat,nlon),dtype=np.float32)
        dinput[DIFFNAME] = np.zeros((ntime,nlevels,nlat,nlon),dtype=np.float32)
        #-- model levels in reverse order
        dinput[LEVELNAME] = lev[::-1].copy()
        #-- extract time and time units
        dinput[TIMENAME] = np.copy(fid1.variables[TIMENAME][:])
        TIME_UNITS = fid1.variables[TIMENAME].units
        dinput[LONNAME] = lon.copy()
        dinput[LATNAME] = lat.copy()

        if MODEL in ('MERRA-2'):
            #-- extract date from monthly files
            MOD,YEAR,MONTH = np.array(rx.findall(fi).pop(), dtype=np.float64)
            #-- output monthly filename
            FILE = os.path.join(ddir,output_file_format.format(MOD,YEAR,MONTH))
            #-- read surface pressure
            surface_pressure = np.copy(fid1.variables[VARNAME][:])
        elif MODEL in ('ERA-Interim','ERA5'):
            #-- extract year from file name
            YEAR, = np.array(rx.findall(fi),dtype=np.int64)
            #-- output yearly filename
            FILE = os.path.join(ddir,output_file_format.format(YEAR))
            #-- read input surface pressure data
            pressure_file = input_pressure_file.format(YEAR)
            with netCDF4.Dataset(os.path.join(ddir,pressure_file),'r') as fid2:
                surface_pressure = np.copy(fid2.variables[VARNAME][:])

        #-- iterate over dates
        for t in range(ntime):
            #-- check dimensions for expver slice
            if (fid1.variables[VARNAME].ndim == 5):
                t_time,q_time = ncdf_expver(fid1,t,TNAME,QNAME)
            else:
                #-- temperature and specific humidity
                #-- reverse layers so bottom=0
                t_time = fid1.variables[TNAME][t,::-1,:,:]
                q_time = fid1.variables[QNAME][t,::-1,:,:]
            #-- calculate geopotential over model levels
            geopotential_height = np.empty((nlat,nlon),dtype=np.float32)
            #-- start with surface geopotential converted to units (m^2/s^2)
            geopotential_height[:,:] = geopotential*GRAVITY
            #-- Integrate the model layers in the atmosphere
            for k in range(nlevels):
                #-- calculate virtual temperature
                virtual_temp = (1.0 + 0.609133*q_time[k,:,:])*t_time[k,:,:]
                #-- calculate numerator and denominator for pressure ratio
                Pnum = A[k] + B[k]*surface_pressure[t,:,:]
                if ((k+1) == nlevels):
                    Pdom = 0.1
                else:
                    Pdom = A[k+1] + B[k+1]*surface_pressure[t,:,:]
                #-- add level to geopotential_levels
                geopotential_height[:,:] += R_dry*virtual_temp*np.log(Pnum/Pdom)
                #-- save level to output variable and convert to output units
                dinput[ZNAME][t,k,:,:] = geopotential_height/GRAVITY
                #-- calculate pressure difference between levels (at interfaces)
                Plower = AI[k] + BI[k]*surface_pressure[t,:,:]
                Pupper = AI[k+1] + BI[k+1]*surface_pressure[t,:,:]
                dinput[DIFFNAME][t,k,:,:] = Pupper - Plower

        #-- save to file
        ncdf_geopotential_write(dinput, fill_value, FILENAME=FILE, ZNAME=ZNAME,
            LEVELNAME=LEVELNAME, DIFFNAME=DIFFNAME, LONNAME=LONNAME,
            LATNAME=LATNAME, TIMENAME=TIMENAME, TIME_UNITS=TIME_UNITS,
            TIME_LONGNAME=TIME_LONGNAME, UNITS=UNITS)
        #-- set the permissions level of the output file to MODE
        os.chmod(FILE, MODE)
        #-- clear dinput dictionary variable
        dinput = None
        #-- close the input netCDF4 file
        fid1.close()

#-- PURPOSE: Compute the Specific Humidity from parameters (Bolton 1980)
#-- http://cires1.colorado.edu/~voemel/vp.html
#-- https://www.eol.ucar.edu/projects/ceop/dm/documents/refdata_report/eqns.html
#-- https://github.com/NCAR/ncl/blob/master/ni/src/lib/nfpfort/mixhum_ptrh.f
def calculate_specific_humidity(P, T, RH):
    #-- ratio of the molecular weights of water vapor to dry air
    epsilon = 0.622
    #-- calibration pressure and temperature
    pc = 6.112
    tc = 243.5
    #-- calculate the saturation vapor pressure in mb
    Es = pc * np.exp((17.67 * T)/(T + tc))
    #-- calculate the vapor pressure in mb
    Ev = Es * (RH/100.0)
    #-- calculate the dew point temperature
    Td = np.log(Ev/pc) * tc/(17.67 - np.log(Ev/pc))
    #-- calculate the specific humidity
    Q = (epsilon * Ev)/(P/100.0 - (0.378 * Ev))
    return (Q,Td)

#-- PURPOSE: extract temperature and specific humidity variables
#-- from a 5d netCDF4 dataset
#-- ERA5 expver dimension (denotes mix of ERA5 and ERA5T)
def ncdf_expver(fileID, slice, TNAME, QNAME):
    ntime,nexp,nlevel,nlat,nlon = fileID.variables[TNAME].shape
    fill_value = fileID.variables[TNAME]._FillValue
    #-- reduced temperature and specific humidity for time
    temperature = np.ma.zeros((nlevel,nlat,nlon))
    temperature.fill_value = fill_value
    humidity = np.ma.zeros((nlevel,nlat,nlon))
    humidity.fill_value = fill_value
    #-- iterate over expver slices to find valid outputs
    for j in range(nexp):
        #-- check if any are valid for expver
        if np.any(fileID.variables[TNAME][slice,j,:,:,:]):
            #-- reverse layers so bottom=0
            temperature[:,:,:] = fileID.variables[TNAME][slice,j,::-1,:,:]
            humidity[:,:,:] = fileID.variables[QNAME][slice,j,::-1,:,:]
    #-- update mask variables
    temperature.mask = (temperature.data == temperature.fill_value)
    humidity.mask = (humidity.data == humidity.fill_value)
    #-- return the reduced temperature and specific humidity variables
    return (temperature,humidity)

#-- PURPOSE: read reanalysis invariant parameters (geopotential,lat,lon)
def ncdf_invariant(FILENAME,LONNAME,LATNAME,ZNAME):
    with netCDF4.Dataset(FILENAME,'r') as fileID:
        geopotential = fileID.variables[ZNAME][:].squeeze()
        longitude = fileID.variables[LONNAME][:].copy()
        latitude = fileID.variables[LATNAME][:].copy()
    return (geopotential,longitude,latitude)

#-- PURPOSE: read reanalysis coordinate parameters
#-- reverse order to go from surface to top-of-atmosphere
def ncdf_coordinates(FILENAME,LEVELNAME,ANAME,BNAME,AINTERFACE,BINTERFACE):
    with netCDF4.Dataset(FILENAME,'r') as fileID:
        levels = fileID.variables[LEVELNAME][:].copy()
        A = fileID.variables[ANAME][::-1].copy()
        B = fileID.variables[BNAME][::-1].copy()
        AI = fileID.variables[AINTERFACE][::-1].copy()
        BI = fileID.variables[BINTERFACE][::-1].copy()
    return (levels,A,B,AI,BI)

#-- PURPOSE: write output geopotential fields data to file
def ncdf_geopotential_write(dinput, fill_value, FILENAME=None, ZNAME=None,
    DIFFNAME=None, LEVELNAME=None, LONNAME=None, LATNAME=None, TIMENAME=None,
    TIME_UNITS=None, TIME_LONGNAME=None, UNITS=None):
    #-- opening NetCDF file for writing
    fileID = netCDF4.Dataset(FILENAME, 'w', format="NETCDF4")

    #-- Defining the NetCDF dimensions
    for key in [LONNAME,LATNAME,TIMENAME,LEVELNAME]:
        fileID.createDimension(key, len(dinput[key]))

    #-- defining the NetCDF variables
    nc = {}
    nc[LATNAME]=fileID.createVariable(LATNAME,dinput[LATNAME].dtype,(LATNAME,))
    nc[LONNAME]=fileID.createVariable(LONNAME,dinput[LONNAME].dtype,(LONNAME,))
    nc[TIMENAME]=fileID.createVariable(TIMENAME,dinput[TIMENAME].dtype,(TIMENAME,))
    nc[LEVELNAME]=fileID.createVariable(LEVELNAME,dinput[LEVELNAME].dtype,(LEVELNAME,))
    nc[DIFFNAME] = fileID.createVariable(DIFFNAME, dinput[DIFFNAME].dtype,
        (TIMENAME,LEVELNAME,LATNAME,LONNAME,), fill_value=fill_value, zlib=True)
    nc[ZNAME] = fileID.createVariable(ZNAME, dinput[ZNAME].dtype,
        (TIMENAME,LEVELNAME,LATNAME,LONNAME,), fill_value=fill_value, zlib=True)
    #-- filling NetCDF variables
    for key,val in dinput.items():
        nc[key][:] = dinput[key].copy()
        dinput[key] = None

    #-- Defining attributes for longitude and latitude
    nc[LONNAME].long_name = 'Longitude'
    nc[LONNAME].units = 'degrees_east'
    nc[LATNAME].long_name = 'Latitude'
    nc[LATNAME].units = 'degrees_north'
    #-- Defining attributes for time
    nc[TIMENAME].units = TIME_UNITS
    nc[TIMENAME].long_name = TIME_LONGNAME
    #-- Definining attributes for model levels
    nc[LEVELNAME].long_name = 'Model_Level_Number'
    #-- Defining attributes for geopotential height
    nc[ZNAME].long_name = 'Geopotential_Heights_on_Model_Levels'
    nc[ZNAME].units = UNITS
    #-- Defining attributes for pressure differences
    nc[DIFFNAME].long_name = 'Pressure_Differences_between_Levels'
    nc[DIFFNAME].units = 'Pa'

    #-- Output NetCDF structure information
    logging.info(os.path.basename(FILENAME))
    logging.info(list(fileID.variables.keys()))

    #-- Closing the NetCDF file
    fileID.close()
    #-- clear nc dictionary variable
    nc = None

#-- Main program that calls reanalysis_geopotential_heights()
def main():
    #-- Read the system arguments listed after the program
    parser = argparse.ArgumentParser(
        description="""Reads temperature and specific humidity data
            to calculate geopotential height and pressure difference
            fields at half levels from reanalysis
            """,
        fromfile_prefix_chars="@"
    )
    parser.convert_arg_line_to_args = utilities.convert_arg_line_to_args
    #-- command line parameters
    choices = ['ERA-Interim','ERA5','MERRA-2']
    parser.add_argument('model',
        metavar='MODEL', type=str, nargs='+',
        default=['ERA5','MERRA-2'], choices=choices,
        help='Reanalysis Model')
    #-- working data directory
    parser.add_argument('--directory','-D',
        type=lambda p: os.path.abspath(os.path.expanduser(p)),
        default=os.getcwd(),
        help='Working data directory')
    #-- years to run
    now = time.gmtime()
    parser.add_argument('--year','-Y',
        type=int, nargs='+', default=range(2000,now.tm_year+1),
        help='Years of model outputs to run')
    #-- print information about each input and output file
    parser.add_argument('--verbose','-V',
        default=False, action='store_true',
        help='Verbose output of run')
    #-- permissions mode of the local directories and files (number in octal)
    parser.add_argument('--mode','-M',
        type=lambda x: int(x,base=8), default=0o775,
        help='Permission mode of directories and files')
    args,_ = parser.parse_known_args()

    #-- for each reanalysis model
    for MODEL in args.model:
        #-- run program
        reanalysis_geopotential_heights(args.directory, MODEL, YEAR=args.year,
            VERBOSE=args.verbose, MODE=args.mode)

#-- run main program
if __name__ == '__main__':
    main()


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:light
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.11.3
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # 2021 Oceanography Camp for Girls Saildrone Lesson
# Developed by Nancy Williams, Veronica Tamsitt, Nicola Guisewhite at University of South Florida College of Marine Science

# ## To Do List:
# * remove all fronts except SAF and SBDY for simplicity
# * make a function for plotting instead of copy/pasting the map each time (or not, if we want to keep it simple)
# * add more markdown in the form of instructions, pictures, pulling variables out into their own cell so girls know where they can make changes to the code
# * in figure titles and filenames, change the variables from using the first four characters (currently var[:4]) to instead cutting off at the first underscore
# * try plotting previous 8-day chl-a snapshot to see if it has better coverage for the eddy crossing
# * edit to make it easy to adjust time series x-axis limits
# * Check Veronica's carbon flux calculation is correct (Nancy)

# ## Data Sources:
# * (too big for Binder so had to remove it) Saildrone 1-minute physical and ADCP data available from: https://data.saildrone.com/data/sets/antarctica-circumnavigation-2019
# (login required, so cannot be accessed using an FTP. Will need to download ahead)
# * Saildrone hourly-ish CO2, pH, and physical data available from: https://www.ncei.noaa.gov/access/metadata/landing-page/bin/iso?id=gov.noaa.nodc:0221912
# * Satellite Chlorophyll: https://neo.sci.gsfc.nasa.gov/view.php?datasetId=MY1DMW_CHLORA&year=2019
# * SSH: https://cds.climate.copernicus.eu/cdsapp#!/dataset/satellite-sea-level-global?tab=overview
# (login required for chla and SSH, download ahead of time. Can also be downloaded using motuclient, login also required https://github.com/clstoulouse/motu-client-python)

# +
# Import the tools you need
import os
import numpy as np
import pandas as pd
import datetime as dt
import xarray as xr
import matplotlib.pyplot as plt
import matplotlib.path as mpath
import cartopy.crs as ccrs
import cartopy.feature
from datetime import datetime

# add something
# -

# Set the paths
output_dir = 'Output/'
data_dir = 'Data/'

# Go and download the hourly Saildrone CO2 data and put it in the `Data/` folder
os.chdir(data_dir) # Change the directory to the `Data/` folder
os.getcwd() # Check that you're now in the `Data/` folder
# Curl downloads the data files directly from the web and shows you the status while it works.
# `!` at the beginning of the line tells you that this command is a unix shell command (not python code)
# ! curl -o 32DB20190119_ASV_Saildrone1020_Antarctic_Jan2019_Aug2019.csv https://www.ncei.noaa.gov/data/oceans/ncei/ocads/data/0221912/32DB20190119_ASV_Saildrone1020_Antarctic_Jan2019_Aug2019.csv
os.chdir("..") # Use ".." to move back up one directory now that we've imported the MLD climatology data

# Import the hourly Saildrone CO2 data file
Saildrone_CO2 = pd.read_csv(
    (data_dir + '32DB20190119_ASV_Saildrone1020_Antarctic_Jan2019_Aug2019.csv'),
    header=4,
    na_values=-999,
)

# Create a datetime object
Saildrone_CO2['datetime'] = pd.to_datetime(Saildrone_CO2['Date'] + ' ' + Saildrone_CO2['Time'])
# Check that the Saildrone data was imported correctly
Saildrone_CO2

# Import the Southern Ocean fronts for mapping
stf = pd.read_csv(data_dir + 'fronts/stf.txt', header=None, sep='\s+',
                  na_values='%', names=['lon','lat'])
saf = pd.read_csv(data_dir + 'fronts/saf.txt', header=None, sep='\s+',
                  na_values='%', names=['lon','lat'])
pf = pd.read_csv(data_dir + 'fronts/pf.txt', header=None, sep='\s+',
                 na_values='%', names=['lon','lat'])
saccf = pd.read_csv(data_dir + 'fronts/saccf.txt', header=None, sep='\s+',
                    na_values='%', names=['lon','lat'])
sbdy = pd.read_csv(data_dir + 'fronts/sbdy.txt', header=None, sep='\s+',
                   na_values='%', names=['lon','lat'])

# +
# Plot the Saildrone track on a map

# Make the "bones" of the figure
plt.figure(figsize=(10, 10))
ax = plt.axes(projection=ccrs.SouthPolarStereo())
ax.set_extent([-180, 180, -90, -30],ccrs.PlateCarree())
ax.add_feature(cartopy.feature.LAND)
ax.add_feature(cartopy.feature.OCEAN, color='lightblue')
ax.gridlines()

# Compute a circle in axes coordinates, which we can use as a boundary
# for the map. We can pan/zoom as much as we like - the boundary will be
# permanently circular.
theta = np.linspace(0, 2 * np.pi, 100)
center, radius = [0.5, 0.5], 0.5
verts = np.vstack([np.sin(theta), np.cos(theta)]).T
circle = mpath.Path(verts * radius + center)

# Plot the ACC fronts in various colors
ax.set_boundary(circle, transform=ax.transAxes)
plt.plot(stf['lon'], stf['lat'], color='Red', transform=ccrs.PlateCarree(),
         label = 'Subtropical Front')
plt.plot(saf['lon'], saf['lat'], color='Orange', transform=ccrs.PlateCarree(),
         label = 'Subantarctic Front')
plt.plot(pf['lon'], pf['lat'], color='Yellow', transform=ccrs.PlateCarree(),
         label = 'Polar Front')
plt.plot(saccf['lon'], saccf['lat'], color='Green', transform=ccrs.PlateCarree(),
         label = 'Southern ACC Front')
plt.plot(sbdy['lon'], sbdy['lat'], color='Blue', transform=ccrs.PlateCarree(),
         label = 'Southern Boundary of ACC')

# Plot the Saildrone in black dots
plt.scatter(Saildrone_CO2.Longitude, Saildrone_CO2.Latitude,
           transform=ccrs.PlateCarree(), c='black', s=3, label='Saildrone', zorder=1000)

# Turn on the legend
plt.legend()

# Save the figure in the output folder
plt.title('2019 Saildrone Antarctic Circumnavigation Track')
plt.savefig(output_dir + 'SaildroneMap' + '.jpg') # Changing the suffix will change the format
plt.show()

# +
# Now plot some variable "var" on the map with colored dots
var = 'SST (C)'

# Make the "bones" of the figure
plt.figure(figsize=(10, 10))
ax = plt.axes(projection=ccrs.SouthPolarStereo())
ax.set_extent([-180, 180, -90, -30],ccrs.PlateCarree())
ax.add_feature(cartopy.feature.LAND)
ax.add_feature(cartopy.feature.OCEAN, color='lightblue')
ax.gridlines()

# Compute a circle in axes coordinates, which we can use as a boundary
# for the map. We can pan/zoom as much as we like - the boundary will be
# permanently circular.
theta = np.linspace(0, 2 * np.pi, 100)
center, radius = [0.5, 0.5], 0.5
verts = np.vstack([np.sin(theta), np.cos(theta)]).T
circle = mpath.Path(verts * radius + center)

# Plot the ACC fronts in various colors
ax.set_boundary(circle, transform=ax.transAxes)
plt.plot(stf['lon'], stf['lat'], color='Red', transform=ccrs.PlateCarree(),
         label = 'Subtropical Front')
plt.plot(saf['lon'], saf['lat'], color='Orange', transform=ccrs.PlateCarree(),
         label = 'Subantarctic Front')
plt.plot(pf['lon'], pf['lat'], color='Yellow', transform=ccrs.PlateCarree(),
         label = 'Polar Front')
plt.plot(saccf['lon'], saccf['lat'], color='Green', transform=ccrs.PlateCarree(),
         label = 'Southern ACC Front')
plt.plot(sbdy['lon'], sbdy['lat'], color='Blue', transform=ccrs.PlateCarree(),
         label = 'Southern Boundary of ACC')

# Plot the Saildrone in black dots
plt.scatter(Saildrone_CO2.Longitude, Saildrone_CO2.Latitude,
            c=Saildrone_CO2[var], cmap='bwr',
            transform=ccrs.PlateCarree(), s=5, zorder=1000)

# Turn on the legend
plt.legend()
cb1 = plt.colorbar()

# Save the figure in the output folder
plt.title('2019 Saildrone ' + var[:4])
plt.savefig(output_dir + var[:4] + 'SaildroneMap' + '.jpg') # Changing the suffix will change the format
plt.show()
# -

# Now let's make a map of some satellite data to show the Saildrone crossing an ocean eddy.
#
# First we need to load in a single daily satellite sea surface height data file from Feb 10th 2019, the day the Saildrone crossed a large eddy.

satellite_ssh = xr.open_dataset(data_dir + 'ssh_2019_02_10.nc')

# Now plot the Saildrone path on a map of sea surface height for a region surrounding the Saildrone on Feb 10th

#set plot parameters (contour levels, colormap etc)
levels_1 = np.arange(-1.2,0.8,0.1) #contour levels
cmap_1 = 'viridis' #contour map colormap
c1 = 'black' #Saildrone track color

# +
#finding position of Saildrone on Feb 10
time_index = np.where(Saildrone_CO2['Date']=='02/10/2019')
tlon = Saildrone_CO2.Longitude.values[time_index]
tlat = Saildrone_CO2.Latitude.values[time_index]

#make a contour plot of satellite ssh
xr.plot.contourf(satellite_ssh.adt[0,:,:],levels=levels_1,cmap=cmap_1,size=8,aspect=2)
xr.plot.contour(satellite_ssh.adt[0,:,:],levels=levels_1,colors='k',linewidths=0.75)
plt.xlim(tlon.min()+360-5,tlon.max()+360+5)
plt.ylim(tlat.min()-5,tlat.max()+5)

#add Saildrone track
plt.scatter(Saildrone_CO2.Longitude+360, Saildrone_CO2.Latitude, c=c1, s=3, label='Saildrone', zorder=1000)
plt.legend()

#give the plot a title and save figure in the output folder
plt.title('Saildrone path across an eddy on Feb 10th')
plt.savefig(output_dir + 'Sea_surface_height_Saildrone_Feb10' + '.jpg')

# -

# Ocean eddies can be identified by closed rings of constant absolute dynamic topography (this is the anomaly in sea surface height from average sea level in meters, which represents changes in pressure). You can see the Saildrone's path crossing near the center of an eddy.
#
# We can do the same thing with satellite chlorophyll-a data. The chlorophyll-a data gives an approximate estimate of the relative phytoplankton biomass (in units of mg/m3) at the sea surface in different locations.

#load satellite chl-a data file
satellite_chla = xr.open_dataset(data_dir + 'A20190412019048.L3m_8D_CHL_chlor_a_4km.nc')

# Here you can edit parameters (colors, range etc) for the map

#set plot parameters (contour levels, colormap etc)
levels_1 = np.arange(0,1.0,0.01) #contour levels
cmap_1 = 'YlGnBu' #contour map colormap
c1 = 'black' #Saildrone track color

# +
#make a contour plot of chl-a data
satellite_chla.chlo_a.values[satellite_chla.chlo_a>1000] = np.nan
xr.plot.contourf(satellite_chla.chlo_a, levels = levels_1, cmap=cmap_1,size=8,aspect=2)
plt.xlim(tlon.min()-5,tlon.max()+5)
plt.ylim(tlat.min()-5,tlat.max()+5)

#add Saildrone track
plt.scatter(Saildrone_CO2.Longitude, Saildrone_CO2.Latitude, c=c1, s=3, label='Saildrone', zorder=1000)
plt.legend()

#save figure
plt.title('Saildrone path and chlorophyll-a concentration')
plt.savefig(output_dir + 'Sea_surface_chlorophylla_Saildrone_Feb10' + '.jpg')
# -

# Now we can add the Saildrone data observations on the map to start to see if there is a relationship between the satellite observations and what the Saildrone measured directly. Note that the Saildrone took a few days to cross this region, while the satellite data shown here is a snapshot for a single day, so it can be tricky to compare the two types of data because the Saildrone is moving in space AND time.

#choose which variable to plot
var = 'SST (C)'
#set minimum and maximum colorbar limits
v_min = 6
v_max = 12
#choose colormap for map
cmap_1 = 'viridis'
#choose colormap for Saildrone variable
cmap_2 = 'RdBu_r'

# +
#make a contour plot of satellite ssh
xr.plot.contourf(satellite_ssh.adt[0,:,:],levels=np.arange(-1.2,0.8,0.1),cmap=cmap_1,size=8,aspect=2)
xr.plot.contour(satellite_ssh.adt[0,:,:],levels=np.arange(-1.2,0.8,0.1),colors='black',linewidths=0.75)
plt.xlim(tlon.min()+360-5,tlon.max()+360+5)
plt.ylim(tlat.min()-5,tlat.max()+5)

#add Saildrone data scattered on top
plt.scatter(Saildrone_CO2.Longitude+360, Saildrone_CO2.Latitude, c=Saildrone_CO2[var], s=15, cmap = cmap_2,
            vmin=v_min,vmax=v_max,label='Saildrone', zorder=1000)
plt.legend()
plt.colorbar()

#add title and save figure
plt.title('Saildrone ' + var[:4] + ' across an eddy on Feb 10th')
plt.savefig(output_dir + 'Sea_surface_height_Saildrone_' + var[:4] + '_Feb10' + '.jpg')
# -

# Now that we've plotted some Saildrone and satellite data on maps to see how different ocean variables are related, there are other ways we can look at the relationship between variables.
#
# This includes scatter plots, which is a useful way to compare data from two variables collected at the same time and location to look for a relationship. In our case, we can compare two different variables collected simulatneously by the Saildrone.

# +
#choose two variables from the Saildrone to compare
var1 = 'SST (C)'
var2 = 'pCO2 SW (sat) uatm'

#choose lower and upper limits for the two variables for plotting
var1_min = -2
var1_max = 18
#var2_min = 98
#var2_max = 102

# +
#create scatter plot
plt.figure(figsize=(12,8))
plt.scatter(Saildrone_CO2[var1], Saildrone_CO2[var2], s=10)
plt.xlim(var1_min,var1_max)
#plt.ylim(var2_min,var2_max)
plt.xlabel(var1)
plt.ylabel(var2)
plt.grid()

#add title and save figure
plt.title('Saildrone '+ var1[:4] + ' vs ' + var2[:4])
plt.savefig(output_dir + 'Saildrone_' + var1[:4] + '_vs_' + var2[:4] + '.jpg')
# -

# If we want to look at the relationship between more than two variables, one way we can look at this is by using a third variable to change the color of the scatter plot points.
#
# In this example, the scatter plot shows the same variable 1 and 2 from the Saildrone on the x and y axes as above, but we can choose a third Saildrone variable as the color of the scatter plot points.

# +
#choose a third variable
var3 = 'Latitude'

#set lower and upper limits of variable 3 for plotting
var3_min = -65
var3_max = -40

# +
#create scatter plot
plt.figure(figsize=(12,8))
plt.scatter(Saildrone_CO2[var1], Saildrone_CO2[var2], c=Saildrone_CO2[var3],
            s=10, vmin = var3_min, vmax = var3_max)
plt.xlim(var1_min,var1_max)
#plt.ylim(var2_min,var2_max)
plt.xlabel(var1)
plt.ylabel(var2)
plt.grid()
cbar = plt.colorbar()
cbar.set_label(var3)

#add title and save figure
plt.title('Saildrone '+ var1[:4] + ' vs ' + var2[:4])
plt.savefig(output_dir + 'Saildrone_' + var1[:4] + '_vs_' + var2[:4] + '_vs_' + var3[:4] + '.jpg')
# -

# Plot time series of wind speed and pressure

# +
#input plot parameters

#variables to plot
var1 = 'WSPD (m/s)'
var2 = 'Licor Atm Pressure (hPa)'

#set x axis limits

# +
#plot time series
plt.figure(figsize=(12,5))
ax1 = plt.subplot(211)
ax1.plot(Saildrone_CO2.datetime,Saildrone_CO2[var1])
plt.xlim(Saildrone_CO2.datetime.values[0],Saildrone_CO2.datetime.values[-1])

ax2 = plt.subplot(212)
ax2.plot(Saildrone_CO2.datetime,Saildrone_CO2[var2])
plt.xlim(Saildrone_CO2.datetime.values[0],Saildrone_CO2.datetime.values[-1])
plt.show()
# -
# Next, we can calculate the flux of carbon between the ocean and the atmosphere based on the difference in pCO2 between the atmosphere and the ocean.

# +
#constants for CO2 flux calculation
#ocean/atmosphere variables needed as inputs
T = Saildrone_CO2['SST (C)'] #sea surface temperature
S  = Saildrone_CO2['Salinity'] #sea surface salinity
u = Saildrone_CO2['WSPD (m/s)'] #surface wind speed
dpCO2 = Saildrone_CO2['dpCO2'] #difference between ocean and atmosphere pCO2

#1. Calculate the transfer velocity (Wanninkhof et al. 2014)
#Schmidt number as a function of temperature
Sc = 2116.8-136.25*T  + 4.7353*np.power(T,2) - 0.092307*np.power(T,3) + 0.000755*np.power(T,4)
K = 0.251*(u*u)*np.power((Sc/660),-0.5)
K = K

#2. calculate solubility constant as a function of temperature and salinity
T_K = T + 273.15
K0 = -58.0931 + ( 90.5069*(100.0 /T_K) ) \
    + (22.2940 * (np.log(T_K/100.0))) + (S * (0.027766 +  ( (-0.025888)*(T_K/100.0)) \
    + (0.0050578*( (T_K/100.0)*(T_K/100.0) ) ) ) )
a = np.exp(K0)

#CO2 flux equation
Saildrone_CO2['FCO2'] = 0.24 * K * a * dpCO2  #FCO2 = K*a(dpCO2)
# -

# Let's plot the time series of carbon fluxes together with the time series of wind speed to see how they are related. Sign of FCO2?

# +
#variables to plot
var1 = 'WSPD (m/s)'
var2 = 'FCO2'

#colors
c1 = 'darkblue'
c2 = 'darkorange'

# +
#plot time series
fig, ax1 = plt.subplots(figsize=(12,5))

#y axis 1
ax1.plot(Saildrone_CO2['datetime'],Saildrone_CO2[var1],color=c1)
ax1.set_xlabel('date')
ax1.set_ylabel(var1, color=c1)
ax1.tick_params(axis='y', labelcolor=c1)

#y axis 2
ax2 = ax1.twinx()
ax2.plot(Saildrone_CO2['datetime'],-Saildrone_CO2[var2],color=c2)
ax2.set_xlabel('date')
ax2.set_ylabel(var2, color=c2)
ax2.tick_params(axis='y', labelcolor=c2)

ax2.plot([Saildrone_CO2['datetime'].values[0], Saildrone_CO2['datetime'].values[-1]],[0,0],
         color='black', linewidth=0.5)
plt.xlim(Saildrone_CO2['datetime'][0],Saildrone_CO2['datetime'][1800])
fig.tight_layout()
plt.show()
# -


# -*- coding: utf-8 -*-
"""
HunterLab Dataset
=================

Defines the *HunterLab* illuminants dataset for the
*CIE 1931 2 Degree Standard Observer* and
*CIE 1964 10 Degree Standard Observer*.

The currently implemented data has been extracted from :cite:`HunterLab2008b`,
however you may want to use different data according to the tables given in
:cite:`HunterLab2008c`.

See Also
--------
`Illuminants Jupyter Notebook
<http://nbviewer.jupyter.org/github/colour-science/colour-notebooks/\
blob/master/notebooks/colorimetry/illuminants.ipynb>`_

References
----------
-   :cite:`HunterLab2008b` : HunterLab. (2008). Hunter L,a,b Color Scale.
    Retrieved from http://www.hunterlab.se/wp-content/uploads/2012/11/\
Hunter-L-a-b.pdf
-   :cite:`HunterLab2008c` : HunterLab. (2008). Illuminant Factors in Universal
    Software and EasyMatch Coatings. Retrieved from
    https://support.hunterlab.com/hc/en-us/article_attachments/201437785/\
an02_02.pdf
"""

from __future__ import division, unicode_literals

import numpy as np
from collections import namedtuple

from colour.utilities import CaseInsensitiveMapping

__author__ = 'Colour Developers'
__copyright__ = 'Copyright (C) 2013-2019 - Colour Developers'
__license__ = 'New BSD License - https://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-science@googlegroups.com'
__status__ = 'Production'

__all__ = [
    'HunterLab_Illuminant_Specification',
    'HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER_DATA',
    'HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER',
    'HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER_DATA',
    'HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER',
    'HUNTERLAB_ILLUMINANTS'
]

HunterLab_Illuminant_Specification = namedtuple(
    'HunterLab_Illuminant_Specification', ('name', 'XYZ_n', 'K_ab'))

# yapf: disable
HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER_DATA = (
    ('A', np.array([109.83, 100.00, 35.55]), np.array([185.20, 38.40])),
    ('C', np.array([98.04, 100.00, 118.11]), np.array([175.00, 70.00])),
    ('D50', np.array([96.38, 100.00, 82.45]), np.array([173.51, 58.48])),
    ('D60', np.array([95.23, 100.00, 100.86]), np.array([172.47, 64.72])),
    ('D65', np.array([95.02, 100.00, 108.82]), np.array([172.30, 67.20])),
    ('D75', np.array([94.96, 100.00, 122.53]), np.array([172.22, 71.30])),
    ('FL2', np.array([98.09, 100.00, 67.53]), np.array([175.00, 52.90])),
    ('TL 4', np.array([101.40, 100.00, 65.90]), np.array([178.00, 52.30])),
    ('UL 3000', np.array([107.99, 100.00, 33.91]), np.array([183.70, 37.50])))
# yapf: enable

HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER = (
    CaseInsensitiveMapping({
        x[0]: HunterLab_Illuminant_Specification(*x)
        for x in HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER_DATA
    }))
"""
*Hunter L,a,b* illuminant dataset for *CIE 1931 2 Degree Standard Observer*.

References
----------
:cite:`HunterLab2008b`, :cite:`HunterLab2008c`

HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER :
    CaseInsensitiveMapping
"""

# yapf: disable
HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER_DATA = (
    ('A', np.array([111.16, 100.00, 35.19]), np.array([186.30, 38.20])),
    ('C', np.array([97.30, 100.00, 116.14]), np.array([174.30, 69.40])),
    ('D50', np.array([96.72, 100.00, 81.45]), np.array([173.82, 58.13])),
    ('D60', np.array([95.21, 100.00, 99.60]), np.array([172.45, 64.28])),
    ('D65', np.array([94.83, 100.00, 107.38]), np.array([172.10, 66.70])),
    ('D75', np.array([94.45, 100.00, 120.70]), np.array([171.76, 70.76])),
    ('FL2', np.array([102.13, 100.00, 69.37]), np.array([178.60, 53.60])),
    ('TL 4', np.array([103.82, 100.00, 66.90]), np.array([180.10, 52.70])),
    ('UL 3000', np.array([111.12, 100.00, 35.21]), np.array([186.30, 38.20])))
# yapf: enable

HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER = (
    CaseInsensitiveMapping({
        x[0]: HunterLab_Illuminant_Specification(*x)
        for x in
        HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER_DATA
    }))
"""
*Hunter L,a,b* illuminant dataset for *CIE 1964 10 Degree Standard Observer*.

References
----------
:cite:`HunterLab2008b`, :cite:`HunterLab2008c`

HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER :
    CaseInsensitiveMapping
"""

HUNTERLAB_ILLUMINANTS = CaseInsensitiveMapping({
    'CIE 1931 2 Degree Standard Observer':
        HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER,
    'CIE 1964 10 Degree Standard Observer':
        HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER
})
HUNTERLAB_ILLUMINANTS.__doc__ = """
Aggregated *Hunter L,a,b* illuminant dataset.

References
----------
:cite:`HunterLab2008b`, :cite:`HunterLab2008c`

HUNTERLAB_ILLUMINANTS : CaseInsensitiveMapping
    **{'CIE 1931 2 Degree Standard Observer',
    'CIE 1964 10 Degree Standard Observer'}**

Aliases:

-   'cie_2_1931': 'CIE 1931 2 Degree Standard Observer'
-   'cie_10_1964': 'CIE 1964 10 Degree Standard Observer'
"""
HUNTERLAB_ILLUMINANTS['cie_2_1931'] = (
    HUNTERLAB_ILLUMINANTS['CIE 1931 2 Degree Standard Observer'])
HUNTERLAB_ILLUMINANTS['cie_10_1964'] = (
    HUNTERLAB_ILLUMINANTS['CIE 1964 10 Degree Standard Observer'])


# Manipulating Time-series

Time-series are a key element when assessing solar resource data. In this section, we present several examples to learn how to deal with different formats in the data and few common tasks to prepare our time-series for later analysis, such as down and up-sampling data when we need different temporal resolution than that initially available or interpolating missing values in the data.

The dataset used in the examples of this section is a customized dataset using solar radiation measurements from the Measurement and Instrumentation Data Center (MIDC) of the U.S. National Renewable Energy Laboratory (NREL). The station selected is located at the University of Nevada - Las Vegas (UNLV) and the data used are 1-minute GHI, DHI and DNI measurements for the year 2020 (Andreas and Stoffel, 2006).

In this section, we cover:

- [1 Time-series handling](#timeseries_handling)

- [2 Down and up-sampling time-series data](#timeseries_downup_sampling)

- [3 Interpolating time-series data](#timeseries_interpolation)

- [4 Visualizating time-series data](#timeseries_visualization)

***

## 1 Time-series handling
Datasets often come in different formats depending on the source. Those formats sometimes cannot be used straightaway to build a time-series and may require additional processing steps before building the time-series. For example:

- **What if date and time are in different columns?**

- **What if the year, month, day and time are in separate columns?**

- **How to the define the timestamp format for a particular dataset?**

- **How to deal with timestamp issues, local vs. universal (UTC) time?**

This subsection presents several examples to deal with different formats in which time-series data could come and shows how to build a time-series or *datetime series*, as known in Python, for later analysis. The processing steps to build time-series are based on [pandas library](https://pandas.pydata.org/).

Let's get started!

# Importing the needed libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import pvlib

### 1.1 Build our customized dataset
In order to build the customized dataset for this section, we make use of the I/O tools of the Python library *pvlib* to retrieve the data from the UNLV station in the MIDC. Data from other stations from the MIDC can be also retrieved using this method by adapting the station ID in the query. The different station IDs are available in the [MIDC raw data page](https://midcdmz.nrel.gov/apps/data_api_doc.pl?_idtextlist).

# Dictionary to rename certain variables from the raw data
var_map = {'Global Horiz [W/m^2]': 'ghi',
           'Direct Normal [W/m^2]':'dni',
           'Diffuse Horiz (calc) [W/m^2]':'dhi',
           'Year':'year'}

# Retrieving the raw data from the station
df_ref = pvlib.iotools.read_midc_raw_data_from_nrel('UNLV',                     # Station id
                                                    pd.Timestamp('20200101'),   # Start date YYYYMMDD
                                                    pd.Timestamp('20201231'),   # End date  YYYYMMDD
                                                    variable_map=var_map)       # Variable Map
# Let's have a look to the first 2 rows of the dataset
df_ref.head(2)

# Let's have a look to the last 2 rows of the dataset
df_ref.tail(2)

The dataset is 1-minute resolution data with 21 variables related to meteorological and other relevant data: ambient temperature, wind speed, wind direction, global horizontal irradiance (GHI), direct normal irradiance (DNI), diffuse horizontal irradiance (DHI), zenith and azimuth angles, airmass, among other.

For the examples in this section we will use GHI, DNI and DHI measurements and time-related data.

# Slice desired variables out of the 21 variables provided in the raw data.
df_ref = df_ref[['ghi', 'dni', 'dhi', 'year']]

# Add multiple temporal data to the dataset
df_ref['month'] = df_ref.index.month
df_ref['day'] = df_ref.index.day
df_ref['hour'] = df_ref.index.hour
df_ref['minute'] = df_ref.index.minute
df_ref['date'] = df_ref.index.strftime('%Y-%m-%d')
df_ref['time'] = df_ref.index.strftime('%H:%M:%S')
df_ref['timestamp'] = df_ref.index.strftime('%Y-%m-%d %H:%M:%S%z')

# Epoch format
df_ref['epoch'] = df_ref.index.astype('int64')//1e9

# Reset the Index of the DataFrame
df_ref = df_ref.reset_index(drop=True)

# Let's have a look to the resulting columns of the dataset
df_ref.columns

Let's visualize the first rows of the **customized reference dataframe:**

# First 3 rows in the dataframe
df_ref.head(3)

Now that we have our customized reference dataset of 1-minute irradiance measurements for 2020 and temporal data, we can start building the timeseries in different ways.

### 1.2 Time-series when timestamps are available:

When timestamps are available, the most straightforward way to build the DataFrame with a datetime index is to convert the column with the timestamp into datetime format and set it as index.

Let's see how!

# A new dataframe copy of the reference dataset
df = df_ref.copy()
# Convert the timestamp string into datetime format
df['timestamp'] = pd.to_datetime(df['timestamp'], format='%Y-%m-%d %H:%M:%S%z')
# Set timestamp column as index
df = df.set_index(df['timestamp'])
# See the first 3 rows of the DataFrame with Datetime Index
df.head(3)

The format of the timestamp is specified in the argument 'format' as a string and can be adapted to any case. The available options in Python can be checked in this [link](https://strftime.org/).

Universal Time Coordinated (UTC) is usually the timestamp provided for many solar radiation data networks and platforms like the BSRN, PVGIS, etc. However, data can be also reported in local time like in our example. Timestamps can be converted to other timezones with the funcion *tz_convert*, which can be useful when dealing with data from different databases and locations worldwide:

# Add UTC timestamp from the local time (Pacific Summer Time)
df['timestamp_utc'] = df.index.tz_convert('UTC')
# See the first 3 rows
df.head(3)

The valid timezone strings for other timezones can be found in this [link](https://pvlib-python.readthedocs.io/en/stable/timetimezones.html). When the timezone is not provided as part of the timestamp, the function *tz_localize* can be used to localize the values in a timezone-naive series. *tz_localize* will be used in the next example.

### 1.3 Time-series when date and time are available:

When date and time are available in separate columns, a timestamp can be created in a new column and the new column can then be set as index and localized. Let's have a look how to do that:

# A new dataframe copy of the reference dataset
df = df_ref.copy()
# New column with the date and time
df['datetime'] = df['date'] + 'T' + df['time']
# Convert the new column into datetime format
df['datetime'] = pd.to_datetime(df['datetime'], format='%Y-%m-%dT%H:%M:%S')
# Set the column 'datetime' as index and localize it to its timezone
df = df.set_index(df['datetime']).tz_localize('Etc/GMT+8')
# See the first 3 rows
df.head(3)

### 1.4 Time-series when the time data is split in multiple columns:

If time-related data are split across multiple columns, a timestamp can be created in a new column similarly than in the previous case. Let's imagine our dataset would have the year, month, day, hour, and minute in separate columns. In that case, we could build our time-series as follows:

# A new dataframe copy of the reference dataset
df = df_ref.copy()
# Let's reduce the code lines and define the new string within the 'to_datetime' function
df['datetime'] = pd.to_datetime(df[['year', 'month', 'day', 'hour', 'minute']],
                                format = '%Y-%m-%d%H:%M')
# Set the column 'datetime' as index
df = df.set_index(df['datetime'])
# Localize the datetime series
df.index = df.index.tz_localize('Etc/GMT+8')
# See the first 3 rows
df.head(3)

### 1.5 Time-series when the timestamp is given as epoch (Unix Time)

If the dataset has epoch timestamps, note that the data will have UTC time. However, it can be converted to any timezone using the function *tz_convert*. If there are epoch timestamps, a datetime series can be formed as follows:

# A new dataframe copy of the reference dataset
df = df_ref.copy()
# Convert epoch timestamps to datetime format and localize
df['datetime'] = pd.to_datetime(df['epoch'], unit='s', utc=True)
# Set datetime as index and convert UTC time to local time
df = df.set_index(df['datetime']).tz_convert('Etc/GMT+8')
# See the results
df.head(3)

We have seen how the same DataFrame with *datetimeindex* can be obtained in multiple ways depending on the format of time data provided.

***

## 2 Down and up-sampling time-series data

When assessing solar resource, you may need a different time-resolution than your data for a particular part of the analysis. In those cases, it is possible to **down-sample and up-sample the data at different temporal resolutions** using two different methods within [pandas library](https://pandas.pydata.org/) called *resample* and *asfreq*. Depending on your needs, you will opt for one or the other. Regardless of the method, both of them require a DataFrame with *datetimeindex* either time-aware (localized) or time-naive (not localized).

###  2.1 Method 'asfreq' vs. 'resample'
Let's first create a new DataFrame with only the columns with solar data and see the differences between both methods with examples.

# New DataFrame with 1-minute data and solar data
df_1min = df[['ghi', 'dhi', 'dni']]
# See our new DataFrame
df_1min.head(3)

Let's try to obtain a DataFrame down-sampled with the maximum monthly data with both methods and see the differences. With *asfreq*, it would be the following:

df_1min.asfreq("1M").max()

With *resample* the result would be:

df_1min.resample("1M").max()

It is obvious that the outputs are not the same and that is because the methods work differently. *asfreq* takes the value at the simultaneous stamps given by the frequency argument. See below:

df_1min.asfreq("1M")

Then *.max()* has returned the maximum of each of the columns.

In contrast, *resample* does return the maximum value within the period of time at the specified frequency. *resample* method requires a mathematical operation to perform in the resampled data (the maximum value in our case). Otherwise, it would return a *DatetimeIndexResampler* object without showing any data. See below:

df_1min.resample("1M")

The *resample* method accepts multiple **mathematical and statistical operations**. For example: maximum (max), minimum (min), arithmetic mean (mean), standard deviation (std), median (median), mode (mode), addition (sum), among others.

Both methods allow for multiple **frequencies options**, the available frequency tags within Python can be found [here](https://stackoverflow.com/questions/35339139/where-is-the-documentation-on-pandas-freq-tags).

### 2.2 Down-sampling the data in a time-series

Down-sampling permits turning more frequent values into less frequent. In the context of solar resource and considering our 1-minute resolution dataset, down-sampling can be used for:
- Producing a timeseries of hourly/daily average irradiance.
- Producing a timeseries of maximum daily irradiance.
- Estimating the hourly/daily/monthly sums of irradiation.
- And many more!

Let's implement some of these listed examples!

#### Producing hourly average irradiance from minutely observations

# Resampling to hourly mean values
df_hourly = df_1min.resample("1H").mean()
# Showing the shape of the new DataFrame
df_hourly.shape # returns Rows, Columns

There are 8760 hours in a year. Yet, we can have a look to the first few rows of the DataFrame:

df_hourly.head(12)

A time-series with the maximum irradiance would be similar replacing *'mean()'* with *'max()'*.

#### Producing time-series of monthly total GHI, DHI, DNI irradiation from minutely observations

# Resampling to monthly aggregated values
monthly_energy = df_1min[['ghi', 'dhi', 'dni']].resample("1M").sum()*(1/60)
# See the results expressed in kWh·sqm
monthly_energy/1000

It could be done in similar way for other resolutions (e.g. daily or annual irradiation).

### 2.3 Up-sampling the data in a time-series

Up-sampling permits obtaining more frequent values from less frequent. For solar data, depending on the application up to sub-minutely data could be required and up-sampling is a technique that provides a manner to increase the temporal resolution to adapt it to our needs. For example, turning an hourly time-series into a half-hourly. Let's see an example using both *resample* and *asfreq*.

#### Producing half-hourly irradiance series from hourly observations
Using the DataFrame *df_hourly* created previously, it can be up-sample as follows:

# Using 'resample' method:
df_hourly.resample("30Min").mean().head(10)

# Using 'asfreq' method:
df_hourly.asfreq("30Min").head(10)

Contrary to the case of down-sampling, both *asfreq* and *resample* provide similar results when up-sampling. However, *asfreq* provides additional functionalities to treat the new timestamps without data, i.e. NaN values.

By passing the argument *'method'* with the string *'backfill'* or *'bfill'* uses the next valid observation to fill the NaN value (back filling). If instead, the string *'pad'* or *'ffill'* is given, the method assigns the last valid observation forward to the next valid (forward filling).

Let's see the same example adding this argument:

# Half-hourly up-sample with back filling function
df_hourly.asfreq("30Min", method='bfill').head(10)

We see that the DataFrame now contains the next valid hourly value in the newly obtained half-hourly timestamps of the previous hour. It would take the previous valid hourly value if we used forward filling. For example:

# Half-hourly up-sample with forward-filling function
df_hourly.asfreq("30Min", method='ffill').head(10)

The forward filling option provides the same value for o'clock and half past timestamps within the same hour. In addition to these two ways to complete the NaN values, the method *asfreq* can replace the NaN values with a constant. See below:

# Half-hourly up-sample filling the new timestamps with a constant
df_hourly.asfreq("30Min", fill_value=0).head(10)

The use of the methods *asfreq* or *resample* will depend on your dataset and the analysis you aim to undertake.

***

## 3 Interpolating time-series data

When up-sampling the data series, it can happen that back-filling, forward-filling and constant replacement does not necessarily work for your analysis/application. An alternative approach is interpolating the replacing the NaN values with an interpolated result. Interpolation in Pandas DataFrames with *DatetimeIndex* is done with the *interpolate* method.

The mathematical interpolation method in *interpolate* is defined with the argument called *'method'*. Pandas permits several interpolation methods, such as 'linear', 'cubic', 'quadratic', 'spline', 'polynomial' and others. All the interpolation options can be found in the [documentation](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.interpolate.html) of the *interpolate* method.

Following the previous example, let's implement interpolation in the missing values of the half-hourly timestamps using 'linear', 'cubic' and 'polynomial' methods:

# Up-sample using the 'asfreq' method
df_30min = df_hourly.asfreq("30Min")
# Interpolate missing values (NaN) with linear interpolation
df_linear = df_30min.interpolate(method='linear')
# See the results:
df_linear.head(10)

Similarly, it can be implemented to other methods:

# Interpolate missing values (NaN) with cubic interpolation
df_cubic = df_30min.interpolate(method='cubic')
# See the results:
df_cubic.head(10)

With polynomial interpolation, the degree or order of the polynomial function needs to be defined as an argument:

# Interpolate missing values (NaN) with polynomial interpolation
df_polynomial = df_30min.interpolate(method='polynomial', order=5)
# See the results:
df_polynomial.head(10)

The interpolation of NaN values when up-sampling time-series data can help overcome the issues of using back or forward filling, specially if you aim to up-sample at higher frequencies than the example shown (e.g. 1-hour to 15-minute resolution series). The mathematical methods available for interpolation within Pandas are diverse and cover beyond the most common interpolation functions.

## 4 Visualizing time-series data

It is often useful to visualize the data to grasp insighs and observe trends about the data. This section shows few examples to visualize time-series data.

### 4.1 Plotting a time-series for a day of interest

Below there is an example to visualize a single day of interest. With DataFrames using *DatetimeIndex* it is easy to select a particular day and Pandas interacts with Matplotlib.Pyplot library to plot straight-away.

# Plotting GHI for a given day in the time-series
df_1min['2020-06-01']['ghi'].plot(label='GHI')
plt.ylabel('Irradiance [W/m$^2$]')
plt.xlabel('Local Time [HH:MM]')
plt.legend(loc='best')
plt.show() # Not needed in Jupyter Notebooks but usually required in other IDEs.

We can visualize the effect of using average (*resample*) vs. instantaneous (*asfreq*) measurements when down-sampling our data.

# Plotting GHI for a given day in the time-series
df_1min['2020-06-01']['ghi'].plot(label='1-min data', alpha=0.4) # Reference data
df_1min.asfreq('30Min')['2020-06-01']['ghi'].plot(label='30-min instant.') # Instantaneous 30-min values
df_1min.resample('30Min').mean()['2020-06-01']['ghi'].plot(label='30-min average') # Average 30-min values
plt.title('Average vs. Actual GHI Measurements') # title of the figure
plt.ylabel('Irradiance [W/m$^2$]') # y-axis label
plt.xlabel('Local Time [HH:MM]') # x-axis label
plt.legend(loc='upper left') # insert legend
plt.show() # Not needed in Jupyter Notebook but usually required in other IDEs.

### 4.2 Plotting a time-series for a few consecutive days of interest

Below there is an example to visualize a few consecutive days (e.g. 5 days) of interest. By using ['start date']:['end date'] it is possible to select time ranges easily with a DataFrame having a *DatetimeIndex*.

# Variables to plot
vars = ['ghi', 'dni', 'dhi']
# Create 3 subplots, with shared X and Y axis
fig, axs = plt.subplots(3, sharex=True, sharey=True, figsize=(9,6))
# Add title to the plot
fig.suptitle('Average Hourly Solar Radiation Observations', fontsize=14)

for i in range(3):
    axs[i].plot(df_1min.resample('1H').mean()['2020-06-01':'2020-06-05'][vars[i]], label='Average') # Average hourly
    axs[i].plot(df_1min.resample('1H').max()['2020-06-01':'2020-06-05'][vars[i]], label='Maximum') # Max. hourly
    axs[i].plot(df_1min.resample('1H').min()['2020-06-01':'2020-06-05'][vars[i]], label='Minimum') # Min. hourly
    axs[i].set_title(vars[i].upper()) # Title for each subplot
fig.subplots_adjust(hspace=0.3) # Adjust the white space between the subplots titles
fig.text(0.04, 0.5, 'Irradiance [W/m$^2$]', va='center', rotation='vertical', fontsize=12) # Common Y Axis
fig.text(0.51, 0.04, 'Local Time', ha='center', fontsize=12) # Common X Axis
plt.legend(loc='upper center', ncol=3) # Legend for the last subplot or 'axs[i].legend()' in the loop to a legend to each.
plt.show()

### 4.3 Plotting a time-series for a few non-consecutive days of interest

Below there is an example to visualize a few non-consecutive days of interest, which could be the case when we would like to observe several days scattered throughout the year a single plot. In order to do this, we need to select the day of interest from the DataFrame and then reset its *DatetimeIndex*. For example:

# List of days of interest
days = ['2020-01-01', '2020-03-01', '2020-06-01', '2020-09-01']
# Iterate over the days and plot each of them
for day in days:
    df_day = df_1min.resample('1H').mean()[day]['ghi'].to_frame()  # average hourly of GHI for current day
    df_day = df_day.reset_index(drop=True) # reset its Index to numeric (i.e. 0,1,2,3...)
    plt.plot(df_day, label=day) # plot the current day
plt.title('Average Hourly GHI Measurements for Days of Interest') # title of the figure
plt.xticks(np.arange(0, 25, step=3), np.arange(0, 25, step=3)) # set labels positions and names
plt.ylabel('Irradiance [W/m$^2$]') # y-axis label
plt.xlabel('Local Time') # x-axis label
plt.legend(loc='best') # insert legend
plt.show()

### 4.4 Daily insolation throughout the year

With time-series data, the hourly/daily/monthly insolation (i.e. the sum of accumulated energy) can also be analysed throughout the year with time-series data. For example, below an example to visualize the daily insolation is shown:

# Calculate the daily insolation expressed in kWh·sqm from GHI measurements
daily_energy = (df_1min['ghi'].resample("1D").sum()*(1/60))/1000 # selecting only GHI returns a Pandas Series

# Create time-series plot
daily_energy.plot(figsize=(9,6), legend=False) # plot timeseries
plt.title('Time-series of Daily Insolation')  # add title
plt.ylabel('Energy [kWh/m$^2$]') # add Y-axis label
plt.xlabel('Time') # add X-axis label
plt.show()

Time-series data can also be visualized in other ways, for instance, as a heat map.

# Prepare the data for heat map of hourly insolation
energy_array = pd.DataFrame() # empty DataFrame for the results
for i in range(1,13): # iterate over months
    # select the data in the month and eliminate the datetimeindex
    df_month = daily_energy[daily_energy.index.month==i].reset_index(drop=True)
    # rename the column with the number of the month
    df_month.columns = [str(i)]
    # Append results to the DataFrame
    energy_array = pd.concat([energy_array, df_month], axis=1)
# Transpose to have months in y-axis and days in x-axis
energy_array = energy_array.transpose()
# Rename the columns of the days
energy_array.columns = np.arange(1, 32)

# Plot heat map of daily insolation
months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', # month labels
          'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
plt.figure(figsize=(10, 5))
ax = sns.heatmap(energy_array, cmap='CMRmap', linewidths=0.2, # plot heatmap with Seaborn (sns) library
                xticklabels=2, annot=False,
                cbar_kws={'label': 'Daily Energy [kWh/m$^2$]'})
ax.set_title('Heat Map of Daily Insolation') # add title
ax.set_yticklabels(months,rotation=0) # add the months as tick-labels for the y-axis
ax.set_xticklabels(ax.get_xticklabels(),rotation=0) # add the days as tick-labels for the x-axis
ax.set_xlabel('Day of the Month')
plt.show()

***

## Section summary

This section has shown how to build and work with a time-series in Python with multiple examples:

- We have seen how to prepare a DataFrame with *DatatimeIndex* to be used as a time-series when the timestamps are given in multiple formats in the temporal data and local/UTC time.

- Changes in the temporal resolution of the data can be applied by down and up-sampling the data and the differences between 2 available methods (*asfreq* and *resample*) have been shown with examples and different sampling frequencies.

- The interpolation of missing data in time-series can be used to up-sample the resolution of the data and examples with some methods have been shown.

- Finally, several ideas to visualize data have been presented.

Overall, the possibilities with time-series of solar resource are many. The most useful and suitable analysis and visualizations will be determined by the application and scope of the study.

***

## References

Andreas, A.; Stoffel, T.; (2006). University of Nevada (UNLV):
Las Vegas, Nevada (Data); NREL Report No. DA-5500-56509.
http://dx.doi.org/10.5439/1052548


# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.4'
#       jupytext_version: 1.2.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + {"deletable": true, "editable": true, "cell_type": "markdown"}
#

#

#

#
Computational Seismology

#
Reproducible Papers - Syngine Paper

#

#

#

# + {"deletable": true, "editable": true, "cell_type": "markdown"}
# ---
#
# # Figure 6:  Data Quality - Detecting Wrong Times with Synthetics
#
# This notebook is part of the supplementary materials for the Syngine paper and reproduces figure 6.
#
# This notebook creates the phase relative times figure. Requires matplotlib >= 1.5 and an ObsPy version (>= 1.0) with the syngine client as well as instaseis.
#
# ##### Authors:
# * Lion Krischer ([@krischer](https://github.com/krischer))

# + {"deletable": true, "editable": true}
# %matplotlib inline

# + {"deletable": true, "editable": true}
import obspy
import numpy as np
from obspy.clients.fdsn import Client
from obspy.clients.syngine import Client as SyngineClient

import itertools

import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
plt.style.use("seaborn-paper")

# + {"deletable": true, "editable": true}
c_iris = Client("IRIS")

# + {"deletable": true, "editable": true}
# Get all stations from the Caltech Regional Seismic Network activate
# at the given time.
inv = c_iris.get_stations(level="response", format="xml", channel="BHZ",
                          network="CI", location="  ",
                          startbefore=obspy.UTCDateTime(2012, 1, 1),
                          endafter=obspy.UTCDateTime(2016, 1, 1))
inv.plot(projection="local");
print(inv)

# + {"deletable": true, "editable": true}
# Downloaded from the GCMT page.
# cat = obspy.read_events("./cmtsolutions.txt", format="CMTSOLUTION").filter("latitude > -20", "longitude > 165")

# This is a prototype - we thus are very selective regarding which data we use...
training = ["smi:local/cmtsolution/201510202152A/event",
            "smi:local/cmtsolution/201212212228A/event",
            "smi:local/cmtsolution/201203090709A/event"]
validate = "smi:local/cmtsolution/201401011603A/event"
application = "smi:local/cmtsolution/201501230347A/event"

all_data = []
all_data.extend(training)
all_data.append(validate)
all_data.append(application)

cat.events = [_i for _i in cat.events if _i.resource_id.id in all_data]

print(cat)
cat.plot();

# + {"deletable": true, "editable": true}
# For all of them: download the syngine data 50 seconds before the p-phase
# and 50 seconds after for the 5s ak135f DB.
c_syngine = SyngineClient()

# + {"deletable": true, "editable": true}
# Download all syngine data.
synthetic_data = {}

# Build up bulk request.
bulk = [
    {"networkcode": "CI", "stationcode": _i.code,
     "latitude": _i.latitude, "longitude": _i.longitude} for _i in inv[0]]

# Actually download everything.
for _i, event in enumerate(cat):
    origin = [_i for _i in event.origins if _i.origin_type == "hypocenter"][0]
    mt = event.focal_mechanisms[0].moment_tensor.tensor
    print("Downloading data for event %i of %i ..." % (_i + 1, len(cat)))
    st_syn = c_syngine.get_waveforms_bulk(
        model="ak135f_5s", bulk=bulk, sourcelatitude=origin.latitude,
        sourcelongitude=origin.longitude, sourcedepthinmeters=origin.depth,
        sourcemomenttensor=[mt.m_rr, mt.m_tt, mt.m_pp, mt.m_rt, mt.m_rp, mt.m_tp],
        origintime=origin.time,
        components="Z",
        # Avoid downsampling - just upsample to the data sampling rate.
        dt=1.0 / 40.0,
        kernelwidth=10,
        units="velocity",
        starttime="P-50", endtime="P+50")

    this_event = {"event_object": event, "stream": st_syn}
    synthetic_data[event.resource_id] = this_event
    print("  -> Downloaded %i synthetic traces." % len(st_syn))

# + {"deletable": true, "editable": true}
# Reordering ... I don't want to redownload again...
data = synthetic_data
for value in data.values():
    value["synthetic_stream"] = value["stream"]
    del value["stream"]

# + {"deletable": true, "editable": true}
# Now download all real data.
for _i, value in enumerate(data.values()):
    print("Downloading data for event %i of %i ..." % (_i + 1, len(cat)))
    st_obs = c_iris.get_waveforms_bulk(
        [(tr.stats.network, tr.stats.station, "",
          "BHZ", tr.stats.starttime, tr.stats.endtime) for tr in value["synthetic_stream"]])
    value["observed_stream"] = st_obs
    print("  -> Downloaded %i observed traces." % len(st_obs))

# + {"deletable": true, "editable": true}
# Cleanup - here we make our job easy and just remove all
# stations for which we don't have data for all events.
# Real implementations will of course have to deal with this.

common_stations = set.intersection(*[set((tr.stats.network, tr.stats.station)
                                         for tr in value["observed_stream"])
                                     for value in data.values()])
print(len(common_stations))

for value in data.values():
    value["observed_stream"] = obspy.Stream(traces=[
        tr for tr in value["observed_stream"] if
        (tr.stats.network, tr.stats.station) in common_stations])
    value["synthetic_stream"] = obspy.Stream(traces=[
        tr for tr in value["synthetic_stream"] if
        (tr.stats.network, tr.stats.station) in common_stations])

# + {"deletable": true, "editable": true}
# Create a new inventory object which only contains the common stations.
from copy import deepcopy
filtered_inv = deepcopy(inv)
filtered_inv[0].stations = [_i for _i in filtered_inv[0].stations if ("CI", _i.code) in common_stations]
filtered_inv.plot(projection="local");

# + {"deletable": true, "editable": true}
from obspy.signal.cross_correlation import xcorr_pick_correction

collected_stats = []

good_data = data

# Now calculate the P phase delay for all stations.
for network, station in common_stations:
    print(network, station)
    for event, this_data in good_data.items():
        obs_tr = this_data["observed_stream"].select(network=network, station=station)[0].copy()
        syn_tr = this_data["synthetic_stream"].select(network=network, station=station)[0].copy()

        # Artificially shift the data for two stations for the "application" event.
        if event.id == application:
            if station == "BEL":
                obs_tr.stats.starttime -= 1.0
            if station == "PHL":
                obs_tr.stats.starttime += 1.0

        # p-phase should be centered in the synthetic trace.
        pick = syn_tr.stats.starttime + (syn_tr.stats.endtime - syn_tr.stats.starttime) / 2.0

        # Process data.
        obs_tr.detrend("linear")\
            .taper(max_percentage=0.05, type="hann")\
            .remove_response(inventory=filtered_inv, pre_filt=(0.02, 0.01, 1, 5))\
            .filter("bandpass", freqmin=0.08, freqmax=0.2, corners=4, zerophase=True)\
            ._ltrim(30.0)\
            ._rtrim(30.0)\
            .taper(max_percentage=0.3, type="hann")
        obs_tr.trim(obs_tr.stats.starttime - 50, obs_tr.stats.endtime + 50, pad=True, fill_value=0.0)

        # Also filter the synthetics.
        syn_tr.taper(max_percentage=0.05, type="hann")
        syn_tr.filter("bandpass", freqmin=0.08, freqmax=0.2, corners=4, zerophase=True)
        syn_tr.data = np.require(syn_tr, requirements=["C"])

        # Interpolate both to the same sample points.
        starttime = max(obs_tr.stats.starttime, syn_tr.stats.starttime)
        endtime = min(obs_tr.stats.endtime, syn_tr.stats.endtime)
        npts = int((endtime - starttime) // (1.0 / 40.0) - 1)
        obs_tr.interpolate(sampling_rate=40.0, method="lanczos", a=5,
                           starttime=starttime, npts=npts)
        syn_tr.interpolate(sampling_rate=40.0, method="lanczos", a=5,
                           starttime=starttime, npts=npts)

        print(event.id, network, station)

        # Subsample precision.
        try:
            shift, corr = xcorr_pick_correction(pick, obs_tr, pick, syn_tr, t_before=20,
                                                t_after=20, cc_maxlag=12)
        except:
            shift, corr = None, None
        collected_stats.append({"time_shift": shift, "correlation": corr,
                                "network": network, "station": station, "event": event.id})

        if "processed_observed_stream" not in this_data:
            this_data["processed_observed_stream"] = obspy.Stream()

        this_data["processed_observed_stream"] += obs_tr

        if "processed_synthetic_stream" not in this_data:
            this_data["processed_synthetic_stream"] = obspy.Stream()

        this_data["processed_synthetic_stream"] += syn_tr

# + {"deletable": true, "editable": true}
import pandas
pandas.set_option('display.max_rows', 10)
cs = pandas.DataFrame(collected_stats)
original_cs = pandas.DataFrame(collected_stats)

# + {"deletable": true, "editable": true}
# Now for all events: substract the mean time shift to account for
# unexplained structure and unknown origin time.
cs.loc[:, "time_shift"] = cs.groupby("event").time_shift.transform(lambda df: df - df.mean())
cs = cs.sort_values("station")
cs

# + {"deletable": true, "editable": true}
# Calculate the reference time shift for each station.
reference_shifts = dict(cs.loc[cs.event.isin(training)].groupby("station").time_shift.mean())
reference_shifts

# + {"deletable": true, "editable": true}
import matplotlib.pyplot as plt

def check_event(event, ax=None, ticks=False):
    _ds = cs[cs.event == event]
    mean = _ds.time_shift.mean()

    things = []

    for _i in _ds.itertuples():
        things.append({"station": _i.station,
                       "time_shift": _i.time_shift - mean,
                       "difference_to_mean": (_i.time_shift - mean) - reference_shifts[_i.station]})

    temp = pandas.DataFrame(things).sort_values("station")

    if ax is None:
        plt.figure(figsize=(15, 5))
        ax = plt.gca()
    plt.bar(np.arange(len(_ds)) - 0.4, temp.difference_to_mean, 0.8, color="0.2")
    plt.ylim(-1.4, 1.4)
    if ticks:
        plt.xticks(range(len(_ds)), temp.station, rotation="vertical")
    else:
        plt.xticks(range(len(_ds)), [""] * len(_ds), rotation="vertical")

    plt.xlim(-1, 23)
check_event(application)

# + {"deletable": true, "editable": true}
check_event(validate)

# + {"deletable": true, "editable": true}
check_event(application)

# + {"deletable": true, "editable": true}
def plot_data(event, station, show=True, legend=False):
    val = data[obspy.core.event.ResourceIdentifier(event)]
    obs = val["processed_observed_stream"].select(station=station)[0]
    syn = val["processed_synthetic_stream"].select(station=station)[0]

    plt.plot(obs.times(), obs.data * 1E6, color="0.3", linestyle="--", linewidth=5,
             label="Observed Data")
    plt.plot(syn.times(), syn.data * 1E6, color="0.1", linestyle=":", label="Synthetic Data")

    ts= original_cs[(original_cs.event == validate) &
                    (original_cs.station == "TIN")].time_shift

    plt.plot(syn.times() - float(ts), syn.data * 1E6, color="k", label="Shifted Synthetic Data")
    plt.ylabel("Velocity [$\mu$m/s]")
    plt.xlabel("Relative Time [s]")
    plt.xlim(20, 65)

    plt.xticks([30, 40, 50, 60], ["30", "40", "50", "60"])

    if legend:
        plt.legend(loc="lower left")

    plt.text(0.03, 0.92, "Station: CI.%s" % station, transform=plt.gca().transAxes, va="top")
    if show:
        plt.show()

plot_data(validate, "TIN")
plot_data(validate, "SCI2")
plot_data(validate, "MPP")

# + {"deletable": true, "editable": true}
# Create final complete plot.
import matplotlib.gridspec as gridspec

fig = plt.figure(figsize=(10, 5))

gs1 = gridspec.GridSpec(2, 1, wspace=0, hspace=0.04, left=0, right=0.27, bottom=0.08, top=0.98)
ax1 = fig.add_subplot(gs1[0])
ax2 = fig.add_subplot(gs1[1])

gs2 = gridspec.GridSpec(3, 1, wspace=0, hspace=0, left=0.32, right=0.62, bottom=0.08, top=0.98)
ax3 = fig.add_subplot(gs2[0])
ax4 = fig.add_subplot(gs2[1])
ax5 = fig.add_subplot(gs2[2])

gs3 = gridspec.GridSpec(3, 1, wspace=0, hspace=0, left=0.64, right=0.94, bottom=0.08, top=0.98)
ax6 = fig.add_subplot(gs3[0])
ax7 = fig.add_subplot(gs3[1])
ax8 = fig.add_subplot(gs3[2])

station_lats = [sta.latitude for net in filtered_inv for sta in net]
station_lngs = [sta.longitude for net in filtered_inv for sta in net]

ev_lats = [event.origins[0].latitude for event in cat]
ev_lngs = [event.origins[0].longitude for event in cat]

m_ortho = Basemap(resolution='c', projection='ortho', lat_0=10.,
                  lon_0=-150., ax=ax1)
m_ortho.drawcoastlines(color="0.3", linewidth=1.0)
m_ortho.drawcountries(color="0.8")
m_ortho.fillcontinents(color="0.9")

_xs, _ys = m_ortho(station_lngs, station_lats)
_xe, _ye = m_ortho(ev_lngs, ev_lats)

for sta, ev in itertools.product(zip(station_lngs, station_lats),
                                 zip(ev_lngs, ev_lats)):
    m_ortho.drawgreatcircle(*sta, *ev, linewidth=1.2,
                            color="0.6", alpha=0.5)

m_ortho.scatter(_xs, _ys, marker="v", s=120, zorder=10,
                color="k", edgecolor="w")
m_ortho.scatter(_xe, _ye, marker="o", s=120, zorder=10,
                color="k", edgecolor="w")

deg2m = 2 * np.pi * 6371 * 1000 / 360
m_local = Basemap(projection='aea',
                  resolution="i",
                  area_thresh=1000.0, lat_0=34.9,
                  lon_0=-118,
                  width=deg2m * 8, height=deg2m * 8,
                  ax=ax2)
m_local.drawmapboundary()
m_local.drawrivers(color="0.7")
m_local.drawcoastlines(color="0.3", linewidth=1.4)
m_local.drawcountries(color="0.4", linewidth=1.1)
m_local.fillcontinents(color="0.9")
m_local.drawstates(color="0.5", linewidth=0.9)

_x, _y = m_local(station_lngs, station_lats)
m_local.scatter(_x, _y, marker="v", s=140, zorder=10,
                color="k", edgecolor="w")

# Waveforms.
plt.sca(ax3)
plot_data(validate, "TIN", show=False)
plt.yticks(np.linspace(-1.5, 1.5, 7))

plt.sca(ax4)
plot_data(validate, "SCI2", show=False)
plt.yticks(np.linspace(-2, 2, 9))

plt.sca(ax5)
plot_data(validate, "MPP", show=False, legend=True)
plt.yticks(np.linspace(-2, 2, 9))

plt.sca(ax6)
stations = list(reference_shifts.keys())
delays = [reference_shifts[_i] for _i in stations]
plt.bar(np.arange(len(delays)) - 0.4, delays, 0.8, color="0.2")
plt.xticks(range(len(delays)), [""] * len(delays))
ax6.yaxis.tick_right()
ax6.yaxis.set_label_position("right")
plt.xlim(-1, 23)
plt.ylim(-0.8, 0.7)
plt.yticks([-0.5, 0, 0.5])
plt.ylabel("Average time shift [s]")
plt.text(0.97, 0.90, "Reference Time Shifts", transform=plt.gca().transAxes, va="top", ha="right")

plt.sca(ax7)
check_event(validate, ax=ax7)
ax7.yaxis.tick_right()
ax7.yaxis.set_label_position("right")
plt.yticks(np.linspace(-1.0, 1.0, 5))
plt.ylabel("Time shift deviation [s]")
plt.text(0.97, 0.90, "Validation Event", transform=plt.gca().transAxes, va="top", ha="right")

plt.sca(ax8)
check_event(application, ax=ax8, ticks=True)
ax8.yaxis.tick_right()
ax8.yaxis.set_label_position("right")
plt.yticks(np.linspace(-1.0, 1.0, 5))
plt.ylabel("Time shift deviation [s]")
plt.text(0.97, 0.90, "Test Event", transform=plt.gca().transAxes, va="top", ha="right")

plt.savefig("data_quality.pdf")
plt.show()