#!/usr/bin/env python
u"""
read_cryosat_L2.py
Written by Tyler Sutterley (10/2018)

Reads CryoSat Level-2 data products from baselines A, B and C
Supported CryoSat Modes: LRM, SAR, SARin, FDM, SID, GDR

INPUTS:
	full_filename: full path of CryoSat .DBL file

OUTPUTS:
	Data_1Hz: Time and Orbit Parameters
	Corrections: Elevation Corrections and Flags
	Data_20Hz: Geolocation and Elevation Measurements with Quality Parameters
	METADATA: MPH, SPH and DSD Header data

UPDATE HISTORY:
Updated 10/2018: updated header read functions for python3
Updated 11/2016: added Abs_Orbit and Ascending_Flg to Data_1Hz outputs
	Abs_Orbit should be same as in read_cryosat_ground_tracks.py
	Ascending_Flg can use in surface regression fits following McMillan (2014)
Updated 05/2016: using __future__ print and division functions
Written 03/2016
"""
from __future__ import print_function
from __future__ import division

import os
import re
import numpy as np

#-- PURPOSE: Initiate L2 MDS variables for CryoSat Baselines A and B
def cryosat_baseline_AB(fid,record_size,n_records):
	#-- CryoSat-2 1 Hz data fields (Location Group)
	#-- Time and Orbit Parameters plus Measurement Mode
	L2_1Hz_parameters = {}
	#-- Time: day part
	L2_1Hz_parameters['Day'] = np.zeros((n_records),dtype=np.int32)
	#-- Time: second part
	L2_1Hz_parameters['Second'] = np.zeros((n_records),dtype=np.int32)
	#-- Time: microsecond part
	L2_1Hz_parameters['Micsec'] = np.zeros((n_records),dtype=np.int32)
	#-- SIRAL mode
	L2_1Hz_parameters['Siral_mode'] = np.zeros((n_records),dtype=np.uint64)
	#-- Lat_1Hz: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_1Hz_parameters['Lat_1Hz'] = np.zeros((n_records),dtype=np.int32)
	#-- Lon_1Hz: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_1Hz_parameters['Lon_1Hz'] = np.zeros((n_records),dtype=np.int32)
	#-- Alt_1Hz: packed units (mm, 1e-3 m)
	#-- Altitude of COG above reference ellipsoid (interpolated value)
	L2_1Hz_parameters['Alt_1Hz'] = np.zeros((n_records),dtype=np.int32)
	#-- Mispointing: packed units (millidegrees, 1e-3 degrees)
	L2_1Hz_parameters['Mispointing'] = np.zeros((n_records),dtype=np.int16)
	#-- Number of valid records in the block of twenty that contain data
	#-- Last few records of the last block of a dataset may be blank blocks
	#-- inserted to bring the file up to a multiple of twenty.
	L2_1Hz_parameters['N_valid'] = np.zeros((n_records),dtype=np.int16)

	#-- CryoSat-2 geophysical corrections (External Corrections Group)
	L2_final_corrections = {}
	#-- Dry Tropospheric Correction packed units (mm, 1e-3 m)
	L2_final_corrections['dryTrop'] = np.zeros((n_records),dtype=np.int16)
	#-- Wet Tropospheric Correction packed units (mm, 1e-3 m)
	L2_final_corrections['wetTrop'] = np.zeros((n_records),dtype=np.int16)
	#-- Inverse Barometric Correction packed units (mm, 1e-3 m)
	L2_final_corrections['InvBar'] = np.zeros((n_records),dtype=np.int16)
	#-- Dynamic Atmosphere Correction packed units (mm, 1e-3 m)
	L2_final_corrections['DynAtm'] = np.zeros((n_records),dtype=np.int16)
	#-- Ionospheric Correction packed units (mm, 1e-3 m)
	L2_final_corrections['Iono'] = np.zeros((n_records),dtype=np.int16)
	#-- Sea State Bias Correction packed units (mm, 1e-3 m)
	L2_final_corrections['SSB'] = np.zeros((n_records),dtype=np.int16)
	#-- Ocean tide Correction packed units (mm, 1e-3 m)
	L2_final_corrections['ocTideElv'] = np.zeros((n_records),dtype=np.int16)
	#-- Long period equilibrium ocean tide Correction packed units (mm, 1e-3 m)
	L2_final_corrections['lpeTideElv'] = np.zeros((n_records),dtype=np.int16)
	#-- Ocean loading tide Correction packed units (mm, 1e-3 m)
	L2_final_corrections['olTideElv'] = np.zeros((n_records),dtype=np.int16)
	#-- Solid Earth tide Correction packed units (mm, 1e-3 m)
	L2_final_corrections['seTideElv'] = np.zeros((n_records),dtype=np.int16)
	#-- Geocentric Polar tide Correction packed units (mm, 1e-3 m)
	L2_final_corrections['gpTideElv'] = np.zeros((n_records),dtype=np.int16)
	L2_final_corrections['Spare1'] = np.zeros((n_records),dtype=np.int16)
	#-- Surface Type: Packed in groups of three bits for each of the 20 records
	L2_final_corrections['Surf_type'] = np.zeros((n_records),dtype=np.uint64)
	#-- Mean Sea Surface or Geoid packed units (mm, 1e-3 m)
	L2_final_corrections['MSS_Geoid'] = np.zeros((n_records),dtype=np.int32)
	#-- Ocean Depth/Land Elevation Model (ODLE) packed units (mm, 1e-3 m)
	L2_final_corrections['ODLE'] = np.zeros((n_records),dtype=np.int32)
	#-- Ice Concentration packed units (%/100)
	L2_final_corrections['Ice_conc'] = np.zeros((n_records),dtype=np.int16)
	#-- Snow Depth packed units (mm, 1e-3 m)
	L2_final_corrections['Snow_depth'] = np.zeros((n_records),dtype=np.int16)
	#-- Snow Density packed units (kg/m^3)
	L2_final_corrections['Snow_density'] = np.zeros((n_records),dtype=np.int16)
	L2_final_corrections['Spare2'] = np.zeros((n_records),dtype=np.int16)
	#-- Corrections Status Flag
	L2_final_corrections['C_status'] = np.zeros((n_records),dtype=np.uint32)
	#-- Significant Wave Height (SWH) packed units (mm, 1e-3)
	L2_final_corrections['SWH'] = np.zeros((n_records),dtype=np.int16)
	#-- Wind Speed packed units (mm/s, 1e-3 m/s)
	L2_final_corrections['Wind_speed'] = np.zeros((n_records),dtype=np.uint16)
	L2_final_corrections['Spare3'] = np.zeros((n_records),dtype=np.int16)
	L2_final_corrections['Spare4'] = np.zeros((n_records),dtype=np.int16)
	L2_final_corrections['Spare5'] = np.zeros((n_records),dtype=np.int16)
	L2_final_corrections['Spare6'] = np.zeros((n_records),dtype=np.int16)

	#-- CryoSat-2 20 Hz data fields (Measurement Group)
	#-- Derived from instrument measurement parameters
	n_blocks = 20
	L2_final_measurements = {}
	#-- Delta between the timestamps for 20Hz record and the 1Hz record
	#-- D_time_mics packed units (microseconds)
	L2_final_measurements['D_time_mics'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Lat: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_final_measurements['Lat'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Lon: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_final_measurements['Lon'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Measured elevation above ellipsoid from retracker: packed units (mm, 1e-3 m)
	L2_final_measurements['Elev'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Interpolated Sea Surface Height Anomaly: packed units (mm, 1e-3 m)
	L2_final_measurements['SSHA_interp'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Interpolated Sea Surface Height measurement count
	L2_final_measurements['SSHA_num'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Interpolation quality estimate RSS: packed units (mm, 1e-3 m)
	L2_final_measurements['SSHA_qual'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Sigma Zero Backscatter for retracker: packed units (1e-2 dB)
	L2_final_measurements['Sig0'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Peakiness: packed units (1e-2)
	L2_final_measurements['Peakiness'] = np.zeros((n_records,n_blocks),dtype=np.uint16)
	#-- Freeboard: packed units (mm, 1e-3 m)
	#-- -9999 default value indicates computation has not been performed
	L2_final_measurements['Freeboard'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Number of averaged echoes or beams
	L2_final_measurements['N_avg'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	L2_final_measurements['Spare1'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Quality flags
	L2_final_measurements['Quality_Flg'] = np.zeros((n_records,n_blocks),dtype=np.uint32)
	L2_final_measurements['Spare2'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	L2_final_measurements['Spare3'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	L2_final_measurements['Spare4'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	L2_final_measurements['Spare5'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- for each record in the CryoSat file
	for r in range(n_records):
		#-- CryoSat-2 Location Group for record r
		L2_1Hz_parameters['Day'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_1Hz_parameters['Second'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_1Hz_parameters['Micsec'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_1Hz_parameters['Siral_mode'][r] = np.fromfile(fid,dtype='>u8',count=1)
		L2_1Hz_parameters['Lat_1Hz'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_1Hz_parameters['Lon_1Hz'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_1Hz_parameters['Alt_1Hz'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_1Hz_parameters['Mispointing'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_1Hz_parameters['N_valid'][r] = np.fromfile(fid,dtype='>i2',count=1)
		#-- CryoSat-2 External Corrections Group for record r
		L2_final_corrections['dryTrop'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['wetTrop'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['InvBar'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['DynAtm'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['Iono'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['SSB'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['ocTideElv'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['lpeTideElv'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['olTideElv'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['seTideElv'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['gpTideElv'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['Spare1'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['Surf_type'][r] = np.fromfile(fid,dtype='>u8',count=1)
		L2_final_corrections['MSS_Geoid'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_final_corrections['ODLE'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_final_corrections['Ice_conc'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['Snow_depth'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['Snow_density'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['Spare2'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['C_status'][r] = np.fromfile(fid,dtype='>u4',count=1)
		L2_final_corrections['SWH'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['Wind_speed'][r] = np.fromfile(fid,dtype='>u2',count=1)
		L2_final_corrections['Spare3'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['Spare4'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['Spare5'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_final_corrections['Spare6'][r] = np.fromfile(fid,dtype='>i2',count=1)
		#-- CryoSat-2 Measurements Group for record r and block b
		for b in range(n_blocks):
			L2_final_measurements['D_time_mics'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_final_measurements['Lat'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_final_measurements['Lon'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_final_measurements['Elev'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_final_measurements['SSHA_interp'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_final_measurements['SSHA_num'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_final_measurements['SSHA_qual'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_final_measurements['Sig0'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_final_measurements['Peakiness'][r,b] = np.fromfile(fid,dtype='>u2',count=1)
			L2_final_measurements['Freeboard'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_final_measurements['N_avg'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_final_measurements['Spare1'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_final_measurements['Quality_Flg'][r,b] = np.fromfile(fid,dtype='>u4',count=1)
			L2_final_measurements['Spare2'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_final_measurements['Spare3'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_final_measurements['Spare4'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_final_measurements['Spare5'][r,b] = np.fromfile(fid,dtype='>i2',count=1)

	#-- Bind all the bits of the l2_mds together into a single dictionary
	CS_l2_mds = {}
	CS_l2_mds['Data_1Hz'] = L2_1Hz_parameters
	CS_l2_mds['Corrections'] = L2_final_corrections
	CS_l2_mds['Data_20Hz'] = L2_final_measurements
	#-- return the output dictionary
	return CS_l2_mds

#-- PURPOSE: Initiate L2 MDS variables for CryoSat Baseline C
def cryosat_baseline_C(fid,record_size,n_records):
	#-- CryoSat-2 1 Hz data fields (Location Group)
	#-- Time and Orbit Parameters plus Measurement Mode
	L2_c_1Hz_parameters = {}
	#-- Time: day part
	L2_c_1Hz_parameters['Day'] = np.zeros((n_records),dtype=np.int32)
	#-- Time: second part
	L2_c_1Hz_parameters['Second'] = np.zeros((n_records),dtype=np.int32)
	#-- Time: microsecond part
	L2_c_1Hz_parameters['Micsec'] = np.zeros((n_records),dtype=np.int32)
	#-- SIRAL mode
	L2_c_1Hz_parameters['Siral_mode'] = np.zeros((n_records),dtype=np.uint64)
	#-- Lat_1Hz: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_c_1Hz_parameters['Lat_1Hz'] = np.zeros((n_records),dtype=np.int32)
	#-- Lon_1Hz: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_c_1Hz_parameters['Lon_1Hz'] = np.zeros((n_records),dtype=np.int32)
	#-- Alt_1Hz: packed units (mm, 1e-3 m)
	#-- Altitude of COG above reference ellipsoid (interpolated value)
	L2_c_1Hz_parameters['Alt_1Hz'] = np.zeros((n_records),dtype=np.int32)
	#-- Roll: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_c_1Hz_parameters['Roll'] = np.zeros((n_records),dtype=np.int32)
	#-- Pitch: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_c_1Hz_parameters['Pitch'] = np.zeros((n_records),dtype=np.int32)
	#-- Yaw: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_c_1Hz_parameters['Yaw'] = np.zeros((n_records),dtype=np.int32)
	L2_c_1Hz_parameters['Spare'] = np.zeros((n_records),dtype=np.int16)
	#-- Number of valid records in the block of twenty that contain data
	#-- Last few records of the last block of a dataset may be blank blocks
	#-- inserted to bring the file up to a multiple of twenty.
	L2_c_1Hz_parameters['N_valid'] = np.zeros((n_records),dtype=np.int16)

	#-- CryoSat-2 geophysical corrections (External Corrections Group)
	L2_c_final_corrections = {}
	#-- Dry Tropospheric Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['dryTrop'] = np.zeros((n_records),dtype=np.int16)
	#-- Wet Tropospheric Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['wetTrop'] = np.zeros((n_records),dtype=np.int16)
	#-- Inverse Barometric Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['InvBar'] = np.zeros((n_records),dtype=np.int16)
	#-- Dynamic Atmosphere Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['DynAtm'] = np.zeros((n_records),dtype=np.int16)
	#-- Ionospheric Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['Iono'] = np.zeros((n_records),dtype=np.int16)
	#-- Sea State Bias Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['SSB'] = np.zeros((n_records),dtype=np.int16)
	#-- Ocean tide Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['ocTideElv'] = np.zeros((n_records),dtype=np.int16)
	#-- Long period equilibrium ocean tide Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['lpeTideElv'] = np.zeros((n_records),dtype=np.int16)
	#-- Ocean loading tide Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['olTideElv'] = np.zeros((n_records),dtype=np.int16)
	#-- Solid Earth tide Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['seTideElv'] = np.zeros((n_records),dtype=np.int16)
	#-- Geocentric Polar tide Correction packed units (mm, 1e-3 m)
	L2_c_final_corrections['gpTideElv'] = np.zeros((n_records),dtype=np.int16)
	L2_c_final_corrections['Spare1'] = np.zeros((n_records),dtype=np.int16)
	#-- Surface Type: Packed in groups of three bits for each of the 20 records
	L2_c_final_corrections['Surf_type'] = np.zeros((n_records),dtype=np.uint64)
	#-- Mean Sea Surface or Geoid packed units (mm, 1e-3 m)
	L2_c_final_corrections['MSS_Geoid'] = np.zeros((n_records),dtype=np.int32)
	#-- Ocean Depth/Land Elevation Model (ODLE) packed units (mm, 1e-3 m)
	L2_c_final_corrections['ODLE'] = np.zeros((n_records),dtype=np.int32)
	#-- Ice Concentration packed units (%/100)
	L2_c_final_corrections['Ice_conc'] = np.zeros((n_records),dtype=np.int16)
	#-- Snow Depth packed units (mm, 1e-3 m)
	L2_c_final_corrections['Snow_depth'] = np.zeros((n_records),dtype=np.int16)
	#-- Snow Density packed units (kg/m^3)
	L2_c_final_corrections['Snow_density'] = np.zeros((n_records),dtype=np.int16)
	L2_c_final_corrections['Spare2'] = np.zeros((n_records),dtype=np.int16)
	#-- Corrections Status Flag
	L2_c_final_corrections['C_status'] = np.zeros((n_records),dtype=np.uint32)
	#-- Significant Wave Height (SWH) packed units (mm, 1e-3)
	L2_c_final_corrections['SWH'] = np.zeros((n_records),dtype=np.int16)
	#-- Wind Speed packed units (mm/s, 1e-3 m/s)
	L2_c_final_corrections['Wind_speed'] = np.zeros((n_records),dtype=np.uint16)
	L2_c_final_corrections['Spare3'] = np.zeros((n_records),dtype=np.int16)
	L2_c_final_corrections['Spare4'] = np.zeros((n_records),dtype=np.int16)
	L2_c_final_corrections['Spare5'] = np.zeros((n_records),dtype=np.int16)
	L2_c_final_corrections['Spare6'] = np.zeros((n_records),dtype=np.int16)

	#-- CryoSat-2 20 Hz data fields (Measurement Group)
	#-- Derived from instrument measurement parameters
	n_blocks = 20
	L2_c_final_measurements = {}
	#-- Delta between the timestamps for 20Hz record and the 1Hz record
	#-- D_time_mics packed units (microseconds)
	L2_c_final_measurements['D_time_mics'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Lat: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_c_final_measurements['Lat'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Lon: packed units (0.1 micro-degree, 1e-7 degrees)
	L2_c_final_measurements['Lon'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Measured elevation above ellipsoid from retracker 1: packed units (mm, 1e-3 m)
	L2_c_final_measurements['Elev_1'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Measured elevation above ellipsoid from retracker 2: packed units (mm, 1e-3 m)
	L2_c_final_measurements['Elev_2'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Measured elevation above ellipsoid from retracker 3: packed units (mm, 1e-3 m)
	L2_c_final_measurements['Elev_3'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Sigma Zero Backscatter for retracker 1: packed units (1e-2 dB)
	L2_c_final_measurements['Sig0_1'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Sigma Zero Backscatter for retracker 2: packed units (1e-2 dB)
	L2_c_final_measurements['Sig0_2'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Sigma Zero Backscatter for retracker 3: packed units (1e-2 dB)
	L2_c_final_measurements['Sig0_3'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Freeboard: packed units (mm, 1e-3 m)
	#-- -9999 default value indicates computation has not been performed
	L2_c_final_measurements['Freeboard'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Interpolated Sea Surface Height Anomaly: packed units (mm, 1e-3 m)
	L2_c_final_measurements['SSHA_interp'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Interpolated Sea Surface Height measurement count
	L2_c_final_measurements['SSHA_num'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Interpolation quality estimate RSS: packed units (mm, 1e-3 m)
	L2_c_final_measurements['SSHA_qual'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Peakiness: packed units (1e-2)
	L2_c_final_measurements['Peakiness'] = np.zeros((n_records,n_blocks),dtype=np.uint16)
	#-- Number of averaged echoes or beams
	L2_c_final_measurements['N_avg'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	L2_c_final_measurements['Spare1'] = np.zeros((n_records,n_blocks),dtype=np.int16)
	#-- Quality flags
	L2_c_final_measurements['Quality_Flg'] = np.zeros((n_records,n_blocks),dtype=np.uint32)
	#-- Corrections Application Flag
	L2_c_final_measurements['Corrections_Flg'] = np.zeros((n_records,n_blocks),dtype=np.uint32)
	#-- Quality metric for retracker 1
	L2_c_final_measurements['Quality_1'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Quality metric for retracker 2
	L2_c_final_measurements['Quality_2'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- Quality metric for retracker 3
	L2_c_final_measurements['Quality_3'] = np.zeros((n_records,n_blocks),dtype=np.int32)
	#-- for each record in the CryoSat file
	for r in range(n_records):
		#-- CryoSat-2 Location Group for record r
		L2_c_1Hz_parameters['Day'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_1Hz_parameters['Second'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_1Hz_parameters['Micsec'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_1Hz_parameters['Siral_mode'][r] = np.fromfile(fid,dtype='>u8',count=1)
		L2_c_1Hz_parameters['Lat_1Hz'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_1Hz_parameters['Lon_1Hz'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_1Hz_parameters['Alt_1Hz'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_1Hz_parameters['Roll'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_1Hz_parameters['Pitch'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_1Hz_parameters['Yaw'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_1Hz_parameters['Spare'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_1Hz_parameters['N_valid'][r] = np.fromfile(fid,dtype='>i2',count=1)
		#-- CryoSat-2 External Corrections Group for record r
		L2_c_final_corrections['dryTrop'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['wetTrop'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['InvBar'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['DynAtm'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['Iono'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['SSB'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['ocTideElv'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['lpeTideElv'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['olTideElv'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['seTideElv'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['gpTideElv'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['Spare1'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['Surf_type'][r] = np.fromfile(fid,dtype='>u8',count=1)
		L2_c_final_corrections['MSS_Geoid'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_final_corrections['ODLE'][r] = np.fromfile(fid,dtype='>i4',count=1)
		L2_c_final_corrections['Ice_conc'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['Snow_depth'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['Snow_density'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['Spare2'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['C_status'][r] = np.fromfile(fid,dtype='>u4',count=1)
		L2_c_final_corrections['SWH'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['Wind_speed'][r] = np.fromfile(fid,dtype='>u2',count=1)
		L2_c_final_corrections['Spare3'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['Spare4'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['Spare5'][r] = np.fromfile(fid,dtype='>i2',count=1)
		L2_c_final_corrections['Spare6'][r] = np.fromfile(fid,dtype='>i2',count=1)
		#-- CryoSat-2 Measurements Group for record r and block b
		for b in range(n_blocks):
			L2_c_final_measurements['D_time_mics'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_c_final_measurements['Lat'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_c_final_measurements['Lon'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_c_final_measurements['Elev_1'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_c_final_measurements['Elev_2'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_c_final_measurements['Elev_3'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_c_final_measurements['Sig0_1'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_c_final_measurements['Sig0_2'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_c_final_measurements['Sig0_3'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_c_final_measurements['Freeboard'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_c_final_measurements['SSHA_interp'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_c_final_measurements['SSHA_num'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_c_final_measurements['SSHA_qual'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_c_final_measurements['Peakiness'][r,b] = np.fromfile(fid,dtype='>u2',count=1)
			L2_c_final_measurements['N_avg'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_c_final_measurements['Spare1'][r,b] = np.fromfile(fid,dtype='>i2',count=1)
			L2_c_final_measurements['Quality_Flg'][r,b] = np.fromfile(fid,dtype='>u4',count=1)
			L2_c_final_measurements['Corrections_Flg'][r,b] = np.fromfile(fid,dtype='>u4',count=1)
			L2_c_final_measurements['Quality_1'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_c_final_measurements['Quality_2'][r,b] = np.fromfile(fid,dtype='>i4',count=1)
			L2_c_final_measurements['Quality_3'][r,b] = np.fromfile(fid,dtype='>i4',count=1)

	#-- Bind all the bits of the l2_mds together into a single dictionary
	CS_l2_c_mds = {}
	CS_l2_c_mds['Data_1Hz'] = L2_c_1Hz_parameters
	CS_l2_c_mds['Corrections'] = L2_c_final_corrections
	CS_l2_c_mds['Data_20Hz'] = L2_c_final_measurements
	#-- return the output dictionary
	return CS_l2_c_mds

#-- PURPOSE: Read ASCII Main Product Header (MPH) block from an ESA PDS file
def read_MPH(full_filename):
	#-- read input data file
	with open(full_filename, 'rb') as fid:
		file_contents = fid.read().splitlines()

	#-- Define constant values associated with PDS file formats
	#-- number of text lines in standard MPH
	n_MPH_lines	= 41
	#-- check that first line of header matches PRODUCT
	if not bool(re.match(b'PRODUCT\=\"(.*)(?=\")',file_contents[0])):
		raise IOError('File does not start with a valid PDS MPH')
	#-- read MPH header text
	s_MPH_fields = {}
	for i in range(n_MPH_lines):
		#-- use regular expression operators to read headers
		if bool(re.match(b'(.*?)\=\"(.*)(?=\")',file_contents[i])):
			#-- data fields within quotes
			field,value=re.findall(b'(.*?)\=\"(.*)(?=\")',file_contents[i]).pop()
			s_MPH_fields[field.decode('utf-8')] = value.decode('utf-8').rstrip()
		elif bool(re.match(b'(.*?)\=(.*)',file_contents[i])):
			#-- data fields without quotes
			field,value=re.findall(b'(.*?)\=(.*)',file_contents[i]).pop()
			s_MPH_fields[field.decode('utf-8')] = value.decode('utf-8').rstrip()

	#-- Return block name array to calling function
	return s_MPH_fields

#-- PURPOSE: Read ASCII Specific Product Header (SPH) block from a PDS file
def read_SPH(full_filename,j_sph_size):
	#-- read input data file
	with open(full_filename, 'rb') as fid:
		file_contents = fid.read().splitlines()

	#-- Define constant values associated with PDS file formats
	#-- number of text lines in standard MPH
	n_MPH_lines	= 41
	#-- compile regular expression operator for reading headers
	rx = re.compile(b'(.*?)\=\"?(.*)',re.VERBOSE)
	#-- check first line of header matches SPH_DESCRIPTOR
	if not bool(re.match(b'SPH\_DESCRIPTOR\=',file_contents[n_MPH_lines+1])):
		raise IOError('File does not have a valid PDS DSD')
	#-- read SPH header text (no binary control characters)
	s_SPH_lines = [li for li in file_contents[n_MPH_lines+1:] if rx.match(li)
		and not re.search(b'[^\x20-\x7e]+',li)]

	#-- extract SPH header text
	s_SPH_fields = {}
	c = 0
	while (c < len(s_SPH_lines)):
		#-- check if line is within DS_NAME portion of SPH header
		if bool(re.match(b'DS_NAME',s_SPH_lines[c])):
			#-- add dictionary for DS_NAME
			field,value=re.findall(b'(.*?)\=\"(.*)(?=\")',s_SPH_lines[c]).pop()
			key = value.decode('utf-8').rstrip()
			s_SPH_fields[key] = {}
			for line in s_SPH_lines[c+1:c+7]:
				if bool(re.match(b'(.*?)\=\"(.*)(?=\")',line)):
					#-- data fields within quotes
					dsfield,dsvalue=re.findall(b'(.*?)\=\"(.*)(?=\")',line).pop()
					s_SPH_fields[key][dsfield.decode('utf-8')] = dsvalue.decode('utf-8').rstrip()
				elif bool(re.match(b'(.*?)\=(.*)',line)):
					#-- data fields without quotes
					dsfield,dsvalue=re.findall(b'(.*?)\=(.*)',line).pop()
					s_SPH_fields[key][dsfield.decode('utf-8')] = dsvalue.decode('utf-8').rstrip()
			#-- add 6 to counter to go to next entry
			c += 6
		#-- use regular expression operators to read headers
		elif bool(re.match(b'(.*?)\=\"(.*)(?=\")',s_SPH_lines[c])):
			#-- data fields within quotes
			field,value=re.findall(b'(.*?)\=\"(.*)(?=\")',s_SPH_lines[c]).pop()
			s_SPH_fields[field.decode('utf-8')] = value.decode('utf-8').rstrip()
		elif bool(re.match(b'(.*?)\=(.*)',s_SPH_lines[c])):
			#-- data fields without quotes
			field,value=re.findall(b'(.*?)\=(.*)',s_SPH_lines[c]).pop()
			s_SPH_fields[field.decode('utf-8')] = value.decode('utf-8').rstrip()
		#-- add 1 to counter to go to next line
		c += 1

	#-- Return block name array to calling function
	return s_SPH_fields

#-- PURPOSE: Read ASCII Data Set Descriptors (DSD) block from a PDS file
def read_DSD(full_filename):
	#-- read input data file
	with open(full_filename, 'rb') as fid:
		file_contents = fid.read().splitlines()

	#-- Define constant values associated with PDS file formats
	#-- number of text lines in standard MPH
	n_MPH_lines	= 41
	#-- number of text lines in a DSD header
	n_DSD_lines = 8

	#-- Level-2 CryoSat DS_NAMES within files
	regex_patterns = []
	regex_patterns.append(b'DS_NAME\="SIR_LRM_L2[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SAR_L2B[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SAR_L2[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_FDM_L2[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SARIL2B[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SARIL2[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SAR_L2B_I[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SAR_L2A[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SIN_L2[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SID_L2[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_LRMIL2[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_LRM_L2_I[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SARIL2A[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SAR_L2A_I[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SAR_L2_I[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SINIL2[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SIN_L2_I[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SIDIL2[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_SID_L2_I[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_GDR_2A[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_GDR_2B[\s+]*"')
	regex_patterns.append(b'DS_NAME\="SIR_GDR_2[\s+]*"')
	#-- find the DSD starting line within the SPH header
	c = 0
	Flag = False
	while ((Flag is False) and (c < len(regex_patterns))):
		#-- find indice within
		indice = [i for i,line in enumerate(file_contents[n_MPH_lines+1:]) if
			re.search(regex_patterns[c],line)]
		if indice:
			Flag = True
		else:
			c+=1
	#-- check that valid indice was found within header
	if not indice:
		raise IOError('Can not find correct DSD field')

	#-- extract s_DSD_fields info
	DSD_START = n_MPH_lines + indice[0] + 1
	s_DSD_fields = {}
	for i in range(DSD_START,DSD_START+n_DSD_lines):
		#-- use regular expression operators to read headers
		if bool(re.match(b'(.*?)\=\"(.*)(?=\")',file_contents[i])):
			#-- data fields within quotes
			field,value=re.findall(b'(.*?)\=\"(.*)(?=\")',file_contents[i]).pop()
			s_DSD_fields[field.decode('utf-8')] = value.decode('utf-8').rstrip()
		elif bool(re.match(b'(.*?)\=(.*)',file_contents[i])):
			#-- data fields without quotes
			field,value=re.findall(b'(.*?)\=(.*)',file_contents[i]).pop()
			s_DSD_fields[field.decode('utf-8')] = value.decode('utf-8').rstrip()

	#-- Return block name array to calling function
	return s_DSD_fields

#-- PURPOSE: read CryoSat Level-2 data
def read_cryosat_L2(full_filename, VERBOSE=False):
	#-- file basename and file extension of input file
	fileBasename,fileExtension=os.path.splitext(os.path.basename(full_filename))

	#-- CryoSat file class
	#-- OFFL (Off Line Processing/Systematic)
	#-- NRT_ (Near Real Time)
	#-- RPRO (ReProcessing)
	#-- TEST (Testing)
	#-- LTA_ (Long Term Archive)
	regex_class = 'OFFL|NRT_|RPRO|TEST|LTA_'
	#-- CryoSat mission products
	#-- SIR_LRM_2 L2 Product from Low Resolution Mode Processing
	#-- SIR_FDM_2 L2 Product from Fast Delivery Marine Mode Processing
	#-- SIR_SIN_2 L2 Product from SAR Interferometric Processing
	#-- SIR_SID_2 L2 Product from SIN Degraded Processing
	#-- SIR_SAR_2 L2 Product from SAR Processing
	#-- SIR_GDR_2 L2 Consolidated Product
	#-- SIR_LRMI2 In-depth L2 Product from LRM Processing
	#-- SIR_SINI2 In-depth L2 Product from SIN Processing
	#-- SIR_SIDI2 In-depth L2 Product from SIN Degraded Process.
	#-- SIR_SARI2 In-depth L2 Product from SAR Processing
	regex_products = ('SIR_LRM_2|SIR_FDM_2|SIR_SIN_2|SIR_SID_2|'
	'SIR_SAR_2|SIR_GDR_2|SIR_LRMI2|SIR_SINI2|SIR_SIDI2|SIR_SARI2')
	#-- CRYOSAT LEVEL-2 PRODUCTS NAMING RULES
	#-- Mission Identifier
	#-- File Class
	#-- File Product
	#-- Validity Start Date and Time
	#-- Validity Stop Date and Time
	#-- Baseline Identifier
	#-- Version Number
	regex_pattern = '(.*?)_({0})_({1})__(\d+T?\d+)_(\d+T?\d+)_(.*?)(\d+)'.format(
		regex_class, regex_products)
	rx = re.compile(regex_pattern, re.VERBOSE)
	#-- extract file information from filename
	MI,CLASS,PRODUCT,START,STOP,BASELINE,VERSION=rx.findall(fileBasename).pop()
	#-- Extract Date information
	start_yr,start_mon,start_day=np.array([START[:4],START[4:6],START[6:8]],dtype=np.uint16)
	start_hh,start_mm,start_ss=np.array([START[-6:-4],START[-4:-2],START[-2:]],dtype=np.uint8)
	stop_yr,stop_mon,stop_day=np.array([STOP[:4],STOP[4:6],STOP[6:8]],dtype=np.uint16)
	stop_hh,stop_mm,stop_ss=np.array([STOP[-6:-4],STOP[-4:-2],STOP[-2:]],dtype=np.uint8)

	#-- Record sizes
	CS_L2_MDS_REC_SIZE = 980
	CS_L2_C_MDS_REC_SIZE = 1392
	#-- check baseline from file to set i_record_size and allocation function
	if (BASELINE == 'C'):
		i_record_size = CS_L2_C_MDS_REC_SIZE
		read_cryosat_variables = cryosat_baseline_C
	else:
		i_record_size = CS_L2_MDS_REC_SIZE
		read_cryosat_variables = cryosat_baseline_AB

	#-- read the input file to get file information
	fid = os.open(os.path.expanduser(full_filename),os.O_RDONLY)
	file_info = os.fstat(fid)
	os.close(fid)

	#-- num DSRs from SPH
	j_num_DSR = np.int32(file_info.st_size//i_record_size)
	#-- print file information
	if VERBOSE:
		print(fileBasename)
		print('{0:d} {1:d} {2:d}'.format(j_num_DSR,file_info.st_size,i_record_size))
		#-- Check if MPH/SPH/DSD headers
		if (j_num_DSR*i_record_size == file_info.st_size):
			print('No Header on file')
			print('The number of DSRs is: {0:d}'.format(j_num_DSR))
		else:
			print('Header on file')

	#-- Check if MPH/SPH/DSD headers
	if (j_num_DSR*i_record_size != file_info.st_size):
		#-- If there are MPH/SPH/DSD headers
		s_MPH_fields = read_MPH(full_filename)
		j_sph_size = np.int32(re.findall('[-+]?\d+',s_MPH_fields['SPH_SIZE']).pop())
		s_SPH_fields = read_SPH(full_filename,j_sph_size)
		#-- extract information from DSD fields
		s_DSD_fields = read_DSD(full_filename)
		#-- extract DS_OFFSET
		j_DS_start = np.int32(re.findall('[-+]?\d+',s_DSD_fields['DS_OFFSET']).pop())
		#-- extract number of DSR in the file
		j_num_DSR = np.int32(re.findall('[-+]?\d+',s_DSD_fields['NUM_DSR']).pop())
		#-- check the record size
		j_DSR_size = np.int32(re.findall('[-+]?\d+',s_DSD_fields['DSR_SIZE']).pop())
		#--  minimum size is start of the read plus number of records to read
		j_check_size = j_DS_start +(j_DSR_size*j_num_DSR)
		if VERBOSE:
			print('The offset of the DSD is: {0:d} bytes'.format(j_DS_start))
			print('The number of DSRs is {0:d}'.format(j_num_DSR))
			print('The size of the DSR is {0:d}'.format(j_DSR_size))
		#-- check if invalid file size
		if (j_check_size > file_info.st_size):
			raise IOError('File size error')
		#-- extract binary data from input CryoSat data file (skip headers)
		fid = open(full_filename, 'rb')
		cryosat_header = fid.read(j_DS_start)
		#-- iterate through CryoSat file and fill output variables
		CS_L2_mds = read_cryosat_variables(fid,i_record_size,j_num_DSR)
		#-- add headers to output dictionary as METADATA
		CS_L2_mds['METADATA'] = {}
		CS_L2_mds['METADATA']['MPH'] = s_MPH_fields
		CS_L2_mds['METADATA']['SPH'] = s_SPH_fields
		CS_L2_mds['METADATA']['DSD'] = s_DSD_fields
		#-- add absolute orbit number to 1Hz data
		CS_L2_mds['Data_1Hz']['Abs_Orbit']=np.zeros((j_num_DSR),dtype=np.uint32)
		CS_L2_mds['Data_1Hz']['Abs_Orbit'][:]=np.uint32(s_MPH_fields['ABS_ORBIT'])
		#-- add ascending/descending flag to 1Hz data (A=ascending,D=descending)
		CS_L2_mds['Data_1Hz']['Ascending_Flg']=np.zeros((j_num_DSR),dtype=np.bool)
		if (s_SPH_fields['ASCENDING_FLAG'] == 'A'):
			CS_L2_mds['Data_1Hz']['Ascending_Flg'][:] = True
		#-- close the input CryoSat binary file
		fid.close()
	else:
		#-- If there are not MPH/SPH/DSD headers
		#-- extract binary data from input CryoSat data file
		fid = open(full_filename, 'rb')
		#-- iterate through CryoSat file and fill output variables
		CS_L2_mds = read_cryosat_variables(fid,i_record_size,j_num_DSR)
		#-- close the input CryoSat binary file
		fid.close()

	#-- return the data and headers
	return CS_L2_mds


# coding: utf-8

# ## Working with Matplotlib
# Introduction:
#
# Matplotlib is a Python 2D plotting library which produces publication quality figures in a variety of hardcopy formats and interactive environments across platforms. Matplotlib can be used in Python scripts, the Python and IPython shell, the jupyter notebook, web application servers, and four graphical user interface toolkits.
#
# ### Here are the main steps we will go through
# * How to save your plot using matplotlib?
# * How to create plot using matplotlib?
# * How to make your graph look pretty?
#
# This is Just a little illustration.
#
#

# #### How to save your plot using matplotlib?
# I am gonna show you how to save your graph, you can refer the docstring for complete information on the various ways it can be used.

# In[1]:

# import matplotlib, numpy
import matplotlib.pyplot as plt
import numpy as np

# In[2]:

#saves plot/figure to image
plt.savefig('name_your_graph.png')

# In[11]:

# we can create simple plot as follows, linear graph
# Prepare the data
linear = np.linspace(0, 10, 100)

# Plot the data, set label and color, b=blue, g=green
plt.plot(linear, linear, label='linear', color='b')

# Add a legend
plt.legend()

# Show the plot
plt.show()

# In[23]:

#by calling
plt.style.available

# In[26]:

x = np.linspace(0, 4 * np.pi)
y = np.sin(x)

plt.plot(x, y)

plt.show()

# In[27]:

plt.style.use('ggplot')

plt.plot(x, y)

plt.show()

# In[32]:

with plt.style.context(('seaborn-darkgrid')):
    plt.plot(x, y, 'r-o')
    plt.show()

# #### Working with text inside your graph

# However, you can use subplots to set up and place your Axes on a regular grid. So that means that in most cases, Axes and subplot are synonymous, they will designate the same thing. When you do call subplot to add Axes to your figure, do so with the add_subplots() function.

# In[14]:

# we can create scatter plot for this
fig = plt.figure()

# Set up Axes
ax = fig.add_subplot(111)

# Scatter the data
ax.scatter(np.linspace(0, 1, 5), np.linspace(0, 5, 5), label='scatter')

# Add a legend
plt.legend()

# Show the plot
plt.show()

# #### How To Change The Size of Figures
#
# Now that you have seen how to initialize a Figure and Axes from scratch, you will also want to know how you can change certain small details that the package sets up for you, such as the figure size.
# Let’s say you don’t to have your figure size to be default size and you want to change this. How do you set the size of your figures manually?

# In[20]:

# easy! you can just call
fig = plt.figure(figsize=(10,8))
ax1 = fig.add_subplot(121)
ax2 = fig.add_subplot(122)

# In[21]:

# let's plot simple bar graph
# Plot the data
ax1.bar([1,2,3],[3,4,5])
ax2.barh([0.5,1,2.5],[0,1,2])

# Show the plot
plt.show()

# In[3]:

x = np.linspace(0, 2, 100)

plt.plot(x, x, label='linear')
plt.plot(x, x**2, label='quadratic')
plt.plot(x, x**3, label='cubic')

plt.xlabel('x label')
plt.ylabel('y label')

plt.title("Simple Plot")

plt.legend()

plt.show()

# In[4]:

#plot data connected by lines
lines = plt.plot(x,x)
plt.show()

# ## More to come soon!


# Business Objectives/Statistical Questions and Feature Selection

*Hayley Boyce, May 12th, 2021*

**Attribution:**

- Tomas Beuzen - Previous BAIT 509 Lecture 6
- Varada Kolhatkar - Heavily guiding me with Feature Selection

# Importing our libraries
import pandas as pd
import altair as alt
import numpy as np
from sklearn.tree import DecisionTreeClassifier
from sklearn.dummy import DummyClassifier, DummyRegressor
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.model_selection import cross_validate, train_test_split
from sklearn import datasets
from sklearn.linear_model import Ridge

import sys
sys.path.append('code/')
from display_tree import display_tree
from plot_classifier import plot_classifier
import matplotlib.pyplot as plt

# Preprocessing and pipeline
from sklearn.impute import SimpleImputer
from sklearn.metrics.pairwise import euclidean_distances
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.compose import make_column_transformer
from sklearn.linear_model import LogisticRegression

from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, StandardScaler, MinMaxScaler

import scipy
from sklearn.model_selection import RandomizedSearchCV

## House Keeping

- Project instructions are out!
- Time in class for groups!
- Assignment 3 was released yesterday!
- More reading in this lecture than usual (sorry!)
- Run code at your own risk (Think section can take up to 20 minutes so if you are running this locally, Start running everything NOW from top to bottom)

## Lecture Learning Objectives

- In the context of supervised learning, form statistical questions  from business questions/objectives.
- Understand the different forms your client may expect you to communicate results.
- Explain the general concept of feature selection.
- Discuss and compare different feature selection methods at a high level.
- Use sklearn's implementation of recursive feature elimination (RFE).
- Implement the forward search algorithm.

## Five Minute Recap/ Lightning Questions

- What is the name of the function used to bound our values between 0 and 1
- What is the name of the function that gives "hard" predictions?
- What is the name of the function that gives "soft" predictions?
- What is the hyperparameter we learned for Ridge and how does it affect the Fundamental Trade-off?
- What is the hyperparameter we learned for Logistic Regression and how does it affect the Fundamental Trade-off?

### Some lingering questions

- How can we start forming good business questions that can be addressed with Machine Learning?
- How to select features for our models?

## Forming statistical questions to answer business objectives

So far you've seen how to solve predictive problems using machine learning but today, we are going to look at the process involved in asking the questions and problems faced by organizations.

Generally, there are four parts of a machine learning analysis. In order from high to low level:

1. **The business question/objective**
2. **The statistical question/objective**
3. **The data and model**
4. **The data product**

Doing a machine learning analysis is about distilling from the highest level to the lowest level. As such, there are three distillations to keep in mind: 1-2, 2-3, and 3-4:

- **1-2 is about asking the right questions**
- **2-3 is about building a useful model**
- **3-4 is about communicating the results**

Note that an analysis isn’t a linear progression through these “steps”; rather, the process is iterative. This is because none of the components are independent. Making progress on any of the three distillations gives you more information as to what the problem is.

We’ll look at each of these distillations in turn.

## (1 - 2) Asking useful statistical questions

- Usually, a company is not served up a machine learning problem, complete with data and a description of the response and predictors.
    - Companies don't exactly know what question is the right question but they do know what they want  to be accomplished.
- Instead, they’re faced with some high-level objective/question that we’ll call the **business question/objective**.
- This question needs refining to a **statistical question/objective** – one that is directly addressable by machine learning.

Example:

- My Capstone Case study.

### Business objectives: examples
- This [altexsoft blog post](https://www.altexsoft.com/blog/business/supervised-learning-use-cases-low-hanging-fruit-in-data-science-for-businesses/) is a great introduction to business use cases of data science/ML
- Examples of business objectives (for which machine learning is a relevant approach)
    - Reduce the amount of spam email received
    - Early prediction of product failure
    - Find undervalued mines
    - Make a transit system more efficient
    - Hire efficient staff

### Refining business objectives to statistical objectives

- Statistical objectives need to be specific
- Remember that supervised learning is about predicting a response $Y$ from predictors $X_1,…,X_p$
- So we need to refine our business objectives to a statistical question(s) we can answer
- This typically involves:
    - Identifying the **response variable** ($Y$) that is most aligned with the business objective.
    - Identifying the **data** (observations + features) that will be used for model development/testing.
    - Note: part of this is the task of feature selection (a topic that we've covered briefly) – but, this is also largely, a human decision based on what we think is more informative, as well as a resource questions (what data is actually available?)

### Statistical objectives: examples
Statistical objectives corresponding to the above business objective examples might be:

| Business Objective | Statistical Question |
| :--- | :--- |
| Reduce the amount of spam email received |

$Y$ = classifying an email as spam/not spam
 $X$ = words present in name and the body of email and other metadata (sender email, time, etc.)
 Cases of spam will be gathered over time as employees identify emails as spam/not spam. The model can be improved as misclassifications are encountered.

| Early prediction of product failure (Kickstarter?) |

$Y$ = classifying a product as faulty/not faulty
 $X$ = Relevant features chosen by an expert
 Data obtained from the test facility

| Find undervalued mines |

$Y$ = total volume of gold and silver at a site
 $X$ =  concentrations of other minerals found in drill samples, geographic information, historical data, etc
 Data obtained from mines where total volumes are already known

| Make a transit system more efficient |

$Y$ = predict the time it takes a bus to travel between set stops
 $X$ = time of day/week/year, weather, etc.
 Use data from company server tracking bus movements

| Hire efficient staff |

$Y$ = predict monthly sales
 $X$ = a personality test, years of work experience, field of experience, etc.
 Use data based on current employees

### Statistical questions are not the full picture!
- Almost always, the business objective is more complex than the statistical question.
- By refining a business objective to a statistical one, we may lose part of the essence of the business objective.
- It’s important to have a sense of the ways in which your statistical objective falls short, and the ways in which it’s on the mark, so that you keep an idea of the big picture.
- For example, predicting whether a new staff hire will be efficient or not is a useful statistical question, but doesn't consider why a company might be attracting certain applicants, how long staff will remain, how staff work together, etc.

### Statistical objectives unrelated to supervised learning

- We are only focussing on statistical questions related to supervised learning and prediction in this course
- But there are other kinds of questions you can ask too
- Consider the following example

**Business objective**: To gain insight into the productivity of two branches of a company.

Examples of statistical questions:

- **Hypothesis testing**: Is the mean number of sick days per employee different between two branches of a company?
    - Supervised learning doesn’t cover testing for differences.
- **Unsupervised learning**: What is the mean sentiment of the internal discussion channels from both branches?
    - There is no data of feature + response here, as required by supervised learning (by definition).
- **Statistical inference**: Estimate the mean difference in monthly revenue generated by both branches, along with how certain you are with that estimate.
    - Supervised learning typically isn’t concerned about communicating how certain your estimate is. (BUT it should and progress is occuring to change this!)

## (2 - 3) Building a useful model
- This is really the main focus of this course, This is the meat/beyond meat patty in your burger!
- This involves using ML algorithms (kNN, loess, decision trees, etc) to build a predictive model from data
- You always should include a baseline model to assess how well the models you build are giving you some leg up.
- A simple model like logistic regression does as well as more complex approaches! At the very least, they can help guide you on what more complex approaches to take next.

## (3 - 4) Communicating results

- So you've distilled your business objectives to a statistical question.
- You've developed a model to answer the statistical question.
- Now your model needs to be delivered and used by others (or your future self)!
- The final delivery is often called "the data product" because it may consist of a variety of things:
    - a report
    - a presentation
    - an app
    - a dashboard
    - a software package/pipeline
- Sometimes the client requests a specific data product -> But note that their suggestion might not always be the best option.
    - Perhaps they request a report and presentation communicating your findings, when a more appropriate product also includes an interactive app that allows them to explore your findings for themselves.
- Either way, the key here is communication. Two import challenges (relevant to your final project):
    - Using appropriate language: there is a lot of jargon in ML, the key is to talk more about the output and the general idea of your model(s), but not machine learning or statistical jargon.
    - Communication with visual design: this is about choosing what visuals are the most effective for communicating. -> Plug: https://viz-learn.mds.ubc.ca/en/
- Usually, the first step is to set up a framework for your data product. For a report, this means outlining what you intend to write about, and where.
- Showing it to your client is useful as a sanity check to see that you’re about to produce something that the client currently sees as being potentially useful.

## Let's Practice

1\. What question is usually more complex?
2\. What model needs to be made for all problems?
3\. In supervised learning, once we have our business objective, part of our statistical question is identifying what?

**True or False:**

4\. When writing your reports, it's important to consider who is reading it.
5\. Sometimes you may need to dig a little to figure out exactly what the client wants.
6\. In supervised learning, we should take into consideration the uncertainty of our models.

```{admonition} Solutions!
:class: dropdown

1. Business question/objective
2. Baseline - Dummy
3. Our target variable
4. True
5. True
6. True

```

## Feature Selection

### Motivation

Remember the curse of dimensionality?

We spoke about this briefly when we discussed $k$-nn and how when we add many different dimensions (features) it can confuse the model with any irrelevant features and the model can disintegrate into predictions no better than random guessing.

Reasons like this are why we need to be careful about which features we include.

**Feature selection** can be described as *finding the features (columns) $X$ that are important for predicting $y$ and removing the features that aren’t.*

Feature selection can be aided using domain knowledge and manually however we can also use tools that help us either tell us:

- which features are most important to a model
- or that can select a number of features that will result with the optimal validation score.

cities_df = pd.read_csv("data/canada_usa_cities.csv")
train_df, test_df = train_test_split(cities_df, test_size=0.2, random_state=123)
X_train, y_train = train_df.drop(columns=["country"], axis=1), train_df["country"]
X_test, y_test = test_df.drop(columns=["country"], axis=1), test_df["country"]

train_df.head()

### Feature importance

Remember our Decision Tree models? Well, we can find out which features are most important in a model using an attribute called `feature_importances_`.

dt_modelku_model = DecisionTreeClassifier(max_depth=5)
dt_model.fit(X_train, y_train)
dt_model.feature_importances_

Here we can see that most of the importance is on the column `latitude`.

X_train.columns

If we graph this, the root of the decision tree will usually reflect the top feature. Just like we can see here:

sys.path.append('code/')
from display_tree import display_tree
display_tree(X_train.columns, dt_model, "imgs/decision_tree")

### New housing data

I know at this point you are probably annoyed and bored of housing data, but good, interesting open-source data is hard to come by. For this example, I really want to show you an example with LOTS of features.

Here is (yet another) housing dataset we acquired from [this GitHub repo](https://github.com/melindaleung/Ames-Iowa-Housing-Dataset), originally created by Dean De Cock.
*(We are using the raw data so we do not need to store it and import it simply from the url.)*

**Attribution:**

The Ames Housing dataset was compiled by Dean De Cock for use in data science education.

His publication can be found [here](http://jse.amstat.org/v19n3/decock.pdf).

ames_df = pd.read_csv('https://raw.githubusercontent.com/melindaleung/Ames-Iowa-Housing-Dataset/master/data/ames%20iowa%20housing.csv', index_col=0)
ames_df.loc[ames_df['SaleCondition'] != 'Normal', 'SaleCondition'] = 'Abnormal'
ames_df

ames_df.info()

Here, we use `.info()` to identify our categorical and numeric features. I split my features and identify my target.

The target variable for this question is `SaleCondition` and we are doing classification.

train_df, test_df = train_test_split(ames_df, test_size=0.2, random_state=77)

X_train = train_df.drop(columns=['SaleCondition', 'PoolQC', 'MiscFeature', 'Alley'])
X_test =  test_df.drop(columns=['SaleCondition', 'PoolQC', 'MiscFeature', 'Alley'])

y_train = train_df['SaleCondition']
y_test = test_df['SaleCondition']

 Note, you should be looking at these individually but I'm being a little lazy here.

numeric_features = X_train.select_dtypes('number').columns.to_list()
categorical_features = X_train.select_dtypes('object').columns.to_list()

Here are our numeric features:

numeric_features

And here are our categorical features:

categorical_features

Next, we need to make our pipelines and column transformer.

We can also cross-validate, but here my main goal is to show you how to get our feature importances from our pipeline!

numeric_pipe = make_pipeline(SimpleImputer(strategy='median'),
                            StandardScaler())
categoric_pipe = make_pipeline(SimpleImputer(strategy="constant", fill_value="missing"),
                            OneHotEncoder(dtype=int, handle_unknown="ignore"))

preprocessor = make_column_transformer((numeric_pipe, numeric_features),
                                       (categoric_pipe, categorical_features))

main_pipe = make_pipeline(preprocessor, LogisticRegression(max_iter=1000))

scores = cross_validate(main_pipe, X_train, y_train, return_train_score=True)

pd.DataFrame(scores).mean()

Once we fit our pipeline outside of `cross_validate()` we can use `coef_` to get our features that contribute to our predictions.

main_pipe.fit(X_train, y_train)
feats_coef = main_pipe.named_steps['logisticregression'].coef_
feats_coef

The problem here, is we don't know which value corresponds to which feature!

Let's first take a look at how many features we have now after preprocessing.

feats_coef.shape

The 282 is refering to the number of features after preprocessing!

Let's get the feature names after preprocessing.

We can obtain the categorical features and combine them with the  numeric features.

cat_feats = preprocessor.named_transformers_['pipeline-2'].named_steps[
    'onehotencoder'].get_feature_names(categorical_features).tolist()

all_feat_names = numeric_features + cat_feats

We can see now that we have the same number of feature names as we do coefficients.

len(all_feat_names)

Let's get them into a dataframe now and sort them:

features_df = pd.DataFrame(data = [all_feat_names,
                                feats_coef.flatten()]).T.rename(columns={0:'feature', 1:'feature_coefs'})
features_df.sort_values('feature_coefs',key= abs, ascending=False)

We can see that `SaleType_New` is the most important feature in our model.

From here we can decide to manually remove some of the columns that are not taken into consideration as much.... or we can instead use a tool to help us!

Enter - **Recursive feature elimination**

### Recursive feature elimination - RFE

We can use feature importances to eliminate unimportant features.

The basic idea with recursive feature elimination is we:
1. We decide $k$ - the number of features to select.
2. Assign importances to features, e.g. by fitting a model and looking at coef_ or feature_importances_.
3. Remove the least important feature.
4. Repeat steps 2-3 until only $k$ features are remaining.

**Note that this is not the same as just removing all the less important features in one shot!**

Let's take a look at how we can do this.

First we import `RFE` from `sklearn.feature_selection`:

from sklearn.feature_selection import RFE

Now instead of simply using `LogisticRegression`, we can wrap it around the `RFE` function and specify how many features we want with `n_features_to_select`.

Here I'm capping the number of features to 30 (an arbitrary number I picked).

This is going to take about 1-2 minutes to run because now, it's recursively removing 1 feature at a time and cross-validating on the final result.

main_pipe = make_pipeline(preprocessor, RFE(LogisticRegression(max_iter=1000),
                                            n_features_to_select=30))

scores = cross_validate(main_pipe, X_train, y_train, return_train_score=True)

pd.DataFrame(scores)

pd.DataFrame(scores).mean()

Looking at this mean validation score compared to when the model was using all the features, we can see it increased a tiny bit!

But now our next question is how do we set $k$? How do we know how many features is the optimal amount... Well, you guessed it! There is a tool for that too!

### RFECV

You can find the optimal number of features using cross-validation with `RFECV` where the optimal $k$ value is selected based on the highest validation score.

You would definitely not want to use the training score! - Why?
 > Because with training score the more features you add the higher the score, this isn't the case with validation score.

We can import `RFECV` from `sklearn.feature_selection` like we did for `RFE`.

from sklearn.feature_selection import RFECV

Instead of `RFE` now we simply use `RFECV` in our pipeline and we do not need to specify the argument `n_features_to_select` like we did with `RFE` since $k$ is selected based on the highest validation score.

(*This is also going to take a couple of minutes*)

main_pipe = make_pipeline(preprocessor, RFECV(LogisticRegression(max_iter=1000), cv=5))

scores = cross_validate(main_pipe, X_train, y_train, return_train_score=True)

Now we have ~91% for our validation score!

pd.DataFrame(scores).mean()

main_pipe.fit(X_train,y_train)
print(main_pipe.named_steps["rfecv"].support_)

print("The number of features selected by RFE: ",
      main_pipe.named_steps["rfecv"].n_features_)

feature_names = all_feat_names
support = main_pipe.named_steps["rfecv"].support_
RFE_selected_feats = np.array(feature_names)[support]
RFE_selected_feats

RFECV selects the features by references their `feature_importances`/`coefs_` as well as the validation score after each feature is removed and seeing if it is increasing.

When a feature is removed and the validation score is no longer increasing, then it stops removing features.

## Forward Selection

Unlike with RFE where we start with all our features and gradually remove the leat important ones, **Forward Selection** is a process where we start with no features and gradually add them!

With RFE we removed the least important feature, whereas with forward selection we add features untill our cross-validation score starts to decreases.

Forward Selection does not guaranty finding the best features set but reduces many problems.
Computationally cheaper (aka faster!)
Overfits less

Forward selection is recently implemented in `sklearn` so please make sure it is up to date! You need version 0.24!

from sklearn.feature_selection import SequentialFeatureSelector

We can import it as follows:

pipe_forward = make_pipeline(preprocessor,
                             SequentialFeatureSelector(LogisticRegression(max_iter=1000),
                                                       direction='forward',
                                                       n_features_to_select=20),
                            LogisticRegression(max_iter=1000))

Running this next cell is going to take a LONG LONG LONG time.

scores = cross_validate(pipe_forward, X_train, y_train,
                        return_train_score=True)
pd.DataFrame(scores).mean()

## Let's Practice

1\. As we increase features, which score will always increase?
2\. Between `RFE` and `RFECV` which one finds the optimal number of features for us?
3\. Which method starts with all our features and iteratively removes them from our model?
4\. Which method starts with no features and iteratively adds features?
5\. Which method does not take into consideration `feature_importances_`/`coefs_` when adding/removing features?

```{admonition} Solutions!
:class: dropdown

1. Training score
2. `RFECV`
3. Recursive Feature Elimination
4. Forward Selection
5. Forward Selection
```

## Extra time? Project time

First -> Project expectations.

Breakout rooms in your project groups!

Use this time to:

- Meet with your team mates;
- Think about a project - choose the data and business objective.
- Propose a statistical objective to address this.
- Also, elaborate on the statistical objective. What’s your plan for the analysis?

## What We've Learned Today

- How to construct a statistical question from a business objective.
- What steps are important in building your analysis.
- How to discover important features in your model.
- the 2 different methods (RFE, Forward selection) to conduct feature selection on your model.