function [alpha_log_all,alpha_log_hc,h_0_threshold,n_soundings] = sounding_new(start_date,end_date,plot_flag,hydro,wet)
% This function computes the refractivity and integrated LOS delays.
% In addition it will estimate the power-law decay cofficient and height at which the relative
% tropopsheric delays are approximately zero. Both coefficients are given based on the full heifght range
% and the lower height range up to hc. Note that alpha is dimensionless and
% h0 is the height in m units. Sounding data needs to be stored in the
% correct format! Read below:
%
% ---- Input ----
% All inputs will be retreived from the aps_parms file. Set the inputs by change
% setparm_aps('Parameter_name',new_value)
%
% Variables include:
% look_angle     Mean look angle of the data in rad. By default this is
%                assumed to be 21 degees.
% lambda         Radar wavelength in m. By default this is assumed to be
%                0.056 cm (C-band).
% h_0_threshold  Consider only soundings that go up till the h0 threshold in km.
%                Being the height at which the differential delay is small.
%                When '0' is specified it is estimated from the data. Note that
% 		 soundings should go up high enough to cover the lower tropopshere (up to 16 km).
%                By default the threshold is set to be 15 km.
% h_thres_hc     Height range [0 h_thres_hc] over which the power-law decay
%                coefficient is estimated computed. By default this is set to be 4 km.
% start_date	 Start date of the sounding period, by default [], full period
%                is considered. Specify as a string in 'yyyymmdd' format.
% end_date       End date of the sounding period, by default [], full period
%                is considered. Specify as a string in 'yyyymmdd' format.
% time_stamp	 Include a time stamp to it. This is a column vector with
%                strings e.g. ['00';'12'] for 00Z and 12Z. Only those files
%                ending with this are considered in the computation.
% sounding_dir	 Optional argument giving directly the full path to the soundings.
%                The files on this path should be the YYYYMMDD_HH.mat files.
% error_promp_flag By default ([] or 'y') this is turn on. Can be usefull to turn of
%                when running in a batch mode. Instead NaN values will be outputed.
%
% Output:
% alpha and h0. The computed sounding delays: refractivity, the (mean) LOS delay [m],
% the (mean) LOS phase delay [rad], and corresponding heights are appended to the existing
% sounding .mat files.
%
%
% NOTE on data format:
% sounding data needs to be stored in a folder called sounding_data,
% within your processing directory. Within this folder, each sounding
% aquisition needs to be stored in as 8 digit .mat files, e.g. YYYYMMDD_HH.mat
% format, with the  pressure (hPa), temperature (degree), relative humidity (%)
% and heights (m) as matlab variables P, T, RH and h.
%
% OPTIONAL:
% - visulize intermediate results (refractivity, delay,
%   mean delay and hc relation with H), by putting plot_flag to 1. By
%   default this is not done.
% - Overwrite all the data to recompute the delays by putting recompute flag to 1
%
%     Copyright (C) 2015  Bekaert David - University of Leeds
%     Email: eedpsb@leeds.ac.uk or davidbekaert.com
%
%     This program is free software; you can redistribute it and/or modify
%     it under the terms of the GNU General Public License as published by
%     the Free Software Foundation; either version 2 of the License, or
%     (at your option) any later version.
%
%     This program is distributed in the hope that it will be useful,
%     but WITHOUT ANY WARRANTY; without even the implied warranty of
%     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
%     GNU General Public License for more details.
%
%     You should have received a copy of the GNU General Public License along
%     with this program; if not, write to the Free Software Foundation, Inc.,
%     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
%
% September 2010 --- Bekaert David
% Modifications:
% 01/11/2011:  DB    Modify from initial version
% 01/12/2011:  DB    Making compatible to other datasets
% 01/11/2012:  DB    Made program more efficient
% 05/11/2012:  DB    Fix bug in loading of the files
% 10/02/2013:  DB    Integrate with other part of toolbox
% 18/03/2013:  DB    Include a time stamp to it.
% 18/03/2013:  DB    Include a start and end time.
% 18/03/2013:  DB    Account for NaN values by removing them for all data columns
% 19/03/2013:  DB    Set the min height range of the delay curve by comparing it to
%                    the median of the height range
% 19/03/2013:  DB    Include a flag to suppress error messages by lack of sounding data and
%                    directly pass NaN value through it.
% 25/03/2013:  DB    Compute the power law coefficient from loglog relation
% 03/04/2013:  DB    Include h0 estimation option from the net delays
% 24/04/2013:  DB    Incorporate in the bigger aps toolbox. Change towards
%                    loglog powerlaw computation.
% 10/05/2013:  DB    Loading the default parameters from the parms_aps file.
% 02/10/2013:  DB    Change filename to sounding and clean the syntax
% 29/10/2013:  DB    Allow the look angle to be a specified as a file
% 29/11/2013:  DB    Get stamps processing flag before lookangle  loading
% 06/11/2014:  DB    Add hydrostatic and wet delay options

% --------- VARIABELS ---------- %
fontsize = 20;                      % fontsize of the figures
save_fig =0;                        % when 1 save figures
recompute = 0;                      % Use earlier computed data unless it is missing
if nargin<3 || isempty(plot_flag)
    plot_flag =0;                       % Control plots of the refractivity, delay and net delay.
end
if nargin<4
   hydro=1;
end
if nargin<5
    wet=1;
end
netdelay_color = [];
curdir = pwd;

hydro
wet

%% Setting the defaults were needed
look_angle = getparm_aps('look_angle');
lambda = getparm_aps('lambda');
h_0_threshold= getparm_aps('sounding_h0');
h_thres_hc= getparm_aps('sounding_h_alpha_thres');

if nargin<1 || isempty(start_date)
    start_date= getparm_aps('sounding_start_date');
end
if nargin<2 || isempty(end_date)
    end_date= getparm_aps('sounding_end_date');
end
time_stamp= getparm_aps('sounding_time_stamp');
sounding_dir= getparm_aps('sounding_dir');
error_promp_flag= getparm_aps('sounding_error_promp');

clear h_0_threshold_default h_thres_hc_default look_angle_default lambda_default

stamps_processed = getparm_aps('stamps_processed');

% checking if the look angle is a file or not
if ischar(look_angle)
	% look angle is specified as a file [rad]
	if strcmp(stamps_processed,'y')
           look_angle = load(look_angle);
	       look_angle = look_angle.la;
           look_angle = mean(look_angle);
	else
	       look_angle = load(look_angle);
	       % use the mean of the look angles
	       look_angle = mean(look_angle);
	end
end

% convert units
theta = look_angle;                       % mean angle of incidence [rad]
h_0_threshold = h_0_threshold*1000;       % put threshold to [m]
h_thres_hc = h_thres_hc*1000;             % put threshold to [m]
clear look_angle

% get height information
if plot_flag==1
    hgt_matfile = getparm_aps('hgt_matfile');
    hgt = load(hgt_matfile);
    max_height = max(hgt.hgt);
    min_height = min(hgt.hgt);
    clear hgt hgt_matfile
end

% Include a while loop. Depending on error message flag, it will break out of the loop and pass
% NaN values as output.
continue_flag =1;
abord_flag = 0;		% when 1 abnormal termination get NaN values instead.
while continue_flag

%% Getting the file list of the sounding data
if isempty(sounding_dir)~=1
	if exist([sounding_dir filesep],'dir')~=7
        error('myApp:argChk', ['The specified filepath of the sounding data does not exist,...  \nAbort,... \n'])
    end
	cd(sounding_dir)
else
	if exist('sounding_data/','dir')~=7
    		error('myApp:argChk', ['There is no sounding_data directory,...  \nAbort,... \n'])
	end
	cd  sounding_data
end
if exist('sounding.list','file')~=2
    % making a list of all the sounding files
    [dummy dummy2] = system('echo sounding_list > sounding.list');
    clear dummy dummy2
    for k=1:size(time_stamp,1)
        command_str = ['ls [0-9]???????_' time_stamp(k,:) '.mat >> sounding.list'];
        [dummy dummy2] = system(command_str);
        clear dummy dummy2
    end
end
temp = tdfread('sounding.list');
[dummy dummy2] = system('rm sounding.list');
clear dummy dummy2
date_list_temp = temp.sounding_list(:,[1:8]);
sounding_list = temp.sounding_list;

% selecting a date range when requested
clear ix

if isempty(start_date)~=1 && isempty(end_date)~=1
	ix = find(datenum(date_list_temp,'yyyymmdd')>=datenum(start_date,'yyyymmdd') & datenum(date_list_temp,'yyyymmdd')<=datenum(end_date,'yyyymmdd'));
	save_name = ['Delay_' start_date '_' end_date];
elseif isempty(start_date)~=1  && isempty(end_date)==1
        ix = find(datenum(date_list_temp,'yyyymmdd')>=datenum(start_date,'yyyymmdd'));
        save_name = ['Delay_' start_date '_end' ];
elseif isempty(start_date)==1 && isempty(end_date)~=1
        ix = find(datenum(date_list_temp,'yyyymmdd')<=datenum(end_date,'yyyymmdd'));
        save_name = ['Delay_start_' end_date];
else
        save_name = ['Delay'];
end
% checking if the save figures folder exist
if save_fig ==1 & exist('figures','dir')~=7
    mkdir('figures')
end

if  isempty(start_date)~=1 || isempty(end_date)~=1
	if isempty(ix)~=1
		date_list_temp = date_list_temp(ix,:);
		sounding_list = sounding_list(ix,:);
	else
		fprintf('No sounding has been acquired in this period \n' )
		continue_flag = 0;
                abord_flag = 1;
                break
	end
	clear temp ix
end

%% Computing refractivity
fprintf('Computing the refractivity \n')
ix_skip_sounding = [];			% in case soundings are rejected along the way
warning('off','MATLAB:load:variableNotFound')
for i=1:size(sounding_list,1)
    temp = load(sounding_list(i,:),'h_min_souding');
    if isfield(temp,'h_min_souding') && recompute==0
        % storing variables for latter processing
        h_range(i,1) = temp.h_min_souding;          % [m]

        if  plot_flag==1
            data = load(sounding_list(i,:),'h_max_souding','Ndelay');
            Ndelay = data.Ndelay;
        else
            data = load(sounding_list(i,:),'h_max_souding');
        end
        h_range(i,2) = data.h_max_souding;          % [m]
        clear data
        skip_sounding = 0;

    else
        load(sounding_list(i,:));
        % This includes variables:
        % - P (pressure in [hPa])
        % - T (temperature in [degrees])
        % - RH (relative humidity in [%])
        % - h (altitude in [m])

        % coping with NaN values in the RH data
        ix1 = find(isnan(RH)==1);
        ix2 = find(isnan(h)==1);
        ix = unique([ix1; ix2]);
        P(ix)=[];
        RH(ix)=[];
        T(ix)=[];
        h(ix)=[];
        clear ix ix1 ix2

        % coping with errors in the data when a double recording was made
        ix_repeat = find(diff(sort(h))==0);
        h(ix_repeat) = [];
        P(ix_repeat)=[];
        RH(ix_repeat)=[];
        T(ix_repeat)=[];
        clear ix_repeat

        % Checking if there is still data left
        if isempty(h)==1 && size(sounding_list,1)==1 && strcmp(error_promp_flag,'y')
             error('myApp:argChk', ['Datafile contains no numeric data. \n'])
        elseif isempty(h)==1
            fprintf([sounding_list(i,:) ' sounding skipped as it containes no numeric data\n'])
            skip_sounding = 1;
            ix_skip_sounding = [ix_skip_sounding ; i];
        else
            skip_sounding = 0;
        end

        % when possible compute the refractivity
        if skip_sounding==1
            % case of no data
            h_range(i,1) = NaN;          % [m]
            h_range(i,2) = NaN;          % [m]
        else
            % saterated pressure es
            Rv = 461.524;                   % [J/K]
            T0 = 273.16;                    % [k]
            L  = 2.5e6;                     % [J/kg]                Latent heat
            e0 = 6.11;                      % [hPa]
            T  = T+273.15;                  % [degree] to [K]       Temperature
            es = e0.*exp(L./Rv.*(1./T0-1./T));

            % partial pressure of water vapour e
            RH = RH./100;                   % [%] to [-]            Relative humidity
            e  = RH.*es;                    % [hPa]

            % Refractivety - Hydrostatic term
            k1 = 77.6;                      % [K/hPa]
            Ndelay.Nhydro = k1.*P./T;              % [K/hPa*hPa/K] = [-]

            % Refractivity - Wet term
            k2_n = 23.3;                    % [K/hPa]
            k3   = 3.75e5;                  % [K^2/hPa]
            Ndelay.Nwet = k2_n.*e./T+k3.*e./T.^2;  % [ppm]

            % refractivety
            Ndelay.N = Ndelay.Nhydro+Ndelay.Nwet;

            % save the heights of the delay as well
            Ndelay.h = h;

            h_min_souding = min(h);
            h_max_souding = max(h);

            % saving the data
            save(sounding_list(i,:),'-append','Ndelay','h_min_souding','h_max_souding')

            % storing variables for latter processing
            h_range(i,1) = h_min_souding;          % [m]
            h_range(i,2) = h_max_souding;          % [m]
            clear h_max_souding h_min_souding

        end
    end

    if floor(i/10)*10 == i
        fprintf([num2str(i) ' refraction dates completed out of ' num2str(size(sounding_list,1)) ' \n']);
    end

    % Test visualizing refractivity with height
    if  plot_flag==1 && skip_sounding~=1
        if i==1
           figure1 = figure('name','Refractivity with height');
           ylabel('Height [km]','fontsize',12)
           xlabel('Refractivity [ppm]','fontsize',12)
           line_colors = hsv(size(sounding_list,1));
           box on
           set(gca,'fontsize',12)
        end
        figure(figure1)
        hold on
        if hydro==1 && wet==0
            if i==1
                title('Refractivity with height','fontsize',12)
            end
            plot(Ndelay.Nhydro,Ndelay.h./1000,'-','color',line_colors(i,:))
        elseif hydro==0 && wet==1
            if i==1
                title('Refractivity with height (hydro)','fontsize',12)
            end
            plot(Ndelay.Nwet,Ndelay.h./1000,'-','color',line_colors(i,:))
        else
            if i==1
                title('Refractivity with height (wet)','fontsize',12)
            end
            plot(Ndelay.N,Ndelay.h./1000,'-','color',line_colors(i,:))
        end
    end
    clear P h RH T Ndelay N
end
clear Rv T0 L e0 es e k1 k2_n k3 i figure1
% remove those sounding acqusitions that have no data coverage
if isempty(ix_skip_sounding)~=1
	sounding_list(ix_skip_sounding,:)=[];
	h_range(ix_skip_sounding,:)=[];
end

%% Computing the height range
fprintf('Computing delay \n')

% defining the height range (equal for the whole dataset)
h_max = min(h_range(:,2));
h_max_mean = mean(h_range(:,2));

if h_0_threshold<=0
  % estimate h0 from the data
  h0_estimate_flag = 1;
  h_0_threshold = 0;			% set the height threshold low such all the delays can be used to compute a net delay
else
  h0_estimate_flag = 0;
end

% check if the delays goes high enough
if h_max<h_0_threshold
    % the set threshold is lower than the lowest value.
    % setting the h_max to h0, but checking the number of soundings that need to be rejected for this.
    ix = find(h_range(:,2)<h_0_threshold);
    n = [1:size(sounding_list,1)]';
    n(ix)=[];
    h_range(ix,:)=[];
    if isempty(ix)==1
        if strcmp(error_promp_flag,'y')
            error('myApp:argChk', ['None of the soundings reach higher than ',num2str(h_max/1000),' km. Try setting h_0_threshold lower. \n'])
        else
            fprintf(['None of the soundings reach higher than ',num2str(h_max/1000),' km. Recommend try to set h_0_threshold lower \n'])
            continue_flag = 0;
            abord_flag = 1;
            break
        end
    end
    clear ix
    if size(n,1)<3 || isempty(n)==1
        if strcmp(error_promp_flag,'y')
            error('myApp:argChk', 'Put h_0_threshold to a lower value. \n')
        else
            fprintf('Try to put h_0_threshold to a lower value \n')
            continue_flag = 0;
            abord_flag = 1;
            break
        end
    end
    if size(n,1)<5
        fprintf(['Only ',num2str(size(n,1)),' soundings are used to compute mean delay \n']);
    end
    sounding_list = sounding_list(n,:);
    h_max = h_0_threshold;
    clear n
elseif h_max>h_0_threshold && h0_estimate_flag==0
    % h0 is lower than the h_max.
    h_max = h_0_threshold;
elseif h_max_mean-h_max >1000 && h0_estimate_flag==1
    % Make h_max larger possible as the h0 value needs to be estimated
    % this will be at the cost of some soundings but they will be included
    % latter on
    h_max = h_max_mean;
    ix = find(h_range(:,2)<h_max);
    h_range(ix,:)=[];
    sounding_list(ix,:) = [];
end

% including a check for the minimum range by comparing it to the median value
h_min_median = median(h_range(:,1));
h_min = max(h_range(:,1));

% removing soundings that do not start low enough
if h_min-h_min_median > 1000
    h_min = h_min_median;

    ix = find(h_range(:,1)>h_min);
    n = [1:size(sounding_list,1)]';
    n(ix)=[];
    h_range(ix,:)=[];
    if isempty(ix)==1
         error('myApp:argChk', ['This should not occur.'])
    end
    clear ix
    if size(n,1)<3 || isempty(n)==1
        error('myApp:argChk', 'Too few soundings are left for mean delay computation. Check for incomplete data files or differences in sounding height ranges \n.')
    end
    if size(n,1)<5
        fprintf(['Only ',num2str(size(n,1)),' soundings are used to compute mean delay \n']);
    end
    sounding_list = sounding_list(n,:);
    clear n
end

fprintf([num2str(size(sounding_list,1)),' soundings are used to compute mean delay \n']);
h_delay = [h_min:1:h_max]';                                              % set to 1 [m] interval
clear h_range h_min h_max

%% computing the integrated refractivity
for i=1:size(sounding_list,1)
    compute_flag=0;
    % checking if it has already been computed. If so use this data at once
    % else compute it
    temp = load(sounding_list(i,:),'h_delay');
    if isfield(temp,'h_delay') && recompute==0
        h_delay_temp=temp.h_delay;
        if h_delay(1)==h_delay_temp(1) && h_delay(end)==h_delay_temp(end)
            if hydro==1 && wet==0
                data = load(sounding_list(i,:),'delay_hydro','phase_delay_hydro');
                if ~isfield(data,'delay_hydro')
                   compute_flag=1;
                else
                    delay = data.delay_hydro;
                    phase_delay = data.phase_delay_hydro;
                end
            elseif hydro==0 && wet==1
                data = load(sounding_list(i,:),'delay_wet','phase_delay_wet');
                if ~isfield(data,'delay_wet')
                    compute_flag=1;
                else
                    delay = data.delay_wet;
                    phase_delay = data.phase_delay_wet;
                end
            else
                data = load(sounding_list(i,:),'delay','phase_delay');
                if ~isfield(data,'delay')
                    compute_flag=1;
                else
                    delay = data.delay;
                    phase_delay = data.phase_delay;
                end
            end

            if compute_flag==0
                % store all delays for later mean LOS delay computation
                delay_matrix(1:length(delay),i) = delay;
                % do the same for the phase delay
                phase_delay_matrix(1:length(delay),i) = phase_delay;
                compute_flag=0;
            end
        else
           % delay needs to be computed
           compute_flag=1;
        end
    else
        compute_flag=1;
    end

    if compute_flag==1
        load(sounding_list(i,:),'Ndelay')

        % interpolation to 1 m interval
        if hydro==1 && wet==0
           N_regular = interp1(Ndelay.h,Ndelay.Nhydro,h_delay,'linear');
        elseif hydro==0 && wet==1
           N_regular = interp1(Ndelay.h,Ndelay.Nwet,h_delay,'linear');
        else
           N_regular = interp1(Ndelay.h,Ndelay.N,h_delay,'linear');
        end

        % integration of refractivity with height (= delay) and projection on the LOS
        delay = zeros([length(N_regular) 1]);                                   % initialize
        delay = flipud(cumsum(flipud(N_regular))./10^6.*cos(theta));            % 10^6 factor is for correction of ppm, theta for LOS projection
        phase_delay = delay*4*pi./lambda;
        clear scale N_regular

        % store all delays for later mean LOS delay computation
        delay_matrix(1:length(delay),i) = delay;
        % do the same for the phase delay
        phase_delay_matrix(1:length(delay),i) = phase_delay;

        % saving the data
        if hydro==1 && wet==0
            delay_hydro = delay;
            phase_delay_hydro = phase_delay;
            save(sounding_list(i,:),'-append','delay_hydro','phase_delay_hydro','h_delay')
        elseif hydro==0 && wet==1
            phase_delay_wet=phase_delay;
            delay_wet = delay;
            save(sounding_list(i,:),'-append','delay_wet','phase_delay_wet','h_delay')
        else
            save(sounding_list(i,:),'-append','delay','phase_delay','h_delay')
        end

    end

    % Test visualizing delay with height
    if plot_flag==1
        if i==1
           figure2 = figure('name','Delay with height');
           xlabel('LOS phase delay [rad]','fontsize',fontsize)
           ylabel('Height [km]','fontsize',fontsize)
           set(gca,'fontsize',fontsize)
        end
        figure(figure2)
        hold on
        plot(phase_delay,h_delay./1000,'-','color',[0.7 0.7 0.7])
        box on

        if i==size(sounding_list,1) && h0_estimate_flag==1
            Ax1 = gca;
            ylimits = get(Ax1,'ylim');
            set(Ax1,'ylim',[0 ylimits(2)])
            % adding a second axis
            box on
            set(gca,'fontsize',fontsize)
            if hydro==1 && wet==0
                title({'LOS hydro delay [m]',' '},'fontsize',fontsize)
                fig_save_name = ['figures' filesep 'Delay_hydro_curves_all.eps'];
            elseif hydro==0 && wet==1
                title({'LOS wet delay [m]',' '},'fontsize',fontsize)
                fig_save_name = ['figures' filesep 'Delay_wet_curves_all.eps'];
            else
                title({'LOS delay [m]',' '},'fontsize',fontsize)
                fig_save_name = ['figures' filesep 'Delay_curves_all.eps'];
            end
            Ax2 = axes('Position',get(Ax1,'Position'),'XAxisLocation','top');
            set(Ax2,'ylim',get(Ax1,'ylim'),'ytick',[],'yticklabel','');
            xlims = get(Ax1,'xlim');
            xtick_new = get(Ax1,'xtick')./xlims(2);
            new_x_label = get(Ax1,'xtick').*lambda./(4*pi);
            xtick_labels = round((new_x_label)*100)./100;
            xtick_labels = num2str(xtick_labels');
            set(Ax2,'xtick',xtick_new,'xticklabel',xtick_labels);
            set(Ax2,'color','none');
            set(Ax2,'fontsize',fontsize)
            box on
            % saving of the figure when requested
            if save_fig==1
                set(figure2,'PaperPositionMode','auto')
                print(figure2,'-depsc','-r150',fig_save_name)
                clear fig_save_name
            end
            clear figure2
        end
    end
    clear Ndelay.N Ndelay.h delay
end
clear i theta

n_soundings = size(delay_matrix,2);

if n_soundings <=3
   continue_flag = 0;
   abord_flag = 1;
   break
end

%% estimating h0 from the delay curves, when required.
% this is based on the net delays and used the std
% as criteria for setting h0
if h0_estimate_flag==1
    n_netdelays_max = 400;
    if (n_soundings*n_soundings-n_soundings)/2<=n_netdelays_max
       	fprintf(['Estimating h0 from all (', num2str((n_soundings*n_soundings-n_soundings)/2),') possible net delay combinations \n'])
    	ix1_random=repmat(1:n_soundings,n_soundings,1);
        ix1_random=single(ix1_random);
    	ix2_random=ix1_random';
    	ix_temp=logical(tril(ones(n_soundings)))&~eye(n_soundings);
    	ix_random=[ix1_random(ix_temp),ix2_random(ix_temp)];
        clear ix1_random ix2_random
    else
        fprintf(['Estimating h0 from (', num2str(n_netdelays_max),') net delay combinations \n'])
        % take 2 random sets of n_netdelays_max
        new_set_search = 1;
        loop_counter = 0;
        while new_set_search
            ix1_random = ceil(rand(n_netdelays_max*5,1)*n_soundings);
            ix2_random = ceil(rand(n_netdelays_max*5,1)*n_soundings);
            ix_random = [ix1_random ix2_random];
            clear ix1_random ix2_random

            % remove those net delays of the same date
            ix_temp = find(ix_random(:,1)-ix_random(:,2)==0);
            ix_random(ix_temp,:)=[];
            clear ix_temp

            % remove repetition of pair combination
            ix_random = unique(ix_random,'rows');

            % check if enough combinations are left otherwize rerun
            if size(ix_random,1)>=n_netdelays_max
               ix_random(n_netdelays_max+1:end,:)=[];
               new_set_search=0;

            end
            loop_counter = loop_counter+1;
            if loop_counter == 50
                fprintf('Having difficulty determining net delay combinations \n')
                keyboard
            end
        end
    end

    % computation of the net delay
    netdelays = delay_matrix(:,ix_random(:,1)) - delay_matrix(:,ix_random(:,2));
    netphasedelays = phase_delay_matrix(:,ix_random(:,1)) - phase_delay_matrix(:,ix_random(:,2));

    if plot_flag==1
        fig_netdelay = figure('name','Net delays');
        if isempty(netdelay_color)
            plot(netdelays,h_delay/1000)
        else
            plot(netdelays,h_delay/1000,'color',netdelay_color)
        end
        max_spacing = max([abs([min(min(netdelays)) max(max(netdelays))])]);
        xlim([-1.1*max_spacing max_spacing*1.1]);
        xlabel('Net LOS delay [m]','fontsize',fontsize)
        ylabel('Height [km]','fontsize',fontsize)
        set(gca,'fontsize',fontsize)

        fig_netphasedelay = figure('name','Net phase delays');
        if isempty(netdelay_color)
            plot(netphasedelays,h_delay/1000)
        else
            plot(netphasedelays,h_delay/1000,'color',netdelay_color)
        end
        max_spacing = max([abs([min(min(netphasedelays)) max(max(netphasedelays))])]);
        xlim([-1.1*max_spacing max_spacing*1.1]);
        xlabel('\Delta\phi_{tropo} [rad]','fontsize',fontsize)
        ylabel('Height [km]','fontsize',fontsize)
        set(gca,'fontsize',fontsize)

    end

    % computation of the netdelay standard deviation
    netdelays_std = std(netdelays,[],2);
    clear netdelays

    % find the height at which the lowest standard deviation becomes larger
    % than the 0.5 cm threshold.
    fprintf(['Estimating h0 from the height at which the std varies more than ' num2str(0.05) ' cm \n'])
    ix = (netdelays_std<=0.0005);
    h_0_threshold = min(h_delay(ix));
    clear ix
    ix = find(h_delay>h_0_threshold);

    % make some sanity checks
    if h_0_threshold<6000
       fprintf(['h0 is estimated to be ' num2str(h_0_threshold/1000) ' km. This is low abord. \n'])
       if strcmp(error_promp_flag,'y')
           error('myApp:argChk', 'Put h_0_threshold to a lower value. \n')
       else
           fprintf('Try setting h_0_threshold manually \n')
           continue_flag = 0;
           abord_flag = 1;
           break
       end
    else
        % plotting the estimate h0 on the netdelay curve
        if plot_flag==1
            figure(fig_netdelay)
            hold on
            plot(get(gca,'xlim'),[h_0_threshold/1000 h_0_threshold/1000],'k--')

            figure(fig_netphasedelay)
            hold on
            plot(get(gca,'xlim'),[h_0_threshold/1000 h_0_threshold/1000],'k--')

            % saving of the figure when requested
            if save_fig==1
                fig_save_name = ['figures' filesep 'NetDelay_curves.eps'];
                set(fig_netdelay,'PaperPositionMode','auto')
                print(fig_netdelay,'-depsc','-r150',fig_save_name)
                clear fig_save_name

                fig_save_name = ['figures' filesep 'NetPhaseDelay_curves.eps'];
                set(fig_netphasedelay,'PaperPositionMode','auto')
                print(fig_netphasedelay,'-depsc','-r150',fig_save_name)
                clear fig_save_name
            end
        end

        fprintf(['h0 is estimated to be ' num2str(h_0_threshold/1000) ' km. \n'])

        % removing all the other variables
        h_delay(ix:end)=[];
        delay_matrix(ix:end,:)=[];
        phase_delay_matrix(ix:end,:)=[];

        % setting the start of the integration to h_0
        delay_matrix = delay_matrix - repmat(delay_matrix(end,:),length(h_delay),1);
        phase_delay_matrix = phase_delay_matrix - repmat(phase_delay_matrix(end,:),length(h_delay),1);

        if plot_flag==1
            figure2 = figure('name','Delay with height');
            plot(phase_delay_matrix,h_delay./1000,'-','color',[0.7 0.7 0.7] )
            xlabel('LOS phase delay [rad]','fontsize',fontsize)
            ylabel('Height [km]','fontsize',fontsize)
            set(gca,'fontsize',fontsize)
            % saving of the figure when requested
            if save_fig==1
                fig_save_name = ['figures' filesep 'Delay_curves.eps'];
                set(figure2,'PaperPositionMode','auto')
                print(figure2,'-depsc','-r150',fig_save_name)
                clear fig_save_name
            end

        end
    end
end

%% Computing mean delay
fprintf('Computing mean delay \n')
delay_mean = mean(delay_matrix,2);                                        % given in [m]
phase_delay_mean = mean(phase_delay_matrix,2);
if plot_flag==1
    % plot mean delay on top of the existing figure
    figure(figure2)
    hold on
    plot(phase_delay_mean,h_delay./1000,'k-','linewidth',2)
    Ax1 = gca;
    ylimits = get(Ax1,'ylim');
    set(Ax1,'ylim',[0 ylimits(2)])
    % adding a second axis
    box on
    set(gca,'fontsize',fontsize)
    title({'LOS delay [m]',' '},'fontsize',fontsize)
    Ax2 = axes('Position',get(Ax1,'Position'),'XAxisLocation','top');
    set(Ax2,'ylim',get(Ax1,'ylim'),'ytick',[],'yticklabel','');
    xlims = get(Ax1,'xlim');
    xtick_new = get(Ax1,'xtick')./xlims(2);
    new_x_label = get(Ax1,'xtick').*lambda./(4*pi);
    xtick_labels = round((new_x_label)*100)./100;
    xtick_labels = num2str(xtick_labels');
    set(Ax2,'xtick',xtick_new,'xticklabel',xtick_labels);
    set(Ax2,'color','none');
    set(Ax2,'fontsize',fontsize)
    % saving of the figure when requested
    if save_fig==1
        fig_save_name = ['figures' filesep  'Delay_curves.eps'];
        set(figure2,'PaperPositionMode','auto')
        print(figure2,'-depsc','-r150',fig_save_name)
        clear fig_save_name
    end
    clear figure2

    % an individual figure for the mean delay
    figure4 = figure('name','Mean delay with height');
    xlabel('LOS phase delay [rad]','fontsize',fontsize)
    ylabel('Height [km]','fontsize',fontsize)
    set(gca,'fontsize',fontsize)
    hold on
    plot(phase_delay_mean,h_delay./1000,'k-','linewidth',2)
    set(gca,'fontsize',fontsize)
end
save([save_name '.mat'],'delay_mean','h_delay','n_soundings','delay_matrix')

%% estimating the power law directly from the log log plot
% computing the loglog powerlaw

h_log = log10(h_0_threshold-h_delay);
phi_delay_log = log10(phase_delay_mean);
delay_log = log10(delay_mean);
A = [h_log ones(size(h_log))];

% based on the full height range
ix_all = find(h_log<=0.001);
A_all = A;
A_all(ix_all,:)=[];
delay_log_temp_all = delay_log;
delay_log_temp_all(ix_all,:)=[];
phase_delay_log_temp_all = phi_delay_log;
phase_delay_log_temp_all(ix_all,:)=[];
coeff_all = inv(A_all'*A_all)*A_all'*delay_log_temp_all;
coeff_all_phase = inv(A_all'*A_all)*A_all'*phase_delay_log_temp_all;
alpha_log_all = coeff_all(1);
K_log_all = exp(coeff_all(2));

% based on the height range till hc_threshold
ix_hc = find(h_log<=0.001 |  h_delay>h_thres_hc);
A_hc = A;
A_hc(ix_hc,:)=[];
delay_log_temp_hc = delay_log;
delay_log_temp_hc(ix_hc,:)=[];
phase_delay_log_temp_hc = phi_delay_log;
phase_delay_log_temp_hc(ix_hc,:)=[];
coeff_hc = inv(A_hc'*A_hc)*A_hc'*delay_log_temp_hc;
coeff_hc_phase = inv(A_hc'*A_hc)*A_hc'*phase_delay_log_temp_hc;
alpha_log_hc = coeff_hc(1);
K_log_hc = exp(coeff_hc(2));

if plot_flag==1
    xlimits = [log10(h_0_threshold-max_height) log10(h_0_threshold+1)];
    % an individual figure for the mean delay
    figure4_1 = figure('name','Log-Log plot of the Mean delay with height');
    ylabel('Log(tropopsheric LOS delay [m])','fontsize',fontsize)
    xlabel('Height [km])','fontsize',fontsize)
    set(gca,'fontsize',fontsize)
    hold on
    plot(h_log,delay_log,'k-','linewidth',2)
    hold on
    plot(A_hc(:,1),A_hc*coeff_hc,'r--','linewidth',2)
    set(gca,'fontsize',fontsize)
    % plotting the powerlaw based on the lower height range
    hold on
    xlim(xlimits)
    ylimits = get(gca,'ylim');
    legend('Mean delay','Power law',2)
    legend boxoff
    set(gca,'xtick',[get(gca,'xtick') log10(h_0_threshold)],'xticklabel',num2str([get(gca,'xtick') log10(h_0_threshold)]'))

    % updating the lables to be heights
    temp_loc = get(gca,'xtick');
    temp = round((h_0_threshold-10.^temp_loc)/1000*100)/100;
    temp_str = num2str(temp');
    set(gca,'xtick',temp_loc,'xticklabel',temp_str)
    box on

    if save_fig==1
        fig_save_name = ['figures' filesep 'loglog_mean_delay.eps'];
        set(figure4_1,'PaperPositionMode','auto')
        print(figure4_1,'-depsc','-r150',fig_save_name)
        clear fig_save_name
    end

    % an individual figure for the mean delay
    figure4_2 = figure('name','Log-Log plot of the Mean phase delay with height');
    ylabel('Log(\phi_{tropo} [rad])','fontsize',fontsize)
    xlabel('Height [km])','fontsize',fontsize)
    set(gca,'fontsize',fontsize)
    hold on
    plot(h_log,phi_delay_log,'k-','linewidth',2)
    hold on
    plot(A_hc(:,1),A_hc*coeff_hc_phase,'r--','linewidth',2)
    set(gca,'fontsize',fontsize)
    % plotting the powerlaw based on the lower height range
    hold on
    xlim(xlimits)
    ylimits = get(gca,'ylim');
    legend('Mean delay','Power law',2)
    legend boxoff
    set(gca,'xtick',[get(gca,'xtick') log10(h_0_threshold)],'xticklabel',num2str([get(gca,'xtick') log10(h_0_threshold)]'))

    % updating the lables to be heights
    temp_loc = get(gca,'xtick');
    temp = round((h_0_threshold-10.^temp_loc)/1000*100)/100;
    temp_str = num2str(temp');
    set(gca,'xtick',temp_loc,'xticklabel',temp_str)
    box on

    if save_fig==1
        fig_save_name = ['figures' filesep 'loglog_mean_phasedelay.eps'];
        set(figure4_2,'PaperPositionMode','auto')
        print(figure4_2,'-depsc','-r150',fig_save_name)
        clear fig_save_name
    end

end

save([save_name '.mat'],'-append','alpha_log_all','alpha_log_hc')
clear ix_all A_all delay_log_temp_all coeff_all ix_hc A_hc delay_log_temp_hc coeff_hc delay_log phi_delay_log h_log

%% evaluating the performance of the estimation by plotting the estimated powerlaw wrt to the mean delay from sounding
if plot_flag==1
    % powerlaw estimated from log log plot full height range
    delay_log_all = K_log_all*(h_0_threshold-h_delay).^alpha_log_all;

    % powerlaw delay from log log plot estimated from the 0 till hc height range
    delay_log_hc = K_log_hc*(h_0_threshold-h_delay).^alpha_log_hc;
    % plot mean delay on top of the existing figure
    hfig_powerlawdelay = figure('name','Fitted power law delay to mean delay curve');
    plot(delay_mean,h_delay./1000,'k-','linewidth',2)
    hold on
    plot(delay_log_hc,h_delay./1000,'r-','linewidth',2)
    hold on
    xlimits = get(gca,'xlim');
    plot([xlimits(1) xlimits(2)],[max_height max_height]./1000,'k--')
    legend('Mean delay','Fitted power law','Max height of the region')
    ylabel('Height [km]','fontsize',fontsize)
    xlabel('LOS delay [m]','fontsize',fontsize)
    set(gca,'fontsize',fontsize)
    box on

    if save_fig==1
        fig_save_name = ['figures' filesep  'MeanDelay_PowerLawDelay.eps'];
        set(hfig_powerlawdelay,'PaperPositionMode','auto')
        print(hfig_powerlawdelay,'-depsc','-r150',fig_save_name)
        clear fig_save_name
    end
    clear hfig_powerlawdelay

end

continue_flag=0;
abord_flag=0;

end

% The while loop was existed because of an error statement.
if abord_flag == 1
	alpha_log_all = NaN;
	alpha_log_hc = NaN;
	n_soundings=NaN;
    h_0_threshold = NaN;
	fprintf('Early termination \n')
else
    h_0_threshold = h_0_threshold/1000;       % put threshold to [km]
	fprintf('DONE \n')
end

cd(curdir)


function [Xc3,tri3,xc3,xp3,dc3,xc_texture,nc3,conf_nc3,Nn3] = Meshing(Xc2,xc2,xp2,Thresh_connect,N_smoothing,om,T,N_x,N_y,fc,cc,kc,alpha_c,fp,cp,kp,alpha_p),

% scaled connection threshold

T_connect = Thresh_connect; 	% scaled threshold

fprintf(1,'Organizing the data...\n');

xp_frac = rem(xp2,1);
xc_frac = rem(xc2(1,:),1);

if std(xp_frac) > std(xc_frac),
    disp('Dense depth map in the image coordinates');
    temporal = 1;
    spatial = 0;
else
    disp('Dense depth map in the cross image and projector frame');
    temporal = 0;
    spatial = 1;
end;

if spatial,

    % Something to fix the organization:
    xpmin = min(xp2);
    xp_ind = round(unique(xp2 - xpmin));
    step_xp = min(diff(xp_ind)); %xp_ind(2) - xp_ind(1);
    xp4 = (xp2 - xpmin)/step_xp + xpmin;

    xpmin = min(xp4);
    xpmax = max(xp4);

    xcmin = min(xc2(2,:));
    xcmax = max(xc2(2,:));

else

    % Something to fix the organization:

    xp4 = xc2(1,:);

    xpmin = min(xp4);
    xpmax = max(xp4);

    xcmin = min(xc2(2,:));
    xcmax = max(xc2(2,:));

end;

Nrow = xcmax - xcmin + 1;
Ncol = xpmax - xpmin + 1;

Xmesh = zeros(Nrow,Ncol);
Ymesh = zeros(Nrow,Ncol);
Zmesh =  zeros(Nrow,Ncol);
Cmesh = zeros(Nrow,Ncol);
xcmesh = zeros(Nrow,Ncol);
ycmesh = zeros(Nrow,Ncol);

ind_col = round(xp4 - xpmin + 1);
ind_row = xc2(2,:) - xcmin + 1;
ind = ind_row + (ind_col-1)*Nrow;
ni = length(ind);

% Good format for ivview:
Xmesh(ind) = Xc2(1,:);
Ymesh(ind) = Xc2(2,:); 		% tries to have it in the right position
Zmesh(ind) = Xc2(3,:); 		% tries to have it in the right position
Cmesh(ind) = ones(1,ni);
xcmesh(ind) = xc2(1,:);
ycmesh(ind) = xc2(2,:);

% Hypothesis on the triangles:

D1 = Cmesh(2:Nrow,1:(Ncol-1)) & Cmesh(1:(Nrow-1),2:Ncol);
F1 = Cmesh(1:(Nrow-1),1:(Ncol-1)) & D1;
F2 = Cmesh(2:Nrow,2:Ncol) & D1;
C1 =  (F1 | F2);

D2 = Cmesh(1:(Nrow-1),1:(Ncol-1)) & Cmesh(2:Nrow,2:Ncol);
F3 = Cmesh(1:(Nrow-1),2:Ncol) & D2;
F4 = Cmesh(2:Nrow,1:(Ncol-1)) & D2;
C2 = (F3 | F4);

Ambi = C1 & C2; 			% ambiguous zones
Div1 = C1 & ~C2; 			% needs to check relative distance of points
Div2 = ~C1 & C2; 			% needs to check relative distance of points

% diagonal measure:

%Dm1 = abs(Zmesh(2:Nrow,1:(Ncol-1)) -  Zmesh(1:(Nrow-1),2:Ncol));

Dm1 =( Xmesh(2:Nrow,1:(Ncol-1)) -  Xmesh(1:(Nrow-1),2:Ncol)).^2 + (Ymesh(2:Nrow,1:(Ncol-1)) -  Ymesh(1:(Nrow-1),2:Ncol)).^2 + (Zmesh(2:Nrow,1:(Ncol-1)) -  Zmesh(1:(Nrow-1),2:Ncol)).^2;

%Dm2 = abs(Zmesh(1:(Nrow-1),1:(Ncol-1)) - Zmesh(2:Nrow,2:Ncol));

Dm2 = (Xmesh(1:(Nrow-1),1:(Ncol-1)) - Xmesh(2:Nrow,2:Ncol)).^2 + (Ymesh(1:(Nrow-1),1:(Ncol-1)) - Ymesh(2:Nrow,2:Ncol)).^2 + (Zmesh(1:(Nrow-1),1:(Ncol-1)) - Zmesh(2:Nrow,2:Ncol)).^2 ;

Div1n = Div1 | ((Dm1 <= Dm2)&Ambi);
Div2n = Div2 | ((Dm2 < Dm1)&Ambi);

Div11 = Div1n & F1;
Div12 = Div1n & F2;
Div21 = Div2n & F3;
Div22 = Div2n & F4;

% look at local difference:

dZ_r = abs(Zmesh(:,2:Ncol)-Zmesh(:,1:(Ncol-1)));
dZ_c = abs(Zmesh(2:Nrow,:)-Zmesh(1:(Nrow-1),:));

Div11 = Div11 & (dZ_r(1:(Nrow-1),:)<T_connect) & (dZ_c(:,1:(Ncol-1))<T_connect); % & (Dm1 < T_connect);
Div12 = Div12 & (dZ_r(2:Nrow,:)<T_connect) & (dZ_c(:,2:Ncol)<T_connect); %& (Dm1 < T_connect);

Div21 = Div21 & (dZ_r(1:(Nrow-1),:)<T_connect) & (dZ_c(:,2:Ncol)<T_connect); % & (Dm2 < T_connect);
Div22 = Div22 & (dZ_r(2:Nrow,:)<T_connect) & (dZ_c(:,1:(Ncol-1))<T_connect); % & (Dm2 < T_connect);

%%% Smoothing:

for i = 1:N_smoothing,

    fprintf(1,'Surface smoothing %d\n',i);

    % first find the neighbor points of every point:

    t = zeros(Nrow,Ncol);
    b = zeros(Nrow,Ncol);
    l = zeros(Nrow,Ncol);
    r = zeros(Nrow,Ncol);
    tr = zeros(Nrow,Ncol);
    br = zeros(Nrow,Ncol);
    tl = zeros(Nrow,Ncol);
    bl = zeros(Nrow,Ncol);

    t(2:Nrow,2:Ncol) = (Div12 | Div21);
    t(2:Nrow,1:(Ncol-1)) = t(2:Nrow,1:(Ncol-1)) | (Div11 | Div22);
    b(1:(Nrow-1),2:Ncol) = (Div12 | Div21);
    b(1:(Nrow-1),1:(Ncol-1)) = b(1:(Nrow-1),1:(Ncol-1)) | (Div11 | Div22);
    r(2:Nrow,1:(Ncol-1)) = (Div12 | Div22);
    r(1:(Nrow-1),1:(Ncol-1)) = r(1:(Nrow-1),1:(Ncol-1)) | (Div11 | Div21);
    l(2:Nrow,2:Ncol) = (Div12 | Div22);
    l(1:(Nrow-1),2:Ncol) = l(1:(Nrow-1),2:Ncol) | (Div11 | Div21);
    tr(2:Nrow,1:(Ncol-1)) = Div11 | Div12;
    br(1:(Nrow-1),1:(Ncol-1)) = Div21 | Div22;
    tl(2:Nrow,2:Ncol) = Div21 | Div22;
    bl(1:(Nrow-1),2:Ncol) = Div11 | Div12;

    Nn = t + b + l + r + tr + br + tl + bl;

    XX = Xmesh; 		     	% zeros(Nrow,2); zeros(2,Ncol+2)];
    YY = Ymesh; 				% zeros(Nrow,2); zeros(2,Ncol+2)];
    ZZ = Zmesh; 				% zeros(Nrow,2); zeros(2,Ncol+2)];

    [is,js] = find(Nn);

    indd = find((is > 1) & (is < Nrow) & (js > 1) & (js < Ncol));

    is = is(indd);
    js = js(indd);

    sm =  is + (js-1)*(Nrow);
    sm_t = is + (js-1)*(Nrow) - 1;
    sm_b = is + (js-1)*(Nrow) + 1;
    sm_r = is + (js)*(Nrow);
    sm_l = is + (js-2)*(Nrow);

    sm_tr =  is + (js)*(Nrow) - 1;
    sm_br =  is + (js)*(Nrow) + 1;
    sm_tl =  is + (js-2)*(Nrow) -1;
    sm_bl =  is + (js-2)*(Nrow) + 1;

    XX(sm) = 0.5 * XX(sm) + 0.5 * ((XX(sm_t).*t(sm)+XX(sm_b).*b(sm)+XX(sm_r).*r(sm)+XX(sm_l).*l(sm)+XX(sm_tr).*tr(sm)+XX(sm_br).*br(sm)+XX(sm_tl).*tl(sm)+XX(sm_bl).*bl(sm))./Nn(sm));

    YY(sm) = 0.5 * YY(sm) + 0.5 * ((YY(sm_t).*t(sm)+YY(sm_b).*b(sm)+YY(sm_r).*r(sm)+YY(sm_l).*l(sm)+YY(sm_tr).*tr(sm)+YY(sm_br).*br(sm)+YY(sm_tl).*tl(sm)+YY(sm_bl).*bl(sm))./Nn(sm));

    ZZ(sm) = 0.5 * ZZ(sm) + 0.5 * ((ZZ(sm_t).*t(sm)+ZZ(sm_b).*b(sm)+ZZ(sm_r).*r(sm)+ZZ(sm_l).*l(sm)+ZZ(sm_tr).*tr(sm)+ZZ(sm_br).*br(sm)+ZZ(sm_tl).*tl(sm)+ZZ(sm_bl).*bl(sm))./Nn(sm));

    Xmesh = XX(1:Nrow,1:Ncol);
    Ymesh = YY(1:Nrow,1:Ncol);
    Zmesh = ZZ(1:Nrow,1:Ncol);

    %%% reconnect after smoothing:

    % diagonal measure:

    Dm1 =( Xmesh(2:Nrow,1:(Ncol-1)) -  Xmesh(1:(Nrow-1),2:Ncol)).^2 + (Ymesh(2:Nrow,1:(Ncol-1)) -  Ymesh(1:(Nrow-1),2:Ncol)).^2 + (Zmesh(2:Nrow,1:(Ncol-1)) -  Zmesh(1:(Nrow-1),2:Ncol)).^2;

    Dm2 = (Xmesh(1:(Nrow-1),1:(Ncol-1)) - Xmesh(2:Nrow,2:Ncol)).^2 + (Ymesh(1:(Nrow-1),1:(Ncol-1)) - Ymesh(2:Nrow,2:Ncol)).^2 + (Zmesh(1:(Nrow-1),1:(Ncol-1)) - Zmesh(2:Nrow,2:Ncol)).^2 ;

    Div1n = Div1 | ((Dm1 <= Dm2)&Ambi);
    Div2n = Div2 | ((Dm2 < Dm1)&Ambi);

    Div11 = Div1n & F1;
    Div12 = Div1n & F2;
    Div21 = Div2n & F3;
    Div22 = Div2n & F4;

    % look at local difference:

    dZ_r = abs(Zmesh(:,2:Ncol)-Zmesh(:,1:(Ncol-1)));
    dZ_c = abs(Zmesh(2:Nrow,:)-Zmesh(1:(Nrow-1),:));

    Div11 = Div11 & (dZ_r(1:(Nrow-1),:)<T_connect) & (dZ_c(:,1:(Ncol-1))<T_connect); % & (Dm1 < T_connect);
    Div12 = Div12 & (dZ_r(2:Nrow,:)<T_connect) & (dZ_c(:,2:Ncol)<T_connect); %& (Dm1 < T_connect);

    Div21 = Div21 & (dZ_r(1:(Nrow-1),:)<T_connect) & (dZ_c(:,2:Ncol)<T_connect); % & (Dm2 < T_connect);
    Div22 = Div22 & (dZ_r(2:Nrow,:)<T_connect) & (dZ_c(:,1:(Ncol-1))<T_connect); % & (Dm2 < T_connect);

end; 				% of smoothing

% At that point the Divij are the final connections

% Find the number of neighbors per points:

t = zeros(Nrow,Ncol);
b = zeros(Nrow,Ncol);
l = zeros(Nrow,Ncol);
r = zeros(Nrow,Ncol);
tr = zeros(Nrow,Ncol);
br = zeros(Nrow,Ncol);
tl = zeros(Nrow,Ncol);
bl = zeros(Nrow,Ncol);

t(2:Nrow,2:Ncol) = (Div12 | Div21);
t(2:Nrow,1:(Ncol-1)) = t(2:Nrow,1:(Ncol-1)) | (Div11 | Div22);
b(1:(Nrow-1),2:Ncol) = (Div12 | Div21);
b(1:(Nrow-1),1:(Ncol-1)) = b(1:(Nrow-1),1:(Ncol-1)) | (Div11 | Div22);
r(2:Nrow,1:(Ncol-1)) = (Div12 | Div22);
r(1:(Nrow-1),1:(Ncol-1)) = r(1:(Nrow-1),1:(Ncol-1)) | (Div11 | Div21);
l(2:Nrow,2:Ncol) = (Div12 | Div22);
l(1:(Nrow-1),2:Ncol) = l(1:(Nrow-1),2:Ncol) | (Div11 | Div21);
tr(2:Nrow,1:(Ncol-1)) = Div11 | Div12;
br(1:(Nrow-1),1:(Ncol-1)) = Div21 | Div22;
tl(2:Nrow,2:Ncol) = Div21 | Div22;
bl(1:(Nrow-1),2:Ncol) = Div11 | Div12;

Nn = t + b + l + r + tr + br + tl + bl;

% build up the matrix of used points: (for renumbering)
% Number of neighbor triangles:

top_left = Div12 + Div21 + Div22;
top_right = Div11 + Div12 + Div22;
bot_left = Div11 + Div12 + Div21;
bot_right = Div11 + Div21 + Div22;

Used_points = zeros(Nrow,Ncol);
Used_points(2:Nrow,2:Ncol) = Used_points(2:Nrow,2:Ncol)+top_left;
Used_points(2:Nrow,1:(Ncol-1)) = Used_points(2:Nrow,1:(Ncol-1))+top_right;
Used_points(1:(Nrow-1),2:Ncol) = Used_points(1:(Nrow-1),2:Ncol)+bot_left;
Used_points(1:(Nrow-1),1:(Ncol-1)) = Used_points(1:(Nrow-1),1:(Ncol-1))+bot_right;

[xc3_2, xp3] = find(Used_points);

ind_points = (xp3-1)*Nrow + xc3_2;

N_vertices = length(ind_points);

Ind_Mat = -ones(Nrow,Ncol);
Ind_Mat(ind_points) = (1:N_vertices)-1;

% Regenere the 3D coordinates, the image location, and the projector coordinates:

Xc3 =  [Xmesh(ind_points) Ymesh(ind_points) Zmesh(ind_points)]';

% number of neighbors:

Nn3 = Nn(ind_points)';

xc3 =  project_points2(Xc3,zeros(3,1),zeros(3,1),fc,cc,kc,alpha_c);% project2(Xc3,eye(3),[0;0;0],fc,cc,kc);

%xc3(2,:) = xc3_2' + xcmin - 1;

%xp3_2D = project_points2(Xc3,om,T,fp,cp,kp,alpha_p);

if spatial,
    xp3 = step_xp * (xp3'-1) + xpmin;
else
    xp3_2D = project_points2(Xc3,om,T,fp,cp,kp,alpha_p);
    xp3 = xp3_2D(1,:);
end;

% Texture coordinates:
xc_texture = [(xc3(1,:)+.5)/(N_x);1-((xc3(2,:)+.5)/(N_y))];

% The boundaries faces (not fully connected):

Div11_u = Div11;
Div12_u = Div12;
Div21_u = Div21;
Div22_u = Div22;

[r11,c11] = find(Div11_u);
[r12,c12] = find(Div12_u);
[r21,c21] = find(Div21_u);
[r22,c22] = find(Div22_u);

% Work with Div11:
ind11_1 =  Nrow*c11+r11;
ind11_2 =  Nrow*(c11-1)+r11;
ind11_3 =  Nrow*(c11-1)+r11+1;

% Work with Div12:
ind12_1 =  Nrow*c12+r12;
ind12_2 =  Nrow*(c12-1)+r12+1;
ind12_3 =  Nrow*c12+r12+1;

% Work with Div21:
ind21_1 =  Nrow*c21+r21;
ind21_2 =  Nrow*(c21-1)+r21;
ind21_3 =  Nrow*c21+r21+1;

% Work with Div22:
ind22_1 =  Nrow*(c22-1)+r22;
ind22_2 =  Nrow*(c22-1)+r22+1;
ind22_3 =  Nrow*c22+r22+1;

% FaceSet
Faces11 = [Ind_Mat(ind11_1) Ind_Mat(ind11_2) Ind_Mat(ind11_3)];
Faces12 = [ Ind_Mat(ind12_1) Ind_Mat(ind12_2) Ind_Mat(ind12_3)];
Faces21 = [Ind_Mat(ind21_1) Ind_Mat(ind21_2) Ind_Mat(ind21_3)];
Faces22 = [ Ind_Mat(ind22_1) Ind_Mat(ind22_2) Ind_Mat(ind22_3)];

Faces = [Faces11;Faces12;Faces21;Faces22]';

N_triangles = size(Faces,2);

% matlab formulation:
tri3 = Faces' + 1;

% Direction of observation:
dc3 = -Xc3;
dc3 = dc3 ./ (ones(3,1) * (sqrt(sum(dc3.^2))));

% use Used_points to keep the number of neighbor triangles:

% compute the surface normals for Div11, Div12, Div21 and Div22, and then
% average them using Used_points as number of them.

nx11 = zeros(Nrow-1,Ncol-1);
ny11 = zeros(Nrow-1,Ncol-1);
nz11 = zeros(Nrow-1,Ncol-1);

nx12 = zeros(Nrow-1,Ncol-1);
ny12 = zeros(Nrow-1,Ncol-1);
nz12 = zeros(Nrow-1,Ncol-1);

nx21 = zeros(Nrow-1,Ncol-1);
ny21 = zeros(Nrow-1,Ncol-1);
nz21 = zeros(Nrow-1,Ncol-1);

nx22 = zeros(Nrow-1,Ncol-1);
ny22 = zeros(Nrow-1,Ncol-1);
nz22 = zeros(Nrow-1,Ncol-1);

u11 = [(Xmesh(ind11_2)-Xmesh(ind11_1))';(Ymesh(ind11_2)-Ymesh(ind11_1))';(Zmesh(ind11_2)-Zmesh(ind11_1))'];
v11 = [(Xmesh(ind11_3)-Xmesh(ind11_1))';(Ymesh(ind11_3)-Ymesh(ind11_1))';(Zmesh(ind11_3)-Zmesh(ind11_1))'];

nx11(r11+(c11-1)*(Nrow-1)) = u11(2,:).*v11(3,:) - u11(3,:).*v11(2,:);
ny11(r11+(c11-1)*(Nrow-1)) = u11(3,:).*v11(1,:) - u11(1,:).*v11(3,:);
nz11(r11+(c11-1)*(Nrow-1)) = u11(1,:).*v11(2,:) - u11(2,:).*v11(1,:);

u12 = [(Xmesh(ind12_2)-Xmesh(ind12_1))';(Ymesh(ind12_2)-Ymesh(ind12_1))';(Zmesh(ind12_2)-Zmesh(ind12_1))'];
v12 = [(Xmesh(ind12_3)-Xmesh(ind12_1))';(Ymesh(ind12_3)-Ymesh(ind12_1))';(Zmesh(ind12_3)-Zmesh(ind12_1))'];

nx12(r12+(c12-1)*(Nrow-1)) = u12(2,:).*v12(3,:) - u12(3,:).*v12(2,:);
ny12(r12+(c12-1)*(Nrow-1)) = u12(3,:).*v12(1,:) - u12(1,:).*v12(3,:);
nz12(r12+(c12-1)*(Nrow-1)) = u12(1,:).*v12(2,:) - u12(2,:).*v12(1,:);

u21 = [(Xmesh(ind21_2)-Xmesh(ind21_1))';(Ymesh(ind21_2)-Ymesh(ind21_1))';(Zmesh(ind21_2)-Zmesh(ind21_1))'];
v21 = [(Xmesh(ind21_3)-Xmesh(ind21_1))';(Ymesh(ind21_3)-Ymesh(ind21_1))';(Zmesh(ind21_3)-Zmesh(ind21_1))'];

nx21(r21+(c21-1)*(Nrow-1)) = u21(2,:).*v21(3,:) - u21(3,:).*v21(2,:);
ny21(r21+(c21-1)*(Nrow-1)) = u21(3,:).*v21(1,:) - u21(1,:).*v21(3,:);
nz21(r21+(c21-1)*(Nrow-1)) = u21(1,:).*v21(2,:) - u21(2,:).*v21(1,:);

u22 = [(Xmesh(ind22_2)-Xmesh(ind22_1))';(Ymesh(ind22_2)-Ymesh(ind22_1))';(Zmesh(ind22_2)-Zmesh(ind22_1))'];
v22 = [(Xmesh(ind22_3)-Xmesh(ind22_1))';(Ymesh(ind22_3)-Ymesh(ind22_1))';(Zmesh(ind22_3)-Zmesh(ind22_1))'];

nx22(r22+(c22-1)*(Nrow-1)) = u22(2,:).*v22(3,:) - u22(3,:).*v22(2,:);
ny22(r22+(c22-1)*(Nrow-1)) = u22(3,:).*v22(1,:) - u22(1,:).*v22(3,:);
nz22(r22+(c22-1)*(Nrow-1)) = u22(1,:).*v22(2,:) - u22(2,:).*v22(1,:);

% Sum all the relevant normal components for each vertice:

nx = zeros(Nrow,Ncol);
ny = zeros(Nrow,Ncol);
nz = zeros(Nrow,Ncol);

nx(1:(Nrow-1),1:(Ncol-1)) = nx11 + nx21 + nx22;
nx(2:Nrow,1:(Ncol-1)) = nx(2:Nrow,1:(Ncol-1)) + nx11 + nx12 + nx22;
nx(1:(Nrow-1),2:Ncol) = nx(1:(Nrow-1),2:Ncol) + nx11 + nx12 + nx21;
nx(2:Nrow,2:Ncol) = nx(2:Nrow,2:Ncol) + nx12 + nx21 + nx22;

ny(1:(Nrow-1),1:(Ncol-1)) = ny11 + ny21 + ny22;
ny(2:Nrow,1:(Ncol-1)) = ny(2:Nrow,1:(Ncol-1)) + ny11 + ny12 + ny22;
ny(1:(Nrow-1),2:Ncol) = ny(1:(Nrow-1),2:Ncol) + ny11 + ny12 + ny21;
ny(2:Nrow,2:Ncol) = ny(2:Nrow,2:Ncol) + ny12 + ny21 + ny22;

nz(1:(Nrow-1),1:(Ncol-1)) = nz11 + nz21 + nz22;
nz(2:Nrow,1:(Ncol-1)) = nz(2:Nrow,1:(Ncol-1)) + nz11 + nz12 + nz22;
nz(1:(Nrow-1),2:Ncol) = nz(1:(Nrow-1),2:Ncol) + nz11 + nz12 + nz21;
nz(2:Nrow,2:Ncol) = nz(2:Nrow,2:Ncol) + nz12 + nz21 + nz22;

% Vertice normals:
nc3 =  [nx(ind_points)';ny(ind_points)';nz(ind_points)'];

% normalization of the normals:

conf_nc3 = sum(nc3.^2);
Norms = sqrt(conf_nc3);

nc3 = nc3 ./ (ones(3,1)*Norms);

% update of the normal components:

nx(ind_points) = nc3(1,:);
ny(ind_points) = nc3(2,:);
nz(ind_points) = nc3(3,:);

%clear Ambi C1 C2 Cmesh D1 D2 Div Div1 Div11 Div11_u Div12 Div12_u Div2 Div21 Div21_u Div22 Div22_u Div1n Div2n Dm1 Dm2 F1 F2 F3 F4 Faces Faces11 Faces12 Faces21 Faces22 Header Ind_Mat Light Ncol Nn Norms Nrow Stripe T_connect Used_points Vertice_norm XX Xc3 Xmesh YY Ymesh ZZ Zmesh b bl bot_left bot_right br c11 c12 c21 c22 coeff dZ_c dZ_r day dot file i ind ind11_1 ind11_2 ind11_3 ind12_1 ind12_2 ind12_3 ind21_3 ind21_1 ind21_2 ind21_3 ind22_1 ind22_2 ind22_3 ind_col ind_row indd is js l ni nx nx11 nx12 nx21 nx22 ny ny11 ny12 ny21 ny22 nz nz11 nz12 nz21 nz22 r r11 r12 r21 r22 rasterimage sm sm_b sm_bl sm_br sm_l sm_r sm_t sm_tl sm_tr t tl top_left top_right tr u11 u12 u21 u22 unshading v v11 v12 v21 v22 Xc3 xc3_2 xc_texture xcmax xcmesh xcmin xp3 xpmax xpmin ycmesh xc3 ind_points


function out = TSTL_predict(y, plen, NNR, stepSize, pmode, embedParams)
% TSTL_predict  Local constant iterative time-series prediction.
%
% References TSTOOL code 'predict', which does local constant iterative
% prediction for scalar data using fast nearest neighbour searching.
% There are four modes available for the prediction output.
%
% TSTOOL: http://www.physik3.gwdg.de/tstool/
%
%---INPUTS:
%
% y, scalar column vector time series
%
% plen, prediction length in samples or as proportion of time series length NNR,
%
% NNR, number of nearest neighbours
%
% stepSize, number of samples to step for each prediction
%
% pmode, prediction mode, four options:
%           (i) 0: output vectors are means of images of nearest neighbours
%           (ii) 1: output vectors are distance-weighted means of images
%                     nearest neighbours
%           (iii) 2: output vectors are calculated using local flow and the
%                    mean of the images of the neighbours
%           (iv) 3: output vectors are calculated using local flow and the
%                    weighted mean of the images of the neighbours
% embedParams, as usual to feed into BF_Embed, except that now you can set
%              to zero to not embed.

% ------------------------------------------------------------------------------
% Copyright (C) 2020, Ben D. Fulcher <ben.d.fulcher@gmail.com>,
% <http://www.benfulcher.com>
%
% If you use this code for your research, please cite the following two papers:
%
% (1) B.D. Fulcher and N.S. Jones, "hctsa: A Computational Framework for Automated
% Time-Series Phenotyping Using Massive Feature Extraction, Cell Systems 5: 527 (2017).
% DOI: 10.1016/j.cels.2017.10.001
%
% (2) B.D. Fulcher, M.A. Little, N.S. Jones, "Highly comparative time-series
% analysis: the empirical structure of time series and their methods",
% J. Roy. Soc. Interface 10(83) 20130048 (2013).
% DOI: 10.1098/rsif.2013.0048
%
% This function is free software: you can redistribute it and/or modify it under
% the terms of the GNU General Public License as published by the Free Software
% Foundation, either version 3 of the License, or (at your option) any later
% version.
%
% This program is distributed in the hope that it will be useful, but WITHOUT
% ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
% FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
% details.
%
% You should have received a copy of the GNU General Public License along with
% this program. If not, see <http://www.gnu.org/licenses/>.
% ------------------------------------------------------------------------------

% ------------------------------------------------------------------------------
%% Check inputs, set defaults
% ------------------------------------------------------------------------------
doPlot = false; % plot outputs to figure (e.g., for debugging)

% (*) Prediction length, plen (the length of the output time series)
if nargin < 2 || isempty(plen)
    plen = 1; % output the same length (proportion)
end
% (proportion set after embedding, as the embedding will lose points
% according to the dimension of the space)

% (*) number of neighest neighbours, NNR
if nargin < 3 || isempty(NNR)
    NNR = 1; % use 1 nearest neighbour
end

% (*) stepSize (in samples)
if nargin < 4 || isempty(stepSize)
    stepSize = 2;
end

% (*) prediction mode, pmode:
if nargin < 5 || isempty(pmode)
    pmode = 0; % output vectors are means of the images of nearest neighbours
end

% (*) embedParams
if nargin < 6 || isempty(embedParams)
    embedParams = {'ac','fnnmar'};
    fprintf(1,'Using default embedding using autocorrelation for tau and Cao''s method for m\n');
end

% ------------------------------------------------------------------------------
%% Embed the scalar time series by time-delay method
% ------------------------------------------------------------------------------
% embedpn = BF_Embed(y,embedParams{1},embedParams{2},2);
% delay = embedpn(1);
% dim = embedpn(2);
if iscell(embedParams)
    s = BF_Embed(y,embedParams{1},embedParams{2},1); % last in
elseif embedParams == 0
    s = signal(y);
end

if ~isa(s,'signal')
    out = NaN; return
end

Ns = length(data(s));
if Ns < 50
    error('This is a very short time series! :(');
end
y = y(1:Ns); % for statistical purposes...
if plen > 0 && plen <= 1
    plen = floor(plen*Ns); % specify a proportion of the time series length
end

% ------------------------------------------------------------------------------
%% Run the code
% ------------------------------------------------------------------------------
try
    rs = predict(s, plen, NNR, stepSize, pmode);
catch
    error('TSTOOL''s predict function didn''t run correctly')
end

y_pred = data(rs);
y_pred1 = y_pred(:,1); % for this embedding dimension (?)

if doPlot
    figure('color','w'); box('on'); view(rs);

    figure('color','w'); box('on');
    hold off; plot(y,'k'), hold on; plot(y_pred1,'m'), hold off;
end

% ------------------------------------------------------------------------------
%% Compare the output to the properties of the true time series
% ------------------------------------------------------------------------------

% actual basic statistical properties
out.pred1mean = mean(y_pred1);
out.pred1std = std(y_pred1);
out.pred1maxc = abs(max(y_pred1)-max(y));
out.pred1maxc = abs(max(y_pred(:))-max(y));
out.pred1minc = abs(min(y_pred1)-min(y));
out.predminc = abs(min(y_pred(:))-min(y));
out.pred1rangec = abs(range(y_pred1)/range(y)-1);

% look at structure in cross correlation function, xcf
% (requires that prediction length the same as the time series itself)
[xcf, lags] = xcorr(y,y_pred1,'coeff');
% plot(lags,xcf);

out.maxabsxcf = max(abs(xcf)); % maximum of cross-correlation function; where it occurs
out.maxabsxcflag = lags(find(abs(xcf) == out.maxabsxcf,1,'first'));
out.maxxcf = max(xcf); % maximum positive cross-correlation
out.maxxcflag = lags(find(xcf == out.maxxcf,1,'first'));
out.meanxcf = mean(xcf);
out.minxcf = min(xcf);
out.stdxcf = std(xcf);

out.pred1_ac1 = CO_AutoCorr(y_pred1,1,'Fourier'); % autocorrelation at lag one of prediction
out.pred1ac1diff = abs(out.pred1_ac1 - CO_AutoCorr(y,1,'Fourier')); % difference in autocorrelations of prediction and original
out.pred1_ac2 = CO_AutoCorr(y_pred1,2,'Fourier'); % autocorrelation at lag one of prediction
out.pred1ac2diff = abs(out.pred1_ac2 - CO_AutoCorr(y,2,'Fourier')); % difference in autocorrelations of prediction and original
out.pred_tau_comp = CO_FirstCrossing(y_pred1,'ac',0,'continuous')/CO_FirstCrossing(y,'ac',0,'continuous'); % ratio of first zero crossings of autocorrelation function

% Autocorrelation structure:
acs_y = CO_AutoCorr(y,1:10,'Fourier');
acs_y_pred1 = CO_AutoCorr(y_pred1,1:10,'Fourier');
out.acs1_10_sumabsdiffpred1 = sum(abs(acs_y - acs_y_pred1));

% mean square residuals: this will likely be a bad measure (as may be out
% of sync)
out.pred1rmsres = sqrt(mean((y-y_pred1).^2));

% align at best positive cross-correlation and then look at residuals
if out.maxxcflag > 0
    y_lagged = y(out.maxxcflag:end);
    Nlag = length(y_lagged);
    y_pred1_lagged = y_pred1(1:Nlag);
elseif out.maxxcflag == 0
    y_lagged = y;
    y_pred1_lagged = y;
    Nlag = length(y);
else % negative
    y_pred1_lagged = y_pred1(-out.maxxcflag:end);
    Nlag = length(y_pred1_lagged);
    y_lagged = y(1:Nlag);
end

% hold off; plot(y_pred1_lagged);
% hold on; plot(y_lagged,'r');
out.Nlagxcorr = Nlag;
res = y_lagged - y_pred1_lagged;
out.bestpred1rmsres = sqrt(mean(res.^2)); % rms residuals
out.ac1bestres = CO_AutoCorr(res,1,'Fourier'); % autocorrelation of residuals

% now look at fraction of points that are within a threshold of each
% other...
fracresfn = @(x) sum(abs(res) < x)/Nlag;
out.fracres005 = fracresfn(0.05); %sum(abs(res)<0.05)/Nlag;
out.fracres01 = fracresfn(0.1); %sum(abs(res)<0.1)/Nlag;
out.fracres02 = fracresfn(0.2); %sum(abs(res)<0.2)/Nlag;
out.fracres03 = fracresfn(0.3); %sum(abs(res)<0.3)/Nlag;
out.fracres05 = fracresfn(0.5); %sum(abs(res)<0.5)/Nlag;

% now look at fraction of points within a circle of (time-measurement) radius
% near a real point in the time series
% this could be done using the closest neighbour of the simulation to the
% real time series

% Could compare different dimensions rather than just the first... find the
% best... etc.

% There are heaps of metrics you could use to compare... Ultimately may be
% able to implement model outputs as rows so as to compare across many
% measures...

end


function [D1]=triSurfSetDist(F1,V1,F2,V2,distMetric)

% function [D1]=triSurfSetDist(F1,V1,F2,V2,distMetric)
% ------------------------------------------------------------------------
%
% Change log:
% 2021/07/22 KMM Removed waitbar for ray-tracing method
% ------------------------------------------------------------------------

%%

switch distMetric
    case 'dist'
        D1=minDist(V1,V2);
    case 'ray'
        [~,~,N1]=patchNormal(F1,V1);

        %Ray trace 1 onto 2
        optionStructRayTrace.tolEps = 1e-6;
        optionStructRayTrace.triSide = 0;
        optionStructRayTrace.rayType = 'ray'; % or line
        optionStructRayTrace.exclusionType = 'normal'; % or exclusive
        optionStructRayTrace.paired=0; % 1 for paired, 0 for non-paired

        [~,indIntersect,d1_all]=triSurfRayTrace(V1,N1,F2,V2,optionStructRayTrace);

        numIntersect=size(indIntersect,1);

        S=sparse(indIntersect(:,1),(1:numIntersect)',1,size(V1,1),numIntersect,numIntersect);
        d1_sparse=sparse(indIntersect(:,1),(1:numIntersect)',abs(d1_all),size(V1,1),numIntersect,numIntersect);

        D1=full(spmin(d1_sparse,[],2,'includenan',S~=0,1));

    case 'dist-ray'
        [D1d]=triSurfSetDist(F1,V1,F2,V2,'dist');
        [D1r]=triSurfSetDist(F1,V1,F2,V2,'ray');
        D1=gnanmin([D1d D1r],[],2);
    case 'near-norm'
        [~,indMin]=minDist(V1,V2);
        [~,~,Nv]=patchNormal(F2,V2);
        N=Nv(indMin,:);
        W=V1-V2(indMin,:);
        D1=abs(dot(N,W,2));
end

%%
% _*GIBBON footer text*_
%
% License: <https://github.com/gibbonCode/GIBBON/blob/master/LICENSE>
%
% GIBBON: The Geometry and Image-based Bioengineering add-On. A toolbox for
% image segmentation, image-based modeling, meshing, and finite element
% analysis.
%
% Copyright (C) 2006-2022 Kevin Mattheus Moerman and the GIBBON contributors
%
% This program is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program.  If not, see <http://www.gnu.org/licenses/>.


function [mustLL, pos_mustLL, mustLL_linear, pos_mustLL_linear] = findMustLLWithGAMS(model, minFluxesW, maxFluxesW, constrOpt, excludedRxns, mustSetFirstOrder, solverName, runID, outputFolder, outputFileName, printExcel, printText, printReport, keepInputs, keepGamsOutputs, verbose)
% This function runs the second step of `optForce`, that is to solve a
% bilevel mixed integer linear programming problem to find a second order
% `MustLL` set.
%
% USAGE:
%
%    [mustLL, pos_mustLL, mustLL_linear, pos_mustLL_linear] = findMustLLWithGAMS(model, minFluxesW, maxFluxesW, varargin)
%
% INPUTS:
%    model:                     (structure) a metabolic model with at least
%                               the following fields:
%
%                                 * .rxns - Reaction IDs in the model
%                                 * .mets - Metabolite IDs in the model
%                                 * .S -    Stoichiometric matrix (sparse)
%                                 * .b -    RHS of `Sv = b` (usually zeros)
%                                 * .c -    Objective coefficients
%                                 * .lb -   Lower bounds for fluxes
%                                 * .ub -   Upper bounds for fluxes
%    minFluxesW:                (double array of size `n_rxns x 1`) minimum
%                               fluxes for each reaction in the model for
%                               wild-type strain. This can be obtained by
%                               running the function `FVAOptForce`. E.g.:
%                               `minFluxesW = [-90; -56];`
%    maxFluxesW:                (double array of size `n_rxns x 1`) maximum
%                               fluxes for each reaction in the model for
%                               wild-type strain. This can be obtained by
%                               running the function `FVA_optForce`. E.g.:
%                               `maxFluxesW = [90; 56];`
%
% OPTIONAL INPUTS:
%    constrOpt:                 (Structure) structure containing
%                               additional contraints. Include here only
%                               reactions whose flux is fixed, i.e.,
%                               reactions whose lower and upper bounds have
%                               the same value. Do not include here
%                               reactions whose lower and upper bounds have
%                               different values. Such contraints should be
%                               defined in the lower and upper bounds of
%                               the model. The structure has the following
%                               fields:
%
%                                 * .rxnList - Reaction list (cell array)
%                                 * .values -  Values for constrained
%                                   reactions (double array)
%                                   E.g.: `struct('rxnList', {{'EX_gluc', 'R75', 'EX_suc'}}, 'values', [-100, 0, 155.5]');`
%    excludedRxns:              (cell array) Reactions to be excluded to
%                               the `MustLL` set. This could be used to avoid
%                               finding transporters or exchange reactions
%                               in the set. Default = empty.
%    mustSetFirstOrder:         (cell array) Reactions that belong to `MustU`
%                               and `MustL` (first order sets). Default = empty.
%    solverName:                (string) Name of the solver used in
%                               GAMS. Default = 'cplex'.
%    runID:                     (string) ID for identifying this run.
%                               Default = ['run' date hour].
%    outputFolder:              (string) name for folder in which
%                               results will be stored. Default =
%                               'OutputsFindMustLL'.
%    outputFileName:            (string) name for files in which
%                               results will be stored. Default =
%                               'MustLLSet'.
%    printExcel:                (double) boolean to describe wheter
%                               data must be printed in an excel file or
%                               not. Default = 1
%    printText:                 (double) boolean to describe wheter
%                               data must be printed in an plaint text file
%                               or not. Default = 1
%    printReport:               (double) 1 to generate a report in a
%                               plain text file. 0 otherwise. Default = 1
%    keepInputs:                (double) 1 to mantain folder with
%                               inputs to run `findMustLL.gms`. 0 otherwise.
%                               Default = 1
%    keepGamsOutputs:           (double) 1 to mantain files returned by
%                               `findMustLL.gms`. 0 otherwise. Default = 1
%    verbose:                   (double) 1 to print results in console.
%                               0 otherwise. Default = 0
%
% OUTPUTS:
%    mustLL:                    (cell array of size number of sets found X
%                               2). Cell array containing the reactions IDs
%                               which belong to the `MustLL` set. Each row
%                               contain a couple of reactions that must
%                               decrease their flux.
%    pos_mustLL:                (double array of size number of sets found
%                               X 2). Double array containing the positions
%                               of each reaction in `mustLL` with regard to
%                               `model.rxns`
%    mustLL_linear:             (cell array of size number of unique
%                               reactions found X 1) Cell array containing
%                               the unique reactions ID which belong to the
%                               `MustLL` Set
%    pos_mustLL_linear:         (double array of size number of unique
%                               reactions found X 1) double array
%                               containing positions for reactions in
%                               `mustLL_linear`. with regard to `model.rxns`
%    outputFileName.xls:        (file) File containing one column
%                               array with identifiers for reactions in
%                               `MustLL`. This file will only be generated if
%                               the user entered `printExcel = 1`. Note that
%                               the user can choose the name of this file
%                               entering the input `outputFileName =
%                               'PutYourOwnFileNameHere';`
%    outputFileName.txt:        (file) File containing one column
%                               array with identifiers for reactions in
%                               `MustLL`. This file will only be generated if
%                               the user entered `printText = 1`. Note that
%                               the user can choose the name of this file
%                               entering the input `outputFileName =
%                               'PutYourOwnFileNameHere';`
%    outputFileName_Info.xls:   (file) File containing one column
%                               array. In each row the user will find a
%                               couple of reactions. Each couple of
%                               reaction was found in one iteration of
%                               `FindMustLL.gms`. This file will only be
%                               generated if the user entered `printExcel = 1`.
%                               Note that the user can choose the name
%                               of this file entering the input
%                               `outputFileName = 'PutYourOwnFileNameHere';`
%    outputFileName_Info.txt:   (file) File containing one column
%                               array. In each row the user will find a
%                               couple of reactions. Each couple of
%                               reaction was found in one iteration of
%                               `FindMustLL.gms`. This file will only be
%                               generated if the user entered `printText = 1`.
%                               Note that the user can choose the name
%                               of this file entering the input
%                               `outputFileName = 'PutYourOwnFileNameHere';`
%    findMustLL.lst:            (file) file autogenerated by GAMS. It
%                               contains information about equations,
%                               variables, parameters as well as
%                               information about the running (values at
%                               each iteration). This file only will be
%                               saved in the output folder if the user
%                               entered `keepGamsOutputs = 1`
%    GtoMLL.gdx:                (file) file containing values for
%                               variables, parameters, etc. which were
%                               found by GAMS when solving `findMustLL.gms`.
%                               This file only will be saved in the output
%                               folder if the user entered `keepInputs = 1`
%
% NOTE:
%
%    This function is based in the GAMS files written by Sridhar
%    Ranganathan which were provided by the research group of Costas D.
%    Maranas. For a detailed description of the `optForce` procedure, please
%    see: `Ranganathan S, Suthers PF, Maranas CD (2010) OptForce: An
%    Optimization Procedure for Identifying All Genetic Manipulations
%    Leading to Targeted Overproductions. PLOS Computational Biology 6(4):
%    e1000744`. https://doi.org/10.1371/journal.pcbi.1000744
%
% .. Author: - Sebastian Mendoza, May 30th 2017, Center for Mathematical Modeling, University of Chile, snmendoz@uc.cl

optionalParameters = {'constrOpt', 'excludedRxns', 'mustSetFirstOrder', 'solverName', 'runID', 'outputFolder', 'outputFileName',  ...
    'printExcel', 'printText', 'printReport', 'keepInputs', 'keepGamsOutputs', 'verbose'};

if (numel(varargin) > 0 && (~ischar(varargin{1}) || ~any(ismember(varargin{1},optionalParameters))))

    tempargin = cell(1,2*(numel(varargin)));
    for i = 1:numel(varargin)

        tempargin{2*(i-1)+1} = optionalParameters{i};
        tempargin{2*(i-1)+2} = varargin{i};
    end
    varargin = tempargin;

end

parser = inputParser();
parser.addRequired('model', @(x) isstruct(x) && isfield(x, 'S') && isfield(model, 'rxns')...
    && isfield(model, 'mets') && isfield(model, 'lb') && isfield(model, 'ub') && isfield(model, 'b')...
    && isfield(model, 'c'))
parser.addRequired('minFluxesW', @isnumeric)
parser.addRequired('maxFluxesW', @isnumeric)
parser.addParamValue('constrOpt', struct('rxnList', {{}}, 'values', []),@ (x) isstruct(x) && isfield(x, 'rxnList') && isfield(x, 'values') ...
    && length(x.rxnList) == length(x.values) && length(intersect(x.rxnList, model.rxns)) == length(x.rxnList))
parser.addParamValue('excludedRxns', {}, @(x) iscell(x) && length(intersect(x, model.rxns)) == length(x))
parser.addParamValue('mustSetFirstOrder', {}, @(x) iscell(x) && length(intersect(x, model.rxns)) == length(x))
solvers = checkGAMSSolvers('MIP');
if isempty(solvers)
    error('there is no GAMS solvers available to solve Mixed Integer Programming problems') ;
else
    if ismember('cplex', lower(solvers))
        defaultSolverName = 'cplex';
    else
        defaultSolverName = lower(solvers(1));
    end
end

parser.addParamValue('solverName', defaultSolverName, @(x) ischar(x))
hour = clock; defaultRunID = ['run-' date '-' num2str(hour(4)) 'h' '-' num2str(hour(5)) 'm'];
parser.addParamValue('runID', defaultRunID, @(x) ischar(x))
parser.addParamValue('outputFolder', 'OutputsFindMustLL', @(x) ischar(x))
parser.addParamValue('outputFileName', 'MustLLSet', @(x) ischar(x))
parser.addParamValue('printExcel', 1, @(x) isnumeric(x) || islogical(x));
parser.addParamValue('printText', 1, @(x) isnumeric(x) || islogical(x));
parser.addParamValue('printReport', 1, @(x) isnumeric(x) || islogical(x));
parser.addParamValue('keepInputs', 1, @(x) isnumeric(x) || islogical(x));
parser.addParamValue('keepGamsOutputs', 1, @(x) isnumeric(x) || islogical(x));
parser.addParamValue('verbose', 1, @(x) isnumeric(x) || islogical(x));

parser.parse(model, minFluxesW, maxFluxesW, varargin{:})
model = parser.Results.model;
minFluxesW = parser.Results.minFluxesW;
maxFluxesW = parser.Results.maxFluxesW;
constrOpt= parser.Results.constrOpt;
excludedRxns= parser.Results.excludedRxns;
mustSetFirstOrder = parser.Results.mustSetFirstOrder;
solverName = parser.Results.solverName;
runID = parser.Results.runID;
outputFolder = parser.Results.outputFolder;
outputFileName = parser.Results.outputFileName;
printExcel = parser.Results.printExcel;
printText = parser.Results.printText;
printReport = parser.Results.printReport;
keepInputs = parser.Results.keepInputs;
keepGamsOutputs = parser.Results.keepGamsOutputs;
verbose = parser.Results.verbose;

% correct size of constrOpt
if ~isempty(constrOpt.rxnList)
    if size(constrOpt.rxnList, 1) > size(constrOpt.rxnList,2); constrOpt.rxnList = constrOpt.rxnList'; end;
    if size(constrOpt.values, 1) > size(constrOpt.values,2); constrOpt.values = constrOpt.values'; end;
end

% first, verify that GAMS is installed in your system
gamsPath = which('gams');
if isempty(gamsPath); error('OptForce: GAMS is not installed in your system. Please install GAMS.'); end;

%name of the function to solve the optimization problem in GAMS
gamsMustLLFunction = 'findMustLL.gms';
%path of that function
pathGamsFunction = which(gamsMustLLFunction);
if isempty(pathGamsFunction); error(['OptForce: ' gamsMustLLFunction ' not in MATLAB path.']); end;
%current path
workingPath = pwd;
%go to the path associate to the ID for this run.
if ~isdir(runID); mkdir(runID); end; cd(runID);

% if the user wants to generate a report.
if printReport
    %create name for file.
    hour = clock;
    reportFileName = ['report-' date '-' num2str(hour(4)) 'h' '-' num2str(hour(5)) 'm.txt'];
    freport = fopen(reportFileName, 'w');
    reportClosed = 0;
    % print date of running.
    fprintf(freport, ['findMustLLWithGAMS.m executed on ' date ' at ' num2str(hour(4)) ':' num2str(hour(5)) '\n\n']);
    % print matlab version.
    fprintf(freport, ['MATLAB: Release R' version('-release') '\n']);
    % print gams version.
    fprintf(freport, ['GAMS: ' regexprep(gamsPath, '\\', '\\\') '\n']);
    % print solver used in GAMS to solve optForce.
    fprintf(freport, ['GAMS solver: ' solverName '\n']);

    %print each of the inputs used in this running.
    fprintf(freport, '\nThe following inputs were used to run OptForce: \n');
    fprintf(freport, '\n------INPUTS------\n');
    %print model.
    fprintf(freport, '\nModel:\n');
    for i = 1:length(model.rxns)
        rxn = printRxnFormula(model, model.rxns{i}, false);
        fprintf(freport, [model.rxns{i} ': ' rxn{1} '\n']);
    end
    %print lower and upper bounds, minimum and maximum values for each of
    %the reactions in wild-type and mutant strain
    fprintf(freport, '\nLB\tUB\tMin_WT\tMax_WT\n');
    for i = 1:length(model.rxns)
        fprintf(freport, '%6.4f\t%6.4f\t%6.4f\t%6.4f\n', model.lb(i), model.ub(i), minFluxesW(i), maxFluxesW(i));
    end

    %print constraints
    fprintf(freport,'\nConstrained reactions:\n');
    for i = 1:length(constrOpt.rxnList)
        fprintf(freport,'%s: fixed in %6.4f\n', constrOpt.rxnList{i}, constrOpt.values(i));
    end

    fprintf(freport, '\nExcluded Reactions:\n');
    for i = 1:length(excludedRxns)
        rxn = printRxnFormula(model, excludedRxns{i}, false);
        fprintf(freport, [excludedRxns{i} ': ' rxn{1} '\n']);
    end

    fprintf(freport, '\nReactions from first order sets(MustU and MustL):\n');
    for i = 1:length(mustSetFirstOrder)
        rxn = printRxnFormula(model, mustSetFirstOrder{i}, false);
        fprintf(freport, [mustSetFirstOrder{i} ': ' rxn{1} '\n']);
    end

    fprintf(freport,'\nrunID(Main Folder): %s \n\noutputFolder: %s \n\noutputFileName: %s \n',...
        runID, outputFolder, outputFileName);

    fprintf(freport,'\nprintExcel: %1.0f \n\nprintText: %1.0f \n\nprintReport: %1.0f \n\nkeepInputs: %1.0f  \n\nkeepGamsOutputs: %1.0f \n\nverbose: %1.0f \n',...
        printExcel, printText, printReport, keepInputs, keepGamsOutputs, verbose);

end

copyfile(pathGamsFunction);

% export inputs for running the optimization problem in GAMS to find the
% MustLL Set
inputFolder = 'InputsMustLL';
exportInputsMustOrder2ToGAMS(model, 'LL', minFluxesW, maxFluxesW, constrOpt, excludedRxns, mustSetFirstOrder, inputFolder)

% create a directory to save results if this don't exist
if ~exist(outputFolder, 'dir')
    mkdir(outputFolder);
end

%run
if verbose
    run = system(['gams ' gamsMustLLFunction ' lo=3 --myroot=' inputFolder '/ --solverName=' solverName ' gdx=GtoMLL']);
else
    run=system(['gams ' gamsMustLLFunction ' --myroot=' inputFolder '/ --solverName=' solverName ' gdx=GtoMLL']);
end

if printReport; fprintf(freport, '\n------RESULTS------\n'); end;

%if user decide not to show inputs files for findMustLL.gms
if ~keepInputs; rmdir(inputFolder, 's'); end;

%if findMustLL.gms was executed correctly "run" should be 0
if run == 0

    if printReport; fprintf(freport, '\nGAMS was executed correctly\n'); end;
    if verbose; fprintf('GAMS was executed correctly\nSummary of information exported by GAMS:\n'); end;
    %show GAMS report in MATLAB console
    if verbose; gdxWhos GtoMLL; end;
    try
        findMustLL.name = 'findMustLL';
        rgdx('GtoMLL', findMustLL); %if do not exist the variable findMustLL in GtoMLL, an error will ocurr.
        if printReport; fprintf(freport, '\nGAMS variables were read by MATLAB correctly\n'); end;
        if verbose; fprintf('GAMS variables were read by MATLAB correctly\n'); end;

        %Using GDXMRW to read solutions found by findMustLL.gms
        %extract matrix 1 found by findMustII.gms. This matrix contains the
        %first reaction in each couple of reactions
        m1.name = 'matrix1';
        m1.compress = 'true';
        m1 = rgdx('GtoMLL', m1);
        uels_m1 = m1.uels{2};

        if ~isempty(uels_m1)
            %if the uel array for m1 is not empty, at least 1 couple of reations was found.
            if printReport; fprintf(freport, '\na MustLL set was found\n'); end;
            if verbose; fprintf('a MustLL set was found\n'); end;

            %find values for matrix 1
            val_m1 = m1.val;
            m1_full = full(sparse(val_m1(:,1), val_m1(:,2:end-1), val_m1(:,3)));

            %find values for matrix 2
            m2.name = 'matrix2';
            m2.compress = 'true';
            m2 = rgdx('GtoMLL', m2);
            uels_m2 = m2.uels{2};
            val_m2 = m2.val;
            m2_full = full(sparse(val_m2(:,1), val_m2(:,2:end-1), val_m2(:,3)));

            %initialize empty array for storing
            n_mustSet = size(m1_full,1);
            mustLL = cell(n_mustSet, 2);
            pos_mustLL = zeros(size(mustLL));
            mustLL_linear = {};

            %write each couple of reactions.
            for i = 1:n_mustSet
                rxn1 = uels_m1(m1_full(i,:) == 1);
                rxn2 = uels_m2(m2_full(i,:) == 1);
                mustLL(i,1) = rxn1;
                mustLL(i,2) = rxn2;
                pos_mustLL(i,1) = find(strcmp(model.rxns, rxn1));
                pos_mustLL(i,2) = find(strcmp(model.rxns, rxn2));
                mustLL_linear = union(mustLL_linear, [rxn1;rxn2]);
            end
            pos_mustLL_linear = cell2mat(arrayfun(@(x)find(strcmp(x, model.rxns)), mustLL_linear, 'UniformOutput', false))';
        else
            %if the uel array for m1 is empty, no couple of reations was found.
            if printReport; fprintf(freport, '\na MustLL set was not found\n'); end;
            if verbose; fprintf('a MustLL set was not found\n'); end;

            %initialize arrays to be returned by this function
            mustLL = {};
            pos_mustLL = [];
            mustLL_linear = {};
            pos_mustLL_linear = [];
        end

        % print info into an excel file if required by the user
        if printExcel
            if  ~isempty(uels_m1)
                currentFolder = pwd;
                cd(outputFolder);
                must = cell(size(mustLL,1), 1);
                for i = 1:size(mustLL, 1)
                    must{i} = strjoin(mustLL(i,:), ' or ');
                end
                xlswrite([outputFileName '_Info'],[{'Reactions'};must]);
                xlswrite(outputFileName, mustLL_linear);
                cd(currentFolder);
                if verbose
                    fprintf(['MustLL set was printed in ' outputFileName '.xls  \n']);
                    fprintf(['MustLL set was also printed in ' outputFileName '_Info.xls  \n']);
                end
                if printReport
                    fprintf(freport, ['\nMustLL set was printed in ' outputFileName '.xls  \n']);
                    fprintf(freport, ['\nMustLL set was printed in ' outputFileName '_Info.xls  \n']);
                end
            else
                if verbose; fprintf('No mustLL set was found. Therefore, no excel file was generated\n'); end;
                if printReport; fprintf(freport, '\nNo mustLL set was found. Therefore, no excel file was generated\n'); end;
            end
        end

        % print info into a plain text file if required by the user
        if printText
            if ~isempty(uels_m1)
                currentFolder = pwd;
                cd(outputFolder);
                f = fopen([outputFileName '_Info.txt'], 'w');
                fprintf(f,'Reactions\n');
                for i = 1:size(mustLL,1)
                    fprintf(f, '%s or %s\n', mustLL{i,1}, mustLL{i,2});
                end
                fclose(f);

                f = fopen([outputFileName '.txt'], 'w');
                for i = 1:length(mustLL_linear)
                    fprintf(f, '%s\n', mustLL_linear{i});
                end
                fclose(f);

                cd(currentFolder);
                if verbose
                    fprintf(['MustLL set was printed in ' outputFileName '.txt  \n']);
                    fprintf(['MustLL set was also printed in ' outputFileName '_Info.txt  \n']);
                end
                if printReport
                    fprintf(freport, ['\nMustLL set was printed in ' outputFileName '.txt  \n']);
                    fprintf(freport, ['\nMustLL set was printed in ' outputFileName '_Info.txt  \n']);
                end

            else
                if verbose; fprintf('No mustLL set was found. Therefore, no plain text file was generated\n'); end;
                if printReport; fprintf(freport, '\nNo mustLL set was found. Therefore, no plain text file was generated\n'); end;
            end
        end

        %close file for saving report
        if printReport; fclose(freport); reportClosed = 1; end;
        if printReport; movefile(reportFileName, outputFolder); end;
        delete(gamsMustLLFunction);

        %remove or move additional files that were generated during running
        if keepGamsOutputs
            if ~isdir(outputFolder); mkdir(outputFolder); end;
            movefile('GtoMLL.gdx', outputFolder);
            movefile(regexprep(gamsMustLLFunction, 'gms', 'lst'), outputFolder);
        else
            delete('GtoMLL.gdx');
            delete(regexprep(gamsMustLLFunction, 'gms', 'lst'));
        end

        %go back to the original path
        cd(workingPath);
    catch
        %GAMS variables were not read correctly by MATLAB
        if verbose; fprintf('GAMS variables were not read by MATLAB corretly\n'); end;
        if printReport && ~reportClosed; fprintf(freport, '\nGAMS variables were not read by MATLAB corretly\n'); fclose(freport); end;
        cd(workingPath);
        error('OptForce: GAMS variables were not read by MATLAB corretly');

    end

    %if findMustLL.gms was not executed correctly "run" should be different from 0
else
    %if GAMS was not executed correcttly
    if printReport && ~reportClosed; fprintf(freport, '\nGAMS was not executed correctly\n'); fclose(freport); end;
    if verbose; fprintf('GAMS was not executed correctly\n'); end;
    cd(workingPath);
    error('OptForce: GAMS was not executed correctly');

end

end