function [Phase_3, Phase_2, Phase_1, UPhase_3] = learn_struct_bnpc(data,node_sizes,epsilon,star)
% G = learn_struct_bnpc(Data,node_sizes,epsilon,star)
%
% Data(i,m) is node i in case m.
% node_sizes and epsilon are optionnals.
% star = 0 to use try_to_separate_B instead of try_to_separate_B_star
%
% see "Learning bayesian Networks from Data: A Efficient Approach Based on Information Theorie"
%     Jie Cheng, David Bell and Weird Liu.
%
% Things to do : rewrite function orient_edges !
%                ! sometimes it causes crashes !
%
% V0.91 : 18 sept 2003 (olivier.francois@insa-rouen.fr)

verbose=1;
%if nargin < 5, mwst=0; end
if nargin < 4, star=1; end
if nargin < 3, epsilon=0.05; end
if nargin < 2, node_sizes=max(data'); end

if verbose
  fprintf('================== phase I : \n');
end
tmp1=cputime;
[Phase_1 II JJ score_mat score_mat2] = phaseI(data, node_sizes, epsilon);
tmp1=cputime-tmp1;

if verbose
  fprintf('Execution time : %2.5f\n',tmp1);
  fprintf('\n================== phase II : \n');
end
tmp1=cputime;
Phase_2 = phaseII(Phase_1, data, node_sizes, epsilon, II, JJ, score_mat);
tmp1=cputime-tmp1;

if verbose
  fprintf('Execution time : %2.5f\n',tmp1);
  fprintf('\n================== phase III : \n');
end
tmp1=cputime;
[Phase_3 UPhase_3]  = phaseIII(Phase_2, data, node_sizes, epsilon, score_mat2, star);
tmp1=cputime-tmp1;
if verbose
  fprintf('Execution time : %2.5f\n',tmp1);
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
function [G, II, JJ, score_mat, sc2] = phaseI(data,node_sizes,alpha)
% [G, II , JJ, score_mat, s2] = phaseI(data,node_sizes,epsilon)
%
% G is an acyclic graph
% [II JJ] is the list of important edges not processed in phase I (for phase II)
% score_mat is the mutual information score matrix
%
% data(i,m) is node i in case m.
% alpha is the significant level for CI tests ( default=0.05 ).
% node_sizes is the vector of sizes ( default=max(data') ).
%
% see "Learning bayesian Networks from Data: A Efficient Approach Based on Information Theorie"
%     Jie Cheng, David Bell and Weird Liu.

% 0.
if nargin < 3, alpha=0.05; end
if nargin < 2, node_sizes=max(data'); end
[N m] = size(data);
score_mat = zeros(N);
edges=0;

% 1.
G = zeros(N);
L=[];

% 2. Use of Chi2 instead of MI ... allow using a confidence level alpha instead of an arbitrary epsilon
for i=1:(N-1)
  for j=(i+1):N
    [I score_mat(i,j)] = cond_indep_chisquare(i,j,[],data,'LRT',alpha,node_sizes);
  end
end
sc2=score_mat;

[tmp ordre]=sort(-score_mat(:));
ordre2=ordre(find(-tmp>alpha));
[II JJ]=ind2sub([N N],ordre2);

pointer=1 ;
fini=length(II);

% 3.
edges=2;
for pointer=1:min(2,fini),
  %fprintf('%d-%d\n',II(pointer),JJ(pointer));
  G(II(pointer),JJ(pointer))=1;
  G(JJ(pointer),II(pointer))=1;
  score_mat(II(pointer),JJ(pointer))=-inf;
end

pointer=min(2,fini);
arret=0;

while pointer<fini & ~arret
  % 4.
  pointer=pointer+1;
  C = ~reachability_graph(G);
  if C(II(pointer),JJ(pointer))
    %fprintf('%d-%d\n',II(pointer),JJ(pointer));
    G(II(pointer),JJ(pointer))=1;
    G(JJ(pointer),II(pointer))=1;
    score_mat(II(pointer),JJ(pointer))=-inf;
    edges=edges+1;
    if edges==N-1
      arret=1;
    end
  end

  % 5.
end

[tmp ordre]=sort(-score_mat(:));
ordre2=ordre(find(-tmp>alpha));
[II JJ]=ind2sub([N N],ordre2);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
function G = phaseII(G1,data,node_sizes,alpha,II, JJ, score_mat)
% G = phaseII(G1,data,node_sizes,epsilon,II,JJ, score_mat)
%
% G1, II, JJ,score_mat are given by phaseI.
% data(i,m) is node i in case m.
% node_sizes is the vector of sizes ( default=max(data') ).
% alpha is the significant level for CI tests ( default=0.05 ).
%
% see "Learning bayesian Networks from Data: A Efficient Approach Based on Information Theorie"
%     Jie Cheng, David Bell and Weird Liu.

st{1}='added';
st{2}='';
% 0.
[N m] = size(data);

G=G1;

% 6.
II=II(end:-1:1);
JJ=JJ(end:-1:1);

pointer=length(II);

while pointer>0
  % 7.
  trysep = try_to_separate_A(G,II(pointer),JJ(pointer),data,alpha,node_sizes);
  %fprintf('Try to separate %d and %d : %s\n',II(pointer),JJ(pointer),st{trysep+1});
  if ~trysep
    G(II(pointer),JJ(pointer))=1;
    G(JJ(pointer),II(pointer))=1;
    %fprintf('%d-%d\n',II(pointer),JJ(pointer));
  end
  % 8.
  pointer=pointer-1;
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
function [G,U] = phaseIII(G1,data,node_sizes,alpha,s2,star)
% [G] = phaseIII(G1,data,node_sizes,alpha,s2,star)
%
% data(i,m) is node i in case m.
% if star~=0, use try_to_separate_B_star instead of try_to_separate_B (default star=1).
% G is the non-oriented graph and G1 the oriented result.
%
% see "Learning bayesian Networks from Data: A Efficient Approach Based on Information Theorie"
%     Jie Cheng, David Bell and Weird Liu.

% 0.
if nargin < 2, disp('Not enough arguments');return; end
if nargin < 3, node_sizes=max(data'); end
if nargin < 4, alpha = 0.05; end
if nargin < 5, star=1; end
G=G1;
N=length(G);
% reachability_matrix of G
M = expm(full(G)) - eye(length(G)); M = (M>0);

% 9.
fprintf('Thinning - separateA\n');

% Edges are examined in the inverse order of their Chi2 (or MI) score
s2(find(~G))=0;
[tmp ordre]=sort(s2(:));
ordre2=ordre(find(tmp>0));
[I J]=ind2sub([N N],ordre2);
ii=1:length(I);
for i=ii,
  %fprintf('%d-%d : ',I(i),J(i));
  G(I(i),J(i))=0;
  G(J(i),I(i))=0;
  trysep = try_to_separate_A(G,I(i),J(i),data,alpha,node_sizes);

  if ~trysep,
    G(I(i),J(i))=1;
    G(J(i),I(i))=1;
    %fprintf(' keep\n');
    %else
    %fprintf('delete\n');
  end
end

% 10.
fprintf('Thinning - separateB'); if star; fprintf('star'); end; fprintf('\n');
s2(find(~G))=0;
[tmp ordre]=sort(s2(:));
ordre2=ordre(find(tmp>0));
[I J]=ind2sub([N N],ordre2);
ii=1:length(I);
for i=ii,
  %fprintf('%d-%d : ',I(i),J(i));
  G(I(i),J(i))=0; G(J(i),I(i))=0;
  if star==0
    trysep = try_to_separate_B(G,I(i),J(i),data,node_sizes,alpha);
  else
    trysep = try_to_separate_B_star(G,I(i),J(i),data,node_sizes,alpha);
  end
  if ~trysep,
    G(I(i),J(i))=1; G(J(i),I(i))=1;
    %fprintf(' keep\n');
    %else
    %fprintf('delete\n');
  end
end

%11.
fprintf('Thinning - orient_edges\n');
U=G;
G=orient_edges(U,data,node_sizes,alpha);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
function [I, N1, N2] = try_to_separate_A(G,node1,node2,data,alpha,node_sizes)
% [I, N1, N2] = try_to_separate_A(G,node1,node2,data,alpha,node_sizess)
%
% G is the current partially directed graph.
% I is a boolean : I=1 <==> separated.
% N1 : neighbors of node1 that are on an adjacency path between node1 and node2 (ditto for N2).
%
% see "Learning bayesian Networks from Data: A Efficient Approach Based on Information Theorie"
%     Jie Cheng, David Bell and Weird Liu.

% 0.
if node1==node2 | length(G)<2
  disp('Error: Check your arguments in try_to_separate_A.');I=0; return
end
N=size(data,1);
if nargin==4, alpha=0.05; node_sizes=max(data'); end
if nargin==5, node_sizes=max(data'); end

% 1.
N1=find(G(node1,:)==1);N01=N1;
GG1=G(setdiff(1:N,node1),setdiff(1:N,node1));
node22=node2-(node2>node1);
% reachability_matrix of GG1
M = expm(full(GG1)) - eye(length(GG1)); M = (M>0);
% N1 is the neighbors of node1 that are on the adjacency between node1 and node2
for i=N1
  j=i-(i>node1);
  if M(j,node22)~=1, N01=setdiff(N01,i); N1=setdiff(N1,i); end
  % 2.
  if ~G(node1,i), N1=setdiff(N1,i); end
end

N2=find(G(node2,:)==1);N02=N2;
GG2=G(setdiff(1:N,node2),setdiff(1:N,node2));
node12=node1-(node2<node1);
% reachability_matrix of GG2
M = expm(full(GG2)) - eye(length(GG2)); M = (M>0);
% N2 is the neighbors of node2 that are on the adjacency between node1 and node2
for i=N2
  j=i-(i>node2);
  if M(node12,j)~=1, N02=setdiff(N02,i); N2=setdiff(N2,i); end
  % 2.
  if ~G(node2,i), N2=setdiff(N2,i); end
end

%fprintf('%d : N1=',node1); fprintf('%d',N1); fprintf('\n');
%fprintf('%d : N2=',node2); fprintf('%d',N2); fprintf('\n');
% 3.
if length(N1)>length(N2), tmp=N1; N1=N2; N2=tmp; clear tmp, end
% 4.
C=N1;
for test=1:2
  if test==2, C=N2; end
  % 5.
  [I v1] = cond_indep_chisquare(node1,node2,C,data,'LRT',alpha,node_sizes);
  if I, %fprintf('%d-%d separated (%2.5f) by C=',node1,node2,v1); fprintf('%d',C); fprintf('\n');
    return,
    %else
    %fprintf('%d-%d not separated (%2.5f) by C=',node1,node2,v1); fprintf('%d',C); fprintf('\n');
  end;

  % 6.
  step6=1;
  while step6
    step6=0;
    if length(C)>=1
      v=[];
      for i=C
	Ci = setdiff(C,i);
	[I(i) v(i)] = cond_indep_chisquare(node1,node2,Ci,data,'LRT',alpha,node_sizes);
      end
      [vm ind] = min(v);

      % 7.
      if I(ind)
	I = 1; return
      else
	if vm < v1
	  v1 = vm;
	  C = setdiff(C,ind);
	  % goto step 6.
	  step6 = 1;
	end
      end
    end

    % 8.
    if test==2, I=0; return, end
  end
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
function I = try_to_separate_B(G,node1,node2,data,node_sizes,alpha,N1,N2)
% I = try_to_separate_B(G,node1,node2,data,node_sizes,epsilon,N1,N2)
%
% G is the current partially directed graph.
% data(i,m), node i in case m.
% node_sizes is the vector of size of the attributs in data ( default max(data') )
% N1 (optionnal) is the neighbors of node1 that are on an adjacency path between node1 and node2 (ditto for N2).
% I is a boolean : I+1 <==> separated.
%
% see "Learning bayesian Networks from Data: A Efficient Approach Based on Information Theorie"
%     Jie Cheng, David Bell and Weird Liu.

% 0.
if node1==node2 | length(G)<2
  disp('Error: Verify your arguments in try_to_separate_B.');I=0; return
end
if nargin < 4,
  disp('Error : not enougth arguments'); I=0; return;
end
N=size(data,1);

% 1.
if nargin < 8
  N1=find(G(node1,:)==1);
  GG1=G(setdiff(1:N,node1),setdiff(1:N,node1));
  node22=node2-(node2>node1);
  M = expm(full(GG1)) - eye(length(GG1)); M = (M>0);
  for i=N1
    j=i-(i>node1);
    if M(j,node22)~=1, N1=setdiff(N1,i); end
  end
  N2=find(G(node2,:)==1);
  GG2=G(setdiff(1:N,node2),setdiff(1:N,node2));
  node12=node1-(node2<node1);
  M = expm(full(GG2)) - eye(length(GG2)); M = (M>0);
  for i=N2
    j=i-(i>node2);
    if M(node12,j)~=1, N2=setdiff(N2,i); end
  end
end
if nargin < 6, alpha=0.05; end
if nargin < 5, node_sizes=max(data'); end
M = expm(full(G)) - eye(length(G)); M = (M>0);

% 2.
N1b=[];
for i=N1
  NN1=find(G(i,:)==1);
  for j=NN1
    if M(i,j)~=1 & ~ismember(j,N1), N1b=union(N1b,j); end
  end
end

% 3.
N2b=[];
for i=N2
  NN2=find(G(i,:)==1);
  for j=NN2
    if M(i,j)~=1 & ~ismember(j,N2), N2b=union(N2b,j); end
  end
end

% 4.
if length(union(N1,N1b)) < length(union(N2,N2b))
  C=union(N1,N1b);
else
  C=union(N2,N2b);
end

% 5.
continu=1;
while continu
  l=length(C);
  %fprintf('%d',continu);
  [I v] = cond_indep_chisquare(node1,node2,C,data,'LRT',alpha,node_sizes);
  if I==1; return, elseif l<2, I=0; return, end

  % 6.
  Cb=C;
  for i=1:l
    Ci=setdiff(C,C(i));
    [I vi] = cond_indep_chisquare(node1,node2,Ci,data,'LRT',alpha,node_sizes);
    e = (v+1)/3;    % e is a small value...
	if I==1,return, elseif vi<v+e, Cb=setdiff(Cb,C(i)); end
  end

  % 7.
  if length(Cb) < l, C=Cb; else continu==0; I=0; return; end
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
function I = try_to_separate_B_star(G,node1,node2,data,node_sizes,alpha,N1,N2)
% I = try_to_separate_B_star(G,node1,node2,data,epsilon,node_sizess)
%
% G is the current partially directed graph.
% I is a boolean.
% N1 is the neighbors of node1 that are on an adjacency path between node1 and node2 (ditto for N2).
%
% see "Learning bayesian Networks from Data: A Efficient Approach Based on Information Theorie"
%     Jie Cheng, David Bell and Weird Liu.

% 0.
if node1==node2 | length(G)<2
  disp('Error: Verify your arguments in try_to_separate_B_star.');I=0; return
end
if nargin < 4,
  disp('Error : not enougth arguments');I=0; return;
end
N=size(data,1);

% 1.
if nargin < 8
  N1=find(G(node1,:)==1);
  GG1=G(setdiff(1:N,node1),setdiff(1:N,node1));
  node22=node2-(node2>node1);
  M = expm(full(GG1)) - eye(length(GG1)); M = (M>0);
  for i=N1
    j=i-(i>node1);
    if M(j,node22)~=1, N1=setdiff(N1,i); end
    % 2.
    if ~G(node1,i), N1=setdiff(N1,i); end
  end
  N2=find(G(node2,:)==1);
  GG2=G(setdiff(1:N,node2),setdiff(1:N,node2));
  node12=node1-(node2<node1);
  M = expm(full(GG2)) - eye(length(GG2)); M = (M>0);
  for i=N2
    j=i-(i>node2);
    if M(node12,j)~=1, N2=setdiff(N2,i); end
    % 2.
    if ~G(node2,i), N2=setdiff(N2,i); end
  end
end
if nargin < 6, alpha=0.05; end
if nargin < 5, node_sizes=max(data'); end

% 3.
if length(N1)>length(N2), tmp=N1; N1=N2; N2=tmp; end

% 4.
C=N1;
l=length(C);
I=0;

% 5.
for test=1:2
  continu=1;
  %test
  if test==2 & ~isempty(N2)
    C=N2; l=length(C); IsInCi=zeros(1,l); IsInCi(l)=1;
  else IsInCi=zeros(1,l);
  end
  s=ones(1,l);
  % Pour tous les sous-ensemble Ci de C :
  while continu & ~isempty(C)
    Ci = setdiff(C.*IsInCi,0);
    [I vi] = cond_indep_chisquare(node1,node2,Ci,data,'LRT',alpha,node_sizes);
    if I, %fprintf('%d-%d separated (%2.5f) by C=',node1,node2,vi); fprintf('%d',Ci); fprintf('\n');
      return,
      %else
      %fprintf('%d-%d not separated (%2.5f) by C=',node1,node2,vi); fprintf('%d',Ci); fprintf('\n');
    end;
    %    if I, return, end

    if IsInCi==s, continu=0;
    else
      IsInCi(l)=IsInCi(l)+1;

      notOK=1; i=l;
      while notOK & i>1
	if IsInCi(i)>s(i), IsInCi(i)=0; IsInCi(i-1)=IsInCi(i-1)+1; else notOK=0; end
	i=i-1;
      end
    end
  end
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
function G1 = orient_edges(G,data,node_sizes,alpha)
% G1 = orient_edges(G,data,node_sizes)
%
% G is the partially directed graph.
% data(i,m), node i in case m.
% node_sizes is the vector of size of the attributs in data ( default max(data') )
%
% see "Learning bayesian Networks from Data: A Efficient Approach Based on Information Theorie"
%     Jie Cheng, David Bell and Weird Liu.

% 0.
if nargin < 4, alpha=0.05; end
if nargin < 3, node_sizes=max(data'); end
if nargin < 2, disp(' Require at least two arguments.'); return, end
N=length(G);
G1=G;

% 1.
[Lnode1 Lnode2]=find(triu(1-triu(G),1)); %[Lnode1 Lnode2]=ind2sub([N N],find(triu(1-triu(G),1)));
for ii = 1:length(Lnode1),
  node1=Lnode1(ii);
  node2=Lnode2(ii);
  %fprintf('%d %d\n',node1,node2);
  N1 = find(G(node1,:)==1);
  N2 = find(G(node2,:)==1);
  if ~isempty(intersect(N1,N2))
    %fprintf('V1='); fprintf('%d ',N1); fprintf('\n');
    %fprintf('V2='); fprintf('%d ',N2); fprintf('\n');
    GG1 = G(setdiff(1:N,node1),setdiff(1:N,node1));
    node22 = node2-(node2>node1);
    % reachability_matrix of GG1
    M = expm(full(GG1)) - eye(length(GG1)); M = (M>0);
    % N1 is the neighbors of node1 that are on the adjacency between node1 and node2
    for i = N1
      j = i-(i>node1);
      if M(j,node22)~=1, N1=setdiff(N1,i);
      end
    end
    GG2 = G(setdiff(1:N,node2),setdiff(1:N,node2));
    node12 = node1-(node2<node1);
    % reachability_matrix of GG2
    M = expm(full(GG2)) - eye(length(GG2)); M = (M>0);
    % N2 is the neighbors of node2 that are on the adjacency between node1 and node2
    for i=N2
      j = i-(i>node2);
      if M(node12,j)~=1, N2=setdiff(N2,i); end
    end
    %fprintf('%d %d\n',node1,node2);
    %fprintf('N1='); fprintf('%d ',N1); fprintf('\n');
    %fprintf('N2='); fprintf('%d ',N2); fprintf('\n');

    % 2.
    M = expm(full(G)) - eye(length(G)); M = (M>0);
    N1b=N1;
    for i=N1
      NN1 = find(G(i,:)==1);
      for j=NN1
	if M(i,j)~=1 & ~ismember(j,N1), N1b=union(N1b,j); end
      end
    end
    %fprintf('N1''='); fprintf('%d ',N1b); fprintf('\n');

    % 3.
    N2b=N2;
    for i=N2
      NN2 = find(G(i,:)==1);
      for j=NN2
	if M(i,j)~=1 & ~ismember(j,N2), N2b=union(N2b,j); end
      end
    end
    %fprintf('N2''='); fprintf('%d ',N2b); fprintf('\n');

    % 4.
    if length(N1b) < length(N2b)
      C = N1b;
    else
      C = N2b;
    end
    %l=length(C);
    %fprintf('C='); fprintf('%d ',C); fprintf('\n');

    % 7.
    step5=1;
    while step5
      step5=0;
      %fprintf('.');
      % 5.
      l=length(C);
      [I v] = cond_indep_chisquare(node1,node2,C,data,'LRT',alpha,node_sizes);
      %fprintf('C='); fprintf('%d ',C);
      %fprintf(': %d %2.5f\n',I,v);

      step8=0;
      if I==1 & v~=0 % v < epsilon
	step8=1;
      else
	if  l==1
	  G1(C,node1)=0; G1(C,node2)=0;
	  fprintf('%d -> %d <- %d\n',node1,C,node2);
	  step8=1;
	end
      end
      %fprintf('%d\n',step8);

      % 6.
      if ~step8
	Cb=C;
	for i=1:l
	  Ci=setdiff(C,C(i));
	  [I vi] = cond_indep_chisquare(node1,node2,Ci,data,'LRT',alpha,node_sizes);
	  % e = (v+1)/3;    % e is a small value...
	  if I==1 % vi < v+e
	    Cb=setdiff(Cb,C(i));
	    if ismember(C(i),N1) & ismember(C(i),N2)
	      G1(C(i),node1)=0; G1(C(i),node2)=0;
	      fprintf('%d -> %d <- %d\n',node1,C(i),node2);
	    end
	    if I==1 % vi < epsilon
	      step8=1;
	    end
	  end
	end % for
      end % if

      % 7.
      if ~step8
	if length(Cb) < length(C), C=Cb; end
	if length(C) > 0, step5==1; end
      end
    end % while step5
    % step8 : passer � la paire de noeud suivant
    %fprintf('\n');
    %else
    %fprintf(' No common neighbor\n');
  end % if
end % for

% 11.
step9=0;
fprintf('Infering directions ');
test = pdag_to_dag(G1);
while ~isdag(test) %& step9<N
  step9=step9+1;
  %if ~isempty(test)
    %fprintf('.');
    % 9.
    for a=1:N, for b=1:N, for c=1:N,
	  if a~=b & b~=c & c~=a
	    if G1(a,b)==1 & G1(b,a)==0
	      %fprintf('%d -> %d ... \n',a,b);
	      if G1(b,c)==1 & G1(c,b)==1
		if G1(a,c)+G1(c,a)==0,
		  G1(c,b)=0;
		  fprintf('%d -> %d (9)\n',b,c);
		end
	      end
	    end
	  end
    end, end, end

    % 10.
    for a=1:N-1, for b=a+1:N
	if G1(a,b)==1 & G1(b,a)==1
	  GGG1=xor(G1,G1'); % matrice des arcs orient�s de G1
	  M = expm(full(GGG1)) - eye(length(GGG1)); M = (M>0);
	  if M(a,b)==1,
	    G1(b,a)=0;
	    fprintf('%d -> %d (10)\n',a,b);
	  end
	end
    end, end

      test = pdag_to_dag(G1);
      if isempty(test), G1=return_one_edge(G1); end
  %else
  %  G1, return,
  %end
end % while 11.
G1 = pdag_to_dag(G1);
fprintf('%d boucles\n',step9);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
function b = isdag(G)
b = sum(sum(G.*G'));        % How many undirected arcs ? (x2)
b=~b & ~isempty(G);
if b
  M = expm(full(G)) - eye(length(G)); M = (M>0);
  b = b & find(sum(sum(eye(length(G)).*M))); % is there no cycle ?
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
function G=return_one_edge(G)
N=length(G); fam=[]; node2=[];
permnode = randperm(5); i=0; %fprintf('rev\n');
while isempty(fam) | isempty(node2) | i==N
 i=i+1;
 node = permnode(i); %node = ceil(rand(1)*N);
 fam=find(G(node,:)==1);
 par=find(G(:,node)==1);
 fam = myunion(fam, par);
 if ~isempty(fam)
  node2 = fam(ceil(rand(1)*length(fam)));
  par2=find(G(:,node)==1);
  if ~isempty(intersect(par2, node)), node2=[]; end
 end
end
if isempty(myintersect(node, par2)),
 G(node, node2)=0;
 G(node2, node)=1;
 fprintf('%d -> %d (Rev)\n',node, node2);
else
 G(node, node2)=1;
 G(node2, node)=0;
 fprintf('%d -> %d (Rev)\n',node2, node);
end


%-------------------------------------------------------------------------------------------------------------------%
%
% IB2d is an Immersed Boundary Code (IB) for solving fully coupled non-linear
% 	fluid-structure interaction models. This version of the code is based off of
%	Peskin's Immersed Boundary Method Paper in Acta Numerica, 2002.
%
% Author: Nicholas A. Battista
% Email:  nick.battista@unc.edu
% Date Created: May 27th, 2015
% Institution: UNC-CH
%
% This code is capable of creating Lagrangian Structures using:
% 	1. Springs
% 	2. Beams (*torsional springs)
% 	3. Target Points
%	4. Muscle-Model (combined Force-Length-Velocity model, "HIll+(Length-Tension)")
%
% One is able to update those Lagrangian Structure parameters, e.g., spring constants, resting %%	lengths, etc
%
% There are a number of built in Examples, mostly used for teaching purposes.
%
% If you would like us to add a specific muscle model, please let Nick (nick.battista@unc.edu) know.
%
%--------------------------------------------------------------------------------------------------------------------%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% FUNCTION: Plots the Lagrangian structure with:
%           (1): the background velocity field
%           (2): the Lagrangian structure itself
%           (3): the vorticity field in a colormap
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

function [loc, diffy] =  please_Plot_Results(ds,X,Y,U,V,vort,uMag,p,C,chiX,chiY,lagPlot,velPlot,vortPlot,pressPlot,uMagPlot,conPlot,firstPrint,loc,diffy,spacing)

%X,Y:  (x,y) values
%U,V:  x-directed, y-directed velocities respectively
%vort: vorticity
%uMag: magnitude of velocity
%p:    pressure

%FLAGS FOR PLOTTING:
%pressPlot - if you want pressure plot = 1
%uMagPlot  - if you want mag. velocity plot = 1
%vortPlot  - if you want vorticity plot = 1
%velPlot   - if you want velocity plot = 1
%lagPlot   - if you want lag. point ONLY plot = 1
%conPlot   - if you want concentration plot =1
%
% Assumption: Assuming chiX and chiY are column vectors
% Assumption: Assuming chiX(i+1)-chiX(i) < .5 and chiY(i+1)-chiY(i) < .5, for all points that don't cross the boundary
%
% This script has been adapted from the Mat-IB code by Hiens and Stockie, 2014.
%

Lx = X(1,end)+X(1,2);
Ly = Y(end,1)+Y(2,1);

%Shift so inside of interval [0,Lx] or [0,Ly]
chiX = mod(chiX,Lx);
chiY = mod(chiY,Ly);

%Connect Geometry Appropriately At Beginning Of Simulation then Store it!
if firstPrint == 1
    diffX = abs([chiX(2:end);chiX(1)]-chiX);
    diffY = abs([chiY(2:end);chiY(1)]-chiY);

    locX = find(diffX > spacing ); % assuming x value can't change more than Lx/2 (without crossing boundary)
    locY = find(diffY > spacing ); % assuming y value can't change more than Ly/2 (without crossing boundary)
    loc =  sort(unique([locX;locY]));

    diffy = sqrt( (chiX(end)-chiX(1) )^2 + ( chiY(end)-chiY(1) )^2 );
end

clf; %Clear previous plots :)

[N1,N2,num_con]=size(C);
figure(1)
numPlots = lagPlot+velPlot+vortPlot+pressPlot+uMagPlot+conPlot+(num_con-1);

ct = 1;

% % % % % PLOTS LAGRANGIAN POINTS ONLY (if selected) % % % % %

if lagPlot == 1
    subplot(1,numPlots,ct)
    axis([0 Lx 0 Ly]);
    title('LAGRANGIAN PTS');
    xlabel('x'); ylabel('y');
    hold all;

    loc = [0;loc;length(chiX)];
    for i=2:length(loc)
       plot(chiX(loc(i-1)+1:loc(i)),chiY(loc(i-1)+1:loc(i)),'m','LineWidth',3);
    end
    if diffy < 5*ds
        xTemp = [chiX(1) chiX(end)];
        yTemp = [chiY(1) chiY(end)];
        plot(xTemp(1:2),yTemp(1:2),'m','LineWidth',3);
    end

    axis square;

    ct=ct+1;
end

% % % % % PLOTS VORTICITY + LAG Pts. (if selected) % % % % %

if vortPlot == 1

    subplot(1,numPlots,ct)
    %
    axis([0 Lx 0 Ly]);
    title('VORTICITY');
    xlabel('x'); ylabel('y');
    hold all;

    %Compute Vorticity and Plot It against Lagrangian Grid!
    x = X(1,:); y = Y(:,1);
    contourf(x,y,flipud(rot90(vort)),10); hold on;

    loc = [0;loc;length(chiX)];
    for i=2:length(loc)
        plot(chiX(loc(i-1)+1:loc(i)),chiY(loc(i-1)+1:loc(i)),'m','LineWidth',3);
    end
    if diffy < 5*ds
        xTemp = [chiX(1) chiX(end)];
        yTemp = [chiY(1) chiY(end)];
        plot(xTemp(1:2),yTemp(1:2),'m','LineWidth',3);
    end

    axis square;

    ct=ct+1;
end

% % % % % PLOTS PRESSURE + LAG Pts. (if selected) % % % % %

if pressPlot == 1

    subplot(1,numPlots,ct)
    %
    axis([0 Lx 0 Ly]);
    title('PRESSURE');
    xlabel('x'); ylabel('y');
    hold all;

    %Use Pressure and Plot It against Lagrangian Grid!
    x = X(1,:); y = Y(:,1);
    contourf(x,y,p,6); hold on;

    loc = [0;loc;length(chiX)];
    for i=2:length(loc)
        plot(chiX(loc(i-1)+1:loc(i)),chiY(loc(i-1)+1:loc(i)),'m','LineWidth',3);
    end
    if diffy < 5*ds
        xTemp = [chiX(1) chiX(end)];
        yTemp = [chiY(1) chiY(end)];
        plot(xTemp(1:2),yTemp(1:2),'m','LineWidth',3);
    end

    axis square;

    ct=ct+1;
end

% % % % % PLOTS MAGNITUDE OF VELOCITY + LAG Pts. (if selected) % % % % %

if uMagPlot == 1

    subplot(1,numPlots,ct)
    %
    axis([0 Lx 0 Ly]);
    title('MAGNITUDE OF VELOCITY');
    xlabel('x'); ylabel('y');
    hold all;

    %Use Mag. Velocity and Plot It against Lagrangian Grid!
    x = X(1,:); y = Y(:,1);
    contourf(x,y,uMag,6); hold on;

    loc = [0;loc;length(chiX)];
    for i=2:length(loc)
        plot(chiX(loc(i-1)+1:loc(i)),chiY(loc(i-1)+1:loc(i)),'m','LineWidth',3);
    end
    if diffy < 5*ds
        xTemp = [chiX(1) chiX(end)];
        yTemp = [chiY(1) chiY(end)];
        plot(xTemp(1:2),yTemp(1:2),'m','LineWidth',3);
    end

    axis square;

    ct=ct+1;
end

% % % % % PLOTS VELOCITY FIELD + LAG Pts. (if selected) % % % % %

if velPlot == 1

    subplot(1,numPlots,ct)
    %
    axis([0 Lx 0 Ly]);
    title('VELOCITY');
    xlabel('x'); ylabel('y');
    hold all;

    quiver(X,Y,U,V); %Print Velocity Field

    loc = [0;loc;length(chiX)];
    for i=2:length(loc)
       plot(chiX(loc(i-1)+1:loc(i)),chiY(loc(i-1)+1:loc(i)),'m','LineWidth',3);
    end
    if diffy < 5*ds
        xTemp = [chiX(1) chiX(end)];
        yTemp = [chiY(1) chiY(end)];
        plot(xTemp(1:2),yTemp(1:2),'m','LineWidth',3);
    end

    axis square;

    ct=ct+1;
end

if conPlot == 1
    % plotting all concentrations
    for ic=1:num_con
        subplot(1,numPlots,ct)
    %
        axis([0 Lx 0 Ly]);
        title(sprintf('Concentration%g',ic));
        xlabel('x'); ylabel('y');
        hold all;

        x = X(1,:); y = Y(:,1);
        pcolor(X,Y,C(:,:,ic));
        shading interp;

        loc = [0;loc;length(chiX)];
        for i=2:length(loc)
            plot(chiX(loc(i-1)+1:loc(i)),chiY(loc(i-1)+1:loc(i)),'m','LineWidth',3);
        end
        if diffy < 5*ds
            xTemp = [chiX(1) chiX(end)];
            yTemp = [chiY(1) chiY(end)];
            plot(xTemp(1:2),yTemp(1:2),'m','LineWidth',3);
        end

        axis square;

        ct=ct+1;
    end
end

drawnow;

hold off;
set(gca,'Box','on');


function [new_nbrs, new_ops, new_nodes, new_topos] = mk_nbrs_of_dag_topo(G0)
% MK_NBRS_OF_DAG_TOPO Make all DAGs that differ from G0 by a single edge deletion, addition or reversal
% [new_nbrs, new_ops, new_nodes, new_topos] = mk_nbrs_of_dag_topo(G0)
%
% new_nbrs{i} is the i'th neighbor of G0.
% new_ops{i} = 'add', 'del', or 'rev' is the operation used to create the i'th neighbor.
% new_nodes(i,1:2) are the head and tail of the operated-on arc.
% new_topos are topological orders of the neighbours.
%
% We implement the fast acyclicity check described by P. Giudici and R. Castelo,
% "Improving MCMC model search for data mining", submitted to J. Machine Learning, 2001.
%
% Written by Qian Diao <qian.diao@intel.com> on 19 Nov 01
% Reference are ..\BNT\graph\mk_nbrs_of_dag.m, ..\BNT\learning\learn_struct_mcmc.m and
% ..\BNT\graph\topological_sort.m
% Copyright Intel 2001
%
new_nbrs = {};
new_ops = {};
new_nodes = [];
new_topos = {};
cs = {};

n = length(G0);
indeg = zeros(1,n);
zero_indeg = []; % a stack of nodes with no parents
for i=1:n
  indeg(i) = length(parents(G0,i));
  cs{i} = children(G0, i);
  if indeg(i)==0
    zero_indeg = [i zero_indeg];
  end
end

dag = G0;
[nbrs, ops, nodes] = mk_nbrs_of_digraph(dag);
A = init_ancestor_matrix(dag);
%assert(acyclic(new_dag));

d1 = 1;
for d = 1:length(ops)
  i = nodes(d, 1); j = nodes(d, 2);
  legal = 0;
  switch ops{d}
   case 'add',
    if A(i,j)==0
      legal = 1;
    end
   case 'del',
    legal = 1;
   case 'rev',
    ps = mysetdiff(parents(dag, j), i);
    % if any(A(ps,i)) then there is a path i -> parent of j -> j
    % so reversing i->j would create a cycle
    legal = ~any(A(ps, i));
  end

  if legal
    tmp = nbrs(:,:,d);
    new_nbrs{d1} = tmp;
    new_ops{d1} =  ops{d};
    new_nodes(d1,1:2) = nodes(d,1:2);

    % obtain the topological orders of neighbour dags
    zero_indeg_nbr = [];
    indeg_nbr = [];
    cs_nbr = [];

    switch ops{d}
     case 'add' % i is a new parent of j
      zero_indeg_nbr = zero_indeg;
      indeg_nbr = indeg;
      if ~isempty(find(zero_indeg == j)) % j is not a root anymore
        zero_indeg_nbr = mysetdiff(zero_indeg, j);
      end
      indeg_nbr(j) = indeg(j)+1;

      t_nbr=1;
      order_nbr = zeros(1,n);
      while ~isempty(zero_indeg_nbr)
        v_nbr = zero_indeg_nbr(1); % pop v
        zero_indeg_nbr = zero_indeg_nbr(2:end);
        order_nbr(t_nbr) = v_nbr;
        t_nbr = t_nbr + 1;
        if v_nbr == i % j is a new child of i
          cs_nbr = sort([j cs{i}]);
        else
          cs_nbr = cs{v_nbr};
        end
        for k = 1:length(cs_nbr)
          c_nbr = cs_nbr(k);
          indeg_nbr(c_nbr) = indeg_nbr(c_nbr) - 1;
          if indeg_nbr(c_nbr) == 0
            zero_indeg_nbr = [c_nbr zero_indeg_nbr]; % push c
          end
        end
      end

     case 'del' % i is not a parent of j anymore
      zero_indeg_nbr = zero_indeg;
      indeg_nbr = indeg;
      if length(parents(tmp, j))==0
        zero_indeg_nbr = -sort(-[zero_indeg, j]); % descending order
      end
      indeg_nbr(j) = indeg(j) - 1;

      t_nbr=1;
      order_nbr = zeros(1,n);
      while ~isempty(zero_indeg_nbr)
        v_nbr = zero_indeg_nbr(1); % pop v
        zero_indeg_nbr = zero_indeg_nbr(2:end);
        order_nbr(t_nbr) = v_nbr;
        t_nbr = t_nbr + 1;
        if v_nbr == i % j is not a child of i anymore
          cs_nbr = mysetdiff(cs{i}, j);
        else
          cs_nbr = cs{v_nbr};
        end
        for k = 1:length(cs_nbr)
          c_nbr = cs_nbr(k);
          indeg_nbr(c_nbr) = indeg_nbr(c_nbr) - 1;
          if indeg_nbr(c_nbr) == 0
            zero_indeg_nbr = [c_nbr zero_indeg_nbr]; % push c
          end
        end
      end

     case 'rev' %i is a new child of j and j is a new parent of i
      zero_indeg_nbr = zero_indeg;
      indeg_nbr = indeg;
      if ~isempty(find(zero_indeg == i))
        zero_indeg_nbr = mysetdiff(zero_indeg_nbr, i);
      end
      if length(parents(tmp, j))==0
        zero_indeg_nbr = -sort(-[zero_indeg_nbr, j]); % decending order
      end
      indeg_nbr(i) = indeg(i)+1;
      indeg_nbr(j) = indeg(j)-1;

      t_nbr=1;
      order_nbr = zeros(1,n);
      while ~isempty(zero_indeg_nbr)
        v_nbr = zero_indeg_nbr(1); % pop v
        zero_indeg_nbr = zero_indeg_nbr(2:end);
        order_nbr(t_nbr) = v_nbr;
        t_nbr = t_nbr + 1;
        cs_nbr = cs{v_nbr};
        if v_nbr == i % j is not a child of i anymore
          cs_nbr = mysetdiff(cs{i}, j);
        end
        if v_nbr == j % i is a new child of j
          cs_nbr = sort([i cs{j}]);
        end
        for k = 1:length(cs_nbr)
          c_nbr = cs_nbr(k);
          indeg_nbr(c_nbr) = indeg_nbr(c_nbr) - 1;
          if indeg_nbr(c_nbr) == 0
            zero_indeg_nbr = [c_nbr zero_indeg_nbr]; % push c
          end
        end
      end
     end

    new_topos{d1} = order_nbr;
    d1 = d1+1;
  end
end

clear nbrs ops nodes;

%%%%%%%%%
function A = update_row(A, j, dag)
% We compute row j of A
A(j, :) = 0;
ps = parents(dag, j);
if ~isempty(ps)
  A(j, ps) = 1;
end
for k=ps(:)'
  anck = find(A(k,:));
  if ~isempty(anck)
    A(j, anck) = 1;
  end
end

%%%%%%%%
function A = init_ancestor_matrix(dag)
order = topological_sort(dag);
A = zeros(length(dag));
for j=order(:)'
  A = update_row(A, j, dag);
end


% testRadiomicsFeaturesWithPyRadiomics.m
%
% Script to compare CERR and pyRadiomics features
%
% APA, 2/22/2018

global planC
indexS = planC{end};

% Wavelet decomposition flag
wavDecompFlg = 1; % 0: original image, 1: wavelet pre-processing

% Specify discretization
fixedMinMaxGrLevFlag = 0; % 1: to input fixed min/max/grLevels, 0: min/max
                          % calculated for the structure and bin width of binWidth.
binWidth = 25;
structNum = 1;
scanNum = 1;
pyradScanType = 'original';

paramS.firstOrderParamS.offsetForEnergy = planC{indexS.scan}(scanNum).scanInfo(1).CTOffset;
paramS.firstOrderParamS.binWidth = binWidth;

% Get structure
[rasterSegments, planC, isError] = getRasterSegments(structNum,planC);
[mask3M, uniqueSlices] = rasterToMask(rasterSegments, scanNum, planC);
scanArray3M = getScanArray(planC{indexS.scan}(scanNum));
scanArray3M = double(scanArray3M) - ...
    planC{indexS.scan}(scanNum).scanInfo(1).CTOffset;
% Wavelet decomposition
if wavDecompFlg == 1
    dirString = 'HHL';
    wavType = 'coif1';
    scanArray3M = flip(scanArray3M,3);
    if mod(size(scanArray3M,3),2) > 0
        scanArray3M(:,:,end+1) = 0*scanArray3M(:,:,1);
    end
    scanArray3M = wavDecom3D(double(scanArray3M),dirString,wavType);
    if mod(size(scanArray3M,3),2) > 0
        scanArray3M = scanArray3M(:,:,1:end-1);
    end
    scanArray3M = flip(scanArray3M,3);
end

SUVvals3M = mask3M.*double(scanArray3M(:,:,uniqueSlices));
[minr, maxr, minc, maxc, mins, maxs] = compute_boundingbox(mask3M);
maskBoundingBox3M = mask3M(minr:maxr,minc:maxc,mins:maxs);

% Assign NaN to image outside mask
volToEval = SUVvals3M(minr:maxr,minc:maxc,mins:maxs);
volToEval(~maskBoundingBox3M) = NaN;

% Get x,y,z grid for the shape features (flip y to make it monotically
% increasing)
[xValsV, yValsV, zValsV] = getUniformScanXYZVals(planC{indexS.scan}(scanNum));
yValsV = fliplr(yValsV);
xValsV = xValsV(minc:maxc);
yValsV = yValsV(minr:maxr);
zValsV = zValsV(mins:maxs);
PixelSpacingX = abs(xValsV(2)-xValsV(1));
PixelSpacingY = abs(yValsV(2)-yValsV(1));
PixelSpacingZ = abs(zValsV(2)-zValsV(1));
VoxelVol = PixelSpacingX*PixelSpacingY*PixelSpacingZ*1000; % convert cm to mm

if fixedMinMaxGrLevFlag
    numGrLevels = paramS.higherOrderParamS.numGrLevels;
    minIntensity = paramS.higherOrderParamS.minIntensity;
    maxIntensity = paramS.higherOrderParamS.maxIntensity;
    % Quantize using the number of bins
    quantizedM = imquantize_cerr(volToEval,numGrLevels,...
        minIntensity,maxIntensity);

else % Quantize using the binwidth
    minIntensity = [];
    maxIntensity = [];
    numGrLevels = [];
    quantizedM = imquantize_cerr(volToEval,numGrLevels,...
        minIntensity,maxIntensity,binwidth);
    numGrLevels = max(quantizedM(:));
    paramS.higherOrderParamS.numGrLevels = numGrLevels;
end

paramS.higherOrderParamS.numGrLevels = numGrLevels;
paramS.higherOrderParamS.patchRadius3dV = [1 1 1];
paramS.higherOrderParamS.imgDiffThresh = 0;

% Number of voxels (used in run percentage calculation)
numVoxels = sum(~isnan(quantizedM(:)));

%% First order features
featureS.firstOrderS = radiomics_first_order_stats...
    (volToEval(logical(maskBoundingBox3M)), VoxelVol, ...
    paramS.firstOrderParamS.offsetForEnergy, paramS.firstOrderParamS.binWidth);
firstOrderS = featureS.firstOrderS;
cerrFirstOrderV = [firstOrderS.energy, firstOrderS.totalEnergy, firstOrderS.interQuartileRange, ...
    firstOrderS.kurtosis+3, firstOrderS.max, firstOrderS.mean, firstOrderS.meanAbsDev, ...
    firstOrderS.median, firstOrderS.medianAbsDev, firstOrderS.min, ...
    firstOrderS.P10, firstOrderS.P90, firstOrderS.interQuartileRange, ...
    firstOrderS.robustMeanAbsDev, firstOrderS.rms, firstOrderS.skewness, ...
    firstOrderS.std, firstOrderS.var, firstOrderS.entropy];
pyradFirstorderNamC = {'Energy', 'TotalEnergy','InterquartileRange','Kurtosis',...
    'Maximum', 'Mean','MeanAbsoluteDeviation','Median','medianAbsDev',...
    'Minimum','10Percentile','90Percentile','InterquartileRange',...
    'RobustMeanAbsoluteDeviation','RootMeanSquared','Skewness',...
    'StandardDeviation','Variance','Entropy'};
pyradFirstorderNamC = strcat([pyradScanType,'_firstorder_'],pyradFirstorderNamC);
pyRadFirstOrderV = [];
for i = 1:length(pyradFirstorderNamC)
    if isfield(teststruct,pyradFirstorderNamC{i})
        pyRadFirstOrderV(i) = teststruct.(pyradFirstorderNamC{i});
    else
        pyRadFirstOrderV(i) = NaN;
    end
end

diffFirstOrderV = (cerrFirstOrderV - pyRadFirstOrderV) ./ cerrFirstOrderV * 100

%% Shape features
featureS.shapeS = getShapeParams(maskBoundingBox3M, ...
        {xValsV, yValsV, zValsV});
shapeS = featureS.shapeS;
cerrShapeV = [shapeS.majorAxis, shapeS.minorAxis, shapeS.leastAxis, ...
    shapeS.flatness, shapeS.elongation, shapeS.max3dDiameter, shapeS.max2dDiameterAxialPlane,...
    shapeS.max2dDiameterSagittalPlane', shapeS.max2dDiameterCoronalPlane, ...
    shapeS.Compactness1, shapeS.Compactness2, shapeS.spherDisprop, ...
    shapeS.sphericity, shapeS.surfToVolRatio/10,...
    shapeS.surfArea*100, shapeS.volume*1000];
pyradShapeNamC = {'MajorAxis', 'MinorAxis', 'LeastAxis', 'Flatness',  'Elongation', ...
    'Maximum3DDiameter', 'Maximum2DDiameterSlice', 'Maximum2DDiameterRow', ...
    'Maximum2DDiameterColumn', 'Compactness1','Compactness2','spherDisprop','Sphericity', ...
    'SurfaceVolumeRatio','SurfaceArea','Volume'};
pyradShapeNamC = strcat([pyradScanType,'_shape_'],pyradShapeNamC);
pyRadShapeV = [];
for i = 1:length(pyradShapeNamC)
    if isfield(teststruct,pyradShapeNamC{i})
        pyRadShapeV(i) = teststruct.(pyradShapeNamC{i});
    else
        pyRadShapeV(i) = NaN;
    end
end
shapeDiffV = (cerrShapeV - pyRadShapeV) ./ cerrShapeV * 100

%% GLCM features

% Define flags
glcmFlagS.energy = 1;
glcmFlagS.jointEntropy = 1;
glcmFlagS.jointMax = 1;
glcmFlagS.jointAvg = 1;
glcmFlagS.jointVar = 1;
glcmFlagS.contrast = 1;
glcmFlagS.invDiffMoment = 1;
glcmFlagS.sumAvg = 1;
glcmFlagS.corr = 1;
glcmFlagS.clustShade = 1;
glcmFlagS.clustProm = 1;
glcmFlagS.haralickCorr = 1;
glcmFlagS.invDiffMomNorm = 1;
glcmFlagS.invDiff = 1;
glcmFlagS.invDiffNorm = 1;
glcmFlagS.invVar = 1;
glcmFlagS.dissimilarity = 1;
glcmFlagS.diffEntropy = 1;
glcmFlagS.diffVar = 1;
glcmFlagS.diffAvg = 1;
glcmFlagS.sumVar = 1;
glcmFlagS.sumEntropy = 1;
glcmFlagS.clustTendency = 1;
glcmFlagS.autoCorr = 1;
glcmFlagS.invDiffMomNorm = 1;
glcmFlagS.firstInfCorr = 1;
glcmFlagS.secondInfCorr = 1;

dirctn      = 1;
cooccurType = 2;
featureS.harFeat3DdirS = get_haralick(dirctn, cooccurType, quantizedM, ...
numGrLevels, glcmFlagS);

% harlCombS = featureS.harFeat3DcombS.CombS;
harlCombS = featureS.harFeat3DdirS.AvgS;
cerrGlcmV = [harlCombS.autoCorr, harlCombS.jointAvg, harlCombS.clustPromin, harlCombS.clustShade, harlCombS.clustTendency, ...
harlCombS.contrast, harlCombS.corr, harlCombS.diffAvg, harlCombS.diffEntropy, harlCombS.diffVar, harlCombS.dissimilarity, ...
harlCombS.energy, harlCombS.jointEntropy, harlCombS.invDiff, harlCombS.invDiffMom, harlCombS.firstInfCorr, ...
harlCombS.secondInfCorr, harlCombS.invDiffMomNorm, harlCombS.invDiffNorm, harlCombS.invVar, ...
harlCombS.sumAvg, harlCombS.sumEntropy, harlCombS.sumVar];

pyradGlcmNamC = {'Autocorrelation', 'JointAverage', 'ClusterProminence', 'ClusterShade',  'ClusterTendency', ...
    'Contrast', 'Correlation', 'DifferenceAverage', 'DifferenceEntropy', 'DifferenceVariance', 'Dissimilarity', ...
    'JointEnergy', 'JointEntropy','Id','Idm', 'Imc1' , ...
    'Imc2', 'Idmn','Idn','InverseVariance', 'sumAverage', 'SumEntropy', 'sumVariance'};

pyradGlcmNamC = strcat([pyradScanType,'_glcm_'],pyradGlcmNamC);
pyRadGlcmV = [];
for i = 1:length(pyradGlcmNamC)
    if isfield(teststruct,pyradGlcmNamC{i})
        pyRadGlcmV(i) = teststruct.(pyradGlcmNamC{i});
    else
        pyRadGlcmV(i) = NaN;
    end
end

glcmDiffV = (cerrGlcmV - pyRadGlcmV) ./ cerrGlcmV * 100

%% RLM features

rlmFlagS.sre = 1;
rlmFlagS.lre = 1;
rlmFlagS.gln = 1;
rlmFlagS.glnNorm = 1;
rlmFlagS.rln = 1;
rlmFlagS.rlnNorm = 1;
rlmFlagS.rp = 1;
rlmFlagS.lglre = 1;
rlmFlagS.hglre = 1;
rlmFlagS.srlgle = 1;
rlmFlagS.srhgle = 1;
rlmFlagS.lrlgle = 1;
rlmFlagS.lrhgle = 1;
rlmFlagS.glv = 1;
rlmFlagS.rlv = 1;
rlmFlagS.re = 1;

numGrLevels = paramS.higherOrderParamS.numGrLevels;

% 3D Run-Length features from combined run length matrix
dirctn      = 1;
rlmType     = 2;
featureS.rlmFeat3DdirS = get_rlm(dirctn, rlmType, quantizedM, ...
    numGrLevels, numVoxels, rlmFlagS);

rlmCombS = featureS.rlmFeat3DdirS.AvgS;
cerrRlmV = [rlmCombS.gln, rlmCombS.glnNorm, rlmCombS.glv, rlmCombS.hglre, rlmCombS.lglre, rlmCombS.lre, rlmCombS.lrhgle, ...
    rlmCombS.lrlgle, rlmCombS.rln, rlmCombS.rlnNorm, rlmCombS.rlv, rlmCombS.rp, ...
    rlmCombS.sre, rlmCombS.srhgle, rlmCombS.srlgle, rlmCombS.re];

pyradRlmNamC = {'GrayLevelNonUniformity', 'GrayLevelNonUniformityNormalized',...
    'GrayLevelVariance', 'HighGrayLevelRunEmphasis',  'LowGrayLevelRunEmphasis', ...
    'LongRunEmphasis', 'LongRunHighGrayLevelEmphasis', 'LongRunLowGrayLevelEmphasis',...
    'RunLengthNonUniformity', 'RunLengthNonUniformityNormalized', 'RunVariance', ...
    'RunPercentage', 'ShortRunEmphasis','ShortRunHighGrayLevelEmphasis', ...
    'ShortRunLowGrayLevelEmphasis','RunEntropy'};

pyradRlmNamC = strcat([pyradScanType,'_glrlm_'],pyradRlmNamC);
pyRadRlmV = [];
for i = 1:length(pyradRlmNamC)
    if isfield(teststruct,pyradRlmNamC{i})
        pyRadRlmV(i) = teststruct.(pyradRlmNamC{i});
    else
        pyRadRlmV(i) = NaN;
    end
end

rlmDiffV = (cerrRlmV - pyRadRlmV) ./ cerrRlmV * 100

%% Size Zone features in 3d

flagS.sae = 1;
flagS.lae = 1;
flagS.gln = 1;
flagS.glv = 1;
flagS.szv = 1;
flagS.glnNorm = 1;
flagS.szn = 1;
flagS.sznNorm = 1;
flagS.zp = 1;
flagS.lglze = 1;
flagS.hglze = 1;
flagS.salgle = 1;
flagS.sahgle = 1;
flagS.lalgle = 1;
flagS.larhgle = 1;
flagS.ze = 1;

szmType = 1; % 1: 3d, 2: 2d
szmM = calcSZM(quantizedM, numGrLevels, szmType);
numVoxels = sum(~isnan(quantizedM(:)));
featureS.szmFeature3dS = szmToScalarFeatures(szmM,numVoxels, szmFlagS);
szmS = featureS.szmFeature3dS;
% cerrSzmV = [szmS.gln, szmS.glnNorm, szmS.glv, szmS.hglre, szmS.lglre, szmS.lre, szmS.lrhgle, ...
%     szmS.lrlgle, szmS.rln, szmS.rlnNorm, szmS.rlv, szmS.rp, ...
%     szmS.sre, szmS.srhgle, szmS.srlgle];
cerrSzmV = [szmS.gln, szmS.glnNorm, szmS.glv, szmS.hglze, szmS.lglze, szmS.lae, szmS.lahgle, ...
    szmS.lalgle, szmS.szn, szmS.sznNorm, szmS.szv, szmS.zp, ...
    szmS.sae, szmS.sahgle, szmS.salgle, szmS.ze];

pyradSzmNamC = {'GrayLevelNonUniformity', 'GrayLevelNonUniformityNormalized',...
    'GrayLevelVariance', 'HighGrayLevelZoneEmphasis',  'LowGrayLevelZoneEmphasis', ...
    'LargeAreaEmphasis', 'LargeAreaHighGrayLevelEmphasis', 'LargeAreaLowGrayLevelEmphasis',...
    'SizeZoneNonUniformity', 'SizeZoneNonUniformityNormalized', 'ZoneVariance', ...
    'ZonePercentage', 'SmallAreaEmphasis','SmallAreaHighGrayLevelEmphasis', ...
    'SmallAreaLowGrayLevelEmphasis','ZoneEntropy'};

pyradSzmNamC = strcat([pyradScanType,'_glszm_'],pyradSzmNamC);
pyRadSzmV = [];
for i = 1:length(pyradSzmNamC)
    if isfield(teststruct,pyradSzmNamC{i})
        pyRadSzmV(i) = teststruct.(pyradSzmNamC{i});
    else
        pyRadSzmV(i) = NaN;
    end
end
szmDiffV = (cerrSzmV - pyRadSzmV) ./ cerrSzmV * 100

%% NGLDM features

patchRadius3dV = paramS.higherOrderParamS.patchRadius3dV;
imgDiffThresh = paramS.higherOrderParamS.imgDiffThresh;
% 3d
ngldmM = calcNGLDM(quantizedM, patchRadius3dV, ...
    numGrLevels, imgDiffThresh);
featureS.ngldmFeatures3dS = ngldmToScalarFeatures(ngldmM,numVoxels);

ngldmS = featureS.ngldmFeatures3dS;
cerrNgldmV = [ngldmS.lde, ngldmS.hde, ngldmS.lgce, ngldmS.hgce, ...
    ngldmS.ldlge, ngldmS.ldhge, ngldmS.hdlge, ngldmS.hdhge, ...
    ngldmS.gln, ngldmS.glnNorm, ngldmS.dcn, ngldmS.dcnNorm,...
    ngldmS.dcp, ngldmS.glv, ngldmS.dcv, ngldmS.entropy, ngldmS.energy];

pyradNgldmNamC = {'SmallDependenceEmphasis', 'LargeDependenceEmphasis',...
    'LowGrayLevelCountEmphasis', 'HighGrayLevelCountEmphasis',  'SmallDependenceLowGrayLevelEmphasis', ...
    'SmallDependenceHighGrayLevelEmphasis', 'LargeDependenceLowGrayLevelEmphasis', ...
    'LargeDependenceHighGrayLevelEmphasis', 'GrayLevelNonUniformity', 'GrayLevelNonUniformityNorm', ...
    'DependenceNonUniformity', 'DependenceNonUniformityNormalized', ...
    'DependencePercentage', 'GrayLevelVariance', 'DependenceVariance', ...
    'DependenceEntropy', 'DependenceEnergy'};

pyradNgldmNamC = strcat([pyradScanType,'_gldm_'],pyradNgldmNamC);
pyRadNgldmV = [];
for i = 1:length(pyradNgldmNamC)
    if isfield(teststruct,pyradNgldmNamC{i})
        pyRadNgldmV(i) = teststruct.(pyradNgldmNamC{i});
    else
        pyRadNgldmV(i) = NaN;
    end
end

ngldmDiffV = (cerrNgldmV - pyRadNgldmV) ./ cerrNgldmV * 100

%% NGTDM features
numGrLevels = paramS.higherOrderParamS.numGrLevels;
patchRadius3dV = paramS.higherOrderParamS.patchRadius3dV;
% 3d
[s,p] = calcNGTDM(quantizedM, patchRadius3dV, numGrLevels);
featureS.ngtdmFeatures3dS = ngtdmToScalarFeatures(s,p,numVoxels);

ngtdmS = featureS.ngtdmFeatures3dS;
cerrNgtdmV = [ngtdmS.busyness ngtdmS.coarseness ngtdmS.complexity ngtdmS.contrast ngtdmS.strength];


function [y]=tt_mvk4(A, x, tol, varargin)

% step_dpow = 0.1; % stepsize for d-power in truncations
% min_dpow = 1; % Minimal d-power for truncation
%
% bot_conv = 0.1; % bottom convergence factor - if better, we can decrease dpow and drank
% top_conv = 0.99; % top convergence factor - if worse, we have to increase dpow and drank

nswp=10;

als_tol_low = 5;
als_iters = 4;

rmax=1000;

verb=1;
kickrank = 10;
 kicktype = 'mr';
% kicktype = 'rand';

% pcatype = 'svd';
pcatype = 'uchol';

y=[];
block_order = [];

for i=1:2:length(varargin)-1
    switch lower(varargin{i})
        case 'nswp'
            nswp=varargin{i+1};
        case 'rmax'
            rmax=varargin{i+1};
        case 'kicktype'
            kicktype=lower(varargin{i+1});
        case 'pcatype'
            pcatype=lower(varargin{i+1});
        case 'y0'
            y=varargin{i+1};
        case 'verb'
            verb=varargin{i+1};
        case 'kickrank'
            kickrank=varargin{i+1};
        case 'step_dpow'
            step_dpow=varargin{i+1};
        case 'min_dpow'
            min_dpow=varargin{i+1};
        case 'bot_conv'
            bot_conv=varargin{i+1};
        case 'top_conv'
            top_conv=varargin{i+1};
        case 'block_order'
            block_order=varargin{i+1};
        case 'als_tol_low'
            als_tol_low=varargin{i+1};
        case 'als_iters'
            als_iters=varargin{i+1};

        otherwise
            error('Unrecognized option: %s\n',varargin{i});
    end
end

rx = x.r;
d = x.d;
tailranks = [false,false];
if (rx(1)~=1)
    e1 = tt_tensor;
    e1.d=1;
    e1.n = [rx(1)];
    e1.r = [1; rx(1)];
    e1.ps = [1; rx(1)^2+1];
    e1.core = reshape(eye(rx(1)), rx(1)^2, 1);
    x = kron(e1, x);

    A = kron(tt_matrix(eye(rx(1))), A);
    tailranks(1)=true;
end;
if (rx(d+1)~=1)
    e1 = tt_tensor;
    e1.d=1;
    e1.n = [rx(d+1)];
    e1.r = [rx(d+1); 1];
    e1.ps = [1; rx(d+1)^2+1];
    e1.core = reshape(eye(rx(d+1)), rx(d+1)^2, 1);
    x = kron(x, e1);

    A = kron(A, tt_matrix(eye(rx(d+1))));
    tailranks(2)=true;
end;

d = x.d;
rx = x.r;
ra = A.r;
n = A.n;
m = A.m;
if (isempty(y))
%     y = tt_rand(n, A.d, kickrank);
    y = tt_ones(n, A.d);
end;

if (isempty(block_order))
    block_order = [+(d), -(d)];
end;

ry = y.r;

cry = core2cell(y);
crA = core2cell(A);
crx = core2cell(x);

phia = cell(d+1,1); phia{1}=1; phia{d+1}=1;

% Orthogonalization
for i=d:-1:2
    cr = cry{i};
    cr = reshape(cr, ry(i), n(i)*ry(i+1));
    [cr, rv]=qr(cr.', 0);
    cr2 = cry{i-1};
    cr2 = reshape(cr2, ry(i-1)*n(i-1), ry(i));
    cr2 = cr2*(rv.');
    ry(i) = size(cr, 2);
    cr = reshape(cr.', ry(i), n(i), ry(i+1));
%     psy(i) = psy(i+1)-ry(i)*n(i)*ry(i+1);
%     cry(psy(i):(psy(i+1)-1))=cr(:);
%     psyo(i) = psyo(i-1)+ry(i-1)*n(i-1)*ry(i);
%     cry(psyo(i-1):(psyo(i)-1)) = cr2(:);
    cry{i-1} = reshape(cr2, ry(i-1), n(i-1), ry(i));
    cry{i} = cr;

    phia{i} = compute_next_Phi(phia{i+1}, cr, crA{i}, crx{i}, 'rl');
%     phia{i} = compute_next_Phi(phia{i+1}, cr, reshape(cra(psa(i):(psa(i+1)-1)), ra(i), n(i), n(i), ra(i+1)), reshape(crx(psx(i):(psx(i+1)-1)), rx(i), n(i), rx(i+1)), 'rl');

end;
% cry = cry(psy(1):(psy(d+1)-1));

last_sweep = false;
swp = 1;
i = 1;

% dy_old = ones(d,1);
dy = zeros(d,1);
max_dy = 0;
% For extra-rank addition
% dpows = ones(d,1)*min_dpow;
% dranks = zeros(d,1);

cur_order = block_order;
order_index = 1;
dir = sign(cur_order(order_index));

% DMRG sweeps
while (swp<=nswp)

    % Extract elements - matrix
    Phi1 = phia{i}; Phi2 = phia{i+1};
%     A1 = reshape(cra(psa(i):(psa(i+1)-1)), ra(i), n(i), n(i), ra(i+1));
%     x1 = reshape(crx(psx(i):(psx(i+1)-1)), rx(i), n(i), rx(i+1));
    x1 = crx{i};
    A1 = crA{i};

    newy = reshape(Phi1, ry(i)*ra(i), rx(i));
    newy = newy*reshape(x1, rx(i), m(i)*rx(i+1));
    newy = reshape(newy, ry(i), ra(i)*m(i), rx(i+1));
    newy = permute(newy, [2, 1, 3]);
    newy = reshape(newy, ra(i)*m(i), ry(i)*rx(i+1));
    A1 = permute(A1, [2, 4, 1, 3]);
    A1 = reshape(A1, n(i)*ra(i+1), ra(i)*m(i));
    newy = A1*newy;
    newy = reshape(newy, n(i), ra(i+1), ry(i), rx(i+1));
    newy = permute(newy, [3, 1, 2, 4]);
    newy = reshape(newy, ry(i)*n(i), ra(i+1)*rx(i+1));
    if (dir>0)&&(strcmp(kicktype, 'mr'))
        newy_save = newy;
    end;
    newy = newy*(reshape(Phi2, ry(i+1), ra(i+1)*rx(i+1)).');
    newy = reshape(newy, ry(i)*n(i)*ry(i+1), 1);

    oldy = reshape(cry{i}, ry(i)*n(i)*ry(i+1), 1);

    dy(i) = norm(newy-oldy)/norm(newy);
    max_dy = max(max_dy, dy(i));
%
%     % The new core does not converge - increase rank
%     if (dy(i)/dy_old(i)>top_conv)&&(dy(i)>tol)
%         dranks(i)=dranks(i)+1;
%         dpows(i)=dpows(i)+step_dpow;
%     end;
%     % The new core converges well - try to decrease rank
%     if (dy(i)/dy_old(i)<bot_conv)||(dy(i)<tol)
%         dranks(i)=max(dranks(i)-1, 0);
%         dpows(i)=max(dpows(i)-step_dpow, min_dpow);
%     end;
%
%     if (last_sweep)
%         dpows(i)=0.5;
%         dranks(i)=0;
%     end;

    % Truncation
    if (dir>0) % left-to-right
        newy = reshape(newy, ry(i)*n(i), ry(i+1));
    else
        newy = reshape(newy, ry(i), n(i)*ry(i+1));
    end;

    if (kickrank>=0)
    [u,s,v]=svd(newy, 'econ');
    s = diag(s);
    r = my_chop2(s, tol/sqrt(d)*norm(s));
%     % Artificial rank increasing
%     r = r+dranks(i);
    r = min(r, numel(s));
    r = min(r, rmax);
    else
        if (dir>0)
            [u,v]=qr(newy, 0);
            v=v';
            r = size(u,2);
            s = ones(r,1);
        else
            [v,u]=qr(newy.', 0);
            v=conj(v);
            u=u.';
            r = size(u,2);
            s = ones(r,1);
        end;
    end;

    if (verb>1)
	fprintf('=mvk4=   block %d{%d}, dy: %3.3e, r: %d\n', i, dir, dy(i), r);
%     elseif (verb>0)
%         fprintf('                                                                          \r')
%         fprintf('=mvk4=   block %d{%d}, dy: %3.3e, r: %d\r', i, dir, dy(i), r);
    end;

    if (dir>0)&&(i<d) % left-to-right, kickrank, etc
        u = u(:,1:r);
        v = v(:,1:r)*diag(s(1:r));

        % kick
        radd = 0; rv = 1;
        if (~last_sweep)&&(kickrank>0)
%             newy = reshape(Phi1, ry(i)*ra(i), rx(i));
%             newy = newy*reshape(crx{i}, rx(i), n(i)*rx(i+1));
%             newy = reshape(newy, ry(i), ra(i)*n(i), rx(i+1));
%             newy = permute(newy, [2, 1, 3]);
%             newy = reshape(newy, ra(i)*n(i), ry(i)*rx(i+1));
%             A1 = permute(crA{i}, [2, 4, 1, 3]);
%             A1 = reshape(A1, n(i)*ra(i+1), ra(i)*n(i));
%             newy = A1*newy;
%             newy = reshape(newy, n(i), ra(i+1), ry(i), rx(i+1));
%             newy = permute(newy, [3, 1, 2, 4]);
%             newy = reshape(newy, ry(i)*n(i), ra(i+1)*rx(i+1));
            if (strcmp(kicktype, 'mr'))
                leftresid = [reshape(u*v', ry(i)*n(i), ry(i+1)), -newy_save];
                if (strcmp(pcatype, 'svd'))
                    [uk,sk,vk]=svd(leftresid, 'econ');
                    uk = uk(:,1:min(kickrank, size(uk,2)));
                else
                    uk = uchol(leftresid.', kickrank*2);
                    uk = uk(:,size(uk,2):-1:max(size(uk,2)-kickrank+1,1));
                end;

%                 if (i+1<d)
%                     newy = reshape(phia{i+3}, ry(i+3)*ra(i+3), rx(i+3));
%                     newy = newy*reshape(crx{i+2}, rx(i+2)*n(i+2), rx(i+3)).';
%                     newy = reshape(newy, ry(i+3), ra(i+3), rx(i+2), n(i+2));
%                     newy = permute(newy, [2, 4, 1, 3]);
%                     newy = reshape(newy, ra(i+3)*n(i+2), ry(i+3)*rx(i+2));
%                     A1 = permute(crA{i+2}, [2, 1, 4, 3]);
%                     A1 = reshape(A1, n(i+2)*ra(i+2), ra(i+3)*n(i+2));
%                     newy = A1*newy;
%                     newy = reshape(newy, n(i+2), ra(i+2), ry(i+3), rx(i+2));
%                     newy = permute(newy, [1, 3, 2, 4]);
%                     newy = reshape(newy, n(i+2)*ry(i+3), ra(i+2)*rx(i+2));
%
%                     oldy = reshape(cry{i+2}, ry(i+2), n(i+2)*ry(i+3)).';
%                     rightresid = [oldy, -newy];
%                     vk = uchol(rightresid.', kickrank*2);
%                     vk = vk(:,size(vk,2):-1:max(size(vk,2)-kickrank+1,1));
%
%                     [oldy,rv]=qr([oldy,vk], 0);
%                     newy = reshape(cry{i+1}, ry(i+1)*n(i+1), ry(i+2));
%                     newy = [newy, zeros(ry(i+1)*n(i+1), size(vk,2))];
%                     newy = newy*(rv.');
%                     ry(i+2) = size(oldy,2);
%                     cry{i+1} = reshape(newy, ry(i+1), n(i+1), ry(i+2));
%                     cry{i+2} = permute(reshape(oldy, n(i+2), ry(i+3), ry(i+2)), [3, 1, 2]);
%                     phia{i+2} = compute_next_Phi(phia{i+3}, cry{i+2}, crA{i+2}, crx{i+2}, 'rl');
%                 end;
            else
                uk = rand(ry(i)*n(i), kickrank);
            end;
            [u,rv]=qr([u,uk], 0);
            radd = size(uk,2);
        end;
%         radd = size(u, 2)+size(uk,2)-r;
        v = [v, zeros(ry(i+1), radd)];
        v = rv*(v');
        r = size(u,2);
%         r = r+radd;
        cr2 = cry{i+1};
        cr2 = reshape(cr2, ry(i+1), n(i+1)*ry(i+2));
        v = v*cr2; % size r+radd, n2, r3

%         y{i} = reshape(u, ry(i), n(i), r);
%         y{i+1} = reshape(v, r, n(i+1), ry(i+2));
        ry(i+1) = r;

%         % Add residual component
%         rescomp = als_rank_one(x, 'A', A, 'f', y);
%         % We have to update ALL phis
%         Rl = [1,1];
%         for j=1:i
%             addbas = rescomp{i};
%             addbas = addbas/norm(addbas);
%             newbas = zeros(ry(i)+1, n(i), ry(i+1)+1);
%             newbas(1:ry(i), :, 1:ry(i+1))=y{i};
%             newbas(ry(i)+1, :, ry(i+1)+1)=addbas;
%             newbas = reshape(newbas, ry(i)+1, n(i)*(ry(i+1)+1));
%             newbas = Rl*newbas;
%             ry(i) = size(Rl, 1);
%             newbas = reshape(newbas, ry(i)*n(i), ry(i+1)+1);
%             % We have to orthogonalize it. approximately, if j<i
%             proj = newbas(:,ry(i+1)+1) - newbas(:,1:ry(i+1))*(newbas(:,1:ry(i+1))'*newbas(:,ry(i+1)+1));
%             if (i<j)&&(norm(proj)<tol/(d^dpows(i)))
%                 % Remove the new vector
%                 newbas2 = newbas(:, 1:ry(i+1));
%                 Rl = [eye(ry(i+1)); zeros(1,ry(i+1))];
%             else
%                 % Reort the new vector to the old one
%                 newbas2 = reort(newbas(:,1:ry(i+1)), newbas(:, ry(i+1)+1));
%                 Rl = newbas2'*newbas;
%             end;
%             radd = size(newbas2, 2)-ry(i+1);
%             y{i} = reshape(newbas2, ry(i), n(i), ry(i+1)+radd);
%             if (radd>0) % update phi
%                 addbas = reshape(newbas2(:, ry(i+1)+1), ry(i), n(i), 1);
%                 newphia = zeros(ry(i+1)+1, ra(i+1), rx(i+1));
%                 newphia(1:ry(i+1),:,:)=phia{i+1};
%                 newphia(ry(i+1)+1,:,:) = compute_next_Phi(phia{i}, addbas, A{i}, x{i}, 'lr');
%                 phia{i+1} = newphia;
%             end;
%         end;
%
%         Rr = [1;1];
%         for j=d:-1:i+2
%             addbas = rescomp{i};
%             addbas = addbas/norm(addbas);
%             newbas = zeros(ry(i)+1, n(i), ry(i+1)+1);
%             newbas(1:ry(i), :, 1:ry(i+1))=y{i};
%             newbas(ry(i)+1, :, ry(i+1)+1)=addbas;
%             newbas = reshape(newbas, (ry(i)+1)*n(i), (ry(i+1)+1));
%             newbas = newbas*Rr;
%             ry(i+1) = size(Rr, 2);
%             newbas = reshape(newbas, ry(i)+1, n(i)*ry(i+1));
%             newbas = newbas.';
%             % We have to orthogonalize it. approximately, if j<i
%             proj = newbas(:, ry(i)+1) - newbas(:,1:ry(i))*(newbas(:,1:ry(i))'*newbas(:,ry(i)+1));
%             if (i<j)&&(norm(proj)<tol/(d^dpows(i)))
%                 % Remove the new vector
%                 newbas2 = newbas(:, 1:ry(i));
%                 Rr = [eye(ry(i)), zeros(ry(i), 1)];
%             else
%                 % Reort the new vector to the old one
%                 newbas2 = reort(newbas(:,1:ry(i)), newbas(:, ry(i)+1));
%                 Rr = newbas2'*newbas;
%                 Rr = Rr.';
%             end;
%             radd = size(newbas2, 2)-ry(i);
%             y{i} = reshape(newbas2.', ry(i)+radd, n(i), ry(i+1));
%             if (radd>0) % update phi
%                 addbas = reshape(newbas2(:, ry(i)+1).', 1, n(i), ry(i+1));
%                 newphia = zeros(ry(i)+1, ra(i), rx(i));
%                 newphia(1:ry(i),:,:)=phia{i};
%                 newphia(ry(i)+1,:,:) = compute_next_Phi(phia{i+1}, addbas, A{i}, x{i}, 'lr');
%                 phia{i} = newphia;
%             end;
%         end;

        u = reshape(u, ry(i), n(i), r);
        v = reshape(v, r, n(i+1), ry(i+2));

        % Recompute phi. Left ones, so permute them appropriately
        phia{i+1} = compute_next_Phi(phia{i}, u, crA{i}, crx{i}, 'lr');

        % Stuff back

        cry{i} = u;
        cry{i+1} = v;
    elseif (dir<0)&&(i>1) % right-to-left
        u = u(:,1:r)*diag(s(1:r));
        v = conj(v(:,1:r));
        % kick
        radd = 0; rv = 1;
%          if (~last_sweep)&&(strcmp(kicktype, 'rand'))&&(kickrank>0)
%  %             newy = reshape(Phi2, ry(i+1)*ra(i+1), rx(i+1));
%  %             newy = newy*reshape(crx{i}, rx(i)*n(i), rx(i+1)).';
%  %             newy = reshape(newy, ry(i+1), ra(i+1), rx(i), n(i));
%  %             newy = permute(newy, [2, 4, 1, 3]);
%  %             newy = reshape(newy, ra(i+1)*n(i), ry(i+1)*rx(i));
%  %             A1 = permute(crA{i}, [1, 2, 4, 3]);
%  %             A1 = reshape(A1, ra(i)*n(i), ra(i+1)*n(i));
%  %             newy = A1*newy;
%  %             newy = reshape(newy, ra(i), n(i), ry(i+1), rx(i));
%  %             newy = permute(newy, [1, 4, 2, 3]);
%  %             newy = reshape(newy, ra(i)*rx(i), n(i)*ry(i+1));
%  %             resid = [reshape(u*v', ry(i), n(i)*ry(i+1)); -newy];
%  %             uk = uchol(resid, kickrank+1);
%  %             uk = uk(:,end:-1:max(end-kickrank+1,1));
%              uk = randn(n(i)*ry(i+1), kickrank);
%              [v,rv]=qr([v,uk], 0);
%              radd = size(uk,2);
%  %             v = reort(v, uk);
%          end;

%  %         radd = size(v, 2)+size(uk,2)-r;
        u = [u, zeros(ry(i), radd)];
        u = u*(rv.');
        cr2 = cry{i-1};
        cr2 = reshape(cr2, ry(i-1)*n(i-1), ry(i));
        u = cr2*u;

%         r = r+radd;
        r = size(v,2);

        u = reshape(u, ry(i-1), n(i-1), r);
        v = reshape(v.', r, n(i), ry(i+1));

        % Recompute phi. Here are right phis
        phia{i} = compute_next_Phi(phia{i+1}, v, crA{i}, crx{i}, 'rl');

        % Stuff back
        ry(i) = r;
        cry{i-1} = u;
        cry{i} = v;
    elseif ((dir>0)&&(i==d))||((dir<0)&&(i==1))
        % Just stuff back the last core
        newy = u(:,1:r)*diag(s(1:r))*(v(:,1:r)');
        newy = reshape(newy, ry(i), n(i), ry(i+1));
        cry{i} = newy;
    end;

    i = i+dir;

    % Reversing, residue check, etc
    cur_order(order_index) = cur_order(order_index) - dir;
    % New direction
    if (cur_order(order_index)==0)
        order_index = order_index+1;

        if (verb>0)
%  	    fprintf('                                                   \r')
            fprintf('=mvk4= sweep %d{%d}, max_dy: %3.3e, erank: %g\n', swp, order_index-1, max_dy, sqrt(ry(1:d)'*(n.*ry(2:d+1))/sum(n)));
        end;

        if (last_sweep)
            break;
        end;

        if (kickrank<0)
            kickrank=kickrank-1;
        end;

        if (max_dy<tol*als_tol_low)&&(kickrank>=0)
            kickrank=-1;
        end;

        if (max_dy<tol)&&(kickrank<=-als_iters)
            last_sweep=true;
            kickrank=0;
        end;

        if (order_index>numel(cur_order)) % New global sweep
            cur_order = block_order;
            order_index = 1;
            %residue
            if (last_sweep)
                cur_order = d-1;
            end;

            max_dy = 0;

%             dy_old = dy;
            swp = swp+1;
        end;

        dir = sign(cur_order(order_index));
        i = i+dir;
    end;
end

if (tailranks(1))
    cry{2} = reshape(cry{1}, n(1), ry(2))*reshape(cry{2}, ry(2), n(2)*ry(3));
    cry{2} = reshape(cry{2}, n(1), n(2), ry(3));
end;
if (tailranks(2))
    cry{d-1} = reshape(cry{d-1}, ry(d-1)*n(d-1), ry(d))*reshape(cry{d}, ry(d), n(d));
    cry{d-1} = reshape(cry{d-1}, ry(d-1), n(d-1), n(d));
    cry = cry(1:d-1);
end;
if (tailranks(1))
    cry = cry(2:end);
end;
y = cell2core(y, cry);

end

function [Phi] = compute_next_Phi(Phi_prev, x, A, y, direction)
% Performs the recurrent Phi (or Psi) matrix computation
% Phi = Phi_prev * (x'Ay).
% If direction is 'lr', computes Psi
% if direction is 'rl', computes Phi
% A can be empty, then only x'y is computed.

if (strcmp(direction, 'rl'))
  % Revert ranks to perform the right-to-left recursion
  x = permute(x, [3, 2, 1]);
  y = permute(y, [3, 2, 1]);
  if (~isempty(A))
    A = permute(A, [4, 2, 3, 1]);
  end
end

rx1 = size(x,1); n = size(x,2); rx2 = size(x,3);
ry1 = size(y,1); m = size(y,2); ry2 = size(y,3);
if (~isempty(A))
  ra1 = size(A,1); ra2 = size(A,4);
else
  ra1 = 1; ra2 = 1;
end

Phi = reshape(Phi_prev, [rx1*ra1, ry1]);
y = reshape(y, [ry1, m*ry2]);
Phi = Phi*y;	% complexity §\mcommentfont$\mathcal{O}(n  r_x r_A r_y^2)$§
Phi = reshape(Phi, [rx1, ra1, m, ry2]);
Phi = permute(Phi, [2, 3, 1, 4]);
if (~isempty(A))
  Phi = reshape(Phi, [ra1*m, rx1*ry2]);
  A = permute(A, [4, 2, 1, 3]);
  A = reshape(A, [ra2*n, ra1*m]);
  Phi = A*Phi;	% complexity §\mcommentfont$\mathcal{O}(n^2  r_x r_A^2 r_y)$§
  Phi = reshape(Phi, [ra2, n, rx1, ry2]);
end
Phi = permute(Phi, [3, 2, 1, 4]);
Phi = reshape(Phi, [rx1*n, ra2*ry2]);
x = reshape(x, [rx1*n, rx2]);
Phi = (x')*Phi;	% complexity §\mcommentfont$\mathcal{O}(n  r_x^2 r_A r_y)$§
if (~isempty(A))
  Phi = reshape(Phi, [rx2, ra2, ry2]);
end
end


classdef prtClassRvmSequential < prtClassRvm
    % prtClassRvmSequential  Relevance vector machine classifier using sequential training
    %
    %   CLASSIFIER = prtClassRvmSequential returns a relevance vector
    %   machine classifier based using sequential training.
    %
    %   CLASSIFIER = prtClassRvmSequential(PROPERTY1, VALUE1, ...)
    %   constructs a prtClassRvmSequential object CLASSIFIER with properties as
    %   specified by PROPERTY/VALUE pairs.
    %
    %    A prtClassRvmSequential object inherits all properties from the abstract class
    %    prtClass. In addition is has the following properties:
    %
    %    kernels                - A cell array of prtKernel objects specifying
    %                             the kernels to use
    %    verbosePlot            - Flag indicating whether or not to plot during
    %                             training
    %    verboseText            - Flag indicating whether or not to output
    %                             verbose updates during training
    %    learningMaxIterations  - The maximum number of iterations
    %
    %    A prtClassRvmSequential also has the following read-only properties:
    %
    %    learningConverged  - Flag indicating if the training converged
    %    beta               - The regression weights, estimated during training
    %    sparseBeta         - The sparse regression weights, estimated during
    %                         training
    %    sparseKernels      - The sparse regression kernels, estimated during
    %                         training
    %
    %   For more information on the algorithm and the above properties, see
    %   the following reference:
    %
    %   Tipping, M. E. and A. C. Faul (2003). Fast marginal likelihood
    %   maximisation for sparse Bayesian models. In C. M. Bishop and B. J.
    %   Frey (Eds.), Proceedings of the Ninth International Workshop on
    %   Artificial Intelligence and Statistics, Key West, FL, Jan 3-6.
    %
    %   prtClassRvmSequential is most useful for datasets with a large
    %   number of observations for which the gram matrix can not be held in
    %   memory. The sequential RVM training algorithm is capable of
    %   operating by generating necessary portions of the gram matrix when
    %   needed. The size of the generated portion of the gram matrix is
    %   determined by the property, largestNumberOfGramColumns. Sequential
    %   RVM training will attempt to generate portions of the gram matrix
    %   that are TraingData.nObservations x largesNumberofGramColums in
    %   size. If the entire gram matrix is this size or smaller it need
    %   only be generated once. Therefore if the entire gram matrix can be
    %   stored in memory, training is much faster. For quickest operation,
    %   largestNumberOfGramColumns should be set as large as possible
    %   without exceeding RAM limitations.
    %
    %    Example:
    %
    %    TestDataSet = prtDataGenUnimodal;      % Create some test and
    %    TrainingDataSet = prtDataGenUnimodal;  % training data classifier
    %    classifier = prtClassRvmSequential('verbosePlot',true); % Create a classifier
    %    classifier = classifier.train(TrainingDataSet);    % Train
    %    classified = run(classifier, TestDataSet);         % Test
    %    % Plot
    %    subplot(2,1,1); classifier.plot;
    %    subplot(2,1,2); [pf,pd] = prtScoreRoc(classified,TestDataSet);
    %    h = plot(pf,pd,'linewidth',3);
    %    title('ROC'); xlabel('Pf'); ylabel('Pd');
    %
    %   See also prtClass, prtClassRvm, prtClassRvnFiguerido,
    %   prtRegressRvmSequential

    properties

        learningPoorlyScaledLikelihoodThreshold = 1e4;
        learningLikelihoodIncreaseThreshold = 1e-6;
        largestNumberOfGramColumns = 5000;
        learningCorrelationRemovalThreshold = 0.99;
        learningFactorRemove = true;   % Remove kernels during train?
        learningRepeatedActionLimit = 25;
    end

    properties(Hidden = true)
        learningResults
    end

    properties (Hidden = true)
        Sigma = [];
    end

    methods
        function Obj = prtClassRvmSequential(varargin)
            Obj = prtUtilAssignStringValuePairs(Obj,varargin{:});
        end

        function Obj = set.learningPoorlyScaledLikelihoodThreshold(Obj,val)
            assert(prtUtilIsPositiveScalar(val),'prt:prtClassRvmSequential:learningPoorlyScaledLikelihoodThreshold','learningPoorlyScaledLikelihoodThreshold must be a positive scalar');
            Obj.learningPoorlyScaledLikelihoodThreshold = val;
        end
        function Obj = set.learningLikelihoodIncreaseThreshold(Obj,val)
            assert(prtUtilIsPositiveScalar(val),'prt:prtClassRvmSequential:learningLikelihoodIncreaseThreshold','learningLikelihoodIncreaseThreshold must be a positive scalar');
            Obj.learningLikelihoodIncreaseThreshold = val;
        end
        function Obj = set.largestNumberOfGramColumns(Obj,val)
            assert(prtUtilIsPositiveScalarInteger(val),'prt:prtClassRvmSequential:largestNumberOfGramColumns','largestNumberOfGramColumns must be a positive integer scalar');
            Obj.largestNumberOfGramColumns = val;
        end
        function Obj = set.learningCorrelationRemovalThreshold(Obj,val)
            assert(prtUtilIsPositiveScalar(val),'prt:prtClassRvmSequential:learningCorrelationRemovalThreshold','learningCorrelationRemovalThreshold must be a positive scalar');
            Obj.learningCorrelationRemovalThreshold = val;
        end
        function Obj = set.learningFactorRemove(Obj,val)
            assert(prtUtilIsLogicalScalar(val),'prt:prtClassRvmSequential:learningFactorRemove','learningFactorRemove must be a logical scalar');
            Obj.learningFactorRemove = val;
        end
        function Obj = set.learningRepeatedActionLimit(Obj,val)
            assert(prtUtilIsPositiveScalarInteger(val),'prt:prtClassRvmSequential:learningRepeatedActionLimit','learningRepeatedActionLimit must be a positive integer scalar');
            Obj.learningRepeatedActionLimit = val;
        end
    end

    methods (Access=protected, Hidden = true)
        function Obj = trainAction(Obj,DataSet)
            %Rvm = trainAction(Rvm,DataSet) (Private; see prtClass\train)

            warningState = warning;
            %warning off MATLAB:nearlySingularMatrix

            y = Obj.getMinusOneOneTargets(DataSet);

            localKernels = Obj.kernels.train(DataSet);
            nBasis = localKernels.nDimensions;

            if false && nBasis <= Obj.largestNumberOfGramColumns
                Obj = trainActionSequentialInMemory(Obj, DataSet, y);
                return
            end

            if Obj.verboseText
                fprintf('Sequential RVM training with %d possible vectors.\n', nBasis);
            end

            % The sometimes we want y [-1 1] but mostly not
            ym11 = y;
            y(y ==-1) = 0;

            Obj.beta = zeros(nBasis,1);

            relevantIndices = false(nBasis,1); % Nobody!
            alpha = inf(nBasis,1); % Nobody!
            forbidden = zeros(nBasis,1); % Will hold who is forbidding you from joining

            % Find first kernel
            kernelCorrs = zeros(nBasis,1);
            nBlocks = ceil(nBasis./ Obj.largestNumberOfGramColumns);
            for iBlock = 1:nBlocks
                cInds = ((iBlock-1)*Obj.largestNumberOfGramColumns+1):min([iBlock*Obj.largestNumberOfGramColumns nBasis]);

                trainedKernelDownSelected = localKernels.retainKernelDimensions(cInds);
                blockPhi = trainedKernelDownSelected.run_OutputDoubleArray(DataSet);
                blockPhiNormalized = bsxfun(@rdivide,blockPhi,sqrt(sum(blockPhi.*blockPhi))); % We have to normalize here

                kernelCorrs(cInds) = abs(blockPhiNormalized'*ym11);
            end
            [maxVal, maxInd] = max(kernelCorrs); %#ok<ASGLU>

            % Make this ind relevant
            relevantIndices(maxInd) = true;
            selectedInds = maxInd;

            % Add things to forbidden list
            initLogical = false(nBasis,1);
            initLogical(maxInd) = true;
            firstKernel = localKernels.retainKernelDimensions(initLogical);
            firstPhiNormalized = firstKernel.run_OutputDoubleArray(DataSet);
            firstPhiNormalized = firstPhiNormalized - mean(firstPhiNormalized);
            firstPhiNormalized = firstPhiNormalized./sqrt(sum(firstPhiNormalized.^2));
            phiCorrs = zeros(nBasis,1);
            for iBlock = 1:nBlocks
                cInds = ((iBlock-1)*Obj.largestNumberOfGramColumns+1):min([iBlock*Obj.largestNumberOfGramColumns nBasis]);

                trainedKernelDownSelected = localKernels.retainKernelDimensions(cInds);
                if nBlocks > 1 % If there is only one block we can keep the one from before
                    blockPhi = trainedKernelDownSelected.run_OutputDoubleArray(DataSet);
                end
                blockPhiDemeanedNormalized = bsxfun(@minus,blockPhi,mean(blockPhi));
                blockPhiDemeanedNormalized = bsxfun(@rdivide,blockPhiDemeanedNormalized,sqrt(sum(blockPhiDemeanedNormalized.*blockPhiDemeanedNormalized))); % We have to normalize here

                phiCorrs(cInds) = blockPhiDemeanedNormalized'*firstPhiNormalized;
            end
            forbidden(phiCorrs > Obj.learningCorrelationRemovalThreshold) = maxInd;

            % Start the actual Process
            if Obj.verboseText
                fprintf('\t Iteration 0: Intialized with vector %d.\n', maxInd);

                nVectorsStringLength = ceil(log10(length(nBasis)))+1;
            end

            if nBlocks == 1
                % If we have only 1 block we only need to do this once.
                trainedKernelDownSelected = localKernels.retainKernelDimensions(true(nBasis,1));
                PhiM = trainedKernelDownSelected.run_OutputDoubleArray(DataSet);
            end

            % Get the first relevant kernel matrix
            trainedKernelDownSelected = localKernels.retainKernelDimensions(relevantIndices);
            cPhi = trainedKernelDownSelected.run_OutputDoubleArray(DataSet);

            repeatedActionCounter = 0;
            for iteration = 1:Obj.learningMaxIterations

                % Store old log Alpha
                logAlphaOld = log(alpha);

                if iteration == 1
                    % Initial estimates
                    % Estimate Sigma, mu etc.

                    logOut = (ym11*0.9+1)/2;
                    mu = cPhi \ log(logOut./(1-logOut));

                    alpha(relevantIndices) = 1./mu.^2;

                    % Laplacian approx. IRLS
                    A = diag(alpha(relevantIndices));

                    [mu, SigmaInvChol, obsNoiseVar] = prtUtilPenalizedIrls(y,cPhi,mu,A);

                    SigmaChol = inv(SigmaInvChol);
                    Obj.Sigma = SigmaChol*SigmaChol'; %#ok<MINV>

                    yHat = 1 ./ (1+exp(-cPhi*mu));
                end

                % Eval additions and subtractions
                Sm = zeros(nBasis,1);
                Qm = zeros(nBasis,1);
                cError = y-yHat;

                cPhiProduct = bsxfun(@times,cPhi,obsNoiseVar);
                for iBlock = 1:nBlocks
                    cInds = ((iBlock-1)*Obj.largestNumberOfGramColumns+1):min([iBlock*Obj.largestNumberOfGramColumns nBasis]);

                    if nBlocks > 1 % If there is only one block we can keep the on from before
                        trainedKernelDownSelected = localKernels.retainKernelDimensions(cInds);
                        PhiM = trainedKernelDownSelected.run_OutputDoubleArray(DataSet);
                    end

                    Sm(cInds) = (obsNoiseVar'*(PhiM.^2)).' - sum((PhiM.'*cPhiProduct*SigmaChol).^2,2);
                    Qm(cInds) = PhiM.'*cError;
                end

                % Find little sm and qm (these are different for relevant vectors)
                sm = Sm;
                qm = Qm;

                cDenom = (alpha(relevantIndices)-Sm(relevantIndices));
                sm(relevantIndices) = alpha(relevantIndices) .* Sm(relevantIndices) ./ cDenom;
                qm(relevantIndices) = alpha(relevantIndices) .* Qm(relevantIndices) ./ cDenom;

                theta = qm.^2 - sm;
                cantBeRelevent = theta < 0;

                % Addition
                addLogLikelihoodChanges = 0.5*( theta./Sm + log(Sm ./ Qm.^2) ); % Eq (27)
                addLogLikelihoodChanges(cantBeRelevent) = 0; % Can't add things that are disallowed by theta
                addLogLikelihoodChanges(relevantIndices) = 0; % Can't add things already in
                addLogLikelihoodChanges(forbidden > 0) = 0; % Can't add things that are forbidden

                % Removal
                removeLogLikelihoodChanges = -0.5*( qm.^2./(sm + alpha) - log(1 + sm./alpha) ); % Eq (37) (I think this is wrong in the paper. The one in the paper uses Si and Qi, I got this based on si and qi (or S and Q in their code), from corrected from analyzing code from http://www.vectoranomaly.com/downloads/downloads.htm)
                removeLogLikelihoodChanges(~relevantIndices) = 0; % Can't remove things not in
                removeLogLikelihoodChanges(imag(removeLogLikelihoodChanges) > 0) = inf;

                % Modify
                updatedAlpha = sm.^2 ./ theta;
                updatedAlphaDiff = 1./updatedAlpha - 1./alpha;
                modifyLogLikelihoodChanges = 0.5*( updatedAlphaDiff.*(Qm.^2) ./ (updatedAlphaDiff.*Sm + 1) - log(1 + Sm.*updatedAlphaDiff) );

                modifyLogLikelihoodChanges(~relevantIndices) = 0; % Can't modify things not in
                modifyLogLikelihoodChanges(cantBeRelevent) = 0; % Can't modify things that technically shouldn't be in (they would get dropped)

                [addChange, bestAddInd] = max(addLogLikelihoodChanges);
                [remChange, bestRemInd] = max(removeLogLikelihoodChanges);
                [modChange, bestModInd] = max(modifyLogLikelihoodChanges);

                if iteration == 1
                    % On the first iteration we don't allow removal
                    [maxChangeVal, actionInd] = max([addChange, nan, modChange]);
                else
                    if remChange > 0 && Obj.learningFactorRemove
                        % Removing is top priority.
                        % If removing increases the likelihood, we have two
                        % options, actually remove that sample or modify that
                        % sample if that is better
                        [maxChangeVal, actionInd] = max([nan remChange, modifyLogLikelihoodChanges(bestRemInd)]);
                    else
                        % Not going to remove, so we would be allowed to modify
                        [maxChangeVal, actionInd] = max([addChange, remChange, modChange]);
                    end
                end

                if maxChangeVal > Obj.learningPoorlyScaledLikelihoodThreshold
                    warning('prtClassRvm:BadKernelMatrix','Kernel matrix is poorly conditioned. Consider modifying your kernels. Optimization Exiting...' );
                    break
                end

                if maxChangeVal < Obj.learningLikelihoodIncreaseThreshold
                    % There are no good options right now. Therefore we
                    % should exit with the previous iteration stats.
                    Obj.learningConverged = true;
                    Obj.learningResults.exitReason = 'No Good Actions';
                    Obj.learningResults.exitValue = maxChangeVal;
                    if Obj.verboseText
                        fprintf('Convergence criterion met, no necessary actions remaining, maximal change in log-likelihood %g\n\n',maxChangeVal);
                    end

                    break;
                end

                switch actionInd
                    case 1
                        cBestInd = bestAddInd;
                    case 2
                        cBestInd = bestRemInd;
                    case 3
                        cBestInd = bestModInd;
                end
                if iteration > 1 && lastAction == actionInd && cBestInd == lastInd
                    repeatedActionCounter = repeatedActionCounter + 1;
                else
                    repeatedActionCounter = 0;
                end

                if repeatedActionCounter >= Obj.learningRepeatedActionLimit
                    if Obj.verboseText
                        fprintf('Exiting... repeating action limit has been reached.\n\n');
                    end
                    return
                end

                if Obj.verboseText
                    actionStrings = {sprintf('Addition: Vector %s has been added.  ', sprintf(sprintf('%%%dd',nVectorsStringLength),bestAddInd));
                                     sprintf('Removal:  Vector %s has been removed.', sprintf(sprintf('%%%dd',nVectorsStringLength), bestRemInd));
                                     sprintf('Update:   Vector %s has been updated.', sprintf(sprintf('%%%dd',nVectorsStringLength), bestModInd));};
                    fprintf('\t Iteration %d: %s Change in log-likelihood %g.\n',iteration, actionStrings{actionInd}, maxChangeVal);
                end

                lastAction = actionInd;
                switch actionInd
                    case 1 % Add

                        relevantIndices(bestAddInd) = true;
                        selectedInds = cat(1,selectedInds,bestAddInd);

                        alpha(bestAddInd) = updatedAlpha(bestAddInd);
                        % Modify Mu
                        % (Penalized IRLS will fix it soon but we need good initialization)

                        trainedKernelDownSelected = localKernels.retainKernelDimensions(bestAddInd);
                        newPhi = trainedKernelDownSelected.run_OutputDoubleArray(DataSet);

                        cFactor = (Obj.Sigma*(cPhi)'*(newPhi.*obsNoiseVar));
                        Sigmaii = 1./(updatedAlpha(bestAddInd) + Sm(bestAddInd));
                        newMu = Sigmaii*Qm(bestAddInd);

                        updatedOldMu = mu - newMu*cFactor;
                        sortedSelected = sort(selectedInds);
                        newMuLocation = find(sortedSelected==bestAddInd);

                        mu = zeros(length(mu)+1,1);
                        mu(setdiff(1:length(mu),newMuLocation)) = updatedOldMu;
                        mu(newMuLocation) = newMu;

                        % Add things to forbidden list
                        newPhiDemeanedNormalized = newPhi - mean(newPhi);
                        newPhiDemeanedNormalized = newPhiDemeanedNormalized./sqrt(sum(newPhiDemeanedNormalized.^2));
                        phiCorrs = zeros(nBasis,1);
                        for iBlock = 1:nBlocks
                            cInds = ((iBlock-1)*Obj.largestNumberOfGramColumns+1):min([iBlock*Obj.largestNumberOfGramColumns nBasis]);

                            if nBlocks > 1
                                trainedKernelDownSelected = localKernels.retainKernelDimensions(cInds);
                                blockPhiDemeanedNormalized = trainedKernelDownSelected.run_OutputDoubleArray(DataSet);

                                blockPhiDemeanedNormalized = bsxfun(@minus,blockPhiDemeanedNormalized,mean(blockPhiDemeanedNormalized));
                                blockPhiDemeanedNormalized = bsxfun(@rdivide,blockPhiDemeanedNormalized,sqrt(sum(blockPhiDemeanedNormalized.*blockPhiDemeanedNormalized))); % We have to normalize here

                            %else we have this from before
                            end

                            phiCorrs(cInds) = blockPhiDemeanedNormalized'*newPhiDemeanedNormalized;
                        end
                        forbidden(phiCorrs > Obj.learningCorrelationRemovalThreshold) = bestAddInd;

                        lastInd = bestAddInd;
                    case 2 % Remove

                        removingInd = sort(selectedInds)==bestRemInd;

                        mu = mu + mu(removingInd) .* Obj.Sigma(:,removingInd) ./ Obj.Sigma(removingInd,removingInd);
                        mu(removingInd) = [];

                        relevantIndices(bestRemInd) = false;
                        selectedInds(selectedInds==bestRemInd) = [];
                        alpha(bestRemInd) = inf;

                        % Anything this guy said is forbidden is now
                        % allowed.
                        forbidden(forbidden == bestRemInd) = 0;

                        lastInd = bestRemInd;

                    case 3 % Modify
                        modifyInd = sort(selectedInds)==bestModInd;

                        alphaChangeInv = 1/(updatedAlpha(bestModInd) - alpha(bestModInd));
                        kappa = 1/(Obj.Sigma(modifyInd,modifyInd) + alphaChangeInv);
                        mu = mu - mu(modifyInd)*kappa*Obj.Sigma(:,modifyInd);

                        alpha(bestModInd) = updatedAlpha(bestModInd);

                        lastInd = bestModInd;
                end

                % At this point relevantIndices and alpha have changes.
                % Now we re-estimate Sigma, mu, and sigma2
                if nBlocks > 1
                    trainedKernelDownSelected = localKernels.retainKernelDimensions(relevantIndices);
                    cPhi = trainedKernelDownSelected.run_OutputDoubleArray(DataSet);
                else
                    cPhi = PhiM(:,relevantIndices);
                end

                % Laplacian approx. IRLS
                A = diag(alpha(relevantIndices));

                if isempty(cPhi)
                    yHat = 0.5*ones(size(y));
                else
                    [mu, SigmaInvChol, obsNoiseVar] = prtUtilPenalizedIrls(y,cPhi,mu,A);

                    SigmaChol = inv(SigmaInvChol);
                    Obj.Sigma = SigmaChol*SigmaChol'; %#ok<MINV>

                    yHat = 1 ./ (1+exp(-cPhi*mu)); % We use a logistic here.
                end

                % Store beta
                Obj.beta = zeros(nBasis,1);
                Obj.beta(relevantIndices) = mu;

                if ~mod(iteration,Obj.verbosePlot)
                    if DataSet.nFeatures == 2
                        Obj.verboseIterationPlot(DataSet,relevantIndices);
                    elseif iteration == 1
                        warning('prt:prtClassRvmSequential','Learning iteration plot can only be produced for training Datasets with 2 features');
                    end
                end

                % Check tolerance
                changeVal = abs(log(alpha)-logAlphaOld);
                changeVal(isnan(changeVal)) = 0; % inf-inf = nan
                if all(changeVal < Obj.learningConvergedTolerance) && iteration > 1
                    Obj.learningConverged = true;
                    Obj.learningResults.exitReason = 'Alpha Not Changing';
                    Obj.learningResults.exitValue = TOL;
                    if Obj.verboseText
                        fprintf('Exiting...Precisions no longer changing appreciably.\n\n');
                    end
                    break;
                end
            end

            if Obj.verboseText && iteration == Obj.learningMaxIterations
                fprintf('Exiting...Convergence not reached before the maximum allowed iterations was reached.\n\n');
            end

            % Make sparse represenation
            Obj.sparseBeta = Obj.beta(relevantIndices,1);
            Obj.sparseKernels = localKernels.retainKernelDimensions(relevantIndices);

            % Very bad training
            if isempty(Obj.sparseBeta)
                warning('prt:prtClassRvm:NoRelevantFeatures','No relevant features were found during training.');
            end

            % Reset warning
            warning(warningState);

        end
    end
end


function [units, data_description] = segtype2units(unitCode)
   %segtype2units   retrieves unit and description based on the segtype code
   %'segtype' in antelope datasets indicate the natural units of the detector
   persistent codeKey
   if isempty(codeKey)
      codeKey = createCodeKey();
   end
   if codeKey.isKey(unitCode)
      details = codeKey(unitCode);
      units = details{1};
      data_description = details{2};
   else
      [units, data_description] = deal('null');
   end

   function SU = createCodeKey()
      %createSegUnits   creates a map  where SU(char) = {units, data_description}
      SU = containers.Map;
      SU('A') = {'nm / sec / sec','acceleration'};
      SU('B') = {'25 mw / m / m','UV (sunburn) index(NOAA)'};
      SU('D') = {'nm', 'displacement'};
      SU('H') = {'Pa','hydroacoustic'};
      SU('I') = {'Pa','infrasound'};
      SU('J') = {'watts','power (Joulses/sec) (UCSD)'};
      SU('K') = {'kPa','generic pressure (UCSB)'};
      SU('M') = {'mm','Wood-Anderson drum recorder'};
      SU('P') = {'mb','barometric pressure'};
      SU('R') = {'mm','rain fall (UCSD)'};
      SU('S') = {'nm / m','strain'};
      SU('T') = {'sec','time'};
      SU('V') = {'nm / sec','velocity'};
      SU('W') = {'watts / m / m', 'insolation'};
      SU('a') = {'deg', 'azimuth'};
      SU('b') = {'bits/ sec', 'bit rate'};
      SU('c') = {'counts', 'dimensionless integer'};
      SU('d') = {'m', 'depth or height (e.g., water)'};
      SU('f') = {'micromoles / sec / m /m', 'photoactive radiation flux'};
      SU('h') = {'pH','hydrogen ion concentration'};
      SU('i') = {'amp','electric curent'};
      SU('m') = {'bitmap','dimensionless bitmap'};
      SU('n') = {'nanoradians','angle (tilt)'};
      SU('o') = {'mg/l','diliution of oxygen (Mark VanScoy)'};
      SU('p') = {'percent','percentage'};
      SU('r') = {'in','rainfall (UCSD)'};
      SU('s') = {'m / sec', 'speed (e.g., wind)'};
      SU('t') = {'C','temperature'};
      SU('u') = {'microsiemens/cm','conductivity'};
      SU('v') = {'volts','electric potential'};
      SU('w') = {'rad / sec', 'rotation rate'};
      SU('-') = {'null','null'};
   end
end