;======================== header ===============================================
;+
; NAME:
;   image_select
; PURPOSE:
;   FUNCTION that selects the good c2 and c3 files and delete the bad ones
;   it keeps a log file of the deleted + reason of refusal
; CATEGORY:
;   cactus ver 4.00 (paperA&A-version, May 2004)
; CALLING SEQUENCE:
;   (meant to be used in batchmode)
; INPUTS:
;
;
; KEYWORD PARAMETERS:
;   /logfile: to print diagnostics to a logfile (default=stdout=screen)
; OUTPUTS:
;
; COMMON BLOCKS:
;   none
; SIDE EFFECTS:
;   unknown
; RESTRICTIONS:
;   unknown
; PROCEDURE:
;
; CALLS:
;
;
; TO DO:
;
;  current version crashes when all tested files are bad
;
; MODIFICATION HISTORY:
;   2002/04/25, ver 1.00, David Berghmans
;               created on the basis of db_c2_select
;   2002/11/21, DB, now doing both c2 and c3
;   2004/01/23, DB & ER, ratio testing added
;                        bug corrected, now moving bad files to archdir/'telescope'/
;   2005/03/17, ER  Extra test on header.datap01 (first percentile of the image)
;   	    	    + rewrite log-file
;   2006/05/05 GRL Re-named cactus_select from image_select; cast 'properly' for SSW
;-
;======================== end header ===========================================
FUNCTION cactus_select, filelist, logfile=logfile, telescope
COMMON CACTUS_ENVS
 ; ------------------------ preamble ------------------------------------
 ; Who am I?
   myname   ='image_select'
 ; main input/output
;   cactusdir=getenv('CACTUSDIR')
;   archdir    = cactusdir+'/archive/'   ; the output directory were we put cleaned images
;   logdir      = cactusdir+'/log/'
   CASE (logfile) OF
	0 : log=-1
	1 :  openw, log, cactuslog+'cactus_log.txt' , /get_lun, /append
	2 :  openw, log, cactuslog+ myname+'_log.txt', /get_lun, /append
    ENDCASE
 ; log information
;   mylogfile=logdir+myname+'.log'
   ; if logfile keyword is set, open the logfile, if not send to screen:
;   IF keyword_set(logfile) THEN openw, log, mylogfile, /get_lun , /append $
;                           ELSE log=-1
 ; say hello
   printf, log, ' >>>> '+myname+'.pro started at '+ systime()
 ; sanity check
   IF (size(telescope))(1) ne 7 THEN BEGIN
      printf, log, 'Sanity check failure: telescope not properly defined'
      exit
   ENDIF
   IF (telescope ne 'c2') and (telescope ne 'c3') and $
      (telescope ne 'C2') and (telescope ne 'C3') THEN BEGIN
      printf, log, 'Sanity check failure: telescope unknown'
      printf, log, '***** telescope=', telescope,' ***************************************'
      exit
   ENDIF
   telescope=STRLOWCASE(telescope)
   printf, log, '***** telescope=', telescope,' ***************************************'
 ;------------------------- Selection criteria -------------------------------
   IF telescope eq 'c2' THEN BEGIN
     ; setting for checking type of image
      nx          = 1024     ; required x size of input images
      ny          = 1024     ; required y size of input images
      filt        ='Orange'  ; required filter wheel position of input images
      polar       ='Clear'   ; required polar wheel position of input images
      lp_num      ='Normal'  ; required lp_num
     ;setting for sanity check
      min_exptime = 25.0
            ; minimal exposure time required
	    ;  (lowest I have ever seen was a corrupted image with 17.0854)
      max_datazer = 186368
            ; maximum number of datazer
    	    ;
 	    ; note that missing blocks in itself do not hurt cactus but a large number of
	    ; missing blocks is an indication for possible corrupteness of the image
    	    ;
            ; (good  image has hdr.datazer EQ 62464 which is the
            ; equivalent of 61 blocks)
      min_dataavg = 2350
            ; minimal data average
   	    ; lowest 'good image' I have ever seen 2461
   	    ; highest 'corrupted image' I have ever seen 2317.65
      min_ratio   =   0.
            ; mimimal value for the ratio dataavg/exptime
	    ; unused
      max_ratio   =   9999.
            ; maximal value for the ratio dataavg/exptime
	    ; unused
      min_datap01 =   850.
      	    ; First percentile of the image is typically 918.
	    ; This extra test is to exclude images which have many
	    ; dark pixelc which are not black
   ENDIF; telescope eq 'c2'
   IF telescope eq 'c3' THEN BEGIN
       ; setting for checking type of image
      nx          = 1024     ; required x size of input images
      ny          = 1024     ; required y size of input images
      filt        ='Clear'   ; required filter wheel position of input images
      polar       ='Clear'   ; required polar wheel position of input images
      lp_num      ='Normal'  ; required lp_num
     ;setting for sanity check
      min_exptime = 18.0
            ; minimal exposure time required
	    ;  (lowest I have ever seen was a corrupted image with 17.0854)
      max_datazer = 186368
            ; maximum number of datazer
    	    ;
            ; note that missing blocks in itself do not hurt cactus but a large number of
	    ; missing blocks is an indication for possible corrupteness of the image
	    ;
	    ; (good  image has hdr.datazer EQ 101376 which is the
            ; equivalent of 99 blocks)
	    ; (120000 threshold removed 78 out of 4206 image in a test)
      min_dataavg = 1800
            ; minimal data average
   	    ; .....unused.....
   	    ; .....unused.....
      min_ratio   =   87.
            ; mimimal value for the ratio dataavg/exptime
	    ; (typical values are between 97 and 108)
      max_ratio   =   118.
            ; maximal value for the ratio dataavg/exptime
	    ; (typical values are between 97 and 108)
      min_datap01 =  365.
      	    ;
	    ; This extra test is to exclude images which have many
	    ; dark pixelc which are not black
    ENDIF; telescope eq 'c3'
    	 Printf, log,  'Quality is 1 if the following conditions are satisfied: '
	 printf, log, '   A : exptime > ',STRTRIM(min_exptime,2)
	 printf, log, '   B : datazer < ',STRTRIM(max_datazer,2)
	 printf, log, '   C : dataavg > ',STRTRIM(min_dataavg,2)
	 printf, log, '   D : dataavg/exptime in [',STRTRIM(min_ratio,2),',',STRTRIM(max_ratio,2),']'
	 printf, log, '   E : datap01 > ',STRTRIM(min_datap01,2)
         printf, log, 'Filename     |    date    |  time    ||Accept||Type |Quality|| A B C D E | '
    	 printf, log, '--------------------------------------------------------------------------'
  ;-----------------------------------------------------------------------------
  ; some variables
    maxnimages=n_elements(filelist) ; this is a max since there some files will be dropped
    good_names = ' ' ;  will contain the selected filenames
    nbad=0
    ntype=0
    nnormal=0
  ; loop over all candidate files
    FOR i= 0,maxnimages-1 DO BEGIN
      filename=filelist[i]
;      printf, log, 'filename   : ', filename
      a=lasco_readfits(filename,hdr, /NO_IMG,/SILENT)
      IF (DATATYPE(hdr) EQ 'STC') THEN BEGIN
       ; check if image is of the correct type
         type_ok = (hdr.naxis1 EQ nx)   and (hdr.naxis2 EQ ny)   and $
                   (hdr.filter EQ filt) and (hdr.polar EQ polar) and $
	           (hdr.lp_num EQ lp_num)
      ; calculate ratio ( of data average to exposure time)
         ratio = hdr.dataavg/hdr.exptime  ; ratio is Floating point number
      ; check if main parameters are within expectations
	 normal=(hdr.exptime GT min_exptime)  and $
	        (hdr.datazer LT max_datazer)  and $
	        (hdr.dataavg GT min_dataavg)  and $
	        (ratio GT min_ratio)  and (ratio LT max_ratio) and $
		(hdr.datap01 GT min_datap01)
       ; keep file or remove it
         IF (type_ok AND normal) THEN good_names= [good_names,filename] $
	                         ELSE BEGIN
;				  spawn, 'mv '+filelist[i]+' '+archdir+telescope+'/'
				  nbad=nbad+1
				  ntype=ntype+(1-type_ok*1)
				  nnormal=nnormal+(1-normal*1)
				 ENDELSE
       ; log info
;	 printf, log, '   obstime       :  '+hdr.date_obs+' '+hdr.time_obs
;	 printf, log, '   Accepted?     :', (type_ok AND normal)
;	 printf, log, '   type ok?      :', type_ok
;        printf, log, '   within specs? :', normal
;	 printf, log, '   exptime = ',(hdr.exptime GT min_exptime) , ' ',hdr.exptime
;	 printf, log, '   datazer = ',(hdr.datazer LT max_datazer) ,  ' ',hdr.datazer
;	 printf, log, '   dataavg = ',(hdr.dataavg GT min_dataavg) ,  ' ',hdr.dataavg
;	 printf, log, '   ratio   = ',(ratio GT min_ratio)   ,  ' ',ratio
;	 printf, log, '   ratio   = ',(ratio LT max_ratio)   ,  ' ',ratio
;	 printf, log, '   datap01 = ',(hdr.datap01 GT min_datap01) ,  ' ',hdr.datap01
         printf, log, hdr.filename,' | ',hdr.date_obs,' | ',STRMID(hdr.time_obs,0,8),' ||  '$
	            , STRTRIM((type_ok AND normal)*1,2),'   ||  ',STRTRIM((type_ok)*1,2),'  |   ',STRTRIM((normal)*1,2),'   || '$
		    , STRTRIM((hdr.exptime GT min_exptime)*1,2) ,' '$
		    , STRTRIM((hdr.datazer LT max_datazer)*1,2) ,' '$
		    , STRTRIM((hdr.dataavg GT min_dataavg)*1,2) ,' '$
		    , STRTRIM(((ratio GT min_ratio) AND (ratio LT max_ratio))*1,2),' ' $
		    , STRTRIM((hdr.datap01 GT min_datap01)*1,2) ,' |'
      ENDIF ELSE printf, log, ' fitsfile error'
  ENDFOR; i
    	 printf, log, '--------------------------------------------------------------------------'
         printf, log, 'Removed                              ||',STRMID(100000000+nbad,7,5),' ||',STRMID(100000000+ntype,7,5),'| ' $
	 ,STRMID(100000000+nnormal,7,5),' || '
  ; chop unused part of array
   if n_elements(good_names) eq 1 then undefine,filelist  else filelist   = good_names[1:*]
  ; ------------------------closing remarks-------------------------------
 ; say goodbye
   printf, log, ' <<<< '+myname+'.pro finished at '+ systime()
 ; close log file if needed
   IF keyword_set(logfile) THEN BEGIN
      close, log
      free_lun, log
   ENDIF
 ; ----------------------------------------------------------------------
 return, filelist
END