-----Original Message-----
From: Jim Sullivan [mailto:sullivan@cluster.bu.edu]
Sent: Tuesday, September 10, 2002 9:43 AM
To: Theodore Fritz
Subject:

I have automated the downloading of level 0 and its processing into BU
level 2 IDL savesets:

PRO GETPWG, CAMMICE=cammice, NOL2=nol2, ORBIT=orbit, ATTITUDE=attitude, $
 DEF=def, STOP=stop, CEPPAD=ceppad
;
; Fetch new POLAR ATTITUDE, ORBIT, CAMMICE, or CEPPAD level zero data from PWG
; Save level zero files locally and make level two savesets
;

lkw = KEYWORD_SET(attitude) OR KEYWORD_SET(cammice) OR KEYWORD_SET(ceppad) OR $
      KEYWORD_SET(orbit)

IF NOT lkw THEN BEGIN
  PRINT,'GETPWG:      Fetch POLAR (compressed) level zero data from PWGdata'
  PRINT,'             and make BU level two savesets'
  PRINT,' Arguments:'
  PRINT,' Keywords:   Only one data set may be selected at at time'
  PRINT,'  ATTITUDE   If set, fetch (predict) attitude data (cdf files)'
  PRINT,'  CAMMICE    If set, fetch CAMMICE level zero data'
  PRINT,'  CEPPAD     If set, fetch CEPPAD level zero data (D)'
  PRINT,'  DEF        If set, fetch definitive orbit/attitude data'
  PRINT,'  NOL2       If set, do not make level two savesets'
  PRINT,'  ORBIT      If set, fetch (predict) orbit data (cdf files)'
  PRINT,' Environment variables:'
  PRINT,'  CAML2PATH  CAMMICE level two path  '+GETENV('CAML2PATH')
  PRINT,'  CAMLZPATH  CAMMICE level zero path '+GETENV('CAMLZPATH')
  PRINT,'  CEPL2PATH  CEPPAD level two path   '+GETENV('CEPL2PATH')
  PRINT,'  CEPLZPATH  CEPPAD level zero path  '+GETENV('CEPLZPATH')
  PRINT,'  ORATPATH   Orbit/attitude path     '+GETENV('ORATPATH')
  PRINT,''
  RETURN
 ENDIF

; Get list of files currently on hand
orat = 0
clzpath = GETENV('CEPLZPATH')
IF KEYWORD_SET(cammice) THEN clzpath=GETENV('CAMLZPATH')
IF KEYWORD_SET(orbit) OR KEYWORD_SET(attitude) THEN BEGIN
  orat = 1
  ; Assumes only one directory in orbit/attitude path
  clzpath = GETENV('ORATPATH')
  IF KEYWORD_SET(def) THEN clzpath=clzpath+'/def' ELSE clzpath=clzpath+'/pre'
  IF KEYWORD_SET(orbit) THEN clzpath=clzpath+'/or' ELSE clzpath=clzpath+'/at'
 ENDIF
IF KEYWORD_SET(ceppad) THEN clzpath=GETENV('CEPLZPATH')
lzpath = STR_SEP(clzpath,':')
; get full list of local files (excluding sfdu files)
kkcnt = 0L
FOR i=0,N_ELEMENTS(lzpath)-1 DO BEGIN
  SPAWN,'\ls -l '+lzpath[i]+'/*',list,COUNT=lcnt,/SH
  kcnt = 0L
  FOR j=0L,lcnt-1 DO BEGIN
    indx = STRPOS(list[j],'sfdu')
    IF orat EQ 1 THEN jndx=STRPOS(list[j],'.dat') ELSE jndx=-1
   IF indx EQ -1 AND jndx EQ -1 AND $
     N_ELEMENTS(STR_SEP(STRCOMPRESS(list[j]),' ')) EQ 9 THEN BEGIN
      fname = STR_SEP(list[j],'/')
      IF kcnt EQ 0 THEN $
       filename=fname[N_ELEMENTS(fname)-1] ELSE $
       filename=[filename,fname[N_ELEMENTS(fname)-1]] 
      kcnt = kcnt + 1L
     ENDIF 
   ENDFOR
  IF kcnt GT 0 THEN IF kkcnt EQ 0 THEN $
   datafile=filename ELSE datafile=[datafile,filename]
  kkcnt = kkcnt + kcnt
  IF N_ELEMENTS(lzpath) GT 1 THEN PRINT,lzpath[i],kcnt
 ENDFOR
; datafile contains the (nonprefixed) file names already held
IF N_ELEMENTS(lzpath) EQ 1 THEN $
 PRINT,kkcnt,' local LZ files in '+lzpath[0] ELSE $
 PRINT,'Total number of local LZ files',N_ELEMENTS(datafile)

IF kkcnt EQ 0 THEN GOTO,Return

; Check PWG ground system over all years since launch for new data files
CALDAT,SYSTIME(/JULIAN),moy,dom,curyear,hod,moh,som
; fetch year by year to mimic PWG storage scheme
FOR year=1996L,LONG(curyear) DO BEGIN
  ; nfile is the number of new (not locally available) files for this year
  nfile = 0L
  ; prepare an ftp command file to get the list of files
  OPENW,lun,/GET_LUN,'/usr/local/home/data/pwgftp.cmd'
  PRINTF,lun,'open pwgdata.gsfc.nasa.gov'
  ; the file .netrc contains all login information
  PRINTF,lun,'cd pub/compressed/po'
  IF KEYWORD_SET(cammice) THEN BEGIN
    PRINTF,lun,'cd cam/lz'
   ENDIF ELSE IF orat EQ 1 THEN BEGIN
    IF KEYWORD_SET(def) THEN PRINTF,lun,'cd def' ELSE PRINTF,lun,'cd pre' 
    IF KEYWORD_SET(orbit) THEN PRINTF,lun,'cd or' ELSE PRINTF,lun,'cd at' 
   ENDIF ELSE BEGIN
    PRINTF,lun,'cd cep/lz'
   ENDELSE
  PRINTF,lun,'cd '+STRING(year,FORMAT='(I4.4)')
  PRINTF,lun,'dir'
  PRINTF,lun,'quit'
  CLOSE,lun
  FREE_LUN,lun
  cmd = '/usr/bin/ftp < /usr/local/home/data/pwgftp.cmd'
  Spawn,cmd,dirlist,COUNT=cnt,/SH
  ; Scan for new files
  FOR i=0L,cnt-1L DO BEGIN
    inline = STRCOMPRESS(dirlist[i])
    list = STR_SEP(inline,' ')
    IF N_ELEMENTS(list) NE 9 THEN GOTO,Skip
    lastfile = list[8]
    ; check file name against current list tabulate new files
    ;  note version numbers are within the file names
    ;  so don't need to check creation dates or checksums
    ;  ORBIT/ATTITUDE files stored ungzipped locally so remove .gz sufffix
    IF orat EQ 1 THEN BEGIN
      indx = STRPOS(list[8],'cdf')
      IF indx EQ -1 THEN GOTO,Skip
      tlist = STRMID(list[8],0,STRPOS(list[8],'.gz')) 
     ENDIF ELSE tlist=list[8]
    indx = WHERE(tlist EQ datafile,icnt)
    IF icnt EQ 0 AND STRLEN(list[8]) GT 26 THEN BEGIN
      IF nfile EQ 0 THEN filelist=list[8] ELSE filelist=[filelist,list[8]]
      nfile = nfile + 1L 
    ENDIF
   Skip:
   ENDFOR
  ; log year, number of files, number of new files, and last file name
  PRINT,year,cnt,nfile,'  '+lastfile
  ; prepare an ftp command file to get the new files for this year
  IF nfile GT 0 THEN BEGIN
    OPENW,lun,/GET_LUN,'/usr/local/home/data/pwgftp.cmd'
    PRINTF,lun,'open pwgdata.gsfc.nasa.gov'
    PRINTF,lun,'cd pub/compressed/po'
    IF KEYWORD_SET(cammice) THEN BEGIN
      PRINTF,lun,'cd cam/lz'
     ENDIF ELSE IF orat EQ 1 THEN BEGIN
      IF KEYWORD_SET(def) THEN PRINTF,lun,'cd def' ELSE PRINTF,lun,'cd pre' 
      IF KEYWORD_SET(orbit) THEN PRINTF,lun,'cd or' ELSE PRINTF,lun,'cd at' 
     ENDIF ELSE BEGIN
      PRINTF,lun,'cd pub/compressed/po/cep/lz'
     ENDELSE
    PRINTF,lun,'cd '+STRING(year,FORMAT='(I4.4)')
    PRINTF,lun,'binary'
    ; stage data through local disk /RAID4
    PRINTF,lun,'lcd /RAID4'
    FOR j=0L,nfile-1 DO PRINTF,lun,'get '+filelist[j]
    PRINTF,lun,'quit'
    CLOSE,lun
    FREE_LUN,lun
    cmd = '/usr/bin/ftp < /usr/local/home/data/pwgftp.cmd'
    Spawn,cmd,result,COUNT=cnt,/SH
    FOR k=0,cnt-1 DO PRINT,result[k]
    ; files contains the list of all new files downloaded
    IF N_ELEMENTS(files) EQ 0 THEN files=filelist ELSE files=[files,filelist]
   ENDIF
 ENDFOR
SPAWN,'\rm /usr/local/home/data/pwgftp.cmd',result,COUNT=cnt,/SH
FOR k=0,cnt-1 DO PRINT,result[k]

; if there are new files process (into BU level-two savesets)
IF N_ELEMENTS(files) GT 0 THEN BEGIN
  IF KEYWORD_SET(cammice) THEN BEGIN
    ; CAMMICE
    SPAWN,'\mv /RAID4/po_*cam*gz '+lzpath[0],result,COUNT=cnt,/SH
    FOR k=0,cnt-1 DO PRINT,result[k]
    caml2path = STR_SEP(GETENV('CAML2PATH'),':')
    FOR k=0L,N_ELEMENTS(files)-1 DO BEGIN
      list = STR_SEP(files[k],'_')
      READS,list[3],yy,mm,dd,FORMAT='(I4,2I2)'
      ydoy = Ydoyymd([yy,mm,dd])
      IF NOT KEYWORD_SET(nol2) THEN $
       Makemicsl2,yy,ydoy(1),ydoy(1),PATH=caml2path[0]
     ENDFOR
   ENDIF ELSE IF orat EQ 1 THEN BEGIN
    ; ORBIT/ATTITUDE
    SPAWN,'gunzip /RAID4/po_*cdf.gz',result,COUNT=cnt,/SH
    FOR k=0L,cnt-1L DO PRINT,result[k]
    SPAWN,'\mv /RAID4/po_*cdf '+lzpath[0],result,COUNT=cnt,/SH
    FOR k=0L,cnt-1L DO PRINT,result[k]
    cepl2path = STR_SEP(GETENV('CEPL2PATH'),':')
    FOR k=0L,N_ELEMENTS(files)-1L DO BEGIN
      list = STR_SEP(files[k],'_')
      READS,list[3],yy,mm,dd,FORMAT='(I4,2I2)'
      IF NOT KEYWORD_SET(nol2) THEN $
       Makepapco,[yy,mm,dd],[yy,mm,dd],LOG=75,/LZ,PATH=cepl2path[0]
     ENDFOR
   ENDIF ELSE BEGIN
    ; CEPPAD
    SPAWN,'gunzip /RAID4/po_*cep*gz',result,COUNT=cnt,/SH
    FOR k=0,cnt-1 DO PRINT,result[k]
    SETENV,'CEPLZPATH=/RAID4'
    cepl2path = STR_SEP(GETENV('CEPL2PATH'),':')
    FOR k=0L,N_ELEMENTS(files)-1 DO BEGIN
      list = STR_SEP(files[k],'_')
      READS,list[3],yy,mm,dd,FORMAT='(I4,2I2)'
      IF NOT KEYWORD_SET(nol2) THEN $
       Makepapco,[yy,mm,dd],[yy,mm,dd],LOG=75,/LZ,PATH=cepl2path[0]
     ENDFOR
    SETENV,'CEPLZPATH='+clzpath
    SPAWN,'gzip /RAID4/po_*cep*',result,COUNT=cnt,/SH
    FOR k=0,cnt-1 DO PRINT,result[k]
    SPAWN,'\mv /RAID4/po_*cep*gz '+lzpath[0],result,COUNT=cnt,/SH
    FOR k=0,cnt-1 DO PRINT,result[k]
   ENDELSE
 ENDIF

Return:
IF KEYWORD_SET(stop) THEN STOP

RETURN
END