Task: Text processing_2







Python: import re
import zipfile
import StringIO

def munge2(readings):

   datePat = re.compile(r'\d{4}-\d{2}-\d{2}')
   valuPat = re.compile(r'[-+]?\d+\.\d+')
   statPat = re.compile(r'-?\d+')
   allOk, totalLines = 0, 0
   datestamps = set([])
   for line in readings:
      totalLines += 1
      fields = line.split('\t')
      date = fields[0]
      pairs = [(fields[i],fields[i+1]) for i in range(1,len(fields),2)]

      lineFormatOk = datePat.match(date) and \
         all( valuPat.match(p[0]) for p in pairs ) and \
         all( statPat.match(p[1]) for p in pairs )
      if not lineFormatOk:
         print 'Bad formatting', line
         continue
		
      if len(pairs)!=24 or any( int(p[1]) < 1 for p in pairs ):
         print 'Missing values', line
         continue

      if date in datestamps:
         print 'Duplicate datestamp', line
         continue
      datestamps.add(date)
      allOk += 1

   print 'Lines with all readings: ', allOk
   print 'Total records: ', totalLines



readings = open('readings.txt','r')
munge2(readings)




Fortran: Crunches a set of hourly data. Starts with a date, then 24 pairs of value,indicator for that day, on one line.
      INTEGER Y,M,D		
      INTEGER GOOD(24,2)	
      REAL*8     V(24,2)	
      CHARACTER*10 DATE(2)	
      INTEGER IT,TI		
      INTEGER NV		
      INTEGER I,NREC,HIC	
      LOGICAL INGOOD		
      INTEGER IN,MSG		
      CHARACTER*666 ACARD	
      IN = 10		
      MSG = 6		
      OPEN (IN,FILE="Readings1.txt", FORM="FORMATTED",	
     1 STATUS ="OLD",ACTION="READ")			
      NV = 0		
      NREC = 0		
      HIC = 0		
      DATE = "snargle"	
      IT = 2		
Chew into the file.
   10 READ (IN,11,END=100,ERR=666) L,ACARD(1:MIN(L,LEN(ACARD)))	
      NREC = NREC + 1		
   11 FORMAT (Q,A)		
      READ (ACARD,12,END=600,ERR=601) Y,M,D	
   12 FORMAT (I4,2(1X,I2))				
      TI = IT			
      IT = 3 - IT		
      DATE(IT) = ACARD(1:10)	
      READ (ACARD(11:L),*,END=600,ERR=601) (V(I,IT),GOOD(I,IT),I = 1,24)	
Comparisons. Should really convert the date to a daynumber, check it by reversion, and then check for + 1 day only.
   20 IF (DATE(IT).EQ.DATE(TI)) THEN	
        IF (ALL(V(:,IT)   .EQ.V(:,TI)) .AND.	
     1      ALL(GOOD(:,IT).EQ.GOOD(:,TI))) THEN	
          WRITE (MSG,21) NREC,DATE(IT),"same."	
   21     FORMAT ("Record",I8," Duplicate date field (",A,"), data ",A)	
         ELSE				
          WRITE (MSG,21) NREC,DATE(IT),"different
        END IF					
      END IF				
      IF (ALL(GOOD(:,IT).GT.0)) NV = NV + 1	
      GO TO 10		

Complaints. Should really distinguish between trouble in the date part and in the data part.
  600 WRITE (MSG,*) '"END" declared - insufficient data?'	
      GO TO 602				
  601 WRITE (MSG,*) '"ERR" declared - improper number format?'	
  602 WRITE (MSG,603) NREC,L,ACARD(1:L)	
  603 FORMAT("Record",I8,", length ",I0," reads ",A)	
      HIC = HIC + 1			
      IF (HIC.LE.12) GO TO 10		
      STOP "Enough distaste."		
  666 WRITE (MSG,101) NREC,"format error
      GO TO 900				

Closedown.
  100 WRITE (MSG,101) NREC,"then end-of-file"	
  101 FORMAT ("Record",I8,": ",A)		
      WRITE (MSG,102) NV	
  102 FORMAT ("  with",I8," having all values good.")	
  900 CLOSE(IN)		
      END