From: jonathan@san-andreas.think.com (Jonathan Goldman)
Newsgroups: comp.infosystems.wais
Subject: Re: Compressed-text type?
Date: 12 Aug 92 14:00:02
Organization: Thinking Machines Corporation, Cambridge MA, USA
NNTP-Posting-Host: san-andreas.quake.think.com
In-reply-to: jason@jazz.cnd.hp.com's message of Wed, 5 Aug 1992 23:51:25 GMT

In article <JASON.92Aug5175124@jazz.cnd.hp.com> jason@jazz.cnd.hp.com (Jason Zions) writes:

   From: jason@jazz.cnd.hp.com (Jason Zions)
   Date: Wed, 5 Aug 1992 23:51:25 GMT

   Has anyone done work on WAIS to permit it to index a large set of compressed
   text files? The idea would be for waisindex to uncompress the text files as
   it read them in; also, when the waisserver sent the file to the client, it
   would send it in compressed form, and the client would uncompress it.

I just did this to the server.  It's quite easy.

   How would I go about doing this?

I had to change two funtions, one in irtfiles.c and the other in futil.c.

In irtfiles.c, replace index_text_file with this:

#define UNCOMPRESSOR "/usr/ucb/zcat"

void index_text_file(filename,
		     separator_function,
		     header_function,
		     date_function,
		     finish_header_function, 
		     type,
		     db,
		     check_for_text_file,
		     check_for_file_already_indexed,
		     word_position, word_pairs)
char* filename;
boolfunc *separator_function;
voidfunc *header_function;
longfunc *date_function;
voidfunc *finish_header_function;
char *type;
database* db;
boolean check_for_text_file;
boolean check_for_file_already_indexed;
boolean word_position, word_pairs;
{
  /* Addes words to the index for a given file.  
   * The function arguments can be NULL which means it would 
   *  always answer NULL.  
   * separator_function is called on every line to see if it 
   *  separates documents.  
   * header_function is called on every line so that a headline 
   *  can be accumulated.  This assumes that it will side effect global 
   *  variables.
   * finish_header_function is called when the document is finished 
   *  (by separator function responding TRUE or EOF) this will return 
   *  the headline string or NULL. 
   *  Presumably finish_header_function will use the
   *  effects of header_function.  finish_header_function 
   *  will only be called once, so it should clear whatever state 
   *  header_function has set.
   * if check_for_text_file then it looks to see if first character
   *  in the file is a printable character.
   * if check_for_file_already_indexed then it looks through the filename 
   *  file to see if the file has not been indexed.  If it has,
   *  then it is checked to see if it is up-to-date. (it does not 
   *  kill the old entry (maybe it should)).
   */

  long filename_id;
  document_table_entry the_document_table_entry;
  long document_id = next_document_id(db);
  FILE* input_stream;
  long file_position_before_line = 0;
  long file_position_before_document = 0;
  long date;
#if (defined(NeXT) || defined(Mach))    
  char tmpFileName[MAX_FILENAME_LEN+1];
#else
  char *tmpFileName = NULL;
#endif				/* NeXT or Mach */


  if(filename != NULL &&
     strlen(filename) > 1 &&
     !strcmp(filename+(strlen(filename)-2), ".Z"))
    /* it's a .Z file.  First, remove the suffix or many things get confused. */
    filename[(strlen(filename)-2)] = 0;
    
  if(probe_file(filename)) {
    input_stream = s_fopen(filename, "r");
  }
  else {
    if(probe_file_possibly_compressed(filename)) {
      char buffer[ 2 * MAX_FILENAME_LEN + 10 ];
#if (defined(NeXT) || defined(Mach))
      tmpnam(tmpFileName);
#else
      tmpFileName = tempnam( "/tmp/", 0 );
#endif				/* NeXT or Mach */
      sprintf(buffer, "%s %s >%s", UNCOMPRESSOR, filename, tmpFileName);
      system(buffer);
      input_stream = s_fopen(tmpFileName,"r");
      unlink (tmpFileName);
    }
  }

  if (NULL == input_stream)

    
    if (NULL == input_stream) { 
      waislog(WLOG_HIGH, WLOG_ERROR, 

	      "File %s does not exist", filename);
      /* then the is not a valid file to be indexed */
      return;
    }
  if(check_for_file_already_indexed){
    time_t time;
    char full_path[MAX_FILENAME_LEN];
    truename(filename, full_path);
    if(true == filename_in_database(full_path, type, &time, db)){
      /* check that it is the same time as this file */
      if(time == file_write_date(filename)){
	waislog(WLOG_HIGH, WLOG_INDEX, 
		"File %s already indexed", filename);
	s_fclose(input_stream);
	return;
      }
    }
  }
    
  /* Make the current filename accessible via global variables.
   * Increment current_filecount so routines can efficiently detect
   * changes in the current file.
   * -- Prentiss Riddle, Rice ONCS, riddle@rice.edu, 5/6/92
   */

  if(current_filename == NULL) current_filename = s_malloc(MAX_FILENAME_LEN+1);

  strncpy(current_filename, filename, MAX_FILENAME_LEN);
  current_filecount++;

  if(check_for_text_file){ 
    /* if we need this to be a text file, check the first character
       for a printable character */
    long ch = fgetc(input_stream);
    /* printf("First character is '%c'\n", ch); */
    if(EOF == ch || (!isprint(ch) && !isspace(ch))){
      s_fclose(input_stream);
      return;
    }
    ungetc(ch, input_stream);
  }
  
  /* write out the filename */
  filename_id = write_filename_table_entry(filename, type, db);
  
  /*  (if (not *drop_table*) (make_drop_table)) maybe put in later */
  
  header_flag_1 = NULL;
  the_document_table_entry.filename_id = filename_id;
  the_document_table_entry.start_character = 0;
  the_document_table_entry.document_length = 0;
  the_document_table_entry.number_of_lines = 0;
  the_document_table_entry.date = 0;

  while(TRUE){
    long line_length;
    boolean newline_terminated;
    char line[MAX_LINE_LENGTH];
    char header[MAX_LINE_LENGTH];
    char* read_line_result;
    boolean eof;

    /* printf("ftell: %ld\n", ftell(input_stream)); */
    /* read a line */
    read_line_result  = fgets(line, MAX_LINE_LENGTH, input_stream);
    beFriendly();
 	
    /* eof = feof(input_stream); */ /* zero means not eof */
    eof	= !read_line_result; 
    
    the_document_table_entry.number_of_lines++;

    header[0] = '\0';		/* set it to the empty string */
    if(eof ||
       ((NULL != separator_function) &&
	separator_function(line))){
  
      /* we are processing a separator, therefore we should
       * finish off the last document, and start a new one
       */
      if(NULL != finish_header_function){
	finish_header_function(header);
      }
      if(0 == strlen(header)){
	char full_path[1000];
	char directory[1000];
	truename(filename, full_path);
	sprintf(header, "%s   %s", pathname_name(full_path),
		pathname_directory(full_path, directory));
      }
      if(the_document_table_entry.date == 0) { /* set date from file */
	struct tm *ftime;
	time_t file_time = file_write_date(filename);
	if(file_time != 0) {
	  if((ftime = localtime(&file_time)) != NULL) {
	    the_document_table_entry.date = 
	      ftime->tm_year*10000 + (ftime->tm_mon+1)*100 + ftime->tm_mday;
	  }
	}
      }

      the_document_table_entry.number_of_lines--; /* dont count separator */
      /* finish off the last */
      finish_document(header, -1, -1, 
		      line, document_id, &the_document_table_entry,
		      eof?	/* if EOF, use file length */
		      file_length(input_stream):file_position_before_line, 
		      file_position_before_document,
		      db, word_position, word_pairs);
      /* initialize the next one */
      the_document_table_entry.filename_id = filename_id;
      the_document_table_entry.start_character = file_position_before_line;
      the_document_table_entry.number_of_lines = 1; /* count separator */
      the_document_table_entry.date = 0;
      file_position_before_document = file_position_before_line;

      document_id = next_document_id(db);	

      if(!eof)
	{			/* not EOF */
	  if(NULL != header_function){
	    header_function(line);
	  }
	  if (date_function != NULL && 
	      (date = date_function(line)) > 0)
	    the_document_table_entry.date = date;
	  line_length = strlen(line);
	  newline_terminated = true;
	}
      else{			/* EOF */
	/* printf("closing the file\n"); */
	s_fclose(input_stream);
	return;
      }
    }
        
    else{		       
      /* not a separator or EOF so process the line */
      long number_of_words;
      if(NULL != header_function) header_function(line);
      if (date_function != NULL && 
	  the_document_table_entry.date == 0 &&
	  (date = date_function(line)) > 0) 
	the_document_table_entry.date = date;


      if(index_contents ) {
	if( _indexable_section) {
	  number_of_words = map_over_words(line, document_id, 1L, 
					   file_position_before_line -
					   file_position_before_document,
					   -1,
					   &line_length, 
					   &newline_terminated,
					   db,	
					   add_word_before_pairs,
					   word_position, 
					   (word_pairs & _form_pairs_section),
					   NULL);
	  if(number_of_words == -1)
	    waislog(WLOG_HIGH, WLOG_ERROR, "map_over_words failed");
	  the_document_table_entry.document_length += number_of_words;
	  len_of_files_since_last_delete += number_of_words;
	  len_of_files_since_last_flush += number_of_words;
	  db->total_word_count += number_of_words;
	}
	else
	  newline_terminated = 0;
      }
    }
    if(newline_terminated)
      file_position_before_line += (line_length + 
				    LENGTH_OF_NEWLINE /* in case of crlf */
				    - 1 /* fgets gets one newline */
				    );
    else
      file_position_before_line = ftell(input_stream);

    
    /* for debugging
       if(file_position_before_line != ftell(input_stream)) {
       waislog(WLOG_LOW, WLOG_INFO, "ftell: %ld, computed ftell: %ld", 
       ftell(input_stream),
       file_position_before_line);
       }
       */	

  }
}

and in futil.c, replace file_write_date with this:

time_t file_write_date(filename)
char* filename;
{ /* os dependent */
#ifdef THINK_C
  return((time_t)0);  /* not implemented yet */
#else
  struct stat *buf = (struct stat*)s_malloc((size_t)sizeof(struct stat));
  time_t mtime;
	
  if(0 != stat(filename, buf)) {
    /* might be the compressed one. */
    char buffer[MAX_FILENAME_LEN+2];
    sprintf(buffer, "%s.Z", filename);
    if (0 != stat(buffer, buf)) {
      panic("could not stat %s", filename);
    }
  }
  mtime =  buf->st_mtime;
  s_free(buf);
  return(mtime);
#endif /* THINK_C */
}
--
Jonny Goldman					Thinking Machines Corporation
jonathan@Think.com				1010 El Camino Real Suite 310
Plebe						Menlo Park, CA 94025
Wide Area Information Servers			415-329-9300x229
Newsgroups: comp.infosystems.wais,comp.lang.postscript,alt.security