From: post@cs.cornell.edu (David H. Post)
Subject: Keyword options added to waisindex
Keywords: WAIS keyword indexing waisindex non-text keywords keyword_file
Organization: Design Research Institute, Cornell University, Ithaca, NY, USA
Date: Wed, 29 Jul 1992 20:42:07 GMT

We've added two options to waisindex for supplying keywords to be indexed
from the command line or a file.  This is very useful for indexing files 
that are not plain text.  From the new man page:

     -keywords "<string>"
               Index (also) the words in <string> for each  docu-
               ment.   This is especially useful with -nocontents
               for indexing graphics images  and  other  non-text
               documents.

     -keyword_file <filename>
               Index (also) the words in the specified file,  for
               each  document.   This  can be used to index files
               that need to be converted  to  plain  text  first.

               For example (-w 132 minimizes line breaks):

                    dvi2tty -w 132 foo.dvi >/tmp/foo.ascii
                    waisindex -d foobase -T DVI   \
                              -keyword_file /tmp/foo.ascii  \
                              -nocontents foo.dvi


The modifications needed are small, and confined to two files, irbuild.c
and irtfiles.c.  Our mods are based on Release 8 b5.  They will probably
appear in the next release from think.com.  Until then, modified sources,
diffs, and man page are available for anonymous ftp from the directory

    wheat.tc.cornell.edu:pub/wais/wais-8-b5-dri-1 

A compressed tar file, wais-8-b5-dri-1.tar.Z (53KB), contains all files.
The diffs are appended to this article.

Let us know if you find these useful.

 --David H Post, post@cs.cornell.edu
   Design Research Institute, Cornell University, Ithaca, New York, USA



diff -c2 irbuild.c.original irbuild.c
*** irbuild.c.original	Thu May 14 19:41:38 1992
--- irbuild.c	Tue Jul 28 15:46:37 1992
***************
*** 16,19 ****
--- 16,24 ----
   */
  
+ 
+ /* Changes by David H Post, Design Research Institute, post@cs.cornell.edu
+  * 92/06/29  Added -keywords and -keyword_file options (see irtfiles.c)
+  */
+ 
  /* Change log:
   * added -stdio option from jik@athena.mit.edu
***************
*** 99,103 ****
  #include "irext.h"
  
! #define INDEXER_DATE "Sun May 10 1992"
  
  /* for reporting errors, in WAIStation it is defined in CRetrievalApp.c */
--- 104,108 ----
  #include "irext.h"
  
! #define INDEXER_DATE "29 Jun 1992"
  
  /* for reporting errors, in WAIStation it is defined in CRetrievalApp.c */
***************
*** 128,131 ****
--- 133,138 ----
    fprintf(stderr,"          [-cmmem  mem%] /* percent of CM memory (CM code only) */\n");
    fprintf(stderr,"          [-T  type] /* type becomes the \"TYPE\" of the document. */\n");
+   fprintf(stderr,"          [-keywords \"<string>\"] /* Keywords to index for each document. */\n");
+   fprintf(stderr,"          [-keyword_file <filename>] /* File of keywords to index. */\n");
    fprintf(stderr,"          [-t	/* format of the file. if none then each file is a document */\n");
    fprintf(stderr,"             text /* simple text files, this is the default */\n");
***************
*** 167,170 ****
--- 174,180 ----
  extern boolean index_contents;
  
+ extern char* keywords;           /* used in irtfiles.c */
+ extern char* keyword_filename;   /* used in irtfiles.c */
+ 
  /* This is the MAIN for building an index.
   */
***************
*** 255,259 ****
      }
      else if(0 == strcmp("-v", next_argument)){
!       fprintf(stderr,"%s: %s\n", command_name, VERSION, INDEXER_DATE);
      }
      else if (0 == strcmp("-stdin", next_argument)) {
--- 265,269 ----
      }
      else if(0 == strcmp("-v", next_argument)){
!       fprintf(stderr,"%s: %s, %s\n", command_name, VERSION, INDEXER_DATE);
      }
      else if (0 == strcmp("-stdin", next_argument)) {
***************
*** 331,334 ****
--- 341,354 ----
        type = next_argument;
        finish_header_function = filename_finish_header_function;
+     }
+     else if(0 == strcmp("-keywords", next_argument)){
+       if(NULL == (next_argument = next_arg(&argc, &argv)))
+ 	panic("Expected -keywords argument string");
+       keywords = next_argument;
+     }
+     else if(0 == strcmp("-keyword_file", next_argument)){
+       if(NULL == (next_argument = next_arg(&argc, &argv)))
+ 	panic("Expected -keyword_file filename");
+       keyword_filename = next_argument;
      }
      else if(0 == strcmp("-contents", next_argument)){



diff -c2 irtfiles.c.original irtfiles.c
*** irtfiles.c.original	Wed May  6 20:31:33 1992
--- irtfiles.c	Mon Jun 29 15:06:09 1992
***************
*** 6,9 ****
--- 6,13 ----
  */
  
+ /* Changes by David H Post, Design Research Institute, post@cs.cornell.edu
+  * 92/06/29  Added -keywords and -keyword_file options (see irbuild.c)
+  */
+ 
  /* Change log:
   * $Log:	irtfiles.c,v $
***************
*** 66,69 ****
--- 70,74 ----
  #define MAX_LINE_LENGTH 1000 /* characters */
  #define extra_weight_for_header 10
+ #define keyword_weight 1
  
  #ifdef UNIX
***************
*** 86,89 ****
--- 91,100 ----
  int  current_filecount = 0;
  
+ /* keywords from command line (set in irbuild.c), used in finish_document */
+ char* keywords = NULL;
+ 
+ /* name of keyword file from command line, used in finish_document */
+ char* keyword_filename = NULL;
+ 
  boolean index_contents = true;
  
***************
*** 487,495 ****
  database* db;
  boolean word_position, word_pairs;
  { long line_length;
    boolean newline_terminated;
    if(0 != strlen(header)){
      /* add weights for the header (if there was one) */
!     long number_of_words =
        map_over_words(header, document_id, 
  		     extra_weight_for_header, 
--- 498,509 ----
  database* db;
  boolean word_position, word_pairs;
+ /* Uses global variables 'keywords' and 'keyword_filename', set in irbuild.c */
  { long line_length;
    boolean newline_terminated;
+   long number_of_words;
+ 
    if(0 != strlen(header)){
      /* add weights for the header (if there was one) */
!     number_of_words =
        map_over_words(header, document_id, 
  		     extra_weight_for_header, 
***************
*** 507,510 ****
--- 521,570 ----
    }
  
+   if(keywords != NULL){
+     /* add keywords from command line to index (if there are any) */
+     number_of_words =
+       map_over_words(keywords, document_id, 
+ 		     keyword_weight, 
+ 		     0,
+ 		     &line_length, 
+ 		     &newline_terminated,
+ 		     db,
+ 		     add_word_before_pairs,
+ 		     word_position, word_pairs);
+     if(number_of_words == -1)
+       waislog(WLOG_HIGH, WLOG_ERROR, "map_over_words failed");
+     db->total_word_count += number_of_words;
+     the_document_table_entry->document_length += number_of_words;
+   }
+ 
+   if(keyword_filename != NULL){
+     /* add keywords from keyword file (if specified on command line) */
+     FILE* keyword_stream = s_fopen(keyword_filename, "r");
+     char line[MAX_LINE_LENGTH];
+     if(NULL == keyword_stream)
+       waislog(WLOG_HIGH, WLOG_ERROR, 
+ 	      "Unable to open keyword file %s", keyword_filename);
+     else  /* read keyword_file, index its contents */
+       while(TRUE){
+ 	/* read a line */
+ 	if( !fgets(line, MAX_LINE_LENGTH, keyword_stream) )
+ 	  break;  /* eof */
+ 	number_of_words =
+ 	  map_over_words(line, document_id, 
+ 			 keyword_weight, 
+ 			 0,
+ 			 &line_length, 
+ 			 &newline_terminated,
+ 			 db,
+ 			 add_word_before_pairs,
+ 			 word_position, word_pairs);
+ 	if(number_of_words == -1)
+ 	  waislog(WLOG_HIGH, WLOG_ERROR, "map_over_words failed");
+ 	db->total_word_count += number_of_words;
+ 	the_document_table_entry->document_length += number_of_words;
+       }
+     s_fclose(keyword_stream);
+   }
+ 
    /* store out the document header here */
    the_document_table_entry->headline_id = 
***************
*** 658,661 ****
--- 718,723 ----
     *  then it is checked to see if it is up-to-date. (it does not 
     *  kill the old entry (maybe it should)).
+    * Words in global string 'keywords' and from file named in global string
+    *  'keyword_filename' (set in irbuild.c) are added to index like header.
     */



diff -c2 version.h.original version.h
*** version.h.original	Sun Mar  1 01:58:31 1992
--- version.h	Tue Jul  7 14:57:31 1992
***************
*** 11,18 ****
   */
  
  #ifndef VERSION_H
  #define VERSION_H
  
! #define VERSION "WAIS release 8 b5"
  
  #endif /* ndef VERSION_H */
--- 11,24 ----
   */
  
+ 
+ /* Changes by David H Post, Design Research Institute, post@cs.cornell.edu
+  * 92/06/29  Added -keywords and -keyword_file options (see irtfiles.c)
+  */
+ 
+ 
  #ifndef VERSION_H
  #define VERSION_H
  
! #define VERSION "WAIS rel. 8 b5 DRI 1"
  
  #endif /* ndef VERSION_H */



Newsgroups: comp.infosystems.wais