From: jik@athena.mit.edu (Jonathan I. Kamens)
Subject: ENHANCEMENT: (8 b5) run-time stop-words and stop-files
Organization: Massachusetts Institute of Technology
Date: Thu, 13 Aug 1992 22:14:55 GMT

This patch modifies the indexer so that the "-stopword" option can be
used to specify an extra word for the stoplist, and "-stopfile" can be
used to specify a list of words for the stoplist.  These words are
added to the built-in stoplist.

I have several databases which have to be rebult regularly, when they
are significantly modified, which have a lot of words in them which I
know are going to have too many entries and therefore be eliminated.
Being able to prevent those words from being indexed gets rid of all
the messages telling me they're being eliminated, and I assume makes
the indexing process a bit faster....

*** 1.47	1992/05/10 14:48:17
--- irbuild.c	1992/08/13 01:21:56
***************
*** 97,102 ****
--- 97,103 ----
  #include "ircfiles.h"
  #include "version.h"
  #include "irext.h"
+ #include "stoplist.h"
  
  #define INDEXER_DATE "Sun May 10 1992"
  
***************
*** 158,163 ****
--- 159,166 ----
    fprintf(stderr,"           | objc /* objective-C .h and .m files */\n");
  #endif /* def NeXT */
    fprintf(stderr,"           | tiff /* tiff files, only indexes the filename */\n");
+   fprintf(stderr,"          [-stopword word] /* word to omit from index */\n");
+   fprintf(stderr,"          [-stopfile file] /* file of words to omit from index */\n");
    fprintf(stderr,"          ] filename filename ...\n");
  }
  
***************
*** 597,602 ****
--- 600,617 ----
  	panic("Don't recognize the '%s' type", next_argument);
        }
      }
+     else if(0 == strcmp("-stopword", next_argument)) {
+       /* a word to add to the list of stop words */
+       if (NULL == (next_argument = next_arg(&argc, &argv)))
+ 	panic("Expected argument to -stopword option");
+       add_stop_word(next_argument);
+     }
+     else if(0 == strcmp("-stopfile", next_argument)) {
+       /* a file of stop words to add */
+       if (NULL == (next_argument = next_arg(&argc, &argv)))
+ 	panic("Expected argument to -stopfile option");
+       add_stop_file(next_argument);
+     }
      else{
        panic("Don't recognize the '%s' option", next_argument);
      }
*** 1.2	1992/02/12 13:50:01
--- stoplist.c	1992/08/13 02:09:50
***************
*** 23,31 ****
  
  #include "stoplist.h"
  #include "cutil.h"
  
! long stoplist_pointer = 0;
! char *stoplist[] = {
    "a",
    "about",
    "above",
--- 23,39 ----
  
  #include "stoplist.h"
  #include "cutil.h"
+ #include "irlex.h"
  
! static char **stoplist_array = 0;
! static long stoplist_pointer = 0;
! static long stoplist_array_size = 0;
! static long stoplist_array_next = 0;
! 
! #define CONSTANT_SIZE (sizeof(constant_stoplist) / sizeof(char *))
! #define STOP_ARRAY_CHUNK 50
! 
! static char *constant_stoplist[] = {
    "a",
    "about",
    "above",
***************
*** 414,435 ****
    "yours",
    "yourself",
    "yourselves",
!   "z", NULL};
  
  void init_stop_list()
  {
    stoplist_pointer = 0;
  }
-  
  
  char *next_stop_word()
  {
!   /* returns NULL if all out */
!   if(stoplist[stoplist_pointer] == NULL)
!     return(NULL);
!   else
!     return(stoplist[stoplist_pointer++]);
  }
  
!   
  
--- 422,488 ----
    "yours",
    "yourself",
    "yourselves",
!   "z"};
  
  void init_stop_list()
  {
    stoplist_pointer = 0;
  }
  
  char *next_stop_word()
  {
!   if (stoplist_pointer < CONSTANT_SIZE) {
!     return(constant_stoplist[stoplist_pointer++]);
!   }
!   else if (stoplist_pointer - CONSTANT_SIZE < stoplist_array_next) {
!     return(stoplist_array[stoplist_pointer++ - CONSTANT_SIZE]);
!   }
!   else {
!     return 0;
!   }
! }
! 
! void add_stop_word(word)
! char *word;
! {
!   char *str;
! 
!   if (stoplist_array_next == stoplist_array_size) {
!     stoplist_array_size += STOP_ARRAY_CHUNK;
!     stoplist_array = stoplist_array ?
! 	 (char **) s_malloc(stoplist_array_size * sizeof(char *)) :
! 	 (char **) s_realloc(stoplist_array,
! 			     stoplist_array_size * sizeof(char *));
!   }
!   str = s_strdup(word);
!   stoplist_array[stoplist_array_next++] = str;
  }
  
! void add_stop_file(file)
! char *file;
! {
!   char word[MAX_WORD_LENGTH + 2];
!   FILE *stop_file;
!   char *ptr;
!   int c;
! 
!   stop_file = fopen(file, "r");
!   if (! stop_file) {
!     panic("Couldn't open stop-word file");
!   }
  
+   while (fgets(word, sizeof(word), stop_file)) {
+     ptr = (char *)index(word, '\n');
+     if (ptr) {
+       *ptr = '\0';
+       add_stop_word(word);
+     }
+     else {
+       while ((c = getc(stop_file)) != EOF && c != '\n') /* empty */;
+       waislog(WLOG_MEDIUM, WLOG_ERROR,
+ 	      "Word %s in stop-word file %s too long, ignored.", word, file);
+     }
+   }
+ 
+   fclose(stop_file);
+ }
*** 1.1	1992/08/12 22:48:16
--- stoplist.h	1992/08/13 02:09:26
***************
*** 9,14 ****
--- 9,16 ----
  #include "cdialect.h"
  
  void init_stop_list _AP((void));
+ void add_stop_word _AP((char *));
+ void add_stop_file _AP((char *));
  
  char *next_stop_word _AP((void));
  
Jonathan Kamens                                         jik@MIT.Edu
MIT Information Systems/Athena              Moderator, news.answers
    (Send correspondence related to the news.answers newsgroup
        {and ONLY correspondence related to the newsgroup}
                 to news-answers-request@MIT.Edu.)
Newsgroups: comp.infosystems.wais