more fact-finding

cl-cleanup.py

@@ -0,0 +1,6 @@
+import parse, sys
+
+soup = parse.parse(sys.argv[1])
+
+with open(sys.argv[1]+"-cleaned.html", "w", encoding="utf-8") as file:
+    file.write(str(soup))

understand.py

@@ -10,29 +10,62 @@ class Document():
         self.doctype = None
         self.summary = None
 
-    def query_doctype(self):
+    def query(self, s):
         try:
-            answer, truncated = self.LLM.send_question(f"""Please read this document carefully, remember all you know about the concepts it touches, and give a brief description of what this document is. Examples: "Textbook about interpersonal psychology", "State standard on oil rig equipment", "User manual for a smartphone app". Write just one line with the description. 
-
-            <document>{self.fad}</document>""")
-
+            answer, truncated = self.LLM.send_question(s)
             return answer.strip()
-
         except  Exception as e:
             print (e.message, e.args)
             return None
+
+
+    def query_doctype(self):
+        return self.query(
+f"""Please read this document carefully. Remember everything you know about the concepts it touches. 
+
+Give a brief description of what this document is. Examples: "Textbook about interpersonal psychology", "State standard on oil rig equipment", "User manual for a smartphone app". 
+
+Write just one line with the description. 
+
+<document>{self.fad}</document>""")
              
     def query_summary(self):
-        try:
-            answer, truncated = self.LLM.send_question(f"""Below is a {self.doctype}. Please read it carefully, remember all you know about the concepts it touches, and give a one-paragraph summary of its content. Write just one paragraph with the summary. 
+        return self.query(
+f"""Below is a document inside <document></document>. It is a {self.doctype}. 
 
-            <document>{self.fad}</document>""")
+Read it carefully. Remember everything you know about the concepts it touches. 
 
-            return answer.strip()
+Give a one-paragraph summary of the document's content. Write just one paragraph with the summary. 
+
+<document>{self.fad}</document>""")
+
+    def query_concepts(self):
+        return self.query(
+f"""Below is a document inside <document></document>. It is a {self.doctype}. 
+                                                       
+Read it carefully. Remember everything you know about the concepts it touches. 
+
+Now, read it again, and compile a list of the main concepts that this document talks about. Output the line "CONCEPTS:" followed by a list of concepts, one per line.
+
+Then, read it one more time, and think about who are the actors implied. Who are the parties that this document is authored by, is governed by, is addressed to? Often, these parties are not explicitly stated in the document, so you will need to use your common sense. See if you can compile a minimal list of the main actors that the document is concerned with. Output the line "ACTORS:" followed by a list of concepts, one per line. 
+
+For example, for an instruction manual for a chainsaw, you might want to output:
+CONCEPTS:
+Power tool
+Safety
+Work area
+Saw chain
+Wood
+...
+ACTORS:
+Chainsaw manufacturer
+Chainsaw user
+Children and bystanders
+...
+
+<document>{self.fad}</document>""")
+        
 
-        except  Exception as e:
-            print (e.message, e.args)
-            return None
 
     def understand(self, filename):
         self.documentSoup = parse.parse(filename)