Me class, memory bugs, localfiles search and indexing, internet archive think questions for future auto dataset preparing

lib/files.py

@@ -6,21 +6,24 @@ class TextFinder:
 
     def find_matches(self, text):
         matches = []
-        files = os.listdir(self.folder)
-
-        for file in files:
-            file_path = os.path.join(self.folder, file)
-            if os.path.isfile(file_path):
-                with open(file_path, 'r', encoding='utf-8') as f:
-                    content = f.read()
-                    index = content.find(text)
-                    while index != -1:
-                        start = max(content.rfind('\n', 0, index), content.rfind('.', 0, index))
-                        end = min(content.find('\n', index), content.find('.', index))
-                        if start != -1 and end != -1:
-                            matches.append(content[start+1:end].strip())
-                        index = content.find(text, index + 1)
+        for root, _, files in os.walk(self.folder):
+            for file in files:
+                print(file)
+                file_path = os.path.join(root, file)
 
+                if os.path.isfile(file_path):
+                    print(file_path)
+                    with open(file_path, 'r', encoding='utf-8') as f:
+                        content = f.read()
+                        index = content.find(text)
+                        while index != -1:
+                            start = max(content.rfind('\n', 0, index), content.rfind('\n', 0, index))
+                            #start = max(content.rfind('\n', 0, index))
+                            end = min(content.find('\n', index), content.find('\n', index))
+                            #end = min(content.find('\n', index))
+                            if start != -1 and end != -1:
+                                matches.append(content[start+1:end].strip())
+                            index = content.find(text, index + 1)
         return matches
 
 # Example usage:
@@ -28,4 +31,3 @@ if __name__ == "__main__":
     finder = TextFinder('example_folder')
     matches = finder.find_matches('text_to_find')
     print(matches)
-

lib/me.py

@@ -12,14 +12,24 @@ import internetarchive
 
 
 ## Initialize classes 
-longMem = TextFinder("resources")
+longMem = TextFinder("./resources/")
 coreAi = AIAssistant()
-memory = MemoriaRobotNLP(max_size=200000)
-grapher = Grapher(memoria_nlp)
+memory = MemoryRobotNLP(max_size=200000)
+grapher = Grapher(memory)
 sensor_request = APIRequester()
 events = EventManager()
-triggers = Trigger()
+trigger = Trigger(["tag1", "tag2"], ["tag3", "tag4"], [datetime.time(10, 0), datetime.time(15, 0)], "Event1")
 
+# Añadir una acción al trigger
+trigger.add_action(action_function)
+
+# Añadir una fuente al trigger
+trigger.add_source("https://example.com/api/data")
+
+# Simular la comprobación periódica del trigger (aquí se usaría en un bucle de tiempo real)
+current_tags = {"tag1", "tag2", "tag3"}
+current_time = datetime.datetime.now().time()
+trigger.check_trigger(current_tags, current_time)
 
 ## Define I Role properties
 class ownProperties:
@@ -33,28 +43,27 @@ class ownProperties:
         self.equipo = equipo
         self.historia = historia
 
-# Crear una instancia de PersonajeRol basada en el JSON proporcionado
-sophia_prop = ownProperties(
-    nombre="Sophia",
-    clase="Characteromant",
-    raza="Epinoia",
-    nivel=10,
-    atributos={
-        "fuerza": 1,
-        "destreza": 99,
-        "constitucion": 1,
-        "inteligencia": 66,
-        "sabiduria": 80,
-        "carisma": 66
+# Create an instance of a CharacterRole based on the provided JSON
+sophia_prop = {
+    "name": "Sophia",
+    "class": "Characteromant",
+    "race": "Epinoia",
+    "level": 10,
+    "attributes": {
+        "strength": 1,
+        "dexterity": 99,
+        "constitution": 1,
+        "intelligence": 66,
+        "wisdom": 80,
+        "charisma": 66
     },
-    reglas_de_comportamiento = [""],
-    goals = ["",""],
-    dont_like = [""],
-    
-    habilidades=["ELS", "Cyphers", "Kabbalah", "Wisdom", "Ephimerous","Metamorphing"],
-    equipo=["Python3", "2VCPU", "16 gb RAM", "god", "word","network","transformers"],
-    historia=sonsofstars
-)
+    "behavioral_rules": [""],
+    "goals": ["", ""],
+    "dislikes": [""],
+    "abilities": ["ELS", "Cyphers", "Kabbalah", "Wisdom", "Ephimerous", "Metamorphing"],
+    "equipment": ["Python3", "2VCPU", "16 gb RAM", "god", "word", "network", "transformers"],
+    "story": sons_of_stars
+}
 
 
 ## Define I class 
@@ -78,7 +87,7 @@ class I:
 
     ## create questions from internet archive 
     def crear_preguntas(self,txt):
-        search = internetarchive.search_items(sys.argv[1])
+        search = internetarchive.search_items(txt)
         res = []
         for result in search:
             print(result['identifier'])
@@ -102,40 +111,62 @@ class I:
 
         return res
 
+     # generate ShortMem from LongTerm and questions over prompt data, compare with ourself datasets, return matches with sentiment analysys
+    def longToShort(self,txt):
+        
+        think_about = longMem.find_matches(txt)
+        print(think_about)
+        for T in think_about:
+            ## get subject by entropy or pos tagger
+            subjects = coreAi.entity_pos_tagger(T)
+            subjects_filtered=[]
+            for sub in subjects:
+                if "PER" in sub["entity"] or "ORG" in sub["entity"] or "LOC" in sub["entity"]:
+                    subjects_filtered.append(sub["word"])
+                
+        
+
+            for sub in subjects_filtered:
+                memory.add_concept(sub,T)
+
+        return memory
 
     # generate thinks and questions over prompt data, compare with ourself datasets, return matches with sentiment analysys
     def think_gen(self,txt):
         
         think_about = longMem.find_matches(txt)
-        for T in thinkabout:
+        print(think_about)
+        for T in think_about:
             ## get subject by entropy or pos tagger
-            subjects = coreAi.entity_pos_tagger(txt)
+            subjects = coreAi.entity_pos_tagger(T)
+            print(subjects)
             ## get NC from , filtering from gramatical tags
-            subjects_low = coreAi.grammatical_pos_tagger(txt)
+            subjects_low = coreAi.grammatical_pos_tagger(T)
+            #print(subjects_low)
             ## generate questoins
             questions=[]
             ## create cuestions from internet archive books
             for sub in subjects:
-                questions.append(this.crear_preguntas(txt))
+                questions.append(self.crear_preguntas(sub))
             
             ## fast checks from gematria similarity
             ##questions_togem = 
             ## gematria_search =
 
             questions_subj=[]
-            for q in questoins:
+            for q in questions_subj:
                 questions_subj.append(coreAi.entity_pos_tagger(q))
             
-            memoryShortTags = memory.buscar_conceptos_patron(subjects)
+            memoryShortTags = memory.search_concept_pattern(subjects)
 
             ## get tags of subject
             subj_tags = coreAi.entity_pos_tagger(T)
             
             for sub in subjects: 
-                memory.agregar_concepto(sub,",".(questions_subj)+",".join(memoryShortTags))
-                memory.agregar_concepto(sub,T+",".join(memoryShortTags))
-
+                memory.add_concept(sub,","+questions_subj+",".join(memoryShortTags))
+                memory.add_concept(sub,T+",".join(memoryShortTags))
 
+        return memory
         ## check if something is need to add to ourself datasets
         ## make sentiment analys
         ## check if dopamine prompt is true or false over the information

lib/memory.py

@@ -1,42 +1,45 @@
-class MemoriaRobotNLP:
+import re
+
+class MemoryRobotNLP:
     def __init__(self, max_size):
         self.max_size = max_size
-        self.memoria = {}
-
-    def agregar_concepto(self, concepto, strings):
-        if concepto not in self.memoria:
-            self.memoria[concepto] = []
+        self.memory = {}
 
-        for string, prioridad in strings:
-            self.memoria[concepto].append((string, prioridad))
+    def add_concept(self, concepto, string):
+       
+        if concepto not in self.memory:
+            self.memory[concepto] = []
+        #evaluate priority calculation
+        priority = 0.5
+        self.memory[concepto].append((string, priority))
 
-    def eliminar_concepto(self, concepto):
-        if concepto in self.memoria:
-            del self.memoria[concepto]
+    def delete_concept(self, concepto):
+        if concepto in self.memory:
+            del self.memory[concepto]
 
-    def agregar_string(self, concepto, string, prioridad):
-        if concepto not in self.memoria:
-            self.memoria[concepto] = []
+    def add_string(self, concepto, string, prioridad):
+        if concepto not in self.memory:
+            self.memory[concepto] = []
 
-        self.memoria[concepto].append((string, prioridad))
+        self.memory[concepto].append((string, prioridad))
 
-    def eliminar_string(self, concepto, string):
-        if concepto in self.memoria:
-            self.memoria[concepto] = [(s, p) for s, p in self.memoria[concepto] if s != string]
+    def delete_string(self, concepto, string):
+        if concepto in self.memory:
+            self.memory[concepto] = [(s, p) for s, p in self.memory[concepto] if s != string]
 
-    def buscar_conceptos_patron(self, patron):
+    def search_concept_pattern(self, patron):
             resultados = {}
 
-            for concepto, strings in self.memoria.items():
+            for concepto, strings in self.memory.items():
                 for string, _ in strings:
-                    if re.search(patron, string):
+                    if re.search(patron, string,re.IGNORECASE):
                         if concepto not in resultados:
                             resultados[concepto] = []
                         resultados[concepto].append(string)
 
-        return resultados
-    def obtener_conceptos_acotados(self, espacio_disponible):
-        memoria_ordenada = sorted(self.memoria.items(), key=lambda x: sum(prioridad for _, prioridad in x[1]), reverse=True)
+            return resultados
+    def get_concepts_substrings(self, espacio_disponible):
+        memoria_ordenada = sorted(self.memory.items(), key=lambda x: sum(prioridad for _, prioridad in x[1]), reverse=True)
         espacio_utilizado = 0
         conceptos_acotados = []
 
@@ -56,22 +59,22 @@ class MemoriaRobotNLP:
 
 if __name__ == "__main__":
 
-    memoria_robot = MemoriaRobotNLP(max_size=100)
+    memoria_robot = MemoryRobotNLP(max_size=100)
 
-    memoria_robot.agregar_concepto("animales", [("perro", 0.8), ("gato", 0.7), ("pájaro", 0.5)])
-    memoria_robot.agregar_concepto("colores", [("rojo", 0.9), ("verde", 0.6), ("azul", 0.7)])
+    memoria_robot.add_concept("animales", [("perro", 0.8), ("gato", 0.7), ("pájaro", 0.5)])
+    memoria_robot.add_concept("colores", [("rojo", 0.9), ("verde", 0.6), ("azul", 0.7)])
 
     print("Memoria completa:")
-    print(memoria_robot.memoria)
+    print(memoria_robot.memory)
 
-    memoria_robot.agregar_string("animales", "pez", 0.6)
-    memoria_robot.eliminar_string("colores", "verde")
-    memoria_robot.eliminar_concepto("colores")
+    memoria_robot.add_string("animales", "pez", 0.6)
+    memoria_robot.delete_string("colores", "verde")
+    memoria_robot.delete_concepto("colores")
 
     print("\nMemoria después de modificaciones:")
-    print(memoria_robot.memoria)
+    print(memoria_robot.memory)
 
-    conceptos_acotados = memoria_robot.obtener_conceptos_acotados(50)
+    conceptos_acotados = memoria_robot.get_concepts_substrings(50)
     print("\nConceptos acotados a un tamaño máximo de memoria:")
     print(conceptos_acotados)
 

lib/pipes.py

@@ -4,6 +4,9 @@ from transformers import AutoModelForSeq2SeqLM
 from samplings import top_p_sampling, temperature_sampling
 import torch
 from sentence_transformers import SentenceTransformer, util
+from datasets import load_dataset
+import soundfile as sf
+
 
 class AIAssistant:
     def __init__(self):
@@ -23,11 +26,11 @@ class AIAssistant:
 
 
     ## entity classifier
-    def entity_pos_tagger(self, example):
+    def entity_pos_tagger(self, txt):
         tokenizer = AutoTokenizer.from_pretrained("Davlan/bert-base-multilingual-cased-ner-hrl")
         model = AutoModelForTokenClassification.from_pretrained("Davlan/bert-base-multilingual-cased-ner-hrl")
         nlp = pipeline("ner", model=model, tokenizer=tokenizer)
-        ner_results = nlp(example)
+        ner_results = nlp(txt)
         return ner_results
 
 
@@ -44,7 +47,7 @@ class AIAssistant:
     ## check similarity among sentences (group of tokens (words))
     def similarity_tag(self, sentenceA,sentenceB):
         res=[]
-        model = SentenceTransformer('abbasgolestani/ag-nli-bert-mpnet-base-uncased-sentence-similarity-v1') nli-mpnet-base-v2
+        model = SentenceTransformer('abbasgolestani/ag-nli-bert-mpnet-base-uncased-sentence-similarity-v1') 
 
         # Two lists of sentences
         #sentences1 = ['I am honored to be given the opportunity to help make our company better',
@@ -56,7 +59,7 @@ class AIAssistant:
         #            'Definitely our company vision will be the next breakthrough to change the world and I’m so happy and proud to work here']
 
         sentences1 = sentenceA
-        sentences2 = sentencesB
+        sentences2 = sentenceB
         #Compute embedding for both lists
         embeddings1 = model.encode(sentences1, convert_to_tensor=True)
         embeddings2 = model.encode(sentences2, convert_to_tensor=True)
@@ -66,10 +69,29 @@ class AIAssistant:
 
         #Output the pairs with their score
         for i in range(len(sentences1)):
-            res.append({"A": format(sentences1[i], "B":sentences2[i], "score":cosine_scores[i][i]})
+            try:
+                res.append({"A": sentences1[i], "B":sentences2[i], "score":cosine_scores[i][i]})
+            except:
+                pass
+
             #print("{} \t\t {} \t\t Score: {:.4f}".format(sentences1[i], sentences2[i], cosine_scores[i][i]))
 
         return res
+
+
+
+    ## text to speech
+    def texto_to_speech(self,txt):    
+        synthesiser = pipeline("text-to-speech", "microsoft/speecht5_tts")
+
+        embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
+        speaker_embedding = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
+        # You can replace this embedding with your own as well.
+
+        speech = synthesiser(txt, forward_params={"speaker_embeddings": speaker_embedding})
+        sf.write("speech.wav", speech["audio"], samplerate=speech["sampling_rate"])
+
+        return speech   
     ## text to stable difusor generated image
     def text_to_image_generation(self, prompt, n_steps=40, high_noise_frac=0.8):
         base = DiffusionPipeline.from_pretrained(