Upload 2 files

model.py

@@ -57,13 +57,13 @@ class DynamicModel(nn.Module):
             for hidden_layer_params in hidden_layers:
                 layers.append(self.create_layer(hidden_layer_params))
 
-        if layer_params.get('memory_augmentation', False):
+        if layer_params.get('memory_augmentation', True):
             layers.append(MemoryAugmentationLayer(layer_params['output_size']))
 
-        if layer_params.get('hybrid_attention', False):
+        if layer_params.get('hybrid_attention', True):
             layers.append(HybridAttentionLayer(layer_params['output_size']))
 
-        if layer_params.get('dynamic_flash_attention', False):
+        if layer_params.get('dynamic_flash_attention', True):
             layers.append(DynamicFlashAttentionLayer(layer_params['output_size']))
 
         return nn.Sequential(*layers)
@@ -143,7 +143,7 @@ def create_model_from_folder(folder_path: str) -> DynamicModel:
         print(f"Warning: Folder {folder_path} does not exist. Creating model with default configuration.")
         return DynamicModel({})
 
-    xml_files_found = True
+    xml_files_found = False
     for root, dirs, files in os.walk(folder_path):
         for file in files:
             if file.endswith('.xml'):
@@ -217,4 +217,4 @@ def main():
         print(f"Epoch {epoch+1}/{num_epochs}, Average Loss: {avg_loss:.4f}")
 
 if __name__ == "__main__":
-    main()
+    main()

model2.py

@@ -0,0 +1,265 @@
+import os
+import xml.etree.ElementTree as ET
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import List, Dict, Any, Optional
+from collections import defaultdict
+from accelerate import Accelerator
+from transformers import AutoTokenizer, AutoModel
+import faiss
+import numpy as np
+
+class DynamicModel(nn.Module):
+    def __init__(self, sections: Dict[str, List[Dict[str, Any]]]):
+        super(DynamicModel, self).__init__()
+        self.sections = nn.ModuleDict()
+
+        if not sections:
+            sections = {
+                'default': [{
+                    'input_size': 128,
+                    'output_size': 256,
+                    'activation': 'relu',
+                    'batch_norm': True,
+                    'dropout': 0.1
+                }]
+            }
+
+        for section_name, layers in sections.items():
+            self.sections[section_name] = nn.ModuleList()
+            for layer_params in layers:
+                print(f"Creating layer in section '{section_name}' with params: {layer_params}")
+                self.sections[section_name].append(self.create_layer(layer_params))
+
+    def create_layer(self, layer_params: Dict[str, Any]) -> nn.Module:
+        layers = []
+        layers.append(nn.Linear(layer_params['input_size'], layer_params['output_size']))
+
+        if layer_params.get('batch_norm', False):
+            layers.append(nn.BatchNorm1d(layer_params['output_size']))
+
+        activation = layer_params.get('activation', 'relu')
+        if activation == 'relu':
+            layers.append(nn.ReLU(inplace=True))
+        elif activation == 'tanh':
+            layers.append(nn.Tanh())
+        elif activation == 'sigmoid':
+            layers.append(nn.Sigmoid())
+        elif activation == 'leaky_relu':
+            layers.append(nn.LeakyReLU(negative_slope=0.01, inplace=True))
+        elif activation == 'elu':
+            layers.append(nn.ELU(alpha=1.0, inplace=True))
+        elif activation is not None:
+            raise ValueError(f"Unsupported activation function: {activation}")
+
+        if dropout_rate := layer_params.get('dropout', 0.0):
+            layers.append(nn.Dropout(p=dropout_rate))
+
+        if hidden_layers := layer_params.get('hidden_layers', []):
+            for hidden_layer_params in hidden_layers:
+                layers.append(self.create_layer(hidden_layer_params))
+
+        if layer_params.get('memory_augmentation', False):
+            layers.append(MemoryAugmentationLayer(layer_params['output_size']))
+
+        if layer_params.get('hybrid_attention', False):
+            layers.append(HybridAttentionLayer(layer_params['output_size']))
+
+        if layer_params.get('dynamic_flash_attention', False):
+            layers.append(DynamicFlashAttentionLayer(layer_params['output_size']))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x: torch.Tensor, section_name: Optional[str] = None) -> torch.Tensor:
+        if section_name is not None:
+            if section_name not in self.sections:
+                raise KeyError(f"Section '{section_name}' not found in model")
+            for layer in self.sections[section_name]:
+                x = layer(x)
+        else:
+            for section_name, layers in self.sections.items():
+                for layer in layers:
+                    x = layer(x)
+        return x
+
+class MemoryAugmentationLayer(nn.Module):
+    def __init__(self, size: int):
+        super(MemoryAugmentationLayer, self).__init__()
+        self.memory = nn.Parameter(torch.randn(size))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return x + self.memory
+
+class HybridAttentionLayer(nn.Module):
+    def __init__(self, size: int):
+        super(HybridAttentionLayer, self).__init__()
+        self.attention = nn.MultiheadAttention(size, num_heads=8)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = x.unsqueeze(1)  # Add sequence dimension
+        attn_output, _ = self.attention(x, x, x)
+        return attn_output.squeeze(1)
+
+class DynamicFlashAttentionLayer(nn.Module):
+    def __init__(self, size: int):
+        super(DynamicFlashAttentionLayer, self).__init__()
+        self.attention = nn.MultiheadAttention(size, num_heads=8)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = x.unsqueeze(1)  # Add sequence dimension
+        attn_output, _ = self.attention(x, x, x)
+        return attn_output.squeeze(1)
+
+def parse_xml_file(file_path: str) -> List[Dict[str, Any]]:
+    tree = ET.parse(file_path)
+    root = tree.getroot()
+
+    layers = []
+    for layer in root.findall('.//layer'):
+        layer_params = {}
+        layer_params['input_size'] = int(layer.get('input_size', 128))
+        layer_params['output_size'] = int(layer.get('output_size', 256))
+        layer_params['activation'] = layer.get('activation', 'relu').lower()
+
+        if layer_params['activation'] not in ['relu', 'tanh', 'sigmoid', 'none']:
+            raise ValueError(f"Unsupported activation function: {layer_params['activation']}")
+
+        if layer_params['input_size'] <= 0 or layer_params['output_size'] <= 0:
+            raise ValueError("Layer dimensions must be positive integers")
+
+        layers.append(layer_params)
+
+    if not layers:
+        layers.append({
+            'input_size': 128,
+            'output_size': 256,
+            'activation': 'relu'
+        })
+
+    return layers
+
+def create_model_from_folder(folder_path: str) -> DynamicModel:
+    sections = defaultdict(list)
+
+    if not os.path.exists(folder_path):
+        print(f"Warning: Folder {folder_path} does not exist. Creating model with default configuration.")
+        return DynamicModel({})
+
+    xml_files_found = False
+    for root, dirs, files in os.walk(folder_path):
+        for file in files:
+            if file.endswith('.xml'):
+                xml_files_found = True
+                file_path = os.path.join(root, file)
+                try:
+                    layers = parse_xml_file(file_path)
+                    section_name = os.path.basename(root).replace('.', '_')
+                    sections[section_name].extend(layers)
+                except Exception as e:
+                    print(f"Error processing {file_path}: {str(e)}")
+
+    if not xml_files_found:
+        print("Warning: No XML files found. Creating model with default configuration.")
+        return DynamicModel({})
+
+    return DynamicModel(dict(sections))
+
+def create_embeddings_and_stores(folder_path: str, model_name: str = "sentence-transformers/all-MiniLM-L6-v2"):
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModel.from_pretrained(model_name)
+
+    vector_store = faiss.IndexFlatL2(384)  # Assuming 384-dimensional embeddings
+    doc_store = []
+
+    for root, dirs, files in os.walk(folder_path):
+        for file in files:
+            if file.endswith('.xml'):
+                file_path = os.path.join(root, file)
+                try:
+                    tree = ET.parse(file_path)
+                    root = tree.getroot()
+                    for elem in root.iter():
+                        if elem.text:
+                            text = elem.text.strip()
+                            inputs = tokenizer(text, return_tensors="pt", truncation=True, padding=True)
+                            with torch.no_grad():
+                                embeddings = model(**inputs).last_hidden_state.mean(dim=1).numpy()
+                            vector_store.add(embeddings)
+                            doc_store.append(text)
+                except Exception as e:
+                    print(f"Error processing {file_path}: {str(e)}")
+
+    return vector_store, doc_store
+
+def query_vector_store(query: str, vector_store, doc_store, model_name: str = "sentence-transformers/all-MiniLM-L6-v2"):
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModel.from_pretrained(model_name)
+
+    inputs = tokenizer(query, return_tensors="pt", truncation=True, padding=True)
+    with torch.no_grad():
+        query_embedding = model(**inputs).last_hidden_state.mean(dim=1).numpy()
+
+    D, I = vector_store.search(query_embedding, k=5)  # Retrieve top 5 documents
+    results = [doc_store[i] for i in I[0]]
+    return results
+
+def main():
+    folder_path = 'data'
+    model = create_model_from_folder(folder_path)
+
+    print(f"Created dynamic PyTorch model with sections: {list(model.sections.keys())}")
+
+    first_section = next(iter(model.sections.keys()))
+    first_layer = model.sections[first_section][0]
+    input_features = first_layer[0].in_features
+
+    sample_input = torch.randn(1, input_features)
+    output = model(sample_input)
+    print(f"Sample output shape: {output.shape}")
+
+    vector_store, doc_store = create_embeddings_and_stores(folder_path)
+
+    accelerator = Accelerator()
+
+    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
+    criterion = nn.CrossEntropyLoss()
+    num_epochs = 10
+
+    dataset = torch.utils.data.TensorDataset(
+        torch.randn(100, input_features),
+        torch.randint(0, 2, (100,))
+    )
+    train_dataloader = torch.utils.data.DataLoader(
+        dataset, 
+        batch_size=16, 
+        shuffle=True
+    )
+
+    model, optimizer, train_dataloader = accelerator.prepare(
+        model, 
+        optimizer, 
+        train_dataloader
+    )
+
+    for epoch in range(num_epochs):
+        model.train()
+        total_loss = 0
+        for batch_idx, (inputs, labels) in enumerate(train_dataloader):
+            optimizer.zero_grad()
+            outputs = model(inputs)
+            loss = criterion(outputs, labels)
+            accelerator.backward(loss)
+            optimizer.step()
+            total_loss += loss.item()
+
+        avg_loss = total_loss / len(train_dataloader)
+        print(f"Epoch {epoch+1}/{num_epochs}, Average Loss: {avg_loss:.4f}")
+
+    # Query the vector store after training
+    user_query = "example query text"
+    results = query_vector_store(user_query, vector_store, doc_store)
+    print(f"Query results: {results}")
+
+if __name__ == "__main__":
+    main()