Repository: Zeeeepa/wiseflow
Pull request #2: Implement Phase 1 of Wiseflow Upgrade Plan
Author: login: Zeeeepa
Labels: name: Review effort 4/5
Created: 2025-04-06T11:08:34Z | Merged: 2025-04-06T11:08:41Z

## Summary by Sourcery

Implement Phase 1 of Wiseflow Upgrade Plan by adding core infrastructure modules for task management, plugin system, data connectors, and processors

New Features:
- Introduced a comprehensive plugin system with base classes for extensible architecture
- Added async task management with support for concurrent data mining tasks
- Created GitHub and web connectors for data collection
- Implemented a reference management system
- Developed a text processing system with focus point extraction

Enhancements:
- Designed modular architecture with abstract base classes for plugins, connectors, and processors
- Implemented flexible configuration and initialization mechanisms
- Added logging and error handling across new modules

Chores:
- Organized core infrastructure into separate modules
- Created base classes for future extensibility


Diff:
diff --git a/core/connectors/__init__.py b/core/connectors/__init__.py
new file mode 100644
index 00000000..20833f65
--- /dev/null
+++ b/core/connectors/__init__.py
@@ -0,0 +1,106 @@
+"""
+Data source connectors for Wiseflow.
+
+This module provides base classes for data source connectors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+
+logger = logging.getLogger(__name__)
+
+class DataItem:
+    """Represents a single item of data collected from a source."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        content: str,
+        metadata: Optional[Dict[str, Any]] = None,
+        url: Optional[str] = None,
+        timestamp: Optional[datetime] = None,
+        content_type: str = "text",
+        language: Optional[str] = None,
+        raw_data: Optional[Any] = None
+    ):
+        """Initialize a data item."""
+        self.source_id = source_id
+        self.content = content
+        self.metadata = metadata or {}
+        self.url = url
+        self.timestamp = timestamp or datetime.now()
+        self.content_type = content_type
+        self.language = language
+        self.raw_data = raw_data
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the data item to a dictionary."""
+        return {
+            "source_id": self.source_id,
+            "content": self.content,
+            "metadata": self.metadata,
+            "url": self.url,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None,
+            "content_type": self.content_type,
+            "language": self.language
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'DataItem':
+        """Create a data item from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
+                
+        return cls(
+            source_id=data["source_id"],
+            content=data["content"],
+            metadata=data.get("metadata", {}),
+            url=data.get("url"),
+            timestamp=timestamp,
+            content_type=data.get("content_type", "text"),
+            language=data.get("language")
+        )
+
+
+class ConnectorBase(PluginBase):
+    """Base class for data source connectors."""
+    
+    name: str = "base_connector"
+    description: str = "Base connector class"
+    source_type: str = "generic"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the connector with optional configuration."""
+        super().__init__(config)
+        self.last_run: Optional[datetime] = None
+        
+    @abstractmethod
+    def collect(self, params: Optional[Dict[str, Any]] = None) -> List[DataItem]:
+        """Collect data from the source."""
+        pass
+    
+    def initialize(self) -> bool:
+        """Initialize the connector. Return True if successful, False otherwise."""
+        try:
+            # Perform any necessary initialization
+            logger.info(f"Initialized connector: {self.name}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize connector {self.name}: {e}")
+            return False
+    
+    def update_last_run(self) -> None:
+        """Update the last run timestamp."""
+        self.last_run = datetime.now()
+        
+    def get_last_run(self) -> Optional[datetime]:
+        """Get the last run timestamp."""
+        return self.last_run
diff --git a/core/connectors/github/__init__.py b/core/connectors/github/__init__.py
new file mode 100644
index 00000000..2177549a
--- /dev/null
+++ b/core/connectors/github/__init__.py
@@ -0,0 +1,291 @@
+"""
+GitHub connector for Wiseflow.
+
+This module provides a connector for GitHub repositories.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import uuid
+from datetime import datetime
+import os
+import requests
+import base64
+import time
+
+from core.plugins.connectors import ConnectorBase, DataItem
+
+logger = logging.getLogger(__name__)
+
+class GitHubConnector(ConnectorBase):
+    """Connector for GitHub repositories."""
+    
+    name: str = "github_connector"
+    description: str = "Connector for GitHub repositories"
+    source_type: str = "github"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the GitHub connector."""
+        super().__init__(config)
+        self.api_token = None
+        self.api_base_url = "https://api.github.com"
+        self.headers = {}
+        
+    def initialize(self) -> bool:
+        """Initialize the connector."""
+        try:
+            # Get API token from config or environment
+            self.api_token = self.config.get("api_token")
+            if not self.api_token and "GITHUB_TOKEN" in os.environ:
+                self.api_token = os.environ["GITHUB_TOKEN"]
+            
+            if not self.api_token:
+                logger.warning("No GitHub API token provided. Rate limits will be restricted.")
+                self.headers = {
+                    "Accept": "application/vnd.github.v3+json"
+                }
+            else:
+                self.headers = {
+                    "Accept": "application/vnd.github.v3+json",
+                    "Authorization": f"token {self.api_token}"
+                }
+            
+            logger.info("Initialized GitHub connector")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize GitHub connector: {e}")
+            return False
+    
+    def collect(self, params: Optional[Dict[str, Any]] = None) -> List[DataItem]:
+        """Collect data from GitHub repositories."""
+        params = params or {}
+        
+        # Get repositories to collect from
+        repos = params.get("repositories", [])
+        if not repos:
+            if self.config.get("repositories"):
+                repos = self.config["repositories"]
+            else:
+                logger.error("No repositories provided for GitHub connector")
+                return []
+        
+        # Determine what to collect
+        collect_readme = params.get("collect_readme", True)
+        collect_issues = params.get("collect_issues", False)
+        collect_code = params.get("collect_code", False)
+        max_issues = params.get("max_issues", 10)
+        
+        results = []
+        
+        for repo in repos:
+            try:
+                # Format: "owner/repo"
+                if "/" not in repo:
+                    logger.warning(f"Invalid repository format: {repo}. Expected 'owner/repo'")
+                    continue
+                
+                logger.info(f"Collecting data from GitHub repository: {repo}")
+                
+                # Collect README
+                if collect_readme:
+                    readme = self._get_readme(repo)
+                    if readme:
+                        results.append(readme)
+                
+                # Collect issues
+                if collect_issues:
+                    issues = self._get_issues(repo, max_issues)
+                    results.extend(issues)
+                
+                # Collect code (basic implementation)
+                if collect_code:
+                    code_files = self._get_code_files(repo)
+                    results.extend(code_files)
+                
+            except Exception as e:
+                logger.error(f"Error collecting data from repository {repo}: {e}")
+        
+        logger.info(f"Collected {len(results)} items from GitHub")
+        return results
+    
+    def _get_readme(self, repo: str) -> Optional[DataItem]:
+        """Get the README file from a repository."""
+        try:
+            url = f"{self.api_base_url}/repos/{repo}/readme"
+            response = requests.get(url, headers=self.headers)
+            
+            if response.status_code == 200:
+                data = response.json()
+                content = base64.b64decode(data["content"]).decode("utf-8")
+                
+                return DataItem(
+                    source_id=f"github_readme_{uuid.uuid4().hex[:8]}",
+                    content=content,
+                    metadata={
+                        "repo": repo,
+                        "type": "readme",
+                        "path": data.get("path", ""),
+                        "sha": data.get("sha", ""),
+                        "size": data.get("size", 0),
+                        "url": data.get("html_url", "")
+                    },
+                    url=data.get("html_url", ""),
+                    content_type="text/markdown"
+                )
+            else:
+                logger.warning(f"Failed to get README for {repo}: {response.status_code}")
+                return None
+        except Exception as e:
+            logger.error(f"Error getting README for {repo}: {e}")
+            return None
+    
+    def _get_issues(self, repo: str, max_issues: int = 10) -> List[DataItem]:
+        """Get issues from a repository."""
+        results = []
+        try:
+            url = f"{self.api_base_url}/repos/{repo}/issues"
+            params = {
+                "state": "all",
+                "per_page": max_issues
+            }
+            response = requests.get(url, headers=self.headers, params=params)
+            
+            if response.status_code == 200:
+                issues = response.json()
+                
+                for issue in issues:
+                    # Skip pull requests
+                    if "pull_request" in issue:
+                        continue
+                    
+                    content = f"# {issue['title']}\n\n{issue['body']}"
+                    
+                    item = DataItem(
+                        source_id=f"github_issue_{issue['number']}_{uuid.uuid4().hex[:8]}",
+                        content=content,
+                        metadata={
+                            "repo": repo,
+                            "type": "issue",
+                            "number": issue["number"],
+                            "state": issue["state"],
+                            "created_at": issue["created_at"],
+                            "updated_at": issue["updated_at"],
+                            "labels": [label["name"] for label in issue.get("labels", [])],
+                            "user": issue["user"]["login"] if "user" in issue else "",
+                            "comments": issue.get("comments", 0)
+                        },
+                        url=issue["html_url"],
+                        content_type="text/markdown",
+                        timestamp=datetime.strptime(issue["created_at"], "%Y-%m-%dT%H:%M:%SZ") if "created_at" in issue else None
+                    )
+                    results.append(item)
+            else:
+                logger.warning(f"Failed to get issues for {repo}: {response.status_code}")
+        except Exception as e:
+            logger.error(f"Error getting issues for {repo}: {e}")
+        
+        return results
+    
+    def _get_code_files(self, repo: str, path: str = "", max_files: int = 5) -> List[DataItem]:
+        """Get code files from a repository."""
+        results = []
+        try:
+            url = f"{self.api_base_url}/repos/{repo}/contents/{path}"
+            response = requests.get(url, headers=self.headers)
+            
+            if response.status_code == 200:
+                contents = response.json()
+                
+                # If it's a single file
+                if not isinstance(contents, list):
+                    if contents.get("type") == "file" and contents.get("size", 0) < 1000000:  # Skip large files
+                        try:
+                            content = base64.b64decode(contents["content"]).decode("utf-8")
+                            
+                            item = DataItem(
+                                source_id=f"github_code_{uuid.uuid4().hex[:8]}",
+                                content=content,
+                                metadata={
+                                    "repo": repo,
+                                    "type": "code",
+                                    "path": contents["path"],
+                                    "sha": contents["sha"],
+                                    "size": contents["size"],
+                                    "name": contents["name"]
+                                },
+                                url=contents["html_url"],
+                                content_type=self._get_content_type(contents["name"])
+                            )
+                            results.append(item)
+                        except Exception as e:
+                            logger.error(f"Error processing file {contents.get('path')}: {e}")
+                    return results
+                
+                # Process directory contents
+                files_processed = 0
+                for item in contents:
+                    if files_processed >= max_files:
+                        break
+                    
+                    if item["type"] == "file" and item["size"] < 1000000:  # Skip large files
+                        # Get file content
+                        file_url = f"{self.api_base_url}/repos/{repo}/contents/{item['path']}"
+                        file_response = requests.get(file_url, headers=self.headers)
+                        
+                        if file_response.status_code == 200:
+                            file_data = file_response.json()
+                            try:
+                                content = base64.b64decode(file_data["content"]).decode("utf-8")
+                                
+                                data_item = DataItem(
+                                    source_id=f"github_code_{uuid.uuid4().hex[:8]}",
+                                    content=content,
+                                    metadata={
+                                        "repo": repo,
+                                        "type": "code",
+                                        "path": item["path"],
+                                        "sha": item["sha"],
+                                        "size": item["size"],
+                                        "name": item["name"]
+                                    },
+                                    url=item["html_url"],
+                                    content_type=self._get_content_type(item["name"])
+                                )
+                                results.append(data_item)
+                                files_processed += 1
+                                
+                                # Respect rate limits
+                                time.sleep(0.5)
+                            except Exception as e:
+                                logger.error(f"Error processing file {item['path']}: {e}")
+            else:
+                logger.warning(f"Failed to get contents for {repo}/{path}: {response.status_code}")
+        except Exception as e:
+            logger.error(f"Error getting code files for {repo}/{path}: {e}")
+        
+        return results
+    
+    def _get_content_type(self, filename: str) -> str:
+        """Get the content type based on the file extension."""
+        ext = os.path.splitext(filename)[1].lower()
+        
+        content_types = {
+            ".py": "text/python",
+            ".js": "text/javascript",
+            ".ts": "text/typescript",
+            ".html": "text/html",
+            ".css": "text/css",
+            ".json": "application/json",
+            ".md": "text/markdown",
+            ".txt": "text/plain",
+            ".c": "text/c",
+            ".cpp": "text/cpp",
+            ".h": "text/c-header",
+            ".java": "text/java",
+            ".go": "text/go",
+            ".rs": "text/rust",
+            ".rb": "text/ruby",
+            ".php": "text/php"
+        }
+        
+        return content_types.get(ext, "text/plain")
diff --git a/core/connectors/web/__init__.py b/core/connectors/web/__init__.py
new file mode 100644
index 00000000..6264feb9
--- /dev/null
+++ b/core/connectors/web/__init__.py
@@ -0,0 +1,101 @@
+"""
+Web connector for Wiseflow.
+
+This module provides a connector for web sources using crawl4ai.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import uuid
+from datetime import datetime
+import os
+
+from core.plugins.connectors import ConnectorBase, DataItem
+from core.crawl4ai.async_webcrawler import AsyncWebCrawler
+from core.crawl4ai.content_scraping_strategy import ContentScrapingStrategy
+from core.crawl4ai.async_configs import AsyncConfigs
+
+logger = logging.getLogger(__name__)
+
+class WebConnector(ConnectorBase):
+    """Connector for web sources."""
+    
+    name: str = "web_connector"
+    description: str = "Connector for web sources using crawl4ai"
+    source_type: str = "web"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the web connector."""
+        super().__init__(config)
+        self.crawler: Optional[AsyncWebCrawler] = None
+        
+    def initialize(self) -> bool:
+        """Initialize the connector."""
+        try:
+            # Create the crawler
+            configs = AsyncConfigs()
+            
+            # Apply custom configurations if provided
+            if self.config.get("max_depth"):
+                configs.max_depth = self.config["max_depth"]
+            if self.config.get("max_pages"):
+                configs.max_pages = self.config["max_pages"]
+            if self.config.get("timeout"):
+                configs.timeout = self.config["timeout"]
+            if self.config.get("user_agent"):
+                configs.user_agent = self.config["user_agent"]
+            
+            # Create the crawler
+            self.crawler = AsyncWebCrawler(configs)
+            logger.info(f"Initialized web connector with config: {configs}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize web connector: {e}")
+            return False
+    
+    def collect(self, params: Optional[Dict[str, Any]] = None) -> List[DataItem]:
+        """Collect data from web sources."""
+        params = params or {}
+        urls = params.get("urls", [])
+        if not urls:
+            if self.config.get("urls"):
+                urls = self.config["urls"]
+            else:
+                logger.error("No URLs provided for web connector")
+                return []
+        
+        if not self.crawler:
+            if not self.initialize():
+                return []
+        
+        results = []
+        for url in urls:
+            try:
+                # Crawl the URL
+                logger.info(f"Crawling URL: {url}")
+                crawl_result = self.crawler.crawl(url)
+                
+                # Process the crawl result
+                for page_url, page_data in crawl_result.items():
+                    content = page_data.get("content", "")
+                    metadata = {
+                        "title": page_data.get("title", ""),
+                        "links": page_data.get("links", []),
+                        "crawl_time": datetime.now().isoformat()
+                    }
+                    
+                    # Create a data item
+                    item = DataItem(
+                        source_id=f"web_{uuid.uuid4().hex[:8]}",
+                        content=content,
+                        metadata=metadata,
+                        url=page_url,
+                        content_type="text/html",
+                        language=page_data.get("language")
+                    )
+                    results.append(item)
+            except Exception as e:
+                logger.error(f"Error crawling URL {url}: {e}")
+        
+        logger.info(f"Collected {len(results)} items from web sources")
+        return results
diff --git a/core/plugins/__init__.py b/core/plugins/__init__.py
new file mode 100644
index 00000000..f6f92172
--- /dev/null
+++ b/core/plugins/__init__.py
@@ -0,0 +1,135 @@
+"""
+Plugin system for Wiseflow.
+
+This module provides the base classes and utilities for the plugin system.
+"""
+
+from typing import Dict, List, Type, Any, Optional
+import importlib
+import os
+import logging
+import json
+import inspect
+from abc import ABC, abstractmethod
+
+logger = logging.getLogger(__name__)
+
+class PluginBase(ABC):
+    """Base class for all plugins."""
+    
+    name: str = "base_plugin"
+    description: str = "Base plugin class"
+    version: str = "0.1.0"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the plugin with optional configuration."""
+        self.config = config or {}
+        self.is_enabled = True
+        
+    @abstractmethod
+    def initialize(self) -> bool:
+        """Initialize the plugin. Return True if successful, False otherwise."""
+        pass
+    
+    def enable(self) -> None:
+        """Enable the plugin."""
+        self.is_enabled = True
+        logger.info(f"Plugin {self.name} enabled")
+        
+    def disable(self) -> None:
+        """Disable the plugin."""
+        self.is_enabled = False
+        logger.info(f"Plugin {self.name} disabled")
+    
+    def get_config(self) -> Dict[str, Any]:
+        """Get the plugin configuration."""
+        return self.config
+    
+    def set_config(self, config: Dict[str, Any]) -> None:
+        """Set the plugin configuration."""
+        self.config = config
+        
+    def __str__(self) -> str:
+        """Return a string representation of the plugin."""
+        return f"{self.name} (v{self.version}): {self.description}"
+
+
+class PluginManager:
+    """Manager for loading and managing plugins."""
+    
+    def __init__(self, plugins_dir: str = "plugins"):
+        """Initialize the plugin manager."""
+        self.plugins_dir = plugins_dir
+        self.plugins: Dict[str, PluginBase] = {}
+        
+    def discover_plugins(self) -> List[str]:
+        """Discover available plugins in the plugins directory."""
+        plugin_modules = []
+        
+        # Walk through the plugins directory
+        for root, dirs, files in os.walk(self.plugins_dir):
+            for file in files:
+                if file.endswith(".py") and not file.startswith("__"):
+                    # Convert file path to module path
+                    rel_path = os.path.relpath(os.path.join(root, file), start=os.path.dirname(self.plugins_dir))
+                    module_path = os.path.splitext(rel_path)[0].replace(os.path.sep, ".")
+                    plugin_modules.append(module_path)
+        
+        return plugin_modules
+    
+    def load_plugin(self, module_path: str) -> Optional[PluginBase]:
+        """Load a plugin from a module path."""
+        try:
+            module = importlib.import_module(module_path)
+            
+            # Find plugin classes in the module
+            for name, obj in inspect.getmembers(module):
+                if (inspect.isclass(obj) and 
+                    issubclass(obj, PluginBase) and 
+                    obj is not PluginBase):
+                    
+                    # Create an instance of the plugin
+                    plugin = obj()
+                    self.plugins[plugin.name] = plugin
+                    logger.info(f"Loaded plugin: {plugin}")
+                    return plugin
+                    
+            logger.warning(f"No plugin class found in module: {module_path}")
+            return None
+            
+        except Exception as e:
+            logger.error(f"Failed to load plugin from {module_path}: {e}")
+            return None
+    
+    def load_all_plugins(self) -> Dict[str, PluginBase]:
+        """Discover and load all available plugins."""
+        plugin_modules = self.discover_plugins()
+        
+        for module_path in plugin_modules:
+            self.load_plugin(module_path)
+            
+        return self.plugins
+    
+    def get_plugin(self, name: str) -> Optional[PluginBase]:
+        """Get a plugin by name."""
+        return self.plugins.get(name)
+    
+    def initialize_plugin(self, name: str, config: Optional[Dict[str, Any]] = None) -> bool:
+        """Initialize a plugin with optional configuration."""
+        plugin = self.get_plugin(name)
+        if plugin:
+            if config:
+                plugin.set_config(config)
+            return plugin.initialize()
+        return False
+    
+    def initialize_all_plugins(self, configs: Optional[Dict[str, Dict[str, Any]]] = None) -> Dict[str, bool]:
+        """Initialize all loaded plugins with optional configurations."""
+        results = {}
+        configs = configs or {}
+        
+        for name, plugin in self.plugins.items():
+            config = configs.get(name)
+            results[name] = self.initialize_plugin(name, config)
+            
+        return results
diff --git a/core/plugins/processors/__init__.py b/core/plugins/processors/__init__.py
new file mode 100644
index 00000000..6a35d49c
--- /dev/null
+++ b/core/plugins/processors/__init__.py
@@ -0,0 +1,103 @@
+"""
+Processors for Wiseflow.
+
+This module provides base classes for data processors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+from core.connectors import DataItem
+
+logger = logging.getLogger(__name__)
+
+class ProcessedData:
+    """Represents processed data from a processor."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        processed_content: str,
+        original_item: Optional[DataItem] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        timestamp: Optional[datetime] = None
+    ):
+        """Initialize processed data."""
+        self.source_id = source_id
+        self.processed_content = processed_content
+        self.original_item = original_item
+        self.metadata = metadata or {}
+        self.timestamp = timestamp or datetime.now()
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the processed data to a dictionary."""
+        return {
+            "source_id": self.source_id,
+            "processed_content": self.processed_content,
+            "original_item": self.original_item.to_dict() if self.original_item else None,
+            "metadata": self.metadata,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedData':
+        """Create processed data from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
+                
+        original_item = None
+        if data.get("original_item"):
+            original_item = DataItem.from_dict(data["original_item"])
+                
+        return cls(
+            source_id=data["source_id"],
+            processed_content=data["processed_content"],
+            original_item=original_item,
+            metadata=data.get("metadata", {}),
+            timestamp=timestamp
+        )
+
+
+class ProcessorBase(PluginBase):
+    """Base class for data processors."""
+    
+    name: str = "base_processor"
+    description: str = "Base processor class"
+    processor_type: str = "generic"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the processor with optional configuration."""
+        super().__init__(config)
+        
+    @abstractmethod
+    def process(self, data_item: DataItem, params: Optional[Dict[str, Any]] = None) -> ProcessedData:
+        """Process a data item."""
+        pass
+    
+    def initialize(self) -> bool:
+        """Initialize the processor. Return True if successful, False otherwise."""
+        try:
+            # Perform any necessary initialization
+            logger.info(f"Initialized processor: {self.name}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize processor {self.name}: {e}")
+            return False
+    
+    def batch_process(self, data_items: List[DataItem], params: Optional[Dict[str, Any]] = None) -> List[ProcessedData]:
+        """Process multiple data items."""
+        results = []
+        for item in data_items:
+            try:
+                result = self.process(item, params)
+                results.append(result)
+            except Exception as e:
+                logger.error(f"Error processing item {item.source_id}: {e}")
+        return results
diff --git a/core/plugins/processors/text/__init__.py b/core/plugins/processors/text/__init__.py
new file mode 100644
index 00000000..35e6e306
--- /dev/null
+++ b/core/plugins/processors/text/__init__.py
@@ -0,0 +1,157 @@
+"""
+Text processors for Wiseflow.
+
+This module provides processors for text data.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import json
+import os
+
+from core.plugins.processors import ProcessorBase, ProcessedData
+from core.plugins.connectors import DataItem
+from core.llms.litellm_wrapper import litellm_llm
+
+logger = logging.getLogger(__name__)
+
+class FocusPointProcessor(ProcessorBase):
+    """Processor that extracts information based on focus points using LLMs."""
+    
+    name: str = "focus_point_processor"
+    description: str = "Extracts information based on focus points using LLMs"
+    processor_type: str = "text"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the focus point processor."""
+        super().__init__(config)
+        self.focus_points = self.config.get("focus_points", [])
+        
+    def initialize(self) -> bool:
+        """Initialize the processor."""
+        try:
+            # Load focus points if not provided in config
+            if not self.focus_points and self.config.get("focus_points_path"):
+                path = self.config["focus_points_path"]
+                if os.path.exists(path):
+                    with open(path, 'r') as f:
+                        self.focus_points = json.load(f)
+            
+            if not self.focus_points:
+                logger.warning("No focus points provided for focus point processor")
+            
+            logger.info(f"Initialized focus point processor with {len(self.focus_points)} focus points")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize focus point processor: {e}")
+            return False
+    
+    def process(self, data_item: DataItem, params: Optional[Dict[str, Any]] = None) -> ProcessedData:
+        """Process a data item by extracting information based on focus points."""
+        params = params or {}
+        
+        # Get focus points from params or use the ones from config
+        focus_points = params.get("focus_points", self.focus_points)
+        if not focus_points:
+            logger.warning("No focus points provided for processing")
+            return ProcessedData(
+                source_id=data_item.source_id,
+                processed_content="",
+                original_item=data_item,
+                metadata={"error": "No focus points provided"}
+            )
+        
+        # Prepare the prompt
+        prompt = self._create_prompt(data_item.content, focus_points)
+        
+        try:
+            # Process with LLM
+            model = params.get("model", os.environ.get("PRIMARY_MODEL", ""))
+            if not model:
+                raise ValueError("No model specified for processing")
+            
+            messages = [
+                {"role": "system", "content": "You are an expert information extractor."},
+                {"role": "user", "content": prompt}
+            ]
+            
+            response = litellm_llm(messages, model, logger=logger)
+            
+            # Parse the response
+            extracted_info = self._parse_response(response)
+            
+            return ProcessedData(
+                source_id=data_item.source_id,
+                processed_content=extracted_info,
+                original_item=data_item,
+                metadata={
+                    "focus_points": focus_points,
+                    "url": data_item.url,
+                    "title": data_item.metadata.get("title", "")
+                }
+            )
+        except Exception as e:
+            logger.error(f"Error processing item {data_item.source_id}: {e}")
+            return ProcessedData(
+                source_id=data_item.source_id,
+                processed_content="",
+                original_item=data_item,
+                metadata={"error": str(e)}
+            )
+    
+    def _create_prompt(self, content: str, focus_points: List[Dict[str, Any]]) -> str:
+        """Create a prompt for the LLM based on the content and focus points."""
+        focus_points_str = "\n".join([
+            f"{i+1}. {fp.get('focuspoint', '')}: {fp.get('explanation', '')}"
+            for i, fp in enumerate(focus_points)
+        ])
+        
+        prompt = f"""
+You are an expert information extractor. Your task is to analyze the following content and extract information related to specific focus points.
+
+FOCUS POINTS:
+{focus_points_str}
+
+CONTENT:
+{content}
+
+For each focus point, extract any relevant information from the content. If there is no relevant information for a focus point, indicate "No relevant information found".
+
+Format your response as a JSON object with the following structure:
+{{
+  "focus_point_1": {{
+    "relevant": true/false,
+    "information": "extracted information or summary",
+    "confidence": 0-100
+  }},
+  "focus_point_2": {{
+    "relevant": true/false,
+    "information": "extracted information or summary",
+    "confidence": 0-100
+  }},
+  ...
+}}
+
+Only include the JSON object in your response, nothing else.
+"""
+        return prompt
+    
+    def _parse_response(self, response: str) -> Dict[str, Any]:
+        """Parse the LLM response into structured data."""
+        try:
+            # Try to extract JSON from the response
+            response = response.strip()
+            
+            # Find JSON object in the response
+            start_idx = response.find('{')
+            end_idx = response.rfind('}')
+            
+            if start_idx >= 0 and end_idx > start_idx:
+                json_str = response[start_idx:end_idx+1]
+                return json.loads(json_str)
+            
+            # If no JSON found, return the raw response
+            return {"raw_response": response}
+        except Exception as e:
+            logger.error(f"Error parsing LLM response: {e}")
+            return {"error": str(e), "raw_response": response}
diff --git a/core/references/__init__.py b/core/references/__init__.py
new file mode 100644
index 00000000..546ec6d9
--- /dev/null
+++ b/core/references/__init__.py
@@ -0,0 +1,278 @@
+"""
+Reference management for Wiseflow.
+
+This module provides functionality for managing reference materials for focus points.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import os
+import uuid
+import json
+import shutil
+from datetime import datetime
+import requests
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+class Reference:
+    """Represents a reference for a focus point."""
+    
+    def __init__(
+        self,
+        reference_id: str,
+        focus_id: str,
+        reference_type: str,
+        path: str,
+        content: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        timestamp: Optional[datetime] = None
+    ):
+        """Initialize a reference."""
+        self.reference_id = reference_id
+        self.focus_id = focus_id
+        self.reference_type = reference_type
+        self.path = path
+        self.content = content
+        self.metadata = metadata or {}
+        self.timestamp = timestamp or datetime.now()
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the reference to a dictionary."""
+        return {
+            "reference_id": self.reference_id,
+            "focus_id": self.focus_id,
+            "reference_type": self.reference_type,
+            "path": self.path,
+            "content": self.content,
+            "metadata": self.metadata,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'Reference':
+        """Create a reference from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
+                
+        return cls(
+            reference_id=data["reference_id"],
+            focus_id=data["focus_id"],
+            reference_type=data["reference_type"],
+            path=data["path"],
+            content=data.get("content"),
+            metadata=data.get("metadata", {}),
+            timestamp=timestamp
+        )
+
+
+class ReferenceManager:
+    """Manages reference materials for focus points."""
+    
+    def __init__(self, storage_path: str = "references"):
+        """Initialize the reference manager."""
+        self.storage_path = storage_path
+        os.makedirs(storage_path, exist_ok=True)
+        self.references: Dict[str, Reference] = {}
+        self.load_references()
+        
+    def load_references(self) -> None:
+        """Load references from storage."""
+        index_path = os.path.join(self.storage_path, "index.json")
+        if os.path.exists(index_path):
+            try:
+                with open(index_path, 'r') as f:
+                    data = json.load(f)
+                    for ref_data in data:
+                        ref = Reference.from_dict(ref_data)
+                        self.references[ref.reference_id] = ref
+                logger.info(f"Loaded {len(self.references)} references from storage")
+            except Exception as e:
+                logger.error(f"Error loading references: {e}")
+        
+    def save_references(self) -> None:
+        """Save references to storage."""
+        index_path = os.path.join(self.storage_path, "index.json")
+        try:
+            with open(index_path, 'w') as f:
+                json.dump([ref.to_dict() for ref in self.references.values()], f, indent=2)
+            logger.info(f"Saved {len(self.references)} references to storage")
+        except Exception as e:
+            logger.error(f"Error saving references: {e}")
+    
+    def add_file_reference(self, focus_id: str, file_path: str) -> Optional[Reference]:
+        """Add a file reference to a focus point."""
+        try:
+            # Generate a unique reference ID
+            reference_id = str(uuid.uuid4())
+            
+            # Create a copy of the file in the storage directory
+            filename = os.path.basename(file_path)
+            storage_dir = os.path.join(self.storage_path, focus_id)
+            os.makedirs(storage_dir, exist_ok=True)
+            
+            dest_path = os.path.join(storage_dir, f"{reference_id}_{filename}")
+            shutil.copy2(file_path, dest_path)
+            
+            # Extract content from the file
+            content = self._extract_content_from_file(file_path)
+            
+            # Create the reference
+            reference = Reference(
+                reference_id=reference_id,
+                focus_id=focus_id,
+                reference_type="file",
+                path=dest_path,
+                content=content,
+                metadata={
+                    "original_path": file_path,
+                    "filename": filename
+                }
+            )
+            
+            # Add to references
+            self.references[reference_id] = reference
+            self.save_references()
+            
+            return reference
+        except Exception as e:
+            logger.error(f"Error adding file reference: {e}")
+            return None
+    
+    def add_web_reference(self, focus_id: str, url: str) -> Optional[Reference]:
+        """Add a web reference to a focus point."""
+        try:
+            # Generate a unique reference ID
+            reference_id = str(uuid.uuid4())
+            
+            # Create a storage directory
+            storage_dir = os.path.join(self.storage_path, focus_id)
+            os.makedirs(storage_dir, exist_ok=True)
+            
+            # Parse the URL to get a filename
+            parsed_url = urlparse(url)
+            filename = os.path.basename(parsed_url.path) or "webpage"
+            if not filename.endswith(".html"):
+                filename += ".html"
+            
+            dest_path = os.path.join(storage_dir, f"{reference_id}_{filename}")
+            
+            # Download the content
+            content = self._download_web_content(url)
+            
+            # Save the content
+            with open(dest_path, 'w', encoding='utf-8') as f:
+                f.write(content)
+            
+            # Create the reference
+            reference = Reference(
+                reference_id=reference_id,
+                focus_id=focus_id,
+                reference_type="web",
+                path=dest_path,
+                content=content,
+                metadata={
+                    "url": url,
+                    "domain": parsed_url.netloc
+                }
+            )
+            
+            # Add to references
+            self.references[reference_id] = reference
+            self.save_references()
+            
+            return reference
+        except Exception as e:
+            logger.error(f"Error adding web reference: {e}")
+            return None
+    
+    def add_text_reference(self, focus_id: str, content: str, name: str = "text_reference") -> Optional[Reference]:
+        """Add a text reference to a focus point."""
+        try:
+            # Generate a unique reference ID
+            reference_id = str(uuid.uuid4())
+            
+            # Create a storage directory
+            storage_dir = os.path.join(self.storage_path, focus_id)
+            os.makedirs(storage_dir, exist_ok=True)
+            
+            # Create a filename
+            filename = f"{name}.txt"
+            dest_path = os.path.join(storage_dir, f"{reference_id}_{filename}")
+            
+            # Save the content
+            with open(dest_path, 'w', encoding='utf-8') as f:
+                f.write(content)
+            
+            # Create the reference
+            reference = Reference(
+                reference_id=reference_id,
+                focus_id=focus_id,
+                reference_type="text",
+                path=dest_path,
+                content=content,
+                metadata={
+                    "name": name
+                }
+            )
+            
+            # Add to references
+            self.references[reference_id] = reference
+            self.save_references()
+            
+            return reference
+        except Exception as e:
+            logger.error(f"Error adding text reference: {e}")
+            return None
+    
+    def get_reference(self, reference_id: str) -> Optional[Reference]:
+        """Get a reference by ID."""
+        return self.references.get(reference_id)
+    
+    def get_references_by_focus(self, focus_id: str) -> List[Reference]:
+        """Get all references for a focus point."""
+        return [ref for ref in self.references.values() if ref.focus_id == focus_id]
+    
+    def delete_reference(self, reference_id: str) -> bool:
+        """Delete a reference."""
+        if reference_id in self.references:
+            reference = self.references[reference_id]
+            
+            # Delete the file
+            try:
+                if os.path.exists(reference.path):
+                    os.remove(reference.path)
+            except Exception as e:
+                logger.error(f"Error deleting reference file: {e}")
+            
+            # Remove from references
+            del self.references[reference_id]
+            self.save_references()
+            
+            return True
+        return False
+    
+    def _extract_content_from_file(self, file_path: str) -> str:
+        """Extract content from a file."""
+        try:
+            # Simple text extraction for now
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                return f.read()
+        except Exception as e:
+            logger.error(f"Error extracting content from file: {e}")
+            return ""
+    
+    def _download_web_content(self, url: str) -> str:
+        """Download content from a URL."""
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+            return response.text
+        except Exception as e:
+            logger.error(f"Error downloading web content: {e}")
+            return ""
diff --git a/core/run_task_new.py b/core/run_task_new.py
new file mode 100644
index 00000000..7105905c
--- /dev/null
+++ b/core/run_task_new.py
@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+from pathlib import Path
+from dotenv import load_dotenv
+env_path = Path(__file__).parent / '.env'
+if env_path.exists():
+    load_dotenv(env_path)
+
+import logging
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+import asyncio
+import os
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+
+from core.general_process import main_process, wiseflow_logger, pb
+from core.task import AsyncTaskManager, Task, create_task_id
+from core.plugins import PluginManager
+from core.plugins.connectors import ConnectorBase
+from core.plugins.processors import ProcessorBase
+
+# Configure the maximum number of concurrent tasks
+MAX_CONCURRENT_TASKS = int(os.environ.get("MAX_CONCURRENT_TASKS", "4"))
+
+# Initialize the task manager
+task_manager = AsyncTaskManager(max_workers=MAX_CONCURRENT_TASKS)
+
+# Initialize the plugin manager
+plugin_manager = PluginManager(plugins_dir="core")
+
+async def load_plugins():
+    """Load and initialize all plugins."""
+    wiseflow_logger.info("Loading plugins...")
+    plugins = plugin_manager.load_all_plugins()
+    wiseflow_logger.info(f"Loaded {len(plugins)} plugins")
+    
+    # Initialize plugins with configurations
+    configs = {}  # Load configurations from database or config files
+    results = plugin_manager.initialize_all_plugins(configs)
+    
+    for name, success in results.items():
+        if success:
+            wiseflow_logger.info(f"Initialized plugin: {name}")
+        else:
+            wiseflow_logger.error(f"Failed to initialize plugin: {name}")
+    
+    return plugins
+
+async def process_focus_point(focus: Dict[str, Any], sites: List[Dict[str, Any]]) -> Any:
+    """Process a focus point using the main_process function."""
+    wiseflow_logger.info(f"Processing focus point: {focus['focuspoint']}")
+    return await main_process(focus, sites)
+
+async def schedule_task():
+    """Schedule and manage data mining tasks."""
+    # Load plugins
+    await load_plugins()
+    
+    while True:
+        wiseflow_logger.info("Checking for active focus points...")
+        
+        # Get active focus points
+        focus_points = pb.read('focus_points', filter='activated=True')
+        sites_record = pb.read('sites')
+        
+        for focus in focus_points:
+            # Check if there's already a task running for this focus point
+            existing_tasks = task_manager.get_tasks_by_focus(focus["id"])
+            active_tasks = [t for t in existing_tasks if t.status in ["pending", "running"]]
+            
+            if active_tasks:
+                wiseflow_logger.info(f"Focus point {focus['focuspoint']} already has active tasks")
+                continue
+            
+            # Get sites for this focus point
+            sites = [_record for _record in sites_record if _record['id'] in focus.get('sites', [])]
+            
+            # Create a new task
+            task_id = create_task_id()
+            auto_shutdown = focus.get("auto_shutdown", False)
+            
+            task = Task(
+                task_id=task_id,
+                focus_id=focus["id"],
+                function=process_focus_point,
+                args=(focus, sites),
+                auto_shutdown=auto_shutdown
+            )
+            
+            # Submit the task
+            wiseflow_logger.info(f"Submitting task {task_id} for focus point: {focus['focuspoint']}")
+            await task_manager.submit_task(task)
+        
+        # Wait before checking again
+        wiseflow_logger.info("Waiting for 60 seconds before checking for new tasks...")
+        await asyncio.sleep(60)
+
+async def main():
+    """Main entry point."""
+    try:
+        wiseflow_logger.info("Starting Wiseflow with concurrent task management...")
+        await schedule_task()
+    except KeyboardInterrupt:
+        wiseflow_logger.info("Shutting down...")
+        await task_manager.shutdown()
+    except Exception as e:
+        wiseflow_logger.error(f"Error in main loop: {e}")
+        await task_manager.shutdown()
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/core/task/__init__.py b/core/task/__init__.py
new file mode 100644
index 00000000..d07fa375
--- /dev/null
+++ b/core/task/__init__.py
@@ -0,0 +1,260 @@
+"""
+Task management for Wiseflow.
+
+This module provides task management and concurrency support.
+"""
+
+from typing import Dict, List, Any, Optional, Union, Callable, Tuple
+import logging
+import threading
+import asyncio
+import uuid
+from datetime import datetime
+import time
+from concurrent.futures import ThreadPoolExecutor, Future
+
+logger = logging.getLogger(__name__)
+
+class Task:
+    """Represents a data mining task."""
+    
+    def __init__(
+        self,
+        task_id: str,
+        focus_id: str,
+        function: Callable,
+        args: Tuple = (),
+        kwargs: Dict[str, Any] = None,
+        auto_shutdown: bool = False
+    ):
+        """Initialize a task."""
+        self.task_id = task_id
+        self.focus_id = focus_id
+        self.function = function
+        self.args = args
+        self.kwargs = kwargs or {}
+        self.auto_shutdown = auto_shutdown
+        self.status = "pending"
+        self.start_time: Optional[datetime] = None
+        self.end_time: Optional[datetime] = None
+        self.result: Any = None
+        self.error: Optional[Exception] = None
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the task to a dictionary."""
+        return {
+            "task_id": self.task_id,
+            "focus_id": self.focus_id,
+            "status": self.status,
+            "auto_shutdown": self.auto_shutdown,
+            "start_time": self.start_time.isoformat() if self.start_time else None,
+            "end_time": self.end_time.isoformat() if self.end_time else None,
+            "error": str(self.error) if self.error else None
+        }
+
+
+class TaskManager:
+    """Manages concurrent data mining tasks."""
+    
+    def __init__(self, max_workers: int = 4):
+        """Initialize the task manager."""
+        self.max_workers = max_workers
+        self.executor = ThreadPoolExecutor(max_workers=max_workers)
+        self.tasks: Dict[str, Task] = {}
+        self.futures: Dict[str, Future] = {}
+        self.lock = threading.Lock()
+        
+    def submit_task(self, task: Task) -> Future:
+        """Submit a task for execution."""
+        with self.lock:
+            # Create a wrapper function to handle task status updates
+            def task_wrapper(*args, **kwargs):
+                try:
+                    task.status = "running"
+                    task.start_time = datetime.now()
+                    logger.info(f"Task {task.task_id} started")
+                    
+                    # Execute the task
+                    result = task.function(*args, **kwargs)
+                    
+                    task.status = "completed"
+                    task.end_time = datetime.now()
+                    task.result = result
+                    logger.info(f"Task {task.task_id} completed")
+                    
+                    # Auto-shutdown if enabled
+                    if task.auto_shutdown:
+                        logger.info(f"Auto-shutdown enabled for task {task.task_id}")
+                        # Implement auto-shutdown logic here
+                    
+                    return result
+                except Exception as e:
+                    task.status = "failed"
+                    task.end_time = datetime.now()
+                    task.error = e
+                    logger.error(f"Task {task.task_id} failed: {e}")
+                    raise
+            
+            # Submit the task to the executor
+            future = self.executor.submit(task_wrapper, *task.args, **task.kwargs)
+            self.tasks[task.task_id] = task
+            self.futures[task.task_id] = future
+            
+            # Add a callback to clean up completed tasks
+            future.add_done_callback(lambda f: self._task_completed(task.task_id))
+            
+            return future
+    
+    def _task_completed(self, task_id: str) -> None:
+        """Handle task completion."""
+        with self.lock:
+            if task_id in self.futures:
+                # Get the result or exception
+                future = self.futures[task_id]
+                try:
+                    future.result()  # This will raise any exception that occurred
+                except Exception as e:
+                    logger.error(f"Task {task_id} failed with exception: {e}")
+    
+    def get_task(self, task_id: str) -> Optional[Task]:
+        """Get a task by ID."""
+        return self.tasks.get(task_id)
+    
+    def get_tasks_by_focus(self, focus_id: str) -> List[Task]:
+        """Get all tasks for a focus point."""
+        return [task for task in self.tasks.values() if task.focus_id == focus_id]
+    
+    def get_all_tasks(self) -> List[Task]:
+        """Get all tasks."""
+        return list(self.tasks.values())
+    
+    def cancel_task(self, task_id: str) -> bool:
+        """Cancel a task."""
+        with self.lock:
+            if task_id in self.futures:
+                future = self.futures[task_id]
+                cancelled = future.cancel()
+                if cancelled:
+                    task = self.tasks[task_id]
+                    task.status = "cancelled"
+                    task.end_time = datetime.now()
+                    logger.info(f"Task {task_id} cancelled")
+                return cancelled
+            return False
+    
+    def shutdown(self, wait: bool = True) -> None:
+        """Shutdown the task manager."""
+        self.executor.shutdown(wait=wait)
+        logger.info("Task manager shutdown")
+
+
+class AsyncTaskManager:
+    """Manages concurrent asynchronous data mining tasks."""
+    
+    def __init__(self, max_workers: int = 4):
+        """Initialize the async task manager."""
+        self.max_workers = max_workers
+        self.semaphore = asyncio.Semaphore(max_workers)
+        self.tasks: Dict[str, Task] = {}
+        self.futures: Dict[str, asyncio.Task] = {}
+        self.lock = asyncio.Lock()
+        
+    async def submit_task(self, task: Task) -> asyncio.Task:
+        """Submit a task for execution."""
+        async with self.lock:
+            # Create a wrapper function to handle task status updates
+            async def task_wrapper(*args, **kwargs):
+                async with self.semaphore:
+                    try:
+                        task.status = "running"
+                        task.start_time = datetime.now()
+                        logger.info(f"Task {task.task_id} started")
+                        
+                        # Execute the task
+                        result = await task.function(*args, **kwargs)
+                        
+                        task.status = "completed"
+                        task.end_time = datetime.now()
+                        task.result = result
+                        logger.info(f"Task {task.task_id} completed")
+                        
+                        # Auto-shutdown if enabled
+                        if task.auto_shutdown:
+                            logger.info(f"Auto-shutdown enabled for task {task.task_id}")
+                            # Implement auto-shutdown logic here
+                        
+                        return result
+                    except Exception as e:
+                        task.status = "failed"
+                        task.end_time = datetime.now()
+                        task.error = e
+                        logger.error(f"Task {task.task_id} failed: {e}")
+                        raise
+            
+            # Create an asyncio task
+            future = asyncio.create_task(task_wrapper(*task.args, **task.kwargs))
+            self.tasks[task.task_id] = task
+            self.futures[task.task_id] = future
+            
+            # Add a callback to clean up completed tasks
+            future.add_done_callback(lambda f: asyncio.create_task(self._task_completed(task.task_id)))
+            
+            return future
+    
+    async def _task_completed(self, task_id: str) -> None:
+        """Handle task completion."""
+        async with self.lock:
+            if task_id in self.futures:
+                # Get the result or exception
+                future = self.futures[task_id]
+                try:
+                    await future  # This will raise any exception that occurred
+                except Exception as e:
+                    logger.error(f"Task {task_id} failed with exception: {e}")
+    
+    def get_task(self, task_id: str) -> Optional[Task]:
+        """Get a task by ID."""
+        return self.tasks.get(task_id)
+    
+    def get_tasks_by_focus(self, focus_id: str) -> List[Task]:
+        """Get all tasks for a focus point."""
+        return [task for task in self.tasks.values() if task.focus_id == focus_id]
+    
+    def get_all_tasks(self) -> List[Task]:
+        """Get all tasks."""
+        return list(self.tasks.values())
+    
+    def cancel_task(self, task_id: str) -> bool:
+        """Cancel a task."""
+        if task_id in self.futures:
+            future = self.futures[task_id]
+            future.cancel()
+            task = self.tasks[task_id]
+            task.status = "cancelled"
+            task.end_time = datetime.now()
+            logger.info(f"Task {task_id} cancelled")
+            return True
+        return False
+    
+    async def shutdown(self) -> None:
+        """Shutdown the task manager."""
+        async with self.lock:
+            for task_id, future in self.futures.items():
+                if not future.done():
+                    future.cancel()
+            
+            # Wait for all tasks to complete or be cancelled
+            for future in self.futures.values():
+                try:
+                    await future
+                except asyncio.CancelledError:
+                    pass
+                except Exception:
+                    pass
+            
+            logger.info("Async task manager shutdown")
+
+
+def create_task_id() -> str:
+    """Create a unique task ID."""
+    return f"task_{uuid.uuid4().hex[:8]}"


Reviews:
user: login: sourcery-ai[bot]
body: Hey @Zeeeepa - I've reviewed your changes - here's some feedback:

**Overall Comments**:
- Consider adding a mechanism to handle plugin versioning and updates to ensure compatibility and access to the latest features.
- It would be beneficial to implement a more robust error handling strategy, especially around external API calls and file operations, to prevent the application from crashing.

<details>
<summary>Here's what I looked at during the review</summary>

- 🟡 **General issues**: 2 issues found
- 🟢 **Security**: all looks good
- 🟢 **Testing**: all looks good
- 🟢 **Complexity**: all looks good
- 🟢 **Documentation**: all looks good
</details>

***

<details>
<summary>Sourcery is free for open source - if you like our reviews please consider sharing them ✨</summary>

- [X](https://twitter.com/intent/tweet?text=I%20just%20got%20an%20instant%20code%20review%20from%20%40SourceryAI%2C%20and%20it%20was%20brilliant%21%20It%27s%20free%20for%20open%20source%20and%20has%20a%20free%20trial%20for%20private%20code.%20Check%20it%20out%20https%3A//sourcery.ai)
- [Mastodon](https://mastodon.social/share?text=I%20just%20got%20an%20instant%20code%20review%20from%20%40SourceryAI%2C%20and%20it%20was%20brilliant%21%20It%27s%20free%20for%20open%20source%20and%20has%20a%20free%20trial%20for%20private%20code.%20Check%20it%20out%20https%3A//sourcery.ai)
- [LinkedIn](https://www.linkedin.com/sharing/share-offsite/?url=https://sourcery.ai)
- [Facebook](https://www.facebook.com/sharer/sharer.php?u=https://sourcery.ai)

</details>

<sub>
Help me be more useful! Please click 👍 or 👎 on each comment and I'll use the feedback to improve your reviews.
</sub>
state: APPROVED
author_association: NONE
submitted_at: 2025-04-06T11:10:07Z
user: login: korbit-ai[bot]
body: ### Review by Korbit AI

###### Korbit automatically attempts to detect when you fix issues in new commits.

| Category | Issue | Status |
|----------|-------|----------|
| <sub>![Error Handling](https://img.shields.io/badge/Error%20Handling-ea580c)</sub> | Silent Error Suppression in Timestamp Parsing ▹ [view]( https://github.com/Zeeeepa/wiseflow/pull/2#discussion_r2030118327 ) ||
| <sub>![Logging](https://img.shields.io/badge/Logging-4f46e5)</sub> | Missing logging requirements in abstract method ▹ [view]( https://github.com/Zeeeepa/wiseflow/pull/2#discussion_r2030118328 ) ||
| <sub>![Documentation](https://img.shields.io/badge/Documentation-7c3aed)</sub> | Incomplete DataItem initialization docstring ▹ [view]( https://github.com/Zeeeepa/wiseflow/pull/2#discussion_r2030118329 ) ||
| <sub>![Documentation](https://img.shields.io/badge/Documentation-7c3aed)</sub> | Insufficient collect method documentation ▹ [view]( https://github.com/Zeeeepa/wiseflow/pull/2#discussion_r2030118330 ) ||
| <sub>![Documentation](https://img.shields.io/badge/Documentation-7c3aed)</sub> | Incomplete Constructor Documentation ▹ [view]( https://github.com/Zeeeepa/wiseflow/pull/2#discussion_r2030118332 ) ||
| <sub>![Documentation](https://img.shields.io/badge/Documentation-7c3aed)</sub> | Abstract Method Contract Missing ▹ [view]( https://github.com/Zeeeepa/wiseflow/pull/2#discussion_r2030118334 ) ||
| <sub>![Error Handling](https://img.shields.io/badge/Error%20Handling-ea580c)</sub> | Silent Error Swallowing in Batch Processing ▹ [view]( https://github.com/Zeeeepa/wiseflow/pull/2#discussion_r2030118336 ) ||
| <sub>![Error Handling](https://img.shields.io/badge/Error%20Handling-ea580c)</sub> | Silent Timestamp Parse Error ▹ [view]( https://github.com/Zeeeepa/wiseflow/pull/2#discussion_r2030118338 ) ||
| <sub>![Logging](https://img.shields.io/badge/Logging-4f46e5)</sub> | Insufficient batch processing metrics logging ▹ [view]( https://github.com/Zeeeepa/wiseflow/pull/2#discussion_r2030118339 ) ||
| <sub>![Readability](https://img.shields.io/badge/Readability-0284c7)</sub> | Unhelpful Implementation Comment ▹ [view]( https://github.com/Zeeeepa/wiseflow/pull/2#discussion_r2030118342 ) ||






<details>
  <summary>Files scanned</summary>
&#8203;



| File Path | Reviewed |
| ------------------ | -------------- |
| core/connectors/__init__.py | ✅ |
| core/plugins/processors/__init__.py | ✅ |
| core/connectors/web/__init__.py | ✅ |
| core/run_task_new.py | ✅ |
| core/plugins/__init__.py | ✅ |
| core/plugins/processors/text/__init__.py | ✅ |
| core/task/__init__.py | ✅ |
| core/references/__init__.py | ✅ |
| core/connectors/github/__init__.py | ✅ |


> Explore our [documentation](https://docs.korbit.ai) to understand the languages and file types [we support](https://docs.korbit.ai/configuration/supported-languages) and the [files we ignore](https://docs.korbit.ai/configuration/ignoring-files).

</details>




> Need a new review? Comment `/korbit-review` on this PR and I'll review your latest changes.
>
> <details>
>   <summary>Korbit Guide: Usage and Customization</summary>
>
> ## Interacting with Korbit
> - You can manually ask Korbit to review your PR using the `/korbit-review` command in a comment at the root of your PR.
> - You can ask Korbit to generate a new PR description using the `/korbit-generate-pr-description` command in any comment on your PR.
> - Too many Korbit comments? I can resolve all my comment threads if you use the `/korbit-resolve` command in any comment on your PR.
> - On any given comment that Korbit raises on your pull request, you can have a discussion with Korbit by replying to the comment.
> - Help train Korbit to improve your reviews by giving a 👍 or 👎 on the comments Korbit posts.
>
> ## Customizing Korbit
> - Check out our [docs](https://docs.korbit.ai/configuration/organization-settings) on how you can make Korbit work best for you and your team.
> - Customize Korbit for your organization through the [Korbit Console](https://app.korbit.ai/9c852237-692e-4846-94ac-e139dfda989d/settings).
>
>
> ### Current Korbit Configuration
>
>
>
> <details>
>   <summary>General Settings</summary>
> &#8203;
>
> | Setting          | Value       |
> | ---------------- | ----------- |
> | Review Schedule | Automatic excluding drafts |
> | Max Issue Count | 10 |
> | Automatic PR Descriptions | ✅ |
>
> </details>
>
>
>
> <details>
>   <summary>Issue Categories</summary>
> &#8203;
>
> | Category          | Enabled       |
> | ---------------- | ----------- |
> | Documentation | ✅ |
> | Logging | ✅ |
> | Error Handling | ✅ |
> | Readability | ✅ |
> | Design | ✅ |
> | Performance | ✅ |
> | Security | ✅ |
> | Functionality | ✅ |
>
> </details>
>
>
> ## Feedback and Support
> - [Tell us what you think of Korbit](https://rfn13ctpmt0.typeform.com/to/DOoeLyoN)
> - [Schedule a call with our team](https://meetings.salesloft.com/korbittechnologies/korbitai)
> - Email us @ support@korbit.ai
> </details>

> [!NOTE]
> Korbit Pro is free for open source projects 🎉
> ###### Looking to add Korbit to your team? Get started with a free 2 week trial [here](https://www.korbit.ai?utm_source=github_open_source_messaging)


state: COMMENTED
author_association: NONE
submitted_at: 2025-04-06T11:10:33Z

Review comments:
diff_hunk: @@ -0,0 +1,260 @@
+"""
+Task management for Wiseflow.
+
+This module provides task management and concurrency support.
+"""
+
+from typing import Dict, List, Any, Optional, Union, Callable, Tuple
+import logging
+import threading
+import asyncio
+import uuid
+from datetime import datetime
+import time
+from concurrent.futures import ThreadPoolExecutor, Future
+
+logger = logging.getLogger(__name__)
+
+class Task:
+    """Represents a data mining task."""
+    
+    def __init__(
+        self,
+        task_id: str,
+        focus_id: str,
+        function: Callable,
+        args: Tuple = (),
+        kwargs: Dict[str, Any] = None,
+        auto_shutdown: bool = False
+    ):
+        """Initialize a task."""
+        self.task_id = task_id
+        self.focus_id = focus_id
+        self.function = function
+        self.args = args
+        self.kwargs = kwargs or {}
+        self.auto_shutdown = auto_shutdown
+        self.status = "pending"
+        self.start_time: Optional[datetime] = None
+        self.end_time: Optional[datetime] = None
+        self.result: Any = None
+        self.error: Optional[Exception] = None
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the task to a dictionary."""
+        return {
+            "task_id": self.task_id,
+            "focus_id": self.focus_id,
+            "status": self.status,
+            "auto_shutdown": self.auto_shutdown,
+            "start_time": self.start_time.isoformat() if self.start_time else None,
+            "end_time": self.end_time.isoformat() if self.end_time else None,
+            "error": str(self.error) if self.error else None
+        }
+
+
+class TaskManager:
+    """Manages concurrent data mining tasks."""
+    
+    def __init__(self, max_workers: int = 4):
+        """Initialize the task manager."""
+        self.max_workers = max_workers
+        self.executor = ThreadPoolExecutor(max_workers=max_workers)
+        self.tasks: Dict[str, Task] = {}
+        self.futures: Dict[str, Future] = {}
+        self.lock = threading.Lock()
+        
+    def submit_task(self, task: Task) -> Future:
+        """Submit a task for execution."""
+        with self.lock:
+            # Create a wrapper function to handle task status updates
+            def task_wrapper(*args, **kwargs):
+                try:
+                    task.status = "running"
+                    task.start_time = datetime.now()
+                    logger.info(f"Task {task.task_id} started")
+                    
+                    # Execute the task
+                    result = task.function(*args, **kwargs)
+                    
+                    task.status = "completed"
+                    task.end_time = datetime.now()
+                    task.result = result
+                    logger.info(f"Task {task.task_id} completed")
+                    
+                    # Auto-shutdown if enabled
+                    if task.auto_shutdown:
+                        logger.info(f"Auto-shutdown enabled for task {task.task_id}")
+                        # Implement auto-shutdown logic here
+                    
+                    return result
+                except Exception as e:
+                    task.status = "failed"
+                    task.end_time = datetime.now()
+                    task.error = e
+                    logger.error(f"Task {task.task_id} failed: {e}")
+                    raise
+            
+            # Submit the task to the executor
+            future = self.executor.submit(task_wrapper, *task.args, **task.kwargs)
+            self.tasks[task.task_id] = task
+            self.futures[task.task_id] = future
+            
+            # Add a callback to clean up completed tasks
+            future.add_done_callback(lambda f: self._task_completed(task.task_id))
+            
+            return future
+    
+    def _task_completed(self, task_id: str) -> None:
+        """Handle task completion."""
+        with self.lock:
+            if task_id in self.futures:
+                # Get the result or exception
+                future = self.futures[task_id]
+                try:
+                    future.result()  # This will raise any exception that occurred
+                except Exception as e:
+                    logger.error(f"Task {task_id} failed with exception: {e}")
+    
+    def get_task(self, task_id: str) -> Optional[Task]:
+        """Get a task by ID."""
+        return self.tasks.get(task_id)
+    
+    def get_tasks_by_focus(self, focus_id: str) -> List[Task]:
+        """Get all tasks for a focus point."""
+        return [task for task in self.tasks.values() if task.focus_id == focus_id]
+    
+    def get_all_tasks(self) -> List[Task]:
+        """Get all tasks."""
+        return list(self.tasks.values())
+    
+    def cancel_task(self, task_id: str) -> bool:
+        """Cancel a task."""
+        with self.lock:
+            if task_id in self.futures:
+                future = self.futures[task_id]
+                cancelled = future.cancel()
+                if cancelled:
+                    task = self.tasks[task_id]
+                    task.status = "cancelled"
+                    task.end_time = datetime.now()
+                    logger.info(f"Task {task_id} cancelled")
+                return cancelled
+            return False
+    
+    def shutdown(self, wait: bool = True) -> None:
+        """Shutdown the task manager."""
+        self.executor.shutdown(wait=wait)
+        logger.info("Task manager shutdown")
+
+
+class AsyncTaskManager:
+    """Manages concurrent asynchronous data mining tasks."""
+    
+    def __init__(self, max_workers: int = 4):
+        """Initialize the async task manager."""
+        self.max_workers = max_workers
+        self.semaphore = asyncio.Semaphore(max_workers)
+        self.tasks: Dict[str, Task] = {}
+        self.futures: Dict[str, asyncio.Task] = {}
+        self.lock = asyncio.Lock()
+        
+    async def submit_task(self, task: Task) -> asyncio.Task:
+        """Submit a task for execution."""
+        async with self.lock:
+            # Create a wrapper function to handle task status updates
+            async def task_wrapper(*args, **kwargs):
+                async with self.semaphore:
+                    try:
+                        task.status = "running"
+                        task.start_time = datetime.now()
+                        logger.info(f"Task {task.task_id} started")
+                        
+                        # Execute the task
+                        result = await task.function(*args, **kwargs)
+                        
+                        task.status = "completed"
+                        task.end_time = datetime.now()
+                        task.result = result
+                        logger.info(f"Task {task.task_id} completed")
+                        
+                        # Auto-shutdown if enabled
+                        if task.auto_shutdown:
+                            logger.info(f"Auto-shutdown enabled for task {task.task_id}")
+                            # Implement auto-shutdown logic here
+                        
+                        return result
+                    except Exception as e:
+                        task.status = "failed"
+                        task.end_time = datetime.now()
+                        task.error = e
+                        logger.error(f"Task {task.task_id} failed: {e}")
+                        raise
+            
+            # Create an asyncio task
+            future = asyncio.create_task(task_wrapper(*task.args, **task.kwargs))
+            self.tasks[task.task_id] = task
+            self.futures[task.task_id] = future
+            
+            # Add a callback to clean up completed tasks
+            future.add_done_callback(lambda f: asyncio.create_task(self._task_completed(task.task_id)))
path: core/task/__init__.py
user: login: sourcery-ai[bot]
body: **suggestion:** Review error handling in the asynchronous task callback.

The done callback wraps _task_completed in an asyncio.create_task call, but it may be beneficial to include explicit error handling within the callback to log any issues more clearly.
created_at: 2025-04-06T11:10:06Z
author_association: NONE
diff_hunk: @@ -0,0 +1,157 @@
+"""
+Text processors for Wiseflow.
+
+This module provides processors for text data.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import json
+import os
+
+from core.plugins.processors import ProcessorBase, ProcessedData
+from core.plugins.connectors import DataItem
+from core.llms.litellm_wrapper import litellm_llm
+
+logger = logging.getLogger(__name__)
+
+class FocusPointProcessor(ProcessorBase):
+    """Processor that extracts information based on focus points using LLMs."""
+    
+    name: str = "focus_point_processor"
+    description: str = "Extracts information based on focus points using LLMs"
+    processor_type: str = "text"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the focus point processor."""
+        super().__init__(config)
+        self.focus_points = self.config.get("focus_points", [])
+        
+    def initialize(self) -> bool:
+        """Initialize the processor."""
+        try:
+            # Load focus points if not provided in config
+            if not self.focus_points and self.config.get("focus_points_path"):
+                path = self.config["focus_points_path"]
+                if os.path.exists(path):
+                    with open(path, 'r') as f:
path: core/plugins/processors/text/__init__.py
user: login: sourcery-ai[bot]
body: **suggestion:** Specify file encoding when opening focus points file.

Including an explicit encoding (e.g., 'utf-8') when opening the focus points file can help avoid issues on systems with different default encodings.

```suggestion
                    with open(path, 'r', encoding='utf-8') as f:
```
created_at: 2025-04-06T11:10:06Z
author_association: NONE
diff_hunk: @@ -0,0 +1,101 @@
+"""
+Web connector for Wiseflow.
+
+This module provides a connector for web sources using crawl4ai.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import uuid
+from datetime import datetime
+import os
+
+from core.plugins.connectors import ConnectorBase, DataItem
+from core.crawl4ai.async_webcrawler import AsyncWebCrawler
+from core.crawl4ai.content_scraping_strategy import ContentScrapingStrategy
+from core.crawl4ai.async_configs import AsyncConfigs
+
+logger = logging.getLogger(__name__)
+
+class WebConnector(ConnectorBase):
+    """Connector for web sources."""
+    
+    name: str = "web_connector"
+    description: str = "Connector for web sources using crawl4ai"
+    source_type: str = "web"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the web connector."""
+        super().__init__(config)
+        self.crawler: Optional[AsyncWebCrawler] = None
+        
+    def initialize(self) -> bool:
+        """Initialize the connector."""
+        try:
+            # Create the crawler
+            configs = AsyncConfigs()
+            
+            # Apply custom configurations if provided
+            if self.config.get("max_depth"):
+                configs.max_depth = self.config["max_depth"]
+            if self.config.get("max_pages"):
+                configs.max_pages = self.config["max_pages"]
+            if self.config.get("timeout"):
+                configs.timeout = self.config["timeout"]
+            if self.config.get("user_agent"):
+                configs.user_agent = self.config["user_agent"]
+            
+            # Create the crawler
+            self.crawler = AsyncWebCrawler(configs)
+            logger.info(f"Initialized web connector with config: {configs}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize web connector: {e}")
+            return False
+    
+    def collect(self, params: Optional[Dict[str, Any]] = None) -> List[DataItem]:
+        """Collect data from web sources."""
+        params = params or {}
+        urls = params.get("urls", [])
+        if not urls:
+            if self.config.get("urls"):
+                urls = self.config["urls"]
+            else:
+                logger.error("No URLs provided for web connector")
+                return []
+        
+        if not self.crawler:
+            if not self.initialize():
+                return []
path: core/connectors/web/__init__.py
user: login: sourcery-ai[bot]
body: **suggestion (code-quality):** Merge nested if conditions ([`merge-nested-ifs`](https://docs.sourcery.ai/Reference/Rules-and-In-Line-Suggestions/Python/Default-Rules/merge-nested-ifs))

```suggestion
        if not self.crawler and not self.initialize():
            return []

```

<br/><details><summary>Explanation</summary>Too much nesting can make code difficult to understand, and this is especially
true in Python, where there are no brackets to help out with the delineation of
different nesting levels.

Reading deeply nested code is confusing, since you have to keep track of which
conditions relate to which levels. We therefore strive to reduce nesting where
possible, and the situation where two `if` conditions can be combined using
`and` is an easy win.
</details>
created_at: 2025-04-06T11:10:06Z
author_association: NONE
diff_hunk: @@ -0,0 +1,291 @@
+"""
+GitHub connector for Wiseflow.
+
+This module provides a connector for GitHub repositories.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import uuid
+from datetime import datetime
+import os
+import requests
+import base64
+import time
+
+from core.plugins.connectors import ConnectorBase, DataItem
+
+logger = logging.getLogger(__name__)
+
+class GitHubConnector(ConnectorBase):
+    """Connector for GitHub repositories."""
+    
+    name: str = "github_connector"
+    description: str = "Connector for GitHub repositories"
+    source_type: str = "github"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the GitHub connector."""
+        super().__init__(config)
+        self.api_token = None
+        self.api_base_url = "https://api.github.com"
+        self.headers = {}
+        
+    def initialize(self) -> bool:
+        """Initialize the connector."""
+        try:
+            # Get API token from config or environment
+            self.api_token = self.config.get("api_token")
+            if not self.api_token and "GITHUB_TOKEN" in os.environ:
+                self.api_token = os.environ["GITHUB_TOKEN"]
+            
+            if not self.api_token:
+                logger.warning("No GitHub API token provided. Rate limits will be restricted.")
+                self.headers = {
+                    "Accept": "application/vnd.github.v3+json"
+                }
+            else:
+                self.headers = {
+                    "Accept": "application/vnd.github.v3+json",
+                    "Authorization": f"token {self.api_token}"
+                }
+            
+            logger.info("Initialized GitHub connector")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize GitHub connector: {e}")
+            return False
+    
+    def collect(self, params: Optional[Dict[str, Any]] = None) -> List[DataItem]:
+        """Collect data from GitHub repositories."""
+        params = params or {}
+        
+        # Get repositories to collect from
+        repos = params.get("repositories", [])
+        if not repos:
+            if self.config.get("repositories"):
+                repos = self.config["repositories"]
+            else:
+                logger.error("No repositories provided for GitHub connector")
+                return []
+        
+        # Determine what to collect
+        collect_readme = params.get("collect_readme", True)
+        collect_issues = params.get("collect_issues", False)
+        collect_code = params.get("collect_code", False)
+        max_issues = params.get("max_issues", 10)
+        
+        results = []
+        
+        for repo in repos:
+            try:
+                # Format: "owner/repo"
+                if "/" not in repo:
+                    logger.warning(f"Invalid repository format: {repo}. Expected 'owner/repo'")
+                    continue
+                
+                logger.info(f"Collecting data from GitHub repository: {repo}")
+                
+                # Collect README
+                if collect_readme:
+                    readme = self._get_readme(repo)
path: core/connectors/github/__init__.py
user: login: sourcery-ai[bot]
body: **issue (code-quality):** Use named expression to simplify assignment and conditional ([`use-named-expression`](https://docs.sourcery.ai/Reference/Default-Rules/refactorings/use-named-expression/))
created_at: 2025-04-06T11:10:06Z
author_association: NONE
diff_hunk: @@ -0,0 +1,291 @@
+"""
+GitHub connector for Wiseflow.
+
+This module provides a connector for GitHub repositories.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import uuid
+from datetime import datetime
+import os
+import requests
+import base64
+import time
+
+from core.plugins.connectors import ConnectorBase, DataItem
+
+logger = logging.getLogger(__name__)
+
+class GitHubConnector(ConnectorBase):
+    """Connector for GitHub repositories."""
+    
+    name: str = "github_connector"
+    description: str = "Connector for GitHub repositories"
+    source_type: str = "github"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the GitHub connector."""
+        super().__init__(config)
+        self.api_token = None
+        self.api_base_url = "https://api.github.com"
+        self.headers = {}
+        
+    def initialize(self) -> bool:
+        """Initialize the connector."""
+        try:
+            # Get API token from config or environment
+            self.api_token = self.config.get("api_token")
+            if not self.api_token and "GITHUB_TOKEN" in os.environ:
+                self.api_token = os.environ["GITHUB_TOKEN"]
+            
+            if not self.api_token:
+                logger.warning("No GitHub API token provided. Rate limits will be restricted.")
+                self.headers = {
+                    "Accept": "application/vnd.github.v3+json"
+                }
+            else:
+                self.headers = {
+                    "Accept": "application/vnd.github.v3+json",
+                    "Authorization": f"token {self.api_token}"
+                }
+            
+            logger.info("Initialized GitHub connector")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize GitHub connector: {e}")
+            return False
+    
+    def collect(self, params: Optional[Dict[str, Any]] = None) -> List[DataItem]:
+        """Collect data from GitHub repositories."""
+        params = params or {}
+        
+        # Get repositories to collect from
+        repos = params.get("repositories", [])
+        if not repos:
+            if self.config.get("repositories"):
+                repos = self.config["repositories"]
+            else:
+                logger.error("No repositories provided for GitHub connector")
+                return []
+        
+        # Determine what to collect
+        collect_readme = params.get("collect_readme", True)
+        collect_issues = params.get("collect_issues", False)
+        collect_code = params.get("collect_code", False)
+        max_issues = params.get("max_issues", 10)
+        
+        results = []
+        
+        for repo in repos:
+            try:
+                # Format: "owner/repo"
+                if "/" not in repo:
+                    logger.warning(f"Invalid repository format: {repo}. Expected 'owner/repo'")
+                    continue
+                
+                logger.info(f"Collecting data from GitHub repository: {repo}")
+                
+                # Collect README
+                if collect_readme:
+                    readme = self._get_readme(repo)
+                    if readme:
+                        results.append(readme)
+                
+                # Collect issues
+                if collect_issues:
+                    issues = self._get_issues(repo, max_issues)
+                    results.extend(issues)
+                
+                # Collect code (basic implementation)
+                if collect_code:
+                    code_files = self._get_code_files(repo)
+                    results.extend(code_files)
+                
+            except Exception as e:
+                logger.error(f"Error collecting data from repository {repo}: {e}")
+        
+        logger.info(f"Collected {len(results)} items from GitHub")
+        return results
+    
+    def _get_readme(self, repo: str) -> Optional[DataItem]:
+        """Get the README file from a repository."""
+        try:
+            url = f"{self.api_base_url}/repos/{repo}/readme"
+            response = requests.get(url, headers=self.headers)
+            
+            if response.status_code == 200:
+                data = response.json()
+                content = base64.b64decode(data["content"]).decode("utf-8")
+                
+                return DataItem(
+                    source_id=f"github_readme_{uuid.uuid4().hex[:8]}",
+                    content=content,
+                    metadata={
+                        "repo": repo,
+                        "type": "readme",
+                        "path": data.get("path", ""),
+                        "sha": data.get("sha", ""),
+                        "size": data.get("size", 0),
+                        "url": data.get("html_url", "")
+                    },
+                    url=data.get("html_url", ""),
+                    content_type="text/markdown"
+                )
+            else:
+                logger.warning(f"Failed to get README for {repo}: {response.status_code}")
+                return None
+        except Exception as e:
+            logger.error(f"Error getting README for {repo}: {e}")
+            return None
+    
+    def _get_issues(self, repo: str, max_issues: int = 10) -> List[DataItem]:
+        """Get issues from a repository."""
+        results = []
+        try:
+            url = f"{self.api_base_url}/repos/{repo}/issues"
+            params = {
+                "state": "all",
+                "per_page": max_issues
+            }
+            response = requests.get(url, headers=self.headers, params=params)
+            
+            if response.status_code == 200:
+                issues = response.json()
+                
+                for issue in issues:
+                    # Skip pull requests
+                    if "pull_request" in issue:
+                        continue
+                    
+                    content = f"# {issue['title']}\n\n{issue['body']}"
+                    
+                    item = DataItem(
+                        source_id=f"github_issue_{issue['number']}_{uuid.uuid4().hex[:8]}",
+                        content=content,
+                        metadata={
+                            "repo": repo,
+                            "type": "issue",
+                            "number": issue["number"],
+                            "state": issue["state"],
+                            "created_at": issue["created_at"],
+                            "updated_at": issue["updated_at"],
+                            "labels": [label["name"] for label in issue.get("labels", [])],
+                            "user": issue["user"]["login"] if "user" in issue else "",
+                            "comments": issue.get("comments", 0)
+                        },
+                        url=issue["html_url"],
+                        content_type="text/markdown",
+                        timestamp=datetime.strptime(issue["created_at"], "%Y-%m-%dT%H:%M:%SZ") if "created_at" in issue else None
+                    )
+                    results.append(item)
+            else:
+                logger.warning(f"Failed to get issues for {repo}: {response.status_code}")
+        except Exception as e:
+            logger.error(f"Error getting issues for {repo}: {e}")
+        
+        return results
+    
+    def _get_code_files(self, repo: str, path: str = "", max_files: int = 5) -> List[DataItem]:
path: core/connectors/github/__init__.py
user: login: sourcery-ai[bot]
body: **issue (code-quality):** Low code quality found in GitHubConnector.\_get\_code\_files - 13% ([`low-code-quality`](https://docs.sourcery.ai/Reference/Default-Rules/comments/low-code-quality/))

<br/><details><summary>Explanation</summary>The quality score for this function is below the quality threshold of 25%.
This score is a combination of the method length, cognitive complexity and working memory.

How can you solve this?

It might be worth refactoring this function to make it shorter and more readable.

- Reduce the function length by extracting pieces of functionality out into
  their own functions. This is the most important thing you can do - ideally a
  function should be less than 10 lines.
- Reduce nesting, perhaps by introducing guard clauses to return early.
- Ensure that variables are tightly scoped, so that code using related concepts
  sits together within the function rather than being scattered.</details>
created_at: 2025-04-06T11:10:06Z
author_association: NONE
diff_hunk: @@ -0,0 +1,135 @@
+"""
+Plugin system for Wiseflow.
+
+This module provides the base classes and utilities for the plugin system.
+"""
+
+from typing import Dict, List, Type, Any, Optional
+import importlib
+import os
+import logging
+import json
+import inspect
+from abc import ABC, abstractmethod
+
+logger = logging.getLogger(__name__)
+
+class PluginBase(ABC):
+    """Base class for all plugins."""
+    
+    name: str = "base_plugin"
+    description: str = "Base plugin class"
+    version: str = "0.1.0"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the plugin with optional configuration."""
+        self.config = config or {}
+        self.is_enabled = True
+        
+    @abstractmethod
+    def initialize(self) -> bool:
+        """Initialize the plugin. Return True if successful, False otherwise."""
+        pass
+    
+    def enable(self) -> None:
+        """Enable the plugin."""
+        self.is_enabled = True
+        logger.info(f"Plugin {self.name} enabled")
+        
+    def disable(self) -> None:
+        """Disable the plugin."""
+        self.is_enabled = False
+        logger.info(f"Plugin {self.name} disabled")
+    
+    def get_config(self) -> Dict[str, Any]:
+        """Get the plugin configuration."""
+        return self.config
+    
+    def set_config(self, config: Dict[str, Any]) -> None:
+        """Set the plugin configuration."""
+        self.config = config
+        
+    def __str__(self) -> str:
+        """Return a string representation of the plugin."""
+        return f"{self.name} (v{self.version}): {self.description}"
+
+
+class PluginManager:
+    """Manager for loading and managing plugins."""
+    
+    def __init__(self, plugins_dir: str = "plugins"):
+        """Initialize the plugin manager."""
+        self.plugins_dir = plugins_dir
+        self.plugins: Dict[str, PluginBase] = {}
+        
+    def discover_plugins(self) -> List[str]:
+        """Discover available plugins in the plugins directory."""
+        plugin_modules = []
+        
+        # Walk through the plugins directory
+        for root, dirs, files in os.walk(self.plugins_dir):
+            for file in files:
+                if file.endswith(".py") and not file.startswith("__"):
+                    # Convert file path to module path
+                    rel_path = os.path.relpath(os.path.join(root, file), start=os.path.dirname(self.plugins_dir))
+                    module_path = os.path.splitext(rel_path)[0].replace(os.path.sep, ".")
+                    plugin_modules.append(module_path)
+        
+        return plugin_modules
+    
+    def load_plugin(self, module_path: str) -> Optional[PluginBase]:
+        """Load a plugin from a module path."""
+        try:
+            module = importlib.import_module(module_path)
+            
+            # Find plugin classes in the module
+            for name, obj in inspect.getmembers(module):
+                if (inspect.isclass(obj) and 
+                    issubclass(obj, PluginBase) and 
+                    obj is not PluginBase):
+                    
+                    # Create an instance of the plugin
+                    plugin = obj()
+                    self.plugins[plugin.name] = plugin
+                    logger.info(f"Loaded plugin: {plugin}")
+                    return plugin
+                    
+            logger.warning(f"No plugin class found in module: {module_path}")
+            return None
+            
+        except Exception as e:
+            logger.error(f"Failed to load plugin from {module_path}: {e}")
+            return None
+    
+    def load_all_plugins(self) -> Dict[str, PluginBase]:
+        """Discover and load all available plugins."""
+        plugin_modules = self.discover_plugins()
+        
+        for module_path in plugin_modules:
+            self.load_plugin(module_path)
+            
+        return self.plugins
+    
+    def get_plugin(self, name: str) -> Optional[PluginBase]:
+        """Get a plugin by name."""
+        return self.plugins.get(name)
+    
+    def initialize_plugin(self, name: str, config: Optional[Dict[str, Any]] = None) -> bool:
+        """Initialize a plugin with optional configuration."""
+        plugin = self.get_plugin(name)
+        if plugin:
path: core/plugins/__init__.py
user: login: sourcery-ai[bot]
body: **suggestion (code-quality):** Use named expression to simplify assignment and conditional ([`use-named-expression`](https://docs.sourcery.ai/Reference/Default-Rules/refactorings/use-named-expression/))

```suggestion
        if plugin := self.get_plugin(name):
```
created_at: 2025-04-06T11:10:06Z
author_association: NONE
diff_hunk: @@ -0,0 +1,157 @@
+"""
+Text processors for Wiseflow.
+
+This module provides processors for text data.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import json
+import os
+
+from core.plugins.processors import ProcessorBase, ProcessedData
+from core.plugins.connectors import DataItem
+from core.llms.litellm_wrapper import litellm_llm
+
+logger = logging.getLogger(__name__)
+
+class FocusPointProcessor(ProcessorBase):
+    """Processor that extracts information based on focus points using LLMs."""
+    
+    name: str = "focus_point_processor"
+    description: str = "Extracts information based on focus points using LLMs"
+    processor_type: str = "text"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the focus point processor."""
+        super().__init__(config)
+        self.focus_points = self.config.get("focus_points", [])
+        
+    def initialize(self) -> bool:
+        """Initialize the processor."""
+        try:
+            # Load focus points if not provided in config
+            if not self.focus_points and self.config.get("focus_points_path"):
+                path = self.config["focus_points_path"]
+                if os.path.exists(path):
+                    with open(path, 'r') as f:
+                        self.focus_points = json.load(f)
+            
+            if not self.focus_points:
+                logger.warning("No focus points provided for focus point processor")
+            
+            logger.info(f"Initialized focus point processor with {len(self.focus_points)} focus points")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize focus point processor: {e}")
+            return False
+    
+    def process(self, data_item: DataItem, params: Optional[Dict[str, Any]] = None) -> ProcessedData:
+        """Process a data item by extracting information based on focus points."""
+        params = params or {}
+        
+        # Get focus points from params or use the ones from config
+        focus_points = params.get("focus_points", self.focus_points)
+        if not focus_points:
+            logger.warning("No focus points provided for processing")
+            return ProcessedData(
+                source_id=data_item.source_id,
+                processed_content="",
+                original_item=data_item,
+                metadata={"error": "No focus points provided"}
+            )
+        
+        # Prepare the prompt
+        prompt = self._create_prompt(data_item.content, focus_points)
+        
+        try:
+            # Process with LLM
+            model = params.get("model", os.environ.get("PRIMARY_MODEL", ""))
path: core/plugins/processors/text/__init__.py
user: login: sourcery-ai[bot]
body: **issue (code-quality):** Extract code out into method ([`extract-method`](https://docs.sourcery.ai/Reference/Default-Rules/refactorings/extract-method/))
created_at: 2025-04-06T11:10:06Z
author_association: NONE
diff_hunk: @@ -0,0 +1,157 @@
+"""
+Text processors for Wiseflow.
+
+This module provides processors for text data.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import json
+import os
+
+from core.plugins.processors import ProcessorBase, ProcessedData
+from core.plugins.connectors import DataItem
+from core.llms.litellm_wrapper import litellm_llm
+
+logger = logging.getLogger(__name__)
+
+class FocusPointProcessor(ProcessorBase):
+    """Processor that extracts information based on focus points using LLMs."""
+    
+    name: str = "focus_point_processor"
+    description: str = "Extracts information based on focus points using LLMs"
+    processor_type: str = "text"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the focus point processor."""
+        super().__init__(config)
+        self.focus_points = self.config.get("focus_points", [])
+        
+    def initialize(self) -> bool:
+        """Initialize the processor."""
+        try:
+            # Load focus points if not provided in config
+            if not self.focus_points and self.config.get("focus_points_path"):
+                path = self.config["focus_points_path"]
+                if os.path.exists(path):
+                    with open(path, 'r') as f:
+                        self.focus_points = json.load(f)
+            
+            if not self.focus_points:
+                logger.warning("No focus points provided for focus point processor")
+            
+            logger.info(f"Initialized focus point processor with {len(self.focus_points)} focus points")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize focus point processor: {e}")
+            return False
+    
+    def process(self, data_item: DataItem, params: Optional[Dict[str, Any]] = None) -> ProcessedData:
+        """Process a data item by extracting information based on focus points."""
+        params = params or {}
+        
+        # Get focus points from params or use the ones from config
+        focus_points = params.get("focus_points", self.focus_points)
+        if not focus_points:
+            logger.warning("No focus points provided for processing")
+            return ProcessedData(
+                source_id=data_item.source_id,
+                processed_content="",
+                original_item=data_item,
+                metadata={"error": "No focus points provided"}
+            )
+        
+        # Prepare the prompt
+        prompt = self._create_prompt(data_item.content, focus_points)
+        
+        try:
+            # Process with LLM
+            model = params.get("model", os.environ.get("PRIMARY_MODEL", ""))
+            if not model:
+                raise ValueError("No model specified for processing")
+            
+            messages = [
+                {"role": "system", "content": "You are an expert information extractor."},
+                {"role": "user", "content": prompt}
+            ]
+            
+            response = litellm_llm(messages, model, logger=logger)
+            
+            # Parse the response
+            extracted_info = self._parse_response(response)
+            
+            return ProcessedData(
+                source_id=data_item.source_id,
+                processed_content=extracted_info,
+                original_item=data_item,
+                metadata={
+                    "focus_points": focus_points,
+                    "url": data_item.url,
+                    "title": data_item.metadata.get("title", "")
+                }
+            )
+        except Exception as e:
+            logger.error(f"Error processing item {data_item.source_id}: {e}")
+            return ProcessedData(
+                source_id=data_item.source_id,
+                processed_content="",
+                original_item=data_item,
+                metadata={"error": str(e)}
+            )
+    
+    def _create_prompt(self, content: str, focus_points: List[Dict[str, Any]]) -> str:
+        """Create a prompt for the LLM based on the content and focus points."""
+        focus_points_str = "\n".join([
+            f"{i+1}. {fp.get('focuspoint', '')}: {fp.get('explanation', '')}"
+            for i, fp in enumerate(focus_points)
+        ])
+        
+        prompt = f"""
path: core/plugins/processors/text/__init__.py
user: login: sourcery-ai[bot]
body: **issue (code-quality):** Inline variable that is immediately returned ([`inline-immediately-returned-variable`](https://docs.sourcery.ai/Reference/Default-Rules/refactorings/inline-immediately-returned-variable/))
created_at: 2025-04-06T11:10:06Z
author_association: NONE
diff_hunk: @@ -0,0 +1,278 @@
+"""
+Reference management for Wiseflow.
+
+This module provides functionality for managing reference materials for focus points.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+import logging
+import os
+import uuid
+import json
+import shutil
+from datetime import datetime
+import requests
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+class Reference:
+    """Represents a reference for a focus point."""
+    
+    def __init__(
+        self,
+        reference_id: str,
+        focus_id: str,
+        reference_type: str,
+        path: str,
+        content: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        timestamp: Optional[datetime] = None
+    ):
+        """Initialize a reference."""
+        self.reference_id = reference_id
+        self.focus_id = focus_id
+        self.reference_type = reference_type
+        self.path = path
+        self.content = content
+        self.metadata = metadata or {}
+        self.timestamp = timestamp or datetime.now()
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the reference to a dictionary."""
+        return {
+            "reference_id": self.reference_id,
+            "focus_id": self.focus_id,
+            "reference_type": self.reference_type,
+            "path": self.path,
+            "content": self.content,
+            "metadata": self.metadata,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'Reference':
+        """Create a reference from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
+                
+        return cls(
+            reference_id=data["reference_id"],
+            focus_id=data["focus_id"],
+            reference_type=data["reference_type"],
+            path=data["path"],
+            content=data.get("content"),
+            metadata=data.get("metadata", {}),
+            timestamp=timestamp
+        )
+
+
+class ReferenceManager:
+    """Manages reference materials for focus points."""
+    
+    def __init__(self, storage_path: str = "references"):
+        """Initialize the reference manager."""
+        self.storage_path = storage_path
+        os.makedirs(storage_path, exist_ok=True)
+        self.references: Dict[str, Reference] = {}
+        self.load_references()
+        
+    def load_references(self) -> None:
+        """Load references from storage."""
+        index_path = os.path.join(self.storage_path, "index.json")
+        if os.path.exists(index_path):
+            try:
+                with open(index_path, 'r') as f:
+                    data = json.load(f)
+                    for ref_data in data:
+                        ref = Reference.from_dict(ref_data)
+                        self.references[ref.reference_id] = ref
+                logger.info(f"Loaded {len(self.references)} references from storage")
+            except Exception as e:
+                logger.error(f"Error loading references: {e}")
+        
+    def save_references(self) -> None:
+        """Save references to storage."""
+        index_path = os.path.join(self.storage_path, "index.json")
+        try:
+            with open(index_path, 'w') as f:
+                json.dump([ref.to_dict() for ref in self.references.values()], f, indent=2)
+            logger.info(f"Saved {len(self.references)} references to storage")
+        except Exception as e:
+            logger.error(f"Error saving references: {e}")
+    
+    def add_file_reference(self, focus_id: str, file_path: str) -> Optional[Reference]:
+        """Add a file reference to a focus point."""
+        try:
+            # Generate a unique reference ID
+            reference_id = str(uuid.uuid4())
+            
+            # Create a copy of the file in the storage directory
+            filename = os.path.basename(file_path)
+            storage_dir = os.path.join(self.storage_path, focus_id)
+            os.makedirs(storage_dir, exist_ok=True)
+            
+            dest_path = os.path.join(storage_dir, f"{reference_id}_{filename}")
+            shutil.copy2(file_path, dest_path)
+            
+            # Extract content from the file
+            content = self._extract_content_from_file(file_path)
+            
+            # Create the reference
+            reference = Reference(
+                reference_id=reference_id,
+                focus_id=focus_id,
+                reference_type="file",
+                path=dest_path,
+                content=content,
+                metadata={
+                    "original_path": file_path,
+                    "filename": filename
+                }
+            )
+            
+            # Add to references
+            self.references[reference_id] = reference
+            self.save_references()
+            
+            return reference
+        except Exception as e:
+            logger.error(f"Error adding file reference: {e}")
+            return None
+    
+    def add_web_reference(self, focus_id: str, url: str) -> Optional[Reference]:
+        """Add a web reference to a focus point."""
+        try:
+            # Generate a unique reference ID
+            reference_id = str(uuid.uuid4())
+            
+            # Create a storage directory
+            storage_dir = os.path.join(self.storage_path, focus_id)
+            os.makedirs(storage_dir, exist_ok=True)
+            
+            # Parse the URL to get a filename
+            parsed_url = urlparse(url)
+            filename = os.path.basename(parsed_url.path) or "webpage"
+            if not filename.endswith(".html"):
+                filename += ".html"
+            
+            dest_path = os.path.join(storage_dir, f"{reference_id}_{filename}")
+            
+            # Download the content
+            content = self._download_web_content(url)
+            
+            # Save the content
+            with open(dest_path, 'w', encoding='utf-8') as f:
+                f.write(content)
+            
+            # Create the reference
+            reference = Reference(
+                reference_id=reference_id,
+                focus_id=focus_id,
+                reference_type="web",
+                path=dest_path,
+                content=content,
+                metadata={
+                    "url": url,
+                    "domain": parsed_url.netloc
+                }
+            )
+            
+            # Add to references
+            self.references[reference_id] = reference
+            self.save_references()
+            
+            return reference
+        except Exception as e:
+            logger.error(f"Error adding web reference: {e}")
+            return None
+    
+    def add_text_reference(self, focus_id: str, content: str, name: str = "text_reference") -> Optional[Reference]:
+        """Add a text reference to a focus point."""
+        try:
+            # Generate a unique reference ID
+            reference_id = str(uuid.uuid4())
+            
+            # Create a storage directory
+            storage_dir = os.path.join(self.storage_path, focus_id)
+            os.makedirs(storage_dir, exist_ok=True)
+            
+            # Create a filename
+            filename = f"{name}.txt"
+            dest_path = os.path.join(storage_dir, f"{reference_id}_{filename}")
+            
+            # Save the content
+            with open(dest_path, 'w', encoding='utf-8') as f:
+                f.write(content)
+            
+            # Create the reference
+            reference = Reference(
+                reference_id=reference_id,
+                focus_id=focus_id,
+                reference_type="text",
+                path=dest_path,
+                content=content,
+                metadata={
+                    "name": name
+                }
+            )
+            
+            # Add to references
+            self.references[reference_id] = reference
+            self.save_references()
+            
+            return reference
+        except Exception as e:
+            logger.error(f"Error adding text reference: {e}")
+            return None
+    
+    def get_reference(self, reference_id: str) -> Optional[Reference]:
+        """Get a reference by ID."""
+        return self.references.get(reference_id)
+    
+    def get_references_by_focus(self, focus_id: str) -> List[Reference]:
+        """Get all references for a focus point."""
+        return [ref for ref in self.references.values() if ref.focus_id == focus_id]
+    
+    def delete_reference(self, reference_id: str) -> bool:
+        """Delete a reference."""
+        if reference_id in self.references:
+            reference = self.references[reference_id]
+            
+            # Delete the file
+            try:
+                if os.path.exists(reference.path):
+                    os.remove(reference.path)
+            except Exception as e:
+                logger.error(f"Error deleting reference file: {e}")
+            
+            # Remove from references
+            del self.references[reference_id]
+            self.save_references()
+            
+            return True
+        return False
path: core/references/__init__.py
user: login: sourcery-ai[bot]
body: **suggestion (code-quality):** Add guard clause ([`last-if-guard`](https://docs.sourcery.ai/Reference/Default-Rules/refactorings/last-if-guard/))

```suggestion
        if reference_id not in self.references:
            return False
        reference = self.references[reference_id]

        # Delete the file
        try:
            if os.path.exists(reference.path):
                os.remove(reference.path)
        except Exception as e:
            logger.error(f"Error deleting reference file: {e}")

        # Remove from references
        del self.references[reference_id]
        self.save_references()

        return True
```
created_at: 2025-04-06T11:10:07Z
author_association: NONE
diff_hunk: @@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+from pathlib import Path
+from dotenv import load_dotenv
+env_path = Path(__file__).parent / '.env'
+if env_path.exists():
+    load_dotenv(env_path)
+
+import logging
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+import asyncio
+import os
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+
+from core.general_process import main_process, wiseflow_logger, pb
+from core.task import AsyncTaskManager, Task, create_task_id
+from core.plugins import PluginManager
+from core.plugins.connectors import ConnectorBase
+from core.plugins.processors import ProcessorBase
+
+# Configure the maximum number of concurrent tasks
+MAX_CONCURRENT_TASKS = int(os.environ.get("MAX_CONCURRENT_TASKS", "4"))
+
+# Initialize the task manager
+task_manager = AsyncTaskManager(max_workers=MAX_CONCURRENT_TASKS)
+
+# Initialize the plugin manager
+plugin_manager = PluginManager(plugins_dir="core")
+
+async def load_plugins():
+    """Load and initialize all plugins."""
+    wiseflow_logger.info("Loading plugins...")
+    plugins = plugin_manager.load_all_plugins()
+    wiseflow_logger.info(f"Loaded {len(plugins)} plugins")
+    
+    # Initialize plugins with configurations
+    configs = {}  # Load configurations from database or config files
+    results = plugin_manager.initialize_all_plugins(configs)
+    
+    for name, success in results.items():
+        if success:
+            wiseflow_logger.info(f"Initialized plugin: {name}")
+        else:
+            wiseflow_logger.error(f"Failed to initialize plugin: {name}")
+    
+    return plugins
+
+async def process_focus_point(focus: Dict[str, Any], sites: List[Dict[str, Any]]) -> Any:
+    """Process a focus point using the main_process function."""
+    wiseflow_logger.info(f"Processing focus point: {focus['focuspoint']}")
+    return await main_process(focus, sites)
+
+async def schedule_task():
+    """Schedule and manage data mining tasks."""
+    # Load plugins
+    await load_plugins()
+    
+    while True:
+        wiseflow_logger.info("Checking for active focus points...")
+        
+        # Get active focus points
+        focus_points = pb.read('focus_points', filter='activated=True')
+        sites_record = pb.read('sites')
+        
+        for focus in focus_points:
+            # Check if there's already a task running for this focus point
+            existing_tasks = task_manager.get_tasks_by_focus(focus["id"])
+            active_tasks = [t for t in existing_tasks if t.status in ["pending", "running"]]
path: core/run_task_new.py
user: login: sourcery-ai[bot]
body: **issue (code-quality):** Use named expression to simplify assignment and conditional ([`use-named-expression`](https://docs.sourcery.ai/Reference/Default-Rules/refactorings/use-named-expression/))
created_at: 2025-04-06T11:10:07Z
author_association: NONE
diff_hunk: @@ -0,0 +1,106 @@
+"""
+Data source connectors for Wiseflow.
+
+This module provides base classes for data source connectors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+
+logger = logging.getLogger(__name__)
+
+class DataItem:
+    """Represents a single item of data collected from a source."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        content: str,
+        metadata: Optional[Dict[str, Any]] = None,
+        url: Optional[str] = None,
+        timestamp: Optional[datetime] = None,
+        content_type: str = "text",
+        language: Optional[str] = None,
+        raw_data: Optional[Any] = None
+    ):
+        """Initialize a data item."""
+        self.source_id = source_id
+        self.content = content
+        self.metadata = metadata or {}
+        self.url = url
+        self.timestamp = timestamp or datetime.now()
+        self.content_type = content_type
+        self.language = language
+        self.raw_data = raw_data
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the data item to a dictionary."""
+        return {
+            "source_id": self.source_id,
+            "content": self.content,
+            "metadata": self.metadata,
+            "url": self.url,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None,
+            "content_type": self.content_type,
+            "language": self.language
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'DataItem':
+        """Create a data item from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
path: core/connectors/__init__.py
user: login: korbit-ai[bot]
body: ### Silent Error Suppression in Timestamp Parsing <sub>![category Error Handling](https://img.shields.io/badge/Error%20Handling-ea580c)</sub>

<details>
  <summary>Tell me more</summary>

###### What is the issue?
The error handling silently ignores timestamp parsing errors by using a bare pass statement.

###### Why this matters
When timestamp parsing fails, there's no way to understand why it failed since the error information is completely discarded. This makes debugging timestamp-related issues much harder.

###### Suggested change ∙ *Feature Preview*
```python
try:
    timestamp = datetime.fromisoformat(data["timestamp"])
except (ValueError, TypeError) as e:
    logger.warning(f"Failed to parse timestamp {data.get('timestamp')}: {e}")
```


###### Provide feedback to improve future suggestions
[![Nice Catch](https://img.shields.io/badge/👍%20Nice%20Catch-71BC78)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2b226d9f-f178-49b1-9008-3e3356ddc6d5/upvote) [![Incorrect](https://img.shields.io/badge/👎%20Incorrect-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2b226d9f-f178-49b1-9008-3e3356ddc6d5?what_not_true=true)  [![Not in Scope](https://img.shields.io/badge/👎%20Out%20of%20PR%20scope-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2b226d9f-f178-49b1-9008-3e3356ddc6d5?what_out_of_scope=true) [![Not in coding standard](https://img.shields.io/badge/👎%20Not%20in%20our%20standards-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2b226d9f-f178-49b1-9008-3e3356ddc6d5?what_not_in_standard=true) [![Other](https://img.shields.io/badge/👎%20Other-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2b226d9f-f178-49b1-9008-3e3356ddc6d5)
</details>

<sub>

💬 Looking for more details? Reply to this comment to chat with Korbit.
</sub>

<!--- korbi internal id:171c8641-6a3e-4f7f-a5f4-a41b861cb2cf -->


[](171c8641-6a3e-4f7f-a5f4-a41b861cb2cf)
created_at: 2025-04-06T11:10:32Z
author_association: NONE
diff_hunk: @@ -0,0 +1,106 @@
+"""
+Data source connectors for Wiseflow.
+
+This module provides base classes for data source connectors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+
+logger = logging.getLogger(__name__)
+
+class DataItem:
+    """Represents a single item of data collected from a source."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        content: str,
+        metadata: Optional[Dict[str, Any]] = None,
+        url: Optional[str] = None,
+        timestamp: Optional[datetime] = None,
+        content_type: str = "text",
+        language: Optional[str] = None,
+        raw_data: Optional[Any] = None
+    ):
+        """Initialize a data item."""
+        self.source_id = source_id
+        self.content = content
+        self.metadata = metadata or {}
+        self.url = url
+        self.timestamp = timestamp or datetime.now()
+        self.content_type = content_type
+        self.language = language
+        self.raw_data = raw_data
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the data item to a dictionary."""
+        return {
+            "source_id": self.source_id,
+            "content": self.content,
+            "metadata": self.metadata,
+            "url": self.url,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None,
+            "content_type": self.content_type,
+            "language": self.language
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'DataItem':
+        """Create a data item from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
+                
+        return cls(
+            source_id=data["source_id"],
+            content=data["content"],
+            metadata=data.get("metadata", {}),
+            url=data.get("url"),
+            timestamp=timestamp,
+            content_type=data.get("content_type", "text"),
+            language=data.get("language")
+        )
+
+
+class ConnectorBase(PluginBase):
+    """Base class for data source connectors."""
+    
+    name: str = "base_connector"
+    description: str = "Base connector class"
+    source_type: str = "generic"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the connector with optional configuration."""
+        super().__init__(config)
+        self.last_run: Optional[datetime] = None
+        
+    @abstractmethod
+    def collect(self, params: Optional[Dict[str, Any]] = None) -> List[DataItem]:
+        """Collect data from the source."""
+        pass
path: core/connectors/__init__.py
user: login: korbit-ai[bot]
body: ### Missing logging requirements in abstract method <sub>![category Logging](https://img.shields.io/badge/Logging-4f46e5)</sub>

<details>
  <summary>Tell me more</summary>

###### What is the issue?
The abstract collect method lacks logging guidance for implementing classes

###### Why this matters
Without standardized logging requirements in the base class documentation, derived connectors might implement inconsistent logging patterns, making system-wide monitoring more difficult

###### Suggested change ∙ *Feature Preview*
```python
@abstractmethod
def collect(self, params: Optional[Dict[str, Any]] = None) -> List[DataItem]:
    """Collect data from the source.
    
    Implementation should include:
    - INFO level log when collection starts
    - INFO level log when collection completes with item count
    - WARNING level log for partial failures
    - ERROR level log for complete failures
    """
    pass
```


###### Provide feedback to improve future suggestions
[![Nice Catch](https://img.shields.io/badge/👍%20Nice%20Catch-71BC78)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/4af1b90d-48c6-41d1-9974-4dd84bda14a3/upvote) [![Incorrect](https://img.shields.io/badge/👎%20Incorrect-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/4af1b90d-48c6-41d1-9974-4dd84bda14a3?what_not_true=true)  [![Not in Scope](https://img.shields.io/badge/👎%20Out%20of%20PR%20scope-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/4af1b90d-48c6-41d1-9974-4dd84bda14a3?what_out_of_scope=true) [![Not in coding standard](https://img.shields.io/badge/👎%20Not%20in%20our%20standards-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/4af1b90d-48c6-41d1-9974-4dd84bda14a3?what_not_in_standard=true) [![Other](https://img.shields.io/badge/👎%20Other-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/4af1b90d-48c6-41d1-9974-4dd84bda14a3)
</details>

<sub>

💬 Looking for more details? Reply to this comment to chat with Korbit.
</sub>

<!--- korbi internal id:126bc31f-2b9b-4f06-b276-d35a08681b2f -->


[](126bc31f-2b9b-4f06-b276-d35a08681b2f)
created_at: 2025-04-06T11:10:32Z
author_association: NONE
diff_hunk: @@ -0,0 +1,106 @@
+"""
+Data source connectors for Wiseflow.
+
+This module provides base classes for data source connectors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+
+logger = logging.getLogger(__name__)
+
+class DataItem:
+    """Represents a single item of data collected from a source."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        content: str,
+        metadata: Optional[Dict[str, Any]] = None,
+        url: Optional[str] = None,
+        timestamp: Optional[datetime] = None,
+        content_type: str = "text",
+        language: Optional[str] = None,
+        raw_data: Optional[Any] = None
+    ):
+        """Initialize a data item."""
path: core/connectors/__init__.py
user: login: korbit-ai[bot]
body: ### Incomplete DataItem initialization docstring <sub>![category Documentation](https://img.shields.io/badge/Documentation-7c3aed)</sub>

<details>
  <summary>Tell me more</summary>

###### What is the issue?
The __init__ docstring is too generic and doesn't explain the purpose of each parameter

###### Why this matters
Without parameter descriptions, other developers won't understand what each field represents and how to use them correctly

###### Suggested change ∙ *Feature Preview*
        """Initialize a data item.

        Args:
            source_id: Unique identifier for the data source
            content: The main content of the data item
            metadata: Additional key-value information about the item
            url: Optional source URL where the data was collected
            timestamp: When the data was collected (defaults to now)
            content_type: Format of the content (e.g., 'text', 'html')
            language: ISO language code of the content
            raw_data: Original data before processing
        """


###### Provide feedback to improve future suggestions
[![Nice Catch](https://img.shields.io/badge/👍%20Nice%20Catch-71BC78)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/9bd4e07b-b3bb-4b67-ac98-45aea2630393/upvote) [![Incorrect](https://img.shields.io/badge/👎%20Incorrect-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/9bd4e07b-b3bb-4b67-ac98-45aea2630393?what_not_true=true)  [![Not in Scope](https://img.shields.io/badge/👎%20Out%20of%20PR%20scope-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/9bd4e07b-b3bb-4b67-ac98-45aea2630393?what_out_of_scope=true) [![Not in coding standard](https://img.shields.io/badge/👎%20Not%20in%20our%20standards-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/9bd4e07b-b3bb-4b67-ac98-45aea2630393?what_not_in_standard=true) [![Other](https://img.shields.io/badge/👎%20Other-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/9bd4e07b-b3bb-4b67-ac98-45aea2630393)
</details>

<sub>

💬 Looking for more details? Reply to this comment to chat with Korbit.
</sub>

<!--- korbi internal id:e7e1db44-3cd5-48cb-9ddb-2cf26d8faf8d -->


[](e7e1db44-3cd5-48cb-9ddb-2cf26d8faf8d)
created_at: 2025-04-06T11:10:32Z
author_association: NONE
diff_hunk: @@ -0,0 +1,106 @@
+"""
+Data source connectors for Wiseflow.
+
+This module provides base classes for data source connectors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+
+logger = logging.getLogger(__name__)
+
+class DataItem:
+    """Represents a single item of data collected from a source."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        content: str,
+        metadata: Optional[Dict[str, Any]] = None,
+        url: Optional[str] = None,
+        timestamp: Optional[datetime] = None,
+        content_type: str = "text",
+        language: Optional[str] = None,
+        raw_data: Optional[Any] = None
+    ):
+        """Initialize a data item."""
+        self.source_id = source_id
+        self.content = content
+        self.metadata = metadata or {}
+        self.url = url
+        self.timestamp = timestamp or datetime.now()
+        self.content_type = content_type
+        self.language = language
+        self.raw_data = raw_data
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the data item to a dictionary."""
+        return {
+            "source_id": self.source_id,
+            "content": self.content,
+            "metadata": self.metadata,
+            "url": self.url,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None,
+            "content_type": self.content_type,
+            "language": self.language
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'DataItem':
+        """Create a data item from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
+                
+        return cls(
+            source_id=data["source_id"],
+            content=data["content"],
+            metadata=data.get("metadata", {}),
+            url=data.get("url"),
+            timestamp=timestamp,
+            content_type=data.get("content_type", "text"),
+            language=data.get("language")
+        )
+
+
+class ConnectorBase(PluginBase):
+    """Base class for data source connectors."""
+    
+    name: str = "base_connector"
+    description: str = "Base connector class"
+    source_type: str = "generic"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the connector with optional configuration."""
+        super().__init__(config)
+        self.last_run: Optional[datetime] = None
+        
+    @abstractmethod
+    def collect(self, params: Optional[Dict[str, Any]] = None) -> List[DataItem]:
+        """Collect data from the source."""
path: core/connectors/__init__.py
user: login: korbit-ai[bot]
body: ### Insufficient collect method documentation <sub>![category Documentation](https://img.shields.io/badge/Documentation-7c3aed)</sub>

<details>
  <summary>Tell me more</summary>

###### What is the issue?
The collect method's docstring doesn't explain the method's purpose, parameters, or return value

###### Why this matters
Developers implementing connectors need to understand what params are expected and what the collection process should return

###### Suggested change ∙ *Feature Preview*
        """Collect data from the source.

        Args:
            params: Optional parameters to customize the data collection

        Returns:
            List of DataItem objects containing the collected data

        This method must be implemented by each connector to define
        its specific data collection logic.
        """


###### Provide feedback to improve future suggestions
[![Nice Catch](https://img.shields.io/badge/👍%20Nice%20Catch-71BC78)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2cb76de9-e022-45eb-bbc7-6f13e66ea79c/upvote) [![Incorrect](https://img.shields.io/badge/👎%20Incorrect-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2cb76de9-e022-45eb-bbc7-6f13e66ea79c?what_not_true=true)  [![Not in Scope](https://img.shields.io/badge/👎%20Out%20of%20PR%20scope-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2cb76de9-e022-45eb-bbc7-6f13e66ea79c?what_out_of_scope=true) [![Not in coding standard](https://img.shields.io/badge/👎%20Not%20in%20our%20standards-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2cb76de9-e022-45eb-bbc7-6f13e66ea79c?what_not_in_standard=true) [![Other](https://img.shields.io/badge/👎%20Other-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2cb76de9-e022-45eb-bbc7-6f13e66ea79c)
</details>

<sub>

💬 Looking for more details? Reply to this comment to chat with Korbit.
</sub>

<!--- korbi internal id:f680eeef-99c2-4eb8-8363-bc467e75703d -->


[](f680eeef-99c2-4eb8-8363-bc467e75703d)
created_at: 2025-04-06T11:10:32Z
author_association: NONE
diff_hunk: @@ -0,0 +1,103 @@
+"""
+Processors for Wiseflow.
+
+This module provides base classes for data processors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+from core.connectors import DataItem
+
+logger = logging.getLogger(__name__)
+
+class ProcessedData:
+    """Represents processed data from a processor."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        processed_content: str,
+        original_item: Optional[DataItem] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        timestamp: Optional[datetime] = None
+    ):
+        """Initialize processed data."""
path: core/plugins/processors/__init__.py
user: login: korbit-ai[bot]
body: ### Incomplete Constructor Documentation <sub>![category Documentation](https://img.shields.io/badge/Documentation-7c3aed)</sub>

<details>
  <summary>Tell me more</summary>

###### What is the issue?
The __init__ docstring lacks parameter descriptions and return type information, unlike other methods in the file.

###### Why this matters
Without parameter descriptions, developers need to read the code to understand the purpose of each parameter, reducing code maintainability.

###### Suggested change ∙ *Feature Preview*
"""Initialize processed data.

Args:
    source_id: Unique identifier for the source data
    processed_content: The processed content string
    original_item: Reference to the original DataItem that was processed
    metadata: Additional metadata about the processing
    timestamp: When the processing occurred, defaults to now"""


###### Provide feedback to improve future suggestions
[![Nice Catch](https://img.shields.io/badge/👍%20Nice%20Catch-71BC78)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2034fea5-4ab8-4c4c-903a-c124463928a4/upvote) [![Incorrect](https://img.shields.io/badge/👎%20Incorrect-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2034fea5-4ab8-4c4c-903a-c124463928a4?what_not_true=true)  [![Not in Scope](https://img.shields.io/badge/👎%20Out%20of%20PR%20scope-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2034fea5-4ab8-4c4c-903a-c124463928a4?what_out_of_scope=true) [![Not in coding standard](https://img.shields.io/badge/👎%20Not%20in%20our%20standards-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2034fea5-4ab8-4c4c-903a-c124463928a4?what_not_in_standard=true) [![Other](https://img.shields.io/badge/👎%20Other-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/2034fea5-4ab8-4c4c-903a-c124463928a4)
</details>

<sub>

💬 Looking for more details? Reply to this comment to chat with Korbit.
</sub>

<!--- korbi internal id:716cc619-7ad5-4805-8211-92c85d7df1f3 -->


[](716cc619-7ad5-4805-8211-92c85d7df1f3)
created_at: 2025-04-06T11:10:33Z
author_association: NONE
diff_hunk: @@ -0,0 +1,103 @@
+"""
+Processors for Wiseflow.
+
+This module provides base classes for data processors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+from core.connectors import DataItem
+
+logger = logging.getLogger(__name__)
+
+class ProcessedData:
+    """Represents processed data from a processor."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        processed_content: str,
+        original_item: Optional[DataItem] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        timestamp: Optional[datetime] = None
+    ):
+        """Initialize processed data."""
+        self.source_id = source_id
+        self.processed_content = processed_content
+        self.original_item = original_item
+        self.metadata = metadata or {}
+        self.timestamp = timestamp or datetime.now()
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the processed data to a dictionary."""
+        return {
+            "source_id": self.source_id,
+            "processed_content": self.processed_content,
+            "original_item": self.original_item.to_dict() if self.original_item else None,
+            "metadata": self.metadata,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedData':
+        """Create processed data from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
+                
+        original_item = None
+        if data.get("original_item"):
+            original_item = DataItem.from_dict(data["original_item"])
+                
+        return cls(
+            source_id=data["source_id"],
+            processed_content=data["processed_content"],
+            original_item=original_item,
+            metadata=data.get("metadata", {}),
+            timestamp=timestamp
+        )
+
+
+class ProcessorBase(PluginBase):
+    """Base class for data processors."""
+    
+    name: str = "base_processor"
+    description: str = "Base processor class"
+    processor_type: str = "generic"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the processor with optional configuration."""
+        super().__init__(config)
+        
+    @abstractmethod
+    def process(self, data_item: DataItem, params: Optional[Dict[str, Any]] = None) -> ProcessedData:
+        """Process a data item."""
path: core/plugins/processors/__init__.py
user: login: korbit-ai[bot]
body: ### Abstract Method Contract Missing <sub>![category Documentation](https://img.shields.io/badge/Documentation-7c3aed)</sub>

<details>
  <summary>Tell me more</summary>

###### What is the issue?
The abstract method's docstring doesn't explain the contract that implementing classes must fulfill.

###### Why this matters
Developers implementing this abstract method lack guidance on what their implementation should accomplish.

###### Suggested change ∙ *Feature Preview*
"""Process a data item and return the results.

Args:
    data_item: The data item to process
    params: Optional parameters to customize processing

Returns:
    ProcessedData containing the processing results

Implementing classes must handle the actual data processing logic."""


###### Provide feedback to improve future suggestions
[![Nice Catch](https://img.shields.io/badge/👍%20Nice%20Catch-71BC78)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/7e64f18d-e2a6-487a-acce-f83a59b4f219/upvote) [![Incorrect](https://img.shields.io/badge/👎%20Incorrect-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/7e64f18d-e2a6-487a-acce-f83a59b4f219?what_not_true=true)  [![Not in Scope](https://img.shields.io/badge/👎%20Out%20of%20PR%20scope-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/7e64f18d-e2a6-487a-acce-f83a59b4f219?what_out_of_scope=true) [![Not in coding standard](https://img.shields.io/badge/👎%20Not%20in%20our%20standards-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/7e64f18d-e2a6-487a-acce-f83a59b4f219?what_not_in_standard=true) [![Other](https://img.shields.io/badge/👎%20Other-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/7e64f18d-e2a6-487a-acce-f83a59b4f219)
</details>

<sub>

💬 Looking for more details? Reply to this comment to chat with Korbit.
</sub>

<!--- korbi internal id:eaf0de5d-e593-42b5-b72d-85ed4dc38628 -->


[](eaf0de5d-e593-42b5-b72d-85ed4dc38628)
created_at: 2025-04-06T11:10:33Z
author_association: NONE
diff_hunk: @@ -0,0 +1,103 @@
+"""
+Processors for Wiseflow.
+
+This module provides base classes for data processors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+from core.connectors import DataItem
+
+logger = logging.getLogger(__name__)
+
+class ProcessedData:
+    """Represents processed data from a processor."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        processed_content: str,
+        original_item: Optional[DataItem] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        timestamp: Optional[datetime] = None
+    ):
+        """Initialize processed data."""
+        self.source_id = source_id
+        self.processed_content = processed_content
+        self.original_item = original_item
+        self.metadata = metadata or {}
+        self.timestamp = timestamp or datetime.now()
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the processed data to a dictionary."""
+        return {
+            "source_id": self.source_id,
+            "processed_content": self.processed_content,
+            "original_item": self.original_item.to_dict() if self.original_item else None,
+            "metadata": self.metadata,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedData':
+        """Create processed data from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
+                
+        original_item = None
+        if data.get("original_item"):
+            original_item = DataItem.from_dict(data["original_item"])
+                
+        return cls(
+            source_id=data["source_id"],
+            processed_content=data["processed_content"],
+            original_item=original_item,
+            metadata=data.get("metadata", {}),
+            timestamp=timestamp
+        )
+
+
+class ProcessorBase(PluginBase):
+    """Base class for data processors."""
+    
+    name: str = "base_processor"
+    description: str = "Base processor class"
+    processor_type: str = "generic"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the processor with optional configuration."""
+        super().__init__(config)
+        
+    @abstractmethod
+    def process(self, data_item: DataItem, params: Optional[Dict[str, Any]] = None) -> ProcessedData:
+        """Process a data item."""
+        pass
+    
+    def initialize(self) -> bool:
+        """Initialize the processor. Return True if successful, False otherwise."""
+        try:
+            # Perform any necessary initialization
+            logger.info(f"Initialized processor: {self.name}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize processor {self.name}: {e}")
+            return False
+    
+    def batch_process(self, data_items: List[DataItem], params: Optional[Dict[str, Any]] = None) -> List[ProcessedData]:
+        """Process multiple data items."""
+        results = []
+        for item in data_items:
+            try:
+                result = self.process(item, params)
+                results.append(result)
+            except Exception as e:
+                logger.error(f"Error processing item {item.source_id}: {e}")
path: core/plugins/processors/__init__.py
user: login: korbit-ai[bot]
body: ### Silent Error Swallowing in Batch Processing <sub>![category Error Handling](https://img.shields.io/badge/Error%20Handling-ea580c)</sub>

<details>
  <summary>Tell me more</summary>

###### What is the issue?
The batch_process method silently continues processing after encountering an error, potentially losing important data without notifying the caller.

###### Why this matters
If processing fails for some items, the caller has no way to identify which items failed since the error is only logged. This could lead to data inconsistencies or incomplete processing being treated as successful.

###### Suggested change ∙ *Feature Preview*
Return both successful and failed results to allow proper error handling by the caller:
```python
def batch_process(self, data_items: List[DataItem], params: Optional[Dict[str, Any]] = None) -> Tuple[List[ProcessedData], List[Tuple[str, Exception]]]:
    results = []
    errors = []
    for item in data_items:
        try:
            result = self.process(item, params)
            results.append(result)
        except Exception as e:
            logger.error(f"Error processing item {item.source_id}: {e}")
            errors.append((item.source_id, e))
    return results, errors
```


###### Provide feedback to improve future suggestions
[![Nice Catch](https://img.shields.io/badge/👍%20Nice%20Catch-71BC78)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/c415b899-aa71-4154-9887-c35ae80c0c50/upvote) [![Incorrect](https://img.shields.io/badge/👎%20Incorrect-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/c415b899-aa71-4154-9887-c35ae80c0c50?what_not_true=true)  [![Not in Scope](https://img.shields.io/badge/👎%20Out%20of%20PR%20scope-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/c415b899-aa71-4154-9887-c35ae80c0c50?what_out_of_scope=true) [![Not in coding standard](https://img.shields.io/badge/👎%20Not%20in%20our%20standards-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/c415b899-aa71-4154-9887-c35ae80c0c50?what_not_in_standard=true) [![Other](https://img.shields.io/badge/👎%20Other-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/c415b899-aa71-4154-9887-c35ae80c0c50)
</details>

<sub>

💬 Looking for more details? Reply to this comment to chat with Korbit.
</sub>

<!--- korbi internal id:09bea410-fb62-4e98-8def-9e4a368d4c94 -->


[](09bea410-fb62-4e98-8def-9e4a368d4c94)
created_at: 2025-04-06T11:10:33Z
author_association: NONE
diff_hunk: @@ -0,0 +1,103 @@
+"""
+Processors for Wiseflow.
+
+This module provides base classes for data processors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+from core.connectors import DataItem
+
+logger = logging.getLogger(__name__)
+
+class ProcessedData:
+    """Represents processed data from a processor."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        processed_content: str,
+        original_item: Optional[DataItem] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        timestamp: Optional[datetime] = None
+    ):
+        """Initialize processed data."""
+        self.source_id = source_id
+        self.processed_content = processed_content
+        self.original_item = original_item
+        self.metadata = metadata or {}
+        self.timestamp = timestamp or datetime.now()
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the processed data to a dictionary."""
+        return {
+            "source_id": self.source_id,
+            "processed_content": self.processed_content,
+            "original_item": self.original_item.to_dict() if self.original_item else None,
+            "metadata": self.metadata,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedData':
+        """Create processed data from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
path: core/plugins/processors/__init__.py
user: login: korbit-ai[bot]
body: ### Silent Timestamp Parse Error <sub>![category Error Handling](https://img.shields.io/badge/Error%20Handling-ea580c)</sub>

<details>
  <summary>Tell me more</summary>

###### What is the issue?
The from_dict method silently ignores timestamp parsing errors without logging or providing context about the failure.

###### Why this matters
Silent failure in timestamp parsing could lead to debugging difficulties and inconsistent data states where the timestamp is unexpectedly None.

###### Suggested change ∙ *Feature Preview*
Log the error and include the problematic timestamp value:
```python
except (ValueError, TypeError) as e:
    logger.warning(f"Failed to parse timestamp {data['timestamp']}: {e}")
```


###### Provide feedback to improve future suggestions
[![Nice Catch](https://img.shields.io/badge/👍%20Nice%20Catch-71BC78)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/22e27ee9-41c0-4640-8d3b-b96bf4d9390d/upvote) [![Incorrect](https://img.shields.io/badge/👎%20Incorrect-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/22e27ee9-41c0-4640-8d3b-b96bf4d9390d?what_not_true=true)  [![Not in Scope](https://img.shields.io/badge/👎%20Out%20of%20PR%20scope-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/22e27ee9-41c0-4640-8d3b-b96bf4d9390d?what_out_of_scope=true) [![Not in coding standard](https://img.shields.io/badge/👎%20Not%20in%20our%20standards-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/22e27ee9-41c0-4640-8d3b-b96bf4d9390d?what_not_in_standard=true) [![Other](https://img.shields.io/badge/👎%20Other-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/22e27ee9-41c0-4640-8d3b-b96bf4d9390d)
</details>

<sub>

💬 Looking for more details? Reply to this comment to chat with Korbit.
</sub>

<!--- korbi internal id:f2f067ae-1db0-4972-a7b6-e5847731802c -->


[](f2f067ae-1db0-4972-a7b6-e5847731802c)
created_at: 2025-04-06T11:10:33Z
author_association: NONE
diff_hunk: @@ -0,0 +1,103 @@
+"""
+Processors for Wiseflow.
+
+This module provides base classes for data processors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+from core.connectors import DataItem
+
+logger = logging.getLogger(__name__)
+
+class ProcessedData:
+    """Represents processed data from a processor."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        processed_content: str,
+        original_item: Optional[DataItem] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        timestamp: Optional[datetime] = None
+    ):
+        """Initialize processed data."""
+        self.source_id = source_id
+        self.processed_content = processed_content
+        self.original_item = original_item
+        self.metadata = metadata or {}
+        self.timestamp = timestamp or datetime.now()
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the processed data to a dictionary."""
+        return {
+            "source_id": self.source_id,
+            "processed_content": self.processed_content,
+            "original_item": self.original_item.to_dict() if self.original_item else None,
+            "metadata": self.metadata,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedData':
+        """Create processed data from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
+                
+        original_item = None
+        if data.get("original_item"):
+            original_item = DataItem.from_dict(data["original_item"])
+                
+        return cls(
+            source_id=data["source_id"],
+            processed_content=data["processed_content"],
+            original_item=original_item,
+            metadata=data.get("metadata", {}),
+            timestamp=timestamp
+        )
+
+
+class ProcessorBase(PluginBase):
+    """Base class for data processors."""
+    
+    name: str = "base_processor"
+    description: str = "Base processor class"
+    processor_type: str = "generic"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the processor with optional configuration."""
+        super().__init__(config)
+        
+    @abstractmethod
+    def process(self, data_item: DataItem, params: Optional[Dict[str, Any]] = None) -> ProcessedData:
+        """Process a data item."""
+        pass
+    
+    def initialize(self) -> bool:
+        """Initialize the processor. Return True if successful, False otherwise."""
+        try:
+            # Perform any necessary initialization
+            logger.info(f"Initialized processor: {self.name}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to initialize processor {self.name}: {e}")
+            return False
+    
+    def batch_process(self, data_items: List[DataItem], params: Optional[Dict[str, Any]] = None) -> List[ProcessedData]:
+        """Process multiple data items."""
+        results = []
+        for item in data_items:
+            try:
+                result = self.process(item, params)
+                results.append(result)
+            except Exception as e:
+                logger.error(f"Error processing item {item.source_id}: {e}")
path: core/plugins/processors/__init__.py
user: login: korbit-ai[bot]
body: ### Insufficient batch processing metrics logging <sub>![category Logging](https://img.shields.io/badge/Logging-4f46e5)</sub>

<details>
  <summary>Tell me more</summary>

###### What is the issue?
batch_process() lacks logging of batch processing progress and success metrics

###### Why this matters
Without progress and success rate logging, it's difficult to monitor batch processing performance and identify degradation patterns

###### Suggested change ∙ *Feature Preview*
```python
def batch_process(self, data_items: List[DataItem], params: Optional[Dict[str, Any]] = None) -> List[ProcessedData]:
    total_items = len(data_items)
    logger.info(f"Starting batch processing of {total_items} items")
    results = []
    success_count = 0
    
    for idx, item in enumerate(data_items, 1):
        try:
            result = self.process(item, params)
            results.append(result)
            success_count += 1
            if idx % 100 == 0:  # Log progress every 100 items
                logger.info(f"Processed {idx}/{total_items} items")
        except Exception as e:
            logger.error(f"Error processing item {item.source_id}: {e}")
    
    logger.info(f"Batch processing completed. Success rate: {success_count}/{total_items}")
    return results
```


###### Provide feedback to improve future suggestions
[![Nice Catch](https://img.shields.io/badge/👍%20Nice%20Catch-71BC78)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/ad130d4d-0961-4430-a62b-247d78249d0a/upvote) [![Incorrect](https://img.shields.io/badge/👎%20Incorrect-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/ad130d4d-0961-4430-a62b-247d78249d0a?what_not_true=true)  [![Not in Scope](https://img.shields.io/badge/👎%20Out%20of%20PR%20scope-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/ad130d4d-0961-4430-a62b-247d78249d0a?what_out_of_scope=true) [![Not in coding standard](https://img.shields.io/badge/👎%20Not%20in%20our%20standards-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/ad130d4d-0961-4430-a62b-247d78249d0a?what_not_in_standard=true) [![Other](https://img.shields.io/badge/👎%20Other-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/ad130d4d-0961-4430-a62b-247d78249d0a)
</details>

<sub>

💬 Looking for more details? Reply to this comment to chat with Korbit.
</sub>

<!--- korbi internal id:5314cb24-4579-4f22-82c4-67c8fc5f1603 -->


[](5314cb24-4579-4f22-82c4-67c8fc5f1603)
created_at: 2025-04-06T11:10:33Z
author_association: NONE
diff_hunk: @@ -0,0 +1,103 @@
+"""
+Processors for Wiseflow.
+
+This module provides base classes for data processors.
+"""
+
+from typing import Dict, List, Any, Optional, Union
+from abc import abstractmethod
+import logging
+from datetime import datetime
+
+from core.plugins import PluginBase
+from core.connectors import DataItem
+
+logger = logging.getLogger(__name__)
+
+class ProcessedData:
+    """Represents processed data from a processor."""
+    
+    def __init__(
+        self,
+        source_id: str,
+        processed_content: str,
+        original_item: Optional[DataItem] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        timestamp: Optional[datetime] = None
+    ):
+        """Initialize processed data."""
+        self.source_id = source_id
+        self.processed_content = processed_content
+        self.original_item = original_item
+        self.metadata = metadata or {}
+        self.timestamp = timestamp or datetime.now()
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the processed data to a dictionary."""
+        return {
+            "source_id": self.source_id,
+            "processed_content": self.processed_content,
+            "original_item": self.original_item.to_dict() if self.original_item else None,
+            "metadata": self.metadata,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedData':
+        """Create processed data from a dictionary."""
+        timestamp = None
+        if data.get("timestamp"):
+            try:
+                timestamp = datetime.fromisoformat(data["timestamp"])
+            except (ValueError, TypeError):
+                pass
+                
+        original_item = None
+        if data.get("original_item"):
+            original_item = DataItem.from_dict(data["original_item"])
+                
+        return cls(
+            source_id=data["source_id"],
+            processed_content=data["processed_content"],
+            original_item=original_item,
+            metadata=data.get("metadata", {}),
+            timestamp=timestamp
+        )
+
+
+class ProcessorBase(PluginBase):
+    """Base class for data processors."""
+    
+    name: str = "base_processor"
+    description: str = "Base processor class"
+    processor_type: str = "generic"
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """Initialize the processor with optional configuration."""
+        super().__init__(config)
+        
+    @abstractmethod
+    def process(self, data_item: DataItem, params: Optional[Dict[str, Any]] = None) -> ProcessedData:
+        """Process a data item."""
+        pass
+    
+    def initialize(self) -> bool:
+        """Initialize the processor. Return True if successful, False otherwise."""
+        try:
+            # Perform any necessary initialization
+            logger.info(f"Initialized processor: {self.name}")
+            return True
path: core/plugins/processors/__init__.py
user: login: korbit-ai[bot]
body: ### Unhelpful Implementation Comment <sub>![category Readability](https://img.shields.io/badge/Readability-0284c7)</sub>

<details>
  <summary>Tell me more</summary>

###### What is the issue?
The initialize method contains a placeholder comment that doesn't provide meaningful information.

###### Why this matters
Placeholder comments that don't explain implementation details reduce code clarity and can confuse developers about whether the implementation is complete.

###### Suggested change ∙ *Feature Preview*
Either remove the comment or provide actual implementation details:
```python
def initialize(self) -> bool:
    """Initialize the processor. Return True if successful, False otherwise."""
    try:
        logger.info(f"Initialized processor: {self.name}")
        return True
```


###### Provide feedback to improve future suggestions
[![Nice Catch](https://img.shields.io/badge/👍%20Nice%20Catch-71BC78)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/72b3b183-58a2-411f-ac26-9523c3323e33/upvote) [![Incorrect](https://img.shields.io/badge/👎%20Incorrect-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/72b3b183-58a2-411f-ac26-9523c3323e33?what_not_true=true)  [![Not in Scope](https://img.shields.io/badge/👎%20Out%20of%20PR%20scope-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/72b3b183-58a2-411f-ac26-9523c3323e33?what_out_of_scope=true) [![Not in coding standard](https://img.shields.io/badge/👎%20Not%20in%20our%20standards-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/72b3b183-58a2-411f-ac26-9523c3323e33?what_not_in_standard=true) [![Other](https://img.shields.io/badge/👎%20Other-white)](https://app.korbit.ai/feedback/9c852237-692e-4846-94ac-e139dfda989d/72b3b183-58a2-411f-ac26-9523c3323e33)
</details>

<sub>

💬 Looking for more details? Reply to this comment to chat with Korbit.
</sub>

<!--- korbi internal id:ddb6b894-85c6-4646-aa98-247ef8c452bf -->


[](ddb6b894-85c6-4646-aa98-247ef8c452bf)
created_at: 2025-04-06T11:10:33Z
author_association: NONE

PR comments:
user: login: sourcery-ai[bot]
created_at: 2025-04-06T11:08:38Z
body: <!-- Generated by sourcery-ai[bot]: start review_guide -->

## Reviewer's Guide by Sourcery

This pull request introduces a new plugin-based architecture for Wiseflow, including connectors for GitHub and web sources, a reference management system, a task management system for concurrent data mining, and a text processor for extracting information using LLMs. It also refactors the main task execution logic to use the new task management system.

#### Class diagram for the new GitHubConnector

```mermaid
classDiagram
    class GitHubConnector {
        -api_token: str
        -api_base_url: str
        -headers: Dict[str, str]
        +__init__(config: Optional[Dict[str, Any]] = None)
        +initialize() : bool
        +collect(params: Optional[Dict[str, Any]] = None) : List[DataItem]
        -_get_readme(repo: str) : Optional[DataItem]
        -_get_issues(repo: str, max_issues: int = 10) : List[DataItem]
        -_get_code_files(repo: str, path: str = "", max_files: int = 5) : List[DataItem]
        -_get_content_type(filename: str) : str
    }
    ConnectorBase <|-- GitHubConnector
    note for GitHubConnector "Connector for GitHub repositories"
```

#### Class diagram for the new WebConnector

```mermaid
classDiagram
    class WebConnector {
        -crawler: Optional[AsyncWebCrawler]
        +__init__(config: Optional[Dict[str, Any]] = None)
        +initialize() : bool
        +collect(params: Optional[Dict[str, Any]] = None) : List[DataItem]
    }
    ConnectorBase <|-- WebConnector
    note for WebConnector "Connector for web sources using crawl4ai"
```

#### Class diagram for the new ReferenceManager

```mermaid
classDiagram
    class ReferenceManager {
        -storage_path: str
        -references: Dict[str, Reference]
        +__init__(storage_path: str = "references")
        +load_references() : None
        +save_references() : None
        +add_file_reference(focus_id: str, file_path: str) : Optional[Reference]
        +add_web_reference(focus_id: str, url: str) : Optional[Reference]
        +add_text_reference(focus_id: str, content: str, name: str = "text_reference") : Optional[Reference]
        +get_reference(reference_id: str) : Optional[Reference]
        +get_references_by_focus(focus_id: str) : List[Reference]
        +delete_reference(reference_id: str) : bool
        -_extract_content_from_file(file_path: str) : str
        -_download_web_content(url: str) : str
    }
```

#### Class diagram for the new Reference

```mermaid
classDiagram
    class Reference {
        -reference_id: str
        -focus_id: str
        -reference_type: str
        -path: str
        -content: Optional[str]
        -metadata: Optional[Dict[str, Any]]
        -timestamp: Optional[datetime]
        +__init__(reference_id: str, focus_id: str, reference_type: str, path: str, content: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, timestamp: Optional[datetime] = None)
        +to_dict() : Dict[str, Any]
        +from_dict(data: Dict[str, Any]) : Reference
    }
```

#### Class diagram for the new TaskManager

```mermaid
classDiagram
    class TaskManager {
        -max_workers: int
        -executor: ThreadPoolExecutor
        -tasks: Dict[str, Task]
        -futures: Dict[str, Future]
        -lock: threading.Lock
        +__init__(max_workers: int = 4)
        +submit_task(task: Task) : Future
        -_task_completed(task_id: str) : None
        +get_task(task_id: str) : Optional[Task]
        +get_tasks_by_focus(focus_id: str) : List[Task]
        +get_all_tasks() : List[Task]
        +cancel_task(task_id: str) : bool
        +shutdown(wait: bool = True) : None
    }
```

#### Class diagram for the new AsyncTaskManager

```mermaid
classDiagram
    class AsyncTaskManager {
        -max_workers: int
        -semaphore: asyncio.Semaphore
        -tasks: Dict[str, Task]
        -futures: Dict[str, asyncio.Task]
        -lock: asyncio.Lock
        +__init__(max_workers: int = 4)
        +submit_task(task: Task) : asyncio.Task
        +get_task(task_id: str) : Optional[Task]
        +get_tasks_by_focus(focus_id: str) : List[Task]
        +get_all_tasks() : List[Task]
        +cancel_task(task_id: str) : bool
        +shutdown() : None
        -_task_completed(task_id: str) : None
    }
```

#### Class diagram for the new Task

```mermaid
classDiagram
    class Task {
        -task_id: str
        -focus_id: str
        -function: Callable
        -args: Tuple
        -kwargs: Dict[str, Any]
        -auto_shutdown: bool
        -status: str
        -start_time: Optional[datetime]
        -end_time: Optional[datetime]
        -result: Any
        -error: Optional[Exception]
        +__init__(task_id: str, focus_id: str, function: Callable, args: Tuple = (), kwargs: Dict[str, Any] = None, auto_shutdown: bool = False)
        +to_dict() : Dict[str, Any]
    }
```

#### Class diagram for the new FocusPointProcessor

```mermaid
classDiagram
    class FocusPointProcessor {
        -focus_points: List[Dict[str, Any]]
        +__init__(config: Optional[Dict[str, Any]] = None)
        +initialize() : bool
        +process(data_item: DataItem, params: Optional[Dict[str, Any]] = None) : ProcessedData
        -_create_prompt(content: str, focus_points: List[Dict[str, Any]]) : str
        -_parse_response(response: str) : Dict[str, Any]
    }
    ProcessorBase <|-- FocusPointProcessor
    note for FocusPointProcessor "Extracts information based on focus points using LLMs"
```

#### Class diagram for the new DataItem

```mermaid
classDiagram
    class DataItem {
        -source_id: str
        -content: str
        -metadata: Optional[Dict[str, Any]]
        -url: Optional[str]
        -timestamp: Optional[datetime]
        -content_type: str
        -language: Optional[str]
        -raw_data: Optional[Any]
        +__init__(source_id: str, content: str, metadata: Optional[Dict[str, Any]] = None, url: Optional[str] = None, timestamp: Optional[datetime] = None, content_type: str = "text", language: Optional[str] = None, raw_data: Optional[Any] = None)
        +to_dict() : Dict[str, Any]
        +from_dict(data: Dict[str, Any]) : DataItem
    }
```

#### Class diagram for the new ProcessedData

```mermaid
classDiagram
    class ProcessedData {
        -source_id: str
        -processed_content: str
        -original_item: Optional[DataItem]
        -metadata: Optional[Dict[str, Any]]
        -timestamp: Optional[datetime]
        +__init__(source_id: str, processed_content: str, original_item: Optional[DataItem] = None, metadata: Optional[Dict[str, Any]] = None, timestamp: Optional[datetime] = None)
        +to_dict() : Dict[str, Any]
        +from_dict(data: Dict[str, Any]) : ProcessedData
    }
```

#### Class diagram for the new PluginManager

```mermaid
classDiagram
    class PluginManager {
        -plugins_dir: str
        -plugins: Dict[str, PluginBase]
        +__init__(plugins_dir: str = "plugins")
        +discover_plugins() : List[str]
        +load_plugin(module_path: str) : Optional[PluginBase]
        +load_all_plugins() : Dict[str, PluginBase]
        +get_plugin(name: str) : Optional[PluginBase]
        +initialize_plugin(name: str, config: Optional[Dict[str, Any]] = None) : bool
        +initialize_all_plugins(configs: Optional[Dict[str, Dict[str, Any]]] = None) : Dict[str, bool]
    }
```

#### Class diagram for the new PluginBase

```mermaid
classDiagram
    class PluginBase {
        -name: str
        -description: str
        -version: str
        -config: Dict[str, Any]
        -is_enabled: bool
        +__init__(config: Optional[Dict[str, Any]] = None)
        +initialize() : bool
        +enable() : None
        +disable() : None
        +get_config() : Dict[str, Any]
        +set_config(config: Dict[str, Any]) : None
        +__str__() : str
    }
```

#### Class diagram for the new ConnectorBase

```mermaid
classDiagram
    class ConnectorBase {
        -name: str
        -description: str
        -source_type: str
        -config: Dict[str, Any]
        -last_run: Optional[datetime]
        +__init__(config: Optional[Dict[str, Any]] = None)
        +collect(params: Optional[Dict[str, Any]] = None) : List[DataItem]
        +initialize() : bool
        +update_last_run() : None
        +get_last_run() : Optional[datetime]
    }
    PluginBase <|-- ConnectorBase
```

#### Class diagram for the new ProcessorBase

```mermaid
classDiagram
    class ProcessorBase {
        -name: str
        -description: str
        -processor_type: str
        -config: Dict[str, Any]
        +__init__(config: Optional[Dict[str, Any]] = None)
        +process(data_item: DataItem, params: Optional[Dict[str, Any]] = None) : ProcessedData
        +initialize() : bool
        +batch_process(data_items: List[DataItem], params: Optional[Dict[str, Any]] = None) : List[ProcessedData]
    }
    PluginBase <|-- ProcessorBase
```

### File-Level Changes

| Change | Details | Files |
| ------ | ------- | ----- |
| Implemented a GitHub connector to collect data from GitHub repositories. | <ul><li>Added a new connector class `GitHubConnector` in `core/connectors/github/__init__.py`.</li><li>Implemented `initialize` method to set up the connector with API token and headers.</li><li>Implemented `collect` method to retrieve data from specified repositories, including READMEs, issues, and code files.</li><li>Implemented helper methods `_get_readme`, `_get_issues`, and `_get_code_files` to fetch specific data types from GitHub.</li><li>Added logic to handle API requests, data parsing, and error handling.</li><li>Included logic to determine content type based on file extension.</li></ul> | `core/connectors/github/__init__.py` |
| Implemented a reference management system for focus points. | <ul><li>Added a `Reference` class to represent a reference with attributes like ID, focus ID, type, path, content, metadata, and timestamp in `core/references/__init__.py`.</li><li>Added a `ReferenceManager` class to manage references, including loading, saving, adding, getting, and deleting references.</li><li>Implemented methods to add file, web, and text references, including storing files locally and extracting content.</li><li>Implemented methods to load and save references to a JSON index file.</li><li>Added methods to retrieve references by ID or focus point.</li></ul> | `core/references/__init__.py` |
| Implemented a task management system for concurrent data mining tasks. | <ul><li>Added a `Task` class to represent a data mining task with attributes like ID, focus ID, function, arguments, status, start/end times, result, and error in `core/task/__init__.py`.</li><li>Added a `TaskManager` class to manage tasks using a thread pool executor.</li><li>Implemented methods to submit, get, cancel, and shutdown tasks.</li><li>Implemented an `AsyncTaskManager` class to manage asynchronous tasks using asyncio.</li><li>Added logic to handle task status updates, error handling, and auto-shutdown.</li><li>Added a helper function `create_task_id` to generate unique task IDs.</li></ul> | `core/task/__init__.py` |
| Implemented a text processor for extracting information based on focus points using LLMs. | <ul><li>Added a `FocusPointProcessor` class to extract information based on focus points using LLMs in `core/plugins/processors/text/__init__.py`.</li><li>Implemented `initialize` method to load focus points from config or a file.</li><li>Implemented `process` method to create a prompt, process with LLM, and parse the response.</li><li>Implemented helper methods `_create_prompt` and `_parse_response` to generate prompts and parse LLM responses.</li><li>Added logic to handle errors and return processed data.</li></ul> | `core/plugins/processors/text/__init__.py` |
| Implemented base classes for plugins, connectors, and processors. | <ul><li>Added a `PluginBase` class with basic plugin functionality in `core/plugins/__init__.py`.</li><li>Added a `PluginManager` class to load and manage plugins.</li><li>Implemented methods to discover, load, get, and initialize plugins.</li><li>Added a `ConnectorBase` class with basic connector functionality in `core/connectors/__init__.py`.</li><li>Added a `DataItem` class to represent a single item of data collected from a source.</li><li>Added a `ProcessorBase` class with basic processor functionality in `core/plugins/processors/__init__.py`.</li><li>Added a `ProcessedData` class to represent processed data from a processor.</li></ul> | `core/plugins/__init__.py`<br/>`core/connectors/__init__.py`<br/>`core/plugins/processors/__init__.py` |
| Implemented a web connector to collect data from web sources using crawl4ai. | <ul><li>Added a `WebConnector` class to crawl web pages and extract content in `core/connectors/web/__init__.py`.</li><li>Implemented `initialize` method to set up the connector with crawl4ai configurations.</li><li>Implemented `collect` method to crawl specified URLs and extract data.</li><li>Utilized `AsyncWebCrawler` from `core/crawl4ai` to perform asynchronous web crawling.</li><li>Created `DataItem` instances from the crawled data.</li></ul> | `core/connectors/web/__init__.py` |
| Refactored the main task execution logic to use the new task management system. | <ul><li>Modified `core/run_task_new.py` to use `AsyncTaskManager` for managing concurrent tasks.</li><li>Implemented plugin loading and initialization using `PluginManager`.</li><li>Implemented task scheduling logic to check for active focus points and submit tasks.</li><li>Utilized `process_focus_point` function to process each focus point.</li><li>Added error handling and shutdown logic.</li></ul> | `core/run_task_new.py` |

---

<details>
<summary>Tips and commands</summary>

#### Interacting with Sourcery

- **Trigger a new review:** Comment `@sourcery-ai review` on the pull request.
- **Continue discussions:** Reply directly to Sourcery's review comments.
- **Generate a GitHub issue from a review comment:** Ask Sourcery to create an
  issue from a review comment by replying to it. You can also reply to a
  review comment with `@sourcery-ai issue` to create an issue from it.
- **Generate a pull request title:** Write `@sourcery-ai` anywhere in the pull
  request title to generate a title at any time. You can also comment
  `@sourcery-ai title` on the pull request to (re-)generate the title at any time.
- **Generate a pull request summary:** Write `@sourcery-ai summary` anywhere in
  the pull request body to generate a PR summary at any time exactly where you
  want it. You can also comment `@sourcery-ai summary` on the pull request to
  (re-)generate the summary at any time.
- **Generate reviewer's guide:** Comment `@sourcery-ai guide` on the pull
  request to (re-)generate the reviewer's guide at any time.
- **Resolve all Sourcery comments:** Comment `@sourcery-ai resolve` on the
  pull request to resolve all Sourcery comments. Useful if you've already
  addressed all the comments and don't want to see them anymore.
- **Dismiss all Sourcery reviews:** Comment `@sourcery-ai dismiss` on the pull
  request to dismiss all existing Sourcery reviews. Especially useful if you
  want to start fresh with a new review - don't forget to comment
  `@sourcery-ai review` to trigger a new review!
- **Generate a plan of action for an issue:** Comment `@sourcery-ai plan` on
  an issue to generate a plan of action for it.

#### Customizing Your Experience

Access your [dashboard](https://app.sourcery.ai) to:
- Enable or disable review features such as the Sourcery-generated pull request
  summary, the reviewer's guide, and others.
- Change the review language.
- Add, remove or edit custom review instructions.
- Adjust other review settings.

#### Getting Help

- [Contact our support team](mailto:support@sourcery.ai) for questions or feedback.
- Visit our [documentation](https://docs.sourcery.ai) for detailed guides and information.
- Keep in touch with the Sourcery team by following us on [X/Twitter](https://x.com/SourceryAI), [LinkedIn](https://www.linkedin.com/company/sourcery-ai/) or [GitHub](https://github.com/sourcery-ai).

</details>

<!-- Generated by sourcery-ai[bot]: end review_guide -->
author_association: NONE
user: login: qodo-code-review[bot]
created_at: 2025-04-06T11:08:58Z
body: ## PR Reviewer Guide 🔍

#### (Review updated until commit https://github.com/Zeeeepa/wiseflow/commit/677ff70845492993055752705182f7a986c97edc)


Here are some key observations to aid the review process:

<table>
<tr><td>⏱️&nbsp;<strong>Estimated effort to review</strong>: 4 🔵🔵🔵🔵⚪</td></tr>
<tr><td>🧪&nbsp;<strong>No relevant tests</strong></td></tr>
<tr><td>🔒&nbsp;<strong>Security concerns</strong><br><br>

<strong>Sensitive information exposure:</strong><br> The GitHub connector in `core/connectors/github/__init__.py` (lines 38-51) retrieves API tokens from configuration or environment variables but stores them in plaintext as instance variables. This could expose sensitive tokens if the object is serialized or if memory is dumped. Additionally, the web connector in `core/connectors/web/__init__.py` doesn't implement proper input validation for URLs, which could lead to SSRF attacks if malicious URLs are provided.</td></tr>
<tr><td>⚡&nbsp;<strong>Recommended focus areas for review</strong><br><br>

<details><summary><a href='https://github.com/Zeeeepa/wiseflow/pull/2/files#diff-3c16ebc45cf3cb186d7fcfe9d56b575b441dcfabc37d6b51657fa94b34b945b0R38-R51'><strong>Security Concern</strong></a>

The GitHub connector stores API tokens in plaintext and doesn't validate token permissions. Consider implementing secure token storage and validation.
</summary>

```python
self.api_token = self.config.get("api_token")
if not self.api_token and "GITHUB_TOKEN" in os.environ:
    self.api_token = os.environ["GITHUB_TOKEN"]

if not self.api_token:
    logger.warning("No GitHub API token provided. Rate limits will be restricted.")
    self.headers = {
        "Accept": "application/vnd.github.v3+json"
    }
else:
    self.headers = {
        "Accept": "application/vnd.github.v3+json",
        "Authorization": f"token {self.api_token}"
    }
```

</details>

<details><summary><a href='https://github.com/Zeeeepa/wiseflow/pull/2/files#diff-d7c7550b73d57a1b0f7d854954d9ec0b12cc4b8c2dc2476a6219605b6b7e5a38R108-R145'><strong>File Handling</strong></a>

The reference manager doesn't validate file types or sizes before copying, which could lead to storage issues or security vulnerabilities.
</summary>

```python
def add_file_reference(self, focus_id: str, file_path: str) -> Optional[Reference]:
    """Add a file reference to a focus point."""
    try:
        # Generate a unique reference ID
        reference_id = str(uuid.uuid4())

        # Create a copy of the file in the storage directory
        filename = os.path.basename(file_path)
        storage_dir = os.path.join(self.storage_path, focus_id)
        os.makedirs(storage_dir, exist_ok=True)

        dest_path = os.path.join(storage_dir, f"{reference_id}_{filename}")
        shutil.copy2(file_path, dest_path)

        # Extract content from the file
        content = self._extract_content_from_file(file_path)

        # Create the reference
        reference = Reference(
            reference_id=reference_id,
            focus_id=focus_id,
            reference_type="file",
            path=dest_path,
            content=content,
            metadata={
                "original_path": file_path,
                "filename": filename
            }
        )

        # Add to references
        self.references[reference_id] = reference
        self.save_references()

        return reference
    except Exception as e:
        logger.error(f"Error adding file reference: {e}")
        return None
```

</details>

<details><summary><a href='https://github.com/Zeeeepa/wiseflow/pull/2/files#diff-ea1be7d666f6f663f949f0f054a05af5be473ba194e79ee283248286887cdd4cR152-R202'><strong>Resource Management</strong></a>

The task management system doesn't implement proper resource limits or timeouts, which could lead to resource exhaustion.
</summary>

```python
"""Manages concurrent asynchronous data mining tasks."""

def __init__(self, max_workers: int = 4):
    """Initialize the async task manager."""
    self.max_workers = max_workers
    self.semaphore = asyncio.Semaphore(max_workers)
    self.tasks: Dict[str, Task] = {}
    self.futures: Dict[str, asyncio.Task] = {}
    self.lock = asyncio.Lock()

async def submit_task(self, task: Task) -> asyncio.Task:
    """Submit a task for execution."""
    async with self.lock:
        # Create a wrapper function to handle task status updates
        async def task_wrapper(*args, **kwargs):
            async with self.semaphore:
                try:
                    task.status = "running"
                    task.start_time = datetime.now()
                    logger.info(f"Task {task.task_id} started")

                    # Execute the task
                    result = await task.function(*args, **kwargs)

                    task.status = "completed"
                    task.end_time = datetime.now()
                    task.result = result
                    logger.info(f"Task {task.task_id} completed")

                    # Auto-shutdown if enabled
                    if task.auto_shutdown:
                        logger.info(f"Auto-shutdown enabled for task {task.task_id}")
                        # Implement auto-shutdown logic here

                    return result
                except Exception as e:
                    task.status = "failed"
                    task.end_time = datetime.now()
                    task.error = e
                    logger.error(f"Task {task.task_id} failed: {e}")
                    raise

        # Create an asyncio task
        future = asyncio.create_task(task_wrapper(*task.args, **task.kwargs))
        self.tasks[task.task_id] = task
        self.futures[task.task_id] = future

        # Add a callback to clean up completed tasks
        future.add_done_callback(lambda f: asyncio.create_task(self._task_completed(task.task_id)))

        return future
```

</details>

</td></tr>
</table>

author_association: NONE
user: login: codiumai-pr-agent-free[bot]
created_at: 2025-04-06T11:09:37Z
body: ## PR Code Suggestions ✨
<!-- 677ff70 -->

Latest suggestions up to 677ff70
Explore these optional code suggestions:

<table><thead><tr><td><strong>Category</strong></td><td align=left><strong>Suggestion&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </strong></td><td align=center><strong>Impact</strong></td></tr><tbody><tr><td rowspan=3>Possible issue</td>
<td>



<details><summary>Add exception handling</summary>

___


**The task scheduling loop doesn't handle exceptions that might occur during task <br>submission or focus point processing. If an exception occurs in the loop, it <br>will break the entire scheduling process. Add exception handling to ensure the <br>loop continues running.**

[core/run_task_new.py [49-96]](https://github.com/Zeeeepa/wiseflow/pull/2/files#diff-7c2e167316d5c4fb41944070f296cae821d46ef922376e4d153ee1091da8cc5aR49-R96)

```diff
 async def schedule_task():
     """Schedule and manage data mining tasks."""
     # Load plugins
     await load_plugins()
     
     while True:
-        wiseflow_logger.info("Checking for active focus points...")
-        
-        # Get active focus points
-        focus_points = pb.read('focus_points', filter='activated=True')
-        sites_record = pb.read('sites')
-        
-        for focus in focus_points:
-            # Check if there's already a task running for this focus point
-            existing_tasks = task_manager.get_tasks_by_focus(focus["id"])
-            active_tasks = [t for t in existing_tasks if t.status in ["pending", "running"]]
+        try:
+            wiseflow_logger.info("Checking for active focus points...")
             
-            if active_tasks:
-                wiseflow_logger.info(f"Focus point {focus['focuspoint']} already has active tasks")
-                continue
+            # Get active focus points
+            focus_points = pb.read('focus_points', filter='activated=True')
+            sites_record = pb.read('sites')
             
-            # Get sites for this focus point
-            sites = [_record for _record in sites_record if _record['id'] in focus.get('sites', [])]
+            for focus in focus_points:
+                try:
+                    # Check if there's already a task running for this focus point
+                    existing_tasks = task_manager.get_tasks_by_focus(focus["id"])
+                    active_tasks = [t for t in existing_tasks if t.status in ["pending", "running"]]
+                    
+                    if active_tasks:
+                        wiseflow_logger.info(f"Focus point {focus['focuspoint']} already has active tasks")
+                        continue
+                    
+                    # Get sites for this focus point
+                    sites = [_record for _record in sites_record if _record['id'] in focus.get('sites', [])]
+                    
+                    # Create a new task
+                    task_id = create_task_id()
+                    auto_shutdown = focus.get("auto_shutdown", False)
+                    
+                    task = Task(
+                        task_id=task_id,
+                        focus_id=focus["id"],
+                        function=process_focus_point,
+                        args=(focus, sites),
+                        auto_shutdown=auto_shutdown
+                    )
+                    
+                    # Submit the task
+                    wiseflow_logger.info(f"Submitting task {task_id} for focus point: {focus['focuspoint']}")
+                    await task_manager.submit_task(task)
+                except Exception as e:
+                    wiseflow_logger.error(f"Error processing focus point {focus.get('focuspoint', 'unknown')}: {e}")
             
-            # Create a new task
-            task_id = create_task_id()
-            auto_shutdown = focus.get("auto_shutdown", False)
-            
-            task = Task(
-                task_id=task_id,
-                focus_id=focus["id"],
-                function=process_focus_point,
-                args=(focus, sites),
-                auto_shutdown=auto_shutdown
-            )
-            
-            # Submit the task
-            wiseflow_logger.info(f"Submitting task {task_id} for focus point: {focus['focuspoint']}")
-            await task_manager.submit_task(task)
-        
-        # Wait before checking again
-        wiseflow_logger.info("Waiting for 60 seconds before checking for new tasks...")
-        await asyncio.sleep(60)
+            # Wait before checking again
+            wiseflow_logger.info("Waiting for 60 seconds before checking for new tasks...")
+            await asyncio.sleep(60)
+        except Exception as e:
+            wiseflow_logger.error(f"Error in schedule_task loop: {e}")
+            # Wait a bit before retrying to avoid tight error loops
+            await asyncio.sleep(10)
```


`[To ensure code accuracy, apply this suggestion manually]`

<details><summary>Suggestion importance[1-10]: 9</summary>

__

Why: This suggestion adds critical exception handling to the main task scheduling loop. Without it, any exception would crash the entire scheduling process, making the system unstable and unreliable.

</details></details></td><td align=center>High

</td></tr><tr><td>



<details><summary>Handle API rate limits</summary>

___


**The code doesn't handle potential API rate limits when collecting code files <br>from GitHub. This could lead to request failures during heavy usage. Add rate <br>limiting handling with exponential backoff.**

[core/connectors/github/__init__.py [100-103]](https://github.com/Zeeeepa/wiseflow/pull/2/files#diff-3c16ebc45cf3cb186d7fcfe9d56b575b441dcfabc37d6b51657fa94b34b945b0R100-R103)

```diff
 # Collect code (basic implementation)
 if collect_code:
-    code_files = self._get_code_files(repo)
-    results.extend(code_files)
+    try:
+        code_files = self._get_code_files(repo)
+        results.extend(code_files)
+    except Exception as e:
+        if "rate limit" in str(e).lower():
+            logger.warning(f"GitHub API rate limit reached for {repo}. Waiting before retrying.")
+            time.sleep(60)  # Wait before potential retry
+        else:
+            logger.error(f"Error collecting code files from {repo}: {e}")
```

- [ ] **Apply this suggestion** <!-- /improve --apply_suggestion=1 -->

<details><summary>Suggestion importance[1-10]: 8</summary>

__

Why: This suggestion adds important error handling for GitHub API rate limits, which is critical for reliable operation. Without this, the connector would fail silently when hitting rate limits, potentially missing data.


</details></details></td><td align=center>Medium

</td></tr><tr><td>



<details><summary>Fix JSON parsing vulnerability</summary>

___


**The JSON parsing logic is vulnerable to malformed JSON responses. If the <br>response contains multiple JSON objects or nested braces, the current approach <br>of finding the first '{' and last '}' could extract invalid JSON. Use a more <br>robust parsing approach.**

[core/plugins/processors/text/__init__.py [139-157]](https://github.com/Zeeeepa/wiseflow/pull/2/files#diff-798fc237031a64ac75603bb5019e67cdda1b8e29d0c5924f3c09cf708292c5dfR139-R157)

```diff
 def _parse_response(self, response: str) -> Dict[str, Any]:
     """Parse the LLM response into structured data."""
     try:
         # Try to extract JSON from the response
         response = response.strip()
         
-        # Find JSON object in the response
-        start_idx = response.find('{')
-        end_idx = response.rfind('}')
-        
-        if start_idx >= 0 and end_idx > start_idx:
-            json_str = response[start_idx:end_idx+1]
-            return json.loads(json_str)
+        # First try direct JSON parsing
+        try:
+            return json.loads(response)
+        except json.JSONDecodeError:
+            # Find JSON object in the response
+            start_idx = response.find('{')
+            end_idx = response.rfind('}')
+            
+            if start_idx >= 0 and end_idx > start_idx:
+                json_str = response[start_idx:end_idx+1]
+                try:
+                    return json.loads(json_str)
+                except json.JSONDecodeError:
+                    pass
         
         # If no JSON found, return the raw response
         return {"raw_response": response}
     except Exception as e:
         logger.error(f"Error parsing LLM response: {e}")
         return {"error": str(e), "raw_response": response}
```

- [ ] **Apply this suggestion** <!-- /improve --apply_suggestion=2 -->

<details><summary>Suggestion importance[1-10]: 7</summary>

__

Why: The suggestion improves the robustness of JSON parsing by attempting direct parsing first and adding proper exception handling. This prevents potential errors when processing malformed LLM responses.


</details></details></td><td align=center>Medium

</td></tr>
<tr><td align="center" colspan="2">

- [ ] More <!-- /improve --more_suggestions=true -->

</td><td></td></tr></tbody></table>

___

#### Previous suggestions
<details><summary>Suggestions up to commit 677ff70</summary>
<br><table><thead><tr><td><strong>Category</strong></td><td align=left><strong>Suggestion&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </strong></td><td align=center><strong>Impact</strong></td></tr><tbody><tr><td rowspan=3>Possible issue</td>
<td>



<details><summary>Handle API rate limits</summary>

___


**The code doesn't handle rate limiting properly when collecting code files. <br>GitHub API has strict rate limits, and the current implementation could hit <br>these limits quickly when processing multiple repositories or files.**

[core/connectors/github/__init__.py [100-103]](https://github.com/Zeeeepa/wiseflow/pull/2/files#diff-3c16ebc45cf3cb186d7fcfe9d56b575b441dcfabc37d6b51657fa94b34b945b0R100-R103)

```diff
 # Collect code (basic implementation)
 if collect_code:
-    code_files = self._get_code_files(repo)
-    results.extend(code_files)
+    try:
+        code_files = self._get_code_files(repo)
+        results.extend(code_files)
+    except Exception as e:
+        logger.error(f"Error collecting code files from {repo}: {e}")
+        # Check if it's a rate limit error and handle accordingly
+        if "rate limit" in str(e).lower():
+            logger.warning(f"GitHub API rate limit reached. Waiting before continuing.")
+            time.sleep(60)  # Wait before trying other operations
```

 <!-- /improve --apply_suggestion=0 -->

<details><summary>Suggestion importance[1-10]: 8</summary>

__

Why: This suggestion adds important error handling for GitHub API rate limits, which is critical for reliable operation. Without proper rate limit handling, the connector could fail when processing multiple repositories, making this a high-impact improvement.


</details></details></td><td align=center>Medium

</td></tr><tr><td>



<details><summary>Add task submission error handling</summary>

___


**The task submission doesn't have any error handling. If task submission fails, <br>the error will propagate and potentially crash the entire scheduling loop, <br>preventing other tasks from being processed.**

[core/run_task_new.py [90-92]](https://github.com/Zeeeepa/wiseflow/pull/2/files#diff-7c2e167316d5c4fb41944070f296cae821d46ef922376e4d153ee1091da8cc5aR90-R92)

```diff
 # Submit the task
-wiseflow_logger.info(f"Submitting task {task_id} for focus point: {focus['focuspoint']}")
-await task_manager.submit_task(task)
+try:
+    wiseflow_logger.info(f"Submitting task {task_id} for focus point: {focus['focuspoint']}")
+    await task_manager.submit_task(task)
+except Exception as e:
+    wiseflow_logger.error(f"Failed to submit task for focus point {focus['focuspoint']}: {e}")
+    # Continue with other tasks rather than crashing the scheduler
```

 <!-- /improve --apply_suggestion=1 -->

<details><summary>Suggestion importance[1-10]: 8</summary>

__

Why: This suggestion adds critical error handling for task submission that prevents a single task failure from crashing the entire scheduling loop. This significantly improves system reliability and ensures other tasks can continue processing.


</details></details></td><td align=center>Medium

</td></tr><tr><td>



<details><summary>Fix JSON parsing logic</summary>

___


**The JSON parsing logic is vulnerable to malformed JSON responses. If the LLM <br>returns nested JSON objects or arrays, the current implementation using find() <br>and rfind() could extract an incomplete JSON fragment, leading to parsing <br>errors.**

[core/plugins/processors/text/__init__.py [139-157]](https://github.com/Zeeeepa/wiseflow/pull/2/files#diff-798fc237031a64ac75603bb5019e67cdda1b8e29d0c5924f3c09cf708292c5dfR139-R157)

```diff
 def _parse_response(self, response: str) -> Dict[str, Any]:
     """Parse the LLM response into structured data."""
     try:
         # Try to extract JSON from the response
         response = response.strip()
         
-        # Find JSON object in the response
-        start_idx = response.find('{')
-        end_idx = response.rfind('}')
+        # First try to parse the entire response as JSON
+        try:
+            return json.loads(response)
+        except json.JSONDecodeError:
+            # If that fails, try to extract JSON from the response
+            start_idx = response.find('{')
+            end_idx = response.rfind('}')
+            
+            if start_idx >= 0 and end_idx > start_idx:
+                json_str = response[start_idx:end_idx+1]
+                try:
+                    return json.loads(json_str)
+                except json.JSONDecodeError:
+                    pass
         
-        if start_idx >= 0 and end_idx > start_idx:
-            json_str = response[start_idx:end_idx+1]
-            return json.loads(json_str)
-        
-        # If no JSON found, return the raw response
+        # If no valid JSON found, return the raw response
         return {"raw_response": response}
     except Exception as e:
         logger.error(f"Error parsing LLM response: {e}")
         return {"error": str(e), "raw_response": response}
```

 <!-- /improve --apply_suggestion=2 -->

<details><summary>Suggestion importance[1-10]: 7</summary>

__

Why: The improved code adds more robust JSON parsing that handles nested structures better by attempting to parse the entire response first. This reduces the risk of parsing errors when working with complex LLM responses.


</details></details></td><td align=center>Medium

</td></tr>
<tr><td align="center" colspan="2">

 <!-- /improve --more_suggestions=true -->

</td><td></td></tr></tbody></table>

</details>

author_association: NONE