MultiagentPersonalAssistant

.gitignore

@@ -0,0 +1 @@
+.env

.idea/MultiagentPersonalAssistant.iml

@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="MultiagentPersonalAssistant" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="test" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyDocumentationSettings">

.idea/misc.xml

@@ -3,5 +3,5 @@
   <component name="Black">
     <option name="sdkName" value="MultiagentPersonalAssistant" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="MultiagentPersonalAssistant" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="test" project-jdk-type="Python SDK" />
 </project>

README.md

@@ -30,6 +30,7 @@ This project is a **voice- and text-based AI assistant** powered by **FastAPI**,
   - Supervisor agent
   - Deep research agent
   - Coding agent
+  - Smart home agent
 - **Supervisor tools**:
   - Web search
   - Current time
@@ -42,6 +43,9 @@ This project is a **voice- and text-based AI assistant** powered by **FastAPI**,
 - **Coder tools**:
   - Github toolkit
   - Web search (For docs research)
+- **Smart home tools**:
+  - Get device list
+  - Change device states
 
 ---
 
@@ -118,7 +122,12 @@ It follows a `TypedDict` schema and may contain keys such as:
     openweathermap_api_key: str
     github_token: str
     tavily_api_key: str
-    groq_api_key: str
+    groq_api_key: str,
+    tuya_access_id: str
+    tuya_access_key: str
+    tuya_username: str
+    tuya_password: str
+    tuya_country: str
     clear_history: bool (True/False)
     messages: list (Filled automatically from the database) 
 
@@ -127,6 +136,13 @@ It follows a `TypedDict` schema and may contain keys such as:
 
 This dictionary acts as a single mutable state shared across all agent steps, allowing for data accumulation, tool responses, and message tracking.
 
+
+---
+
+## ⚙️ Tuya Setup
+
+Take a look at the [official instructions](https://developer.tuya.com/en/docs/iot/device-control-best-practice?id=Ka72202tz4m67)
+
 ---
 
 ## 🧠 Built With
@@ -137,6 +153,7 @@ This dictionary acts as a single mutable state shared across all agent steps, al
 - [FastAPI](https://fastapi.tiangolo.com/)
 - [Whisper](https://github.com/openai/whisper)
 - [SQLAlchemy + Async](https://docs.sqlalchemy.org/en/20/orm/extensions/asyncio.html)
+- [Tuya Smart Home](https://github.com/tuya/tuya-iot-python-sdk?tab=readme-ov-file)
 
 ---
 

agent/multi_agent.py

@@ -22,10 +22,14 @@ class State(TypedDict):
     github_token: str
     tavily_api_key: str
     groq_api_key: str
+    tuya_access_id: str
+    tuya_access_key: str
+    tuya_username: str
+    tuya_password: str
+    tuya_country: str
     clear_history: bool
     messages: list
 
-
 class Assistant:
     def __init__(self, state: State):
         self.state = state
@@ -52,7 +56,8 @@ class Assistant:
             config_data = {}
             config_fields = [
                 'assistant_name', 'openweathermap_api_key', 'github_token',
-                'tavily_api_key', 'groq_api_key'
+                'tavily_api_key', 'groq_api_key', 'tuya_access_id', 'tuya_access_key',
+                'tuya_username', 'tuya_password', 'tuya_country'
             ]
 
             for field in config_fields:
@@ -72,6 +77,7 @@ class Assistant:
     def compile_multi_agent_system(self):
         """Create and return the multi-agent system"""
         try:
+            from agent.smart_home_agent import smart_home_agent
             from agent.deep_research_agent import deep_research_agent
             from agent.coder_agent import coder_agent
             from agent.prompts import supervisor_instructions
@@ -86,7 +92,7 @@ class Assistant:
                 output_messages_key="llm_input_messages",
             )
 
-            agents = [coder_agent, deep_research_agent]
+            agents = [coder_agent, deep_research_agent, smart_home_agent]
 
             supervisor = create_supervisor(
                 model=llm_supervisor,

agent/prompts.py

@@ -6,6 +6,13 @@ coder_instructions = f"""You are a coding expert. Follow these steps every time:
 
 """
 
+smart_home_instructions = f"""You are a smart home control expert. Follow these steps every time:\n
+
+1. Use the 'get_device_list' tool to retrieve the list of available devices with their names, ids, and states.\n
+2. Identify the target device(s) based on the user’s request.\n
+3. Use the 'change_device_states' tool to update the states of the selected device(s).\n
+"""
+
 deep_research_instructions = f"""You are a deep research expert. Follow these steps every time:\n
 
 1. Iterate over the given search queries.\n
@@ -53,4 +60,13 @@ def deep_research_system_message(state: dict):
     4. Conduct additional research using available tools.\t
     5. Write an extensive report based on web search results.\n
     6. Critically review your report and cicle over these steps until it is factually correct and very detailed.\n
+    """
+
+def smart_home_system_message(device_list: list):
+    return f"""Your job is to control the smart home.\nYou have access to the following devices:\n
+    {"\n\n".join(
+    f"{i+1}. {device['name']} (id: {device['id']})\nStates:\n" +
+    "\n".join(f"{state}" for state in device['states'])
+    for i, device in enumerate(device_list)
+    )}.\nUse the given tools ('get_device_list', 'change_device_states') to complete the user's tasks.
     """

agent/smart_home_agent.py

@@ -0,0 +1,30 @@
+from agent.prompts import smart_home_instructions, smart_home_system_message
+from langchain_core.messages import HumanMessage, SystemMessage
+from agent.models import llm_agents, llm_peripheral
+from langgraph.prebuilt import create_react_agent
+from langgraph.constants import START, END
+from langgraph.graph import StateGraph
+from agent.tools import smart_home_tools, get_device_list
+
+agent = create_react_agent(
+    llm_agents,
+    tools=smart_home_tools,
+    prompt=smart_home_instructions
+)
+
+def home_control_agent(state: dict):
+    system_message = SystemMessage(smart_home_system_message(get_device_list.invoke({})))
+    state.update(agent.invoke({
+        'messages': [
+            system_message,
+            HumanMessage(state['messages'][-1].content),
+        ]
+    }))
+    return state
+
+graph = StateGraph(dict)
+graph.add_node("home_control_agent", home_control_agent)
+graph.add_edge(START, "home_control_agent")
+graph.add_edge("home_control_agent", END)
+
+smart_home_agent = graph.compile(name="smart_home_agent")

agent/states.py

@@ -10,5 +10,4 @@ class PlanResearch(TypedDict):
 class PlanCodingTask(TypedDict):
     repo_name: str = Field(description="The name of the GitHub repository for the project.")
     private: bool = Field(description="Whether or not the repository is private.", default=False)
-    task_description: str = Field(description="A detailed description of the project for the coder to create.")
-
+    task_description: str = Field(description="A detailed description of the project for the coder to create.")

agent/tools.py

@@ -5,26 +5,61 @@ from langchain_community.tools import WikipediaQueryRun
 from langchain_tavily.tavily_search import TavilySearch
 from timezonefinder import TimezoneFinder
 from langchain_core.tools import tool
+from tuya_iot import TuyaOpenAPI
 from dotenv import load_dotenv
 from geopy import Nominatim
 from github import Github
+import phonenumbers
+import pycountry
 import datetime
 import pytz
 import os
 
 load_dotenv()
 
-wikipedia = WikipediaQueryRun(api_wrapper=WikipediaAPIWrapper())
-yahoo = YahooFinanceNewsTool()
+
+tf = TimezoneFinder()
 web_search = TavilySearch(
     max_results=5,
     topic="general",
     include_answer=True
 )
-tf = TimezoneFinder()
+yahoo = YahooFinanceNewsTool()
 geolocator = Nominatim(user_agent="my_geocoder")
+wikipedia = WikipediaQueryRun(api_wrapper=WikipediaAPIWrapper())
 weekday_mapping = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")
-
+endpoint_map = {
+    "https://openapi.tuyacn.com": [86],
+    "https://openapi.tuyain.com": [91],
+    "https://openapi.tuyaus.com": [1, 51, 52, 54, 55, 56, 57, 58, 81, 82, 239,
+                                   245, 502, 591, 593, 595, 597, 598, 674, 678, 682, 683, 685, 686, 690, 970,
+                                   1340, 1684, 1670, 1671, 1787, 1809, 1829, 1849, 5999, 35818, 64],
+    "https://openapi.tuyaeu.com": [7, 20, 27, 30, 31, 32, 33, 34, 36, 39, 40,
+                                   90, 92, 93, 94, 212, 213, 216, 218, 220, 221, 222, 223, 224, 225, 226, 227,
+                                   228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 240, 241, 242, 243,
+                                   244, 246, 248, 250, 251, 252, 253, 254, 255, 256, 257, 258, 260, 261, 262,
+                                   263, 264, 265, 266, 267, 268, 269, 291, 297, 298, 299, 350, 351, 352, 353,
+                                   354, 355, 356, 357, 358, 359, 370, 371, 372, 373, 374, 375, 376, 377, 378,
+                                   379, 380, 381, 382, 385, 386, 387, 389, 420, 421, 423, 500, 501, 503, 504,
+                                   505, 506, 507, 508, 509, 590, 592, 594, 596, 672, 676, 679, 680, 681, 687,
+                                   688, 689, 691, 692, 960, 961, 962, 964, 965, 966, 967, 968, 971, 972, 973,
+                                   974, 975, 976, 977, 992, 993, 994, 995, 996, 998, 1242, 1246, 1264, 1268,
+                                   1284, 1345, 1441, 1473, 1649, 1664, 1721, 1758, 1767, 1784, 1868, 1869,
+                                   1876, 4779, 61, 880, 41, 43, 44, 45, 46, 47, 48, 49],
+    "https://openapi-sg.iotbing.com": [84, 856, 855, 66, 95, 60, 65, 62, 63,
+                                       673, 670, 675, 677, 852, 853, 886],
+}
+country = pycountry.countries.lookup(os.getenv('TUYA_COUNTRY').title())
+country_code = phonenumbers.country_code_for_region(country.alpha_2)
+for endpoint, country_codes in endpoint_map.items():
+    if country_code in country_codes:
+        os.environ['TUYA_COUNTRY_CODE'] = str(country_code)
+        os.environ['TUYA_ENDPOINT'] = endpoint
+        break
+openapi = TuyaOpenAPI(os.getenv('TUYA_ENDPOINT'), os.getenv('TUYA_ACCESS_ID'), os.getenv('TUYA_ACCESS_KEY'))
+connection = openapi.connect(os.getenv('TUYA_USERNAME'), os.getenv('TUYA_PASSWORD'), os.getenv('TUYA_COUNTRY_CODE'), 'TuyaSmart')
+if not connection['success']:
+    connection = openapi.connect(os.getenv('TUYA_USERNAME'), os.getenv('TUYA_PASSWORD'), os.getenv('TUYA_COUNTRY_CODE'), 'SmartLife')
 @tool
 def create_repo(repo_name: str, private: bool = False):
     """Creates a GitHub repository with the given repo_name."""
@@ -174,6 +209,32 @@ def weather(location: str = None):
     except Exception as e:
         return f"Error getting weather: {str(e)}"
 
+@tool
+def get_device_list():
+    """
+    Get the list of devices bound to the smart home.
+    Each device has id: str, name: str and states: List[dict].
+    States is a list of device states which can be changed.
+    State format: {'code': ..., 'value': ...}
+    """
+    response = openapi.get('/v1.0/users/{}/devices'.format(connection['result']['uid']))['result']
+    devices = []
+    for device in response:
+        devices.append({"id": device['id'], 'name': device['name'], 'states': device['status']})
+    return devices
+
+@tool
+def change_device_states(id:str, commands:list[dict]):
+    """
+    Control the smart home devices by sending commands.
+    Accepts device id:str and a list of commands: List[dict].
+    Commands is a list of changed device states.
+    State format: {'code': ..., 'value': ...}
+    """
+    response = openapi.post('/v1.0/devices/{}/commands'.format(id), {"commands": commands})
+    return "success" if response["success"] else "failure"
+
 coder_tools = [web_search, create_repo, create_branch, commit_file_to_repo, read_file, list_files, list_repos, list_branches, delete_file]
 supervisor_tools = [yahoo, web_search, current_time, weather]
 deep_research_tools = [web_search, yahoo, wikipedia]
+smart_home_tools = [get_device_list, change_device_states]

database_interaction/config.py

@@ -54,6 +54,11 @@ async def load_config_to_env(user_id: str):
         'github_token': 'GITHUB_TOKEN',
         'tavily_api_key': 'TAVILY_API_KEY',
         'groq_api_key': 'GROQ_API_KEY',
+        'tuya_access_id': 'TUYA_ACCESS_ID',
+        'tuya_access_key': 'TUYA_ACCESS_KEY',
+        'tuya_username': 'TUYA_USERNAME',
+        'tuya_password': 'TUYA_PASSWORD',
+        'tuya_country': 'TUYA_COUNTRY',
     }
 
     for config_key, env_key in env_mapping.items():
@@ -65,9 +70,8 @@ async def load_config_to_env(user_id: str):
 async def save_env_to_file():
     """Save current environment variables to .env file"""
     env_vars = [
-        'DATABASE_URL', 'TOKEN', 'ASSISTANT_NAME', 'LOCATION', 'LATITUDE', 'LONGITUDE',
-        'OPENWEATHERMAP_API_KEY', 'GITHUB_TOKEN', 'TAVILY_API_KEY',
-        'GROQ_API_KEY'
+        'DATABASE_URL', 'TOKEN', 'ASSISTANT_NAME', 'LOCATION', 'LATITUDE', 'LONGITUDE','OPENWEATHERMAP_API_KEY', 'GITHUB_TOKEN',
+        'TAVILY_API_KEY', 'GROQ_API_KEY', 'TUYA_ACCESS_ID', 'TUYA_ACCESS_KEY', 'TUYA_USERNAME', 'TUYA_PASSWORD', 'TUYA_COUNTRY',
     ]
 
     # Set default values if not present

database_interaction/models.py

@@ -15,10 +15,15 @@ class User(Base):
 
 class UserConfig(Base):
     __tablename__ = "user_config"
-
     user_id = Column(String, primary_key=True, index=True)
     assistant_name = Column(String, nullable=True)
     openweathermap_api_key = Column(String, nullable=True)
     github_token = Column(String, nullable=True)
     tavily_api_key = Column(String, nullable=True)
-    groq_api_key = Column(String, nullable=True)
+    groq_api_key = Column(String, nullable=True)
+    tuya_access_id = Column(String, nullable=True)
+    tuya_access_key = Column(String, nullable=True)
+    tuya_username = Column(String, nullable=True)
+    tuya_password = Column(String, nullable=True)
+    tuya_country = Column(String, nullable=True)
+    tuya_schema = Column(String, nullable=True)

requirements.txt

@@ -4,6 +4,8 @@ python-dotenv>=1.1.1
 langchain>=0.3.27
 langgraph>=0.6.3
 openai>=1.99.1
+phonenumbers
+pycountry
 timm>=1.0.19
 langchain-groq>=0.3.7
 langgraph-supervisor>=0.0.29

tmp/hyperparams.yaml

@@ -1,57 +0,0 @@
-pretrained_path: speechbrain/lang-id-voxlingua107-ecapa
-
-
-# Feature parameters
-n_mels: 60
-left_frames: 0
-right_frames: 0
-deltas: false
-
-# Number of speakers
-out_n_neurons: 107
-
-# Functions
-compute_features: !new:speechbrain.lobes.features.Fbank
-  n_mels: 60
-  left_frames: 0
-  right_frames: 0
-  deltas: false
-
-embedding_model: !new:speechbrain.lobes.models.ECAPA_TDNN.ECAPA_TDNN
-  input_size: 60
-  channels: [1024, 1024, 1024, 1024, 3072]
-  kernel_sizes: [5, 3, 3, 3, 1]
-  dilations: [1, 2, 3, 4, 1]
-  attention_channels: 128
-  lin_neurons: 256
-
-
-classifier: !new:speechbrain.lobes.models.Xvector.Classifier
-  input_shape: [null, null, 256]
-  activation: !name:torch.nn.LeakyReLU
-  lin_blocks: 1
-  lin_neurons: 512
-  out_neurons: !ref <out_n_neurons>
-
-
-mean_var_norm: !new:speechbrain.processing.features.InputNormalization
-  norm_type: sentence
-  std_norm: false
-
-modules:
-    compute_features: !ref <compute_features>
-    mean_var_norm: !ref <mean_var_norm>
-    embedding_model: !ref <embedding_model>
-    classifier: !ref <classifier>
-
-label_encoder: !new:speechbrain.dataio.encoder.CategoricalEncoder
-
-pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
-    loadables:
-        embedding_model: !ref <embedding_model>
-        classifier: !ref <classifier>
-        label_encoder: !ref <label_encoder>
-    paths:
-        embedding_model: !ref <pretrained_path>/embedding_model.ckpt
-        classifier: !ref <pretrained_path>/classifier.ckpt
-        label_encoder: !ref <pretrained_path>/label_encoder.txt

tmp/label_encoder.ckpt

@@ -1,109 +0,0 @@
-'ab: Abkhazian' => 0
-'af: Afrikaans' => 1
-'am: Amharic' => 2
-'ar: Arabic' => 3
-'as: Assamese' => 4
-'az: Azerbaijani' => 5
-'ba: Bashkir' => 6
-'be: Belarusian' => 7
-'bg: Bulgarian' => 8
-'bn: Bengali' => 9
-'bo: Tibetan' => 10
-'br: Breton' => 11
-'bs: Bosnian' => 12
-'ca: Catalan' => 13
-'ceb: Cebuano' => 14
-'cs: Czech' => 15
-'cy: Welsh' => 16
-'da: Danish' => 17
-'de: German' => 18
-'el: Greek' => 19
-'en: English' => 20
-'eo: Esperanto' => 21
-'es: Spanish' => 22
-'et: Estonian' => 23
-'eu: Basque' => 24
-'fa: Persian' => 25
-'fi: Finnish' => 26
-'fo: Faroese' => 27
-'fr: French' => 28
-'gl: Galician' => 29
-'gn: Guarani' => 30
-'gu: Gujarati' => 31
-'gv: Manx' => 32
-'ha: Hausa' => 33
-'haw: Hawaiian' => 34
-'hi: Hindi' => 35
-'hr: Croatian' => 36
-'ht: Haitian' => 37
-'hu: Hungarian' => 38
-'hy: Armenian' => 39
-'ia: Interlingua' => 40
-'id: Indonesian' => 41
-'is: Icelandic' => 42
-'it: Italian' => 43
-'iw: Hebrew' => 44
-'ja: Japanese' => 45
-'jw: Javanese' => 46
-'ka: Georgian' => 47
-'kk: Kazakh' => 48
-'km: Central Khmer' => 49
-'kn: Kannada' => 50
-'ko: Korean' => 51
-'la: Latin' => 52
-'lb: Luxembourgish' => 53
-'ln: Lingala' => 54
-'lo: Lao' => 55
-'lt: Lithuanian' => 56
-'lv: Latvian' => 57
-'mg: Malagasy' => 58
-'mi: Maori' => 59
-'mk: Macedonian' => 60
-'ml: Malayalam' => 61
-'mn: Mongolian' => 62
-'mr: Marathi' => 63
-'ms: Malay' => 64
-'mt: Maltese' => 65
-'my: Burmese' => 66
-'ne: Nepali' => 67
-'nl: Dutch' => 68
-'nn: Norwegian Nynorsk' => 69
-'no: Norwegian' => 70
-'oc: Occitan' => 71
-'pa: Panjabi' => 72
-'pl: Polish' => 73
-'ps: Pushto' => 74
-'pt: Portuguese' => 75
-'ro: Romanian' => 76
-'ru: Russian' => 77
-'sa: Sanskrit' => 78
-'sco: Scots' => 79
-'sd: Sindhi' => 80
-'si: Sinhala' => 81
-'sk: Slovak' => 82
-'sl: Slovenian' => 83
-'sn: Shona' => 84
-'so: Somali' => 85
-'sq: Albanian' => 86
-'sr: Serbian' => 87
-'su: Sundanese' => 88
-'sv: Swedish' => 89
-'sw: Swahili' => 90
-'ta: Tamil' => 91
-'te: Telugu' => 92
-'tg: Tajik' => 93
-'th: Thai' => 94
-'tk: Turkmen' => 95
-'tl: Tagalog' => 96
-'tr: Turkish' => 97
-'tt: Tatar' => 98
-'uk: Ukrainian' => 99
-'ur: Urdu' => 100
-'uz: Uzbek' => 101
-'vi: Vietnamese' => 102
-'war: Waray' => 103
-'yi: Yiddish' => 104
-'yo: Yoruba' => 105
-'zh: Chinese' => 106
-================
-'starting_index' => 0