# Creates a directory in which to look up available agents import os from typing import List, Optional from src.simuleval_transcoder import SimulevalTranscoder import json import logging logger = logging.getLogger("socketio_server_pubsub") # fmt: off M4T_P0_LANGS = [ "eng", "arb", "ben", "cat", "ces", "cmn", "cym", "dan", "deu", "est", "fin", "fra", "hin", "ind", "ita", "jpn", "kor", "mlt", "nld", "pes", "pol", "por", "ron", "rus", "slk", "spa", "swe", "swh", "tel", "tgl", "tha", "tur", "ukr", "urd", "uzn", "vie", ] # fmt: on class NoAvailableAgentException(Exception): pass class AgentWithInfo: def __init__( self, agent, name: str, modalities: List[str], target_langs: List[str], # Supported dynamic params are defined in StreamingTypes.ts dynamic_params: List[str] = [], description="", has_expressive: Optional[bool] = None, ): self.agent = agent self.has_expressive = has_expressive self.name = name self.description = description self.modalities = modalities self.target_langs = target_langs self.dynamic_params = dynamic_params def get_capabilities_for_json(self): return { "name": self.name, "description": self.description, "modalities": self.modalities, "targetLangs": self.target_langs, "dynamicParams": self.dynamic_params, } @classmethod def load_from_json(cls, config: str): """ Takes in JSON array of models to load in, e.g. [{"name": "s2s_m4t_emma-unity2_multidomain_v0.1", "description": "M4T model that supports simultaneous S2S and S2T", "modalities": ["s2t", "s2s"], "targetLangs": ["en"]}, {"name": "s2s_m4t_expr-emma_v0.1", "description": "ES-EN expressive model that supports S2S and S2T", "modalities": ["s2t", "s2s"], "targetLangs": ["en"]}] """ configs = json.loads(config) agents = [] for config in configs: agent = SimulevalTranscoder.build_agent(config["name"]) agents.append( AgentWithInfo( agent=agent, name=config["name"], modalities=config["modalities"], target_langs=config["targetLangs"], ) ) return agents class SimulevalAgentDirectory: # Available models. These are the directories where the models can be found, and also serve as an ID for the model. seamless_streaming_agent = "SeamlessStreaming" seamless_agent = "Seamless" def __init__(self): self.agents = [] self.did_build_and_add_agents = False def add_agent(self, agent: AgentWithInfo): self.agents.append(agent) def build_agent_if_available(self, model_id, config_name=None): agent = None try: if config_name is not None: agent = SimulevalTranscoder.build_agent( model_id, config_name=config_name, ) else: agent = SimulevalTranscoder.build_agent( model_id, ) except Exception as e: from fairseq2.assets.error import AssetError logger.warning("Failed to build agent %s: %s" % (model_id, e)) if isinstance(e, AssetError): logger.warning( "Please download gated assets and set `gated_model_dir` in the config" ) raise e return agent def build_and_add_agents(self, models_override=None): if self.did_build_and_add_agents: return if models_override is not None: agent_infos = AgentWithInfo.load_from_json(models_override) for agent_info in agent_infos: self.add_agent(agent_info) else: s2s_agent = None if os.environ.get("USE_EXPRESSIVE_MODEL", "0") == "1": logger.info("Building expressive model...") s2s_agent = self.build_agent_if_available( SimulevalAgentDirectory.seamless_agent, config_name="vad_s2st_sc_24khz_main.yaml", ) has_expressive = True else: logger.info("Building non-expressive model...") s2s_agent = self.build_agent_if_available( SimulevalAgentDirectory.seamless_streaming_agent, config_name="vad_s2st_sc_main.yaml", ) has_expressive = False if s2s_agent: self.add_agent( AgentWithInfo( agent=s2s_agent, name=SimulevalAgentDirectory.seamless_streaming_agent, modalities=["s2t", "s2s"], target_langs=M4T_P0_LANGS, dynamic_params=["expressive"], description="multilingual expressive model that supports S2S and S2T", has_expressive=has_expressive, ) ) if len(self.agents) == 0: logger.error( "No agents were loaded. This likely means you are missing the actual model files specified in simuleval_agent_directory." ) self.did_build_and_add_agents = True def get_agent(self, name): for agent in self.agents: if agent.name == name: return agent return None def get_agent_or_throw(self, name): agent = self.get_agent(name) if agent is None: raise NoAvailableAgentException("No agent found with name= %s" % (name)) return agent def get_agents_capabilities_list_for_json(self): return [agent.get_capabilities_for_json() for agent in self.agents]