Sync widgets demo

packages/tasks/README.md

@@ -25,7 +25,7 @@ This package contains the definition files (written in Typescript) for the huggi
 
 - **pipeline types** a.k.a. **task types** (used to determine which widget to display on the model page, and which inference API to run)
 - **default widget inputs** (when they aren't provided in the model card)
-- definitions and UI elements for **third party libraries**.
+- definitions and UI elements for **model libraries** (and soon for **dataset libraries**).
 
 Please add to any of those definitions by opening a PR. Thanks 🔥
 

packages/tasks/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/tasks",
 	"packageManager": "pnpm@8.10.5",
-	"version": "0.2.2",
+	"version": "0.3.2",
 	"description": "List of ML tasks for huggingface.co/tasks",
 	"repository": "https://github.com/huggingface/huggingface.js.git",
 	"publishConfig": {

packages/tasks/src/default-widget-inputs.ts

@@ -1,9 +1,9 @@
 import type { WidgetExample } from "./widget-example";
-import type { PipelineType } from "./pipelines";
+import type { WidgetType } from "./pipelines";
 
 type LanguageCode = string;
 
-type PerLanguageMapping = Map<PipelineType, string[] | WidgetExample[]>;
+type PerLanguageMapping = Map<WidgetType, string[] | WidgetExample[]>;
 
 /// NOTE TO CONTRIBUTORS:
 ///

packages/tasks/src/index.ts

@@ -1,11 +1,11 @@
 export { LIBRARY_TASK_MAPPING_EXCLUDING_TRANSFORMERS } from "./library-to-tasks";
-export { MODEL_LIBRARIES_UI_ELEMENTS } from "./library-ui-elements";
 export { MAPPING_DEFAULT_WIDGET } from "./default-widget-inputs";
 export type { TaskData, TaskDemo, TaskDemoEntry, ExampleRepo } from "./tasks";
 export * from "./tasks";
 export {
 	PIPELINE_DATA,
 	PIPELINE_TYPES,
+	type WidgetType,
 	type PipelineType,
 	type PipelineData,
 	type Modality,
@@ -14,9 +14,10 @@ export {
 	SUBTASK_TYPES,
 	PIPELINE_TYPES_SET,
 } from "./pipelines";
-export { ModelLibrary, ALL_DISPLAY_MODEL_LIBRARY_KEYS } from "./model-libraries";
-export type { ModelLibraryKey } from "./model-libraries";
+export { ALL_DISPLAY_MODEL_LIBRARY_KEYS, ALL_MODEL_LIBRARY_KEYS, MODEL_LIBRARIES_UI_ELEMENTS } from "./model-libraries";
+export type { LibraryUiElement, ModelLibraryKey } from "./model-libraries";
 export type { ModelData, TransformersInfo } from "./model-data";
+export type { SpecialTokensMap, TokenizerConfig } from "./tokenizer-data";
 export type {
 	WidgetExample,
 	WidgetExampleAttribute,
@@ -38,8 +39,7 @@ export type {
 	WidgetExampleOutputText,
 } from "./widget-example";
 export { InferenceDisplayability } from "./model-data";
+export { SPECIAL_TOKENS_ATTRIBUTES } from "./tokenizer-data";
 
 import * as snippets from "./snippets";
 export { snippets };
-
-export type { LibraryUiElement } from "./library-ui-elements";

packages/tasks/src/library-to-tasks.ts

@@ -3,7 +3,7 @@ import type { PipelineType } from "./pipelines";
 
 /**
  * Mapping from library name (excluding Transformers) to its supported tasks.
- * Inference Endpoints (serverless) should be disabled for all other (library, task) pairs beyond this mapping.
+ * Inference API (serverless) should be disabled for all other (library, task) pairs beyond this mapping.
  * As an exception, we assume Transformers supports all inference tasks.
  * This mapping is generated automatically by "python-api-export-tasks" action in huggingface/api-inference-community repo upon merge.
  * Ref: https://github.com/huggingface/api-inference-community/pull/158
@@ -27,7 +27,7 @@ export const LIBRARY_TASK_MAPPING_EXCLUDING_TRANSFORMERS: Partial<Record<ModelLi
 	keras: ["image-classification"],
 	nemo: ["automatic-speech-recognition"],
 	open_clip: ["zero-shot-classification", "zero-shot-image-classification"],
-	paddlenlp: ["conversational", "fill-mask", "summarization", "zero-shot-classification"],
+	paddlenlp: ["fill-mask", "summarization", "zero-shot-classification"],
 	peft: ["text-generation"],
 	"pyannote-audio": ["automatic-speech-recognition"],
 	"sentence-transformers": ["feature-extraction", "sentence-similarity"],

packages/tasks/src/model-data.ts

@@ -1,5 +1,6 @@
 import type { PipelineType } from "./pipelines";
 import type { WidgetExample } from "./widget-example";
+import type { TokenizerConfig } from "./tokenizer-data";
 
 export enum InferenceDisplayability {
 	/**
@@ -53,6 +54,7 @@ export interface ModelData {
 			base_model_name?: string;
 			task_type?: string;
 		};
+		tokenizer?: TokenizerConfig;
 	};
 	/**
 	 * all the model tags

packages/tasks/src/model-libraries-downloads.ts

@@ -0,0 +1,20 @@
+/**
+ * This file contains the (simplified) types used
+ * to represent queries that are made to Elastic
+ * in order to count number of model downloads
+ *
+ * Read this doc about download stats on the Hub:
+ *
+ * https://huggingface.co/docs/hub/models-download-stats
+ *
+ * see also:
+ * https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-bool-query.html
+ */
+
+export type ElasticBoolQueryFilter =
+	// match a single filename
+	| { term?: { path: string } }
+	// match multiple possible filenames
+	| { terms?: { path: string[] } }
+	// match a wildcard
+	| { wildcard?: { path: string } };

packages/tasks/src/{library-ui-elements.ts → model-libraries-snippets.ts}

@@ -1,35 +1,7 @@
 import type { ModelData } from "./model-data";
-import type { ModelLibraryKey } from "./model-libraries";
 
 const TAG_CUSTOM_CODE = "custom_code";
 
-/**
- * Elements configurable by a model library.
- */
-export interface LibraryUiElement {
-	/**
-	 * Name displayed on the main
-	 * call-to-action button on the model page.
-	 */
-	btnLabel: string;
-	/**
-	 * Repo name
-	 */
-	repoName: string;
-	/**
-	 * URL to library's repo
-	 */
-	repoUrl: string;
-	/**
-	 * URL to library's docs
-	 */
-	docsUrl?: string;
-	/**
-	 * Code snippet displayed on model page
-	 */
-	snippets: (model: ModelData) => string[];
-}
-
 function nameWithoutNamespace(modelId: string): string {
 	const splitted = modelId.split("/");
 	return splitted.length === 1 ? splitted[0] : splitted[1];
@@ -37,7 +9,7 @@ function nameWithoutNamespace(modelId: string): string {
 
 //#region snippets
 
-const adapters = (model: ModelData) => [
+export const adapters = (model: ModelData): string[] => [
 	`from adapters import AutoAdapterModel
 
 model = AutoAdapterModel.from_pretrained("${model.config?.adapter_transformers?.model_name}")
@@ -60,14 +32,14 @@ predictor_input = {"passage": "My name is Wolfgang and I live in Berlin", "quest
 predictions = predictor.predict_json(predictor_input)`,
 ];
 
-const allennlp = (model: ModelData) => {
+export const allennlp = (model: ModelData): string[] => {
 	if (model.tags?.includes("question-answering")) {
 		return allennlpQuestionAnswering(model);
 	}
 	return allennlpUnknown(model);
 };
 
-const asteroid = (model: ModelData) => [
+export const asteroid = (model: ModelData): string[] => [
 	`from asteroid.models import BaseModel
 
 model = BaseModel.from_pretrained("${model.id}")`,
@@ -77,7 +49,7 @@ function get_base_diffusers_model(model: ModelData): string {
 	return model.cardData?.base_model?.toString() ?? "fill-in-base-model";
 }
 
-const bertopic = (model: ModelData) => [
+export const bertopic = (model: ModelData): string[] => [
 	`from bertopic import BERTopic
 
 model = BERTopic.load("${model.id}")`,
@@ -112,7 +84,7 @@ pipeline = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}
 pipeline.load_textual_inversion("${model.id}")`,
 ];
 
-const diffusers = (model: ModelData) => {
+export const diffusers = (model: ModelData): string[] => {
 	if (model.tags?.includes("controlnet")) {
 		return diffusers_controlnet(model);
 	} else if (model.tags?.includes("lora")) {
@@ -124,7 +96,7 @@ const diffusers = (model: ModelData) => {
 	}
 };
 
-const espnetTTS = (model: ModelData) => [
+export const espnetTTS = (model: ModelData): string[] => [
 	`from espnet2.bin.tts_inference import Text2Speech
 
 model = Text2Speech.from_pretrained("${model.id}")
@@ -132,7 +104,7 @@ model = Text2Speech.from_pretrained("${model.id}")
 speech, *_ = model("text to generate speech from")`,
 ];
 
-const espnetASR = (model: ModelData) => [
+export const espnetASR = (model: ModelData): string[] => [
 	`from espnet2.bin.asr_inference import Speech2Text
 
 model = Speech2Text.from_pretrained(
@@ -145,7 +117,7 @@ text, *_ = model(speech)[0]`,
 
 const espnetUnknown = () => [`unknown model type (must be text-to-speech or automatic-speech-recognition)`];
 
-const espnet = (model: ModelData) => {
+export const espnet = (model: ModelData): string[] => {
 	if (model.tags?.includes("text-to-speech")) {
 		return espnetTTS(model);
 	} else if (model.tags?.includes("automatic-speech-recognition")) {
@@ -154,7 +126,7 @@ const espnet = (model: ModelData) => {
 	return espnetUnknown();
 };
 
-const fairseq = (model: ModelData) => [
+export const fairseq = (model: ModelData): string[] => [
 	`from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
 
 models, cfg, task = load_model_ensemble_and_task_from_hf_hub(
@@ -162,27 +134,27 @@ models, cfg, task = load_model_ensemble_and_task_from_hf_hub(
 )`,
 ];
 
-const flair = (model: ModelData) => [
+export const flair = (model: ModelData): string[] => [
 	`from flair.models import SequenceTagger
 
 tagger = SequenceTagger.load("${model.id}")`,
 ];
 
-const keras = (model: ModelData) => [
+export const keras = (model: ModelData): string[] => [
 	`from huggingface_hub import from_pretrained_keras
 
 model = from_pretrained_keras("${model.id}")
 `,
 ];
 
-const open_clip = (model: ModelData) => [
+export const open_clip = (model: ModelData): string[] => [
 	`import open_clip
 
 model, preprocess_train, preprocess_val = open_clip.create_model_and_transforms('hf-hub:${model.id}')
 tokenizer = open_clip.get_tokenizer('hf-hub:${model.id}')`,
 ];
 
-const paddlenlp = (model: ModelData) => {
+export const paddlenlp = (model: ModelData): string[] => {
 	if (model.config?.architectures?.[0]) {
 		const architecture = model.config.architectures[0];
 		return [
@@ -206,7 +178,7 @@ const paddlenlp = (model: ModelData) => {
 	}
 };
 
-const pyannote_audio_pipeline = (model: ModelData) => [
+export const pyannote_audio_pipeline = (model: ModelData): string[] => [
 	`from pyannote.audio import Pipeline
   
 pipeline = Pipeline.from_pretrained("${model.id}")
@@ -223,7 +195,7 @@ waveform, sample_rate = Audio().crop("file.wav", excerpt)
 pipeline({"waveform": waveform, "sample_rate": sample_rate})`,
 ];
 
-const pyannote_audio_model = (model: ModelData) => [
+const pyannote_audio_model = (model: ModelData): string[] => [
 	`from pyannote.audio import Model, Inference
 
 model = Model.from_pretrained("${model.id}")
@@ -238,14 +210,14 @@ excerpt = Segment(start=2.0, end=5.0)
 inference.crop("file.wav", excerpt)`,
 ];
 
-const pyannote_audio = (model: ModelData) => {
+export const pyannote_audio = (model: ModelData): string[] => {
 	if (model.tags?.includes("pyannote-audio-pipeline")) {
 		return pyannote_audio_pipeline(model);
 	}
 	return pyannote_audio_model(model);
 };
 
-const tensorflowttsTextToMel = (model: ModelData) => [
+const tensorflowttsTextToMel = (model: ModelData): string[] => [
 	`from tensorflow_tts.inference import AutoProcessor, TFAutoModel
 
 processor = AutoProcessor.from_pretrained("${model.id}")
@@ -253,7 +225,7 @@ model = TFAutoModel.from_pretrained("${model.id}")
 `,
 ];
 
-const tensorflowttsMelToWav = (model: ModelData) => [
+const tensorflowttsMelToWav = (model: ModelData): string[] => [
 	`from tensorflow_tts.inference import TFAutoModel
 
 model = TFAutoModel.from_pretrained("${model.id}")
@@ -261,14 +233,14 @@ audios = model.inference(mels)
 `,
 ];
 
-const tensorflowttsUnknown = (model: ModelData) => [
+const tensorflowttsUnknown = (model: ModelData): string[] => [
 	`from tensorflow_tts.inference import TFAutoModel
 
 model = TFAutoModel.from_pretrained("${model.id}")
 `,
 ];
 
-const tensorflowtts = (model: ModelData) => {
+export const tensorflowtts = (model: ModelData): string[] => {
 	if (model.tags?.includes("text-to-mel")) {
 		return tensorflowttsTextToMel(model);
 	} else if (model.tags?.includes("mel-to-wav")) {
@@ -277,7 +249,7 @@ const tensorflowtts = (model: ModelData) => {
 	return tensorflowttsUnknown(model);
 };
 
-const timm = (model: ModelData) => [
+export const timm = (model: ModelData): string[] => [
 	`import timm
 
 model = timm.create_model("hf_hub:${model.id}", pretrained=True)`,
@@ -319,7 +291,7 @@ model = joblib.load(
 	];
 };
 
-const sklearn = (model: ModelData) => {
+export const sklearn = (model: ModelData): string[] => {
 	if (model.tags?.includes("skops")) {
 		const skopsmodelFile = model.config?.sklearn?.filename;
 		const skopssaveFormat = model.config?.sklearn?.model_format;
@@ -336,29 +308,29 @@ const sklearn = (model: ModelData) => {
 	}
 };
 
-const fastai = (model: ModelData) => [
+export const fastai = (model: ModelData): string[] => [
 	`from huggingface_hub import from_pretrained_fastai
 
 learn = from_pretrained_fastai("${model.id}")`,
 ];
 
-const sampleFactory = (model: ModelData) => [
+export const sampleFactory = (model: ModelData): string[] => [
 	`python -m sample_factory.huggingface.load_from_hub -r ${model.id} -d ./train_dir`,
 ];
 
-const sentenceTransformers = (model: ModelData) => [
+export const sentenceTransformers = (model: ModelData): string[] => [
 	`from sentence_transformers import SentenceTransformer
 
 model = SentenceTransformer("${model.id}")`,
 ];
 
-const setfit = (model: ModelData) => [
+export const setfit = (model: ModelData): string[] => [
 	`from setfit import SetFitModel
 
 model = SetFitModel.from_pretrained("${model.id}")`,
 ];
 
-const spacy = (model: ModelData) => [
+export const spacy = (model: ModelData): string[] => [
 	`!pip install https://huggingface.co/${model.id}/resolve/main/${nameWithoutNamespace(model.id)}-any-py3-none-any.whl
 
 # Using spacy.load().
@@ -370,13 +342,13 @@ import ${nameWithoutNamespace(model.id)}
 nlp = ${nameWithoutNamespace(model.id)}.load()`,
 ];
 
-const span_marker = (model: ModelData) => [
+export const span_marker = (model: ModelData): string[] => [
 	`from span_marker import SpanMarkerModel
 
 model = SpanMarkerModel.from_pretrained("${model.id}")`,
 ];
 
-const stanza = (model: ModelData) => [
+export const stanza = (model: ModelData): string[] => [
 	`import stanza
 
 stanza.download("${nameWithoutNamespace(model.id).replace("stanza-", "")}")
@@ -399,7 +371,7 @@ const speechBrainMethod = (speechbrainInterface: string) => {
 	}
 };
 
-const speechbrain = (model: ModelData) => {
+export const speechbrain = (model: ModelData): string[] => {
 	const speechbrainInterface = model.config?.speechbrain?.interface;
 	if (speechbrainInterface === undefined) {
 		return [`# interface not specified in config.json`];
@@ -419,7 +391,7 @@ model.${speechbrainMethod}("file.wav")`,
 	];
 };
 
-const transformers = (model: ModelData) => {
+export const transformers = (model: ModelData): string[] => {
 	const info = model.transformersInfo;
 	if (!info) {
 		return [`# ⚠️ Type of model unknown`];
@@ -461,7 +433,7 @@ const transformers = (model: ModelData) => {
 	return [autoSnippet];
 };
 
-const transformersJS = (model: ModelData) => {
+export const transformersJS = (model: ModelData): string[] => {
 	if (!model.pipeline_tag) {
 		return [`// ⚠️ Unknown pipeline tag`];
 	}
@@ -492,7 +464,7 @@ const peftTask = (peftTaskType?: string) => {
 	}
 };
 
-const peft = (model: ModelData) => {
+export const peft = (model: ModelData): string[] => {
 	const { base_model_name: peftBaseModel, task_type: peftTaskType } = model.config?.peft ?? {};
 	const pefttask = peftTask(peftTaskType);
 	if (!pefttask) {
@@ -512,14 +484,14 @@ model = PeftModel.from_pretrained(model, "${model.id}")`,
 	];
 };
 
-const fasttext = (model: ModelData) => [
+export const fasttext = (model: ModelData): string[] => [
 	`from huggingface_hub import hf_hub_download
 import fasttext
 
 model = fasttext.load_model(hf_hub_download("${model.id}", "model.bin"))`,
 ];
 
-const stableBaselines3 = (model: ModelData) => [
+export const stableBaselines3 = (model: ModelData): string[] => [
 	`from huggingface_sb3 import load_from_hub
 checkpoint = load_from_hub(
 	repo_id="${model.id}",
@@ -541,9 +513,11 @@ transcriptions = asr_model.transcribe(["file.wav"])`,
 	}
 };
 
-const mlAgents = (model: ModelData) => [`mlagents-load-from-hf --repo-id="${model.id}" --local-dir="./downloads"`];
+export const mlAgents = (model: ModelData): string[] => [
+	`mlagents-load-from-hf --repo-id="${model.id}" --local-dir="./download: string[]s"`,
+];
 
-const sentis = (/* model: ModelData */) => [
+export const sentis = (/* model: ModelData */): string[] => [
 	`string modelName = "[Your model name here].sentis";
 Model model = ModelLoader.Load(Application.streamingAssetsPath + "/" + modelName);
 IWorker engine = WorkerFactory.CreateWorker(BackendType.GPUCompute, model);
@@ -551,14 +525,14 @@ IWorker engine = WorkerFactory.CreateWorker(BackendType.GPUCompute, model);
 `,
 ];
 
-const mlx = (model: ModelData) => [
+export const mlx = (model: ModelData): string[] => [
 	`pip install huggingface_hub hf_transfer
 
-export HF_HUB_ENABLE_HF_TRANSFER=1
+export HF_HUB_ENABLE_HF_TRANS: string[]FER=1
 huggingface-cli download --local-dir ${nameWithoutNamespace(model.id)} ${model.id}`,
 ];
 
-const nemo = (model: ModelData) => {
+export const nemo = (model: ModelData): string[] => {
 	let command: string[] | undefined = undefined;
 	// Resolve the tag to a nemo domain/sub-domain
 	if (model.tags?.includes("automatic-speech-recognition")) {
@@ -568,232 +542,10 @@ const nemo = (model: ModelData) => {
 	return command ?? [`# tag did not correspond to a valid NeMo domain.`];
 };
 
-const pythae = (model: ModelData) => [
+export const pythae = (model: ModelData): string[] => [
 	`from pythae.models import AutoModel
 
 model = AutoModel.load_from_hf_hub("${model.id}")`,
 ];
 
 //#endregion
-
-export const MODEL_LIBRARIES_UI_ELEMENTS: Partial<Record<ModelLibraryKey, LibraryUiElement>> = {
-	"adapter-transformers": {
-		btnLabel: "Adapters",
-		repoName: "adapters",
-		repoUrl: "https://github.com/Adapter-Hub/adapters",
-		docsUrl: "https://huggingface.co/docs/hub/adapters",
-		snippets: adapters,
-	},
-	allennlp: {
-		btnLabel: "AllenNLP",
-		repoName: "AllenNLP",
-		repoUrl: "https://github.com/allenai/allennlp",
-		docsUrl: "https://huggingface.co/docs/hub/allennlp",
-		snippets: allennlp,
-	},
-	asteroid: {
-		btnLabel: "Asteroid",
-		repoName: "Asteroid",
-		repoUrl: "https://github.com/asteroid-team/asteroid",
-		docsUrl: "https://huggingface.co/docs/hub/asteroid",
-		snippets: asteroid,
-	},
-	bertopic: {
-		btnLabel: "BERTopic",
-		repoName: "BERTopic",
-		repoUrl: "https://github.com/MaartenGr/BERTopic",
-		snippets: bertopic,
-	},
-	diffusers: {
-		btnLabel: "Diffusers",
-		repoName: "🤗/diffusers",
-		repoUrl: "https://github.com/huggingface/diffusers",
-		docsUrl: "https://huggingface.co/docs/hub/diffusers",
-		snippets: diffusers,
-	},
-	espnet: {
-		btnLabel: "ESPnet",
-		repoName: "ESPnet",
-		repoUrl: "https://github.com/espnet/espnet",
-		docsUrl: "https://huggingface.co/docs/hub/espnet",
-		snippets: espnet,
-	},
-	fairseq: {
-		btnLabel: "Fairseq",
-		repoName: "fairseq",
-		repoUrl: "https://github.com/pytorch/fairseq",
-		snippets: fairseq,
-	},
-	flair: {
-		btnLabel: "Flair",
-		repoName: "Flair",
-		repoUrl: "https://github.com/flairNLP/flair",
-		docsUrl: "https://huggingface.co/docs/hub/flair",
-		snippets: flair,
-	},
-	keras: {
-		btnLabel: "Keras",
-		repoName: "Keras",
-		repoUrl: "https://github.com/keras-team/keras",
-		docsUrl: "https://huggingface.co/docs/hub/keras",
-		snippets: keras,
-	},
-	mlx: {
-		btnLabel: "MLX",
-		repoName: "MLX",
-		repoUrl: "https://github.com/ml-explore/mlx-examples/tree/main",
-		snippets: mlx,
-	},
-	nemo: {
-		btnLabel: "NeMo",
-		repoName: "NeMo",
-		repoUrl: "https://github.com/NVIDIA/NeMo",
-		snippets: nemo,
-	},
-	open_clip: {
-		btnLabel: "OpenCLIP",
-		repoName: "OpenCLIP",
-		repoUrl: "https://github.com/mlfoundations/open_clip",
-		snippets: open_clip,
-	},
-	paddlenlp: {
-		btnLabel: "paddlenlp",
-		repoName: "PaddleNLP",
-		repoUrl: "https://github.com/PaddlePaddle/PaddleNLP",
-		docsUrl: "https://huggingface.co/docs/hub/paddlenlp",
-		snippets: paddlenlp,
-	},
-	peft: {
-		btnLabel: "PEFT",
-		repoName: "PEFT",
-		repoUrl: "https://github.com/huggingface/peft",
-		snippets: peft,
-	},
-	"pyannote-audio": {
-		btnLabel: "pyannote.audio",
-		repoName: "pyannote-audio",
-		repoUrl: "https://github.com/pyannote/pyannote-audio",
-		snippets: pyannote_audio,
-	},
-	"sentence-transformers": {
-		btnLabel: "sentence-transformers",
-		repoName: "sentence-transformers",
-		repoUrl: "https://github.com/UKPLab/sentence-transformers",
-		docsUrl: "https://huggingface.co/docs/hub/sentence-transformers",
-		snippets: sentenceTransformers,
-	},
-	setfit: {
-		btnLabel: "setfit",
-		repoName: "setfit",
-		repoUrl: "https://github.com/huggingface/setfit",
-		docsUrl: "https://huggingface.co/docs/hub/setfit",
-		snippets: setfit,
-	},
-	sklearn: {
-		btnLabel: "Scikit-learn",
-		repoName: "Scikit-learn",
-		repoUrl: "https://github.com/scikit-learn/scikit-learn",
-		snippets: sklearn,
-	},
-	fastai: {
-		btnLabel: "fastai",
-		repoName: "fastai",
-		repoUrl: "https://github.com/fastai/fastai",
-		docsUrl: "https://huggingface.co/docs/hub/fastai",
-		snippets: fastai,
-	},
-	spacy: {
-		btnLabel: "spaCy",
-		repoName: "spaCy",
-		repoUrl: "https://github.com/explosion/spaCy",
-		docsUrl: "https://huggingface.co/docs/hub/spacy",
-		snippets: spacy,
-	},
-	"span-marker": {
-		btnLabel: "SpanMarker",
-		repoName: "SpanMarkerNER",
-		repoUrl: "https://github.com/tomaarsen/SpanMarkerNER",
-		docsUrl: "https://huggingface.co/docs/hub/span_marker",
-		snippets: span_marker,
-	},
-	speechbrain: {
-		btnLabel: "speechbrain",
-		repoName: "speechbrain",
-		repoUrl: "https://github.com/speechbrain/speechbrain",
-		docsUrl: "https://huggingface.co/docs/hub/speechbrain",
-		snippets: speechbrain,
-	},
-	stanza: {
-		btnLabel: "Stanza",
-		repoName: "stanza",
-		repoUrl: "https://github.com/stanfordnlp/stanza",
-		docsUrl: "https://huggingface.co/docs/hub/stanza",
-		snippets: stanza,
-	},
-	tensorflowtts: {
-		btnLabel: "TensorFlowTTS",
-		repoName: "TensorFlowTTS",
-		repoUrl: "https://github.com/TensorSpeech/TensorFlowTTS",
-		snippets: tensorflowtts,
-	},
-	timm: {
-		btnLabel: "timm",
-		repoName: "pytorch-image-models",
-		repoUrl: "https://github.com/rwightman/pytorch-image-models",
-		docsUrl: "https://huggingface.co/docs/hub/timm",
-		snippets: timm,
-	},
-	transformers: {
-		btnLabel: "Transformers",
-		repoName: "🤗/transformers",
-		repoUrl: "https://github.com/huggingface/transformers",
-		docsUrl: "https://huggingface.co/docs/hub/transformers",
-		snippets: transformers,
-	},
-	"transformers.js": {
-		btnLabel: "Transformers.js",
-		repoName: "transformers.js",
-		repoUrl: "https://github.com/xenova/transformers.js",
-		docsUrl: "https://huggingface.co/docs/hub/transformers-js",
-		snippets: transformersJS,
-	},
-	fasttext: {
-		btnLabel: "fastText",
-		repoName: "fastText",
-		repoUrl: "https://fasttext.cc/",
-		snippets: fasttext,
-	},
-	"sample-factory": {
-		btnLabel: "sample-factory",
-		repoName: "sample-factory",
-		repoUrl: "https://github.com/alex-petrenko/sample-factory",
-		docsUrl: "https://huggingface.co/docs/hub/sample-factory",
-		snippets: sampleFactory,
-	},
-	"stable-baselines3": {
-		btnLabel: "stable-baselines3",
-		repoName: "stable-baselines3",
-		repoUrl: "https://github.com/huggingface/huggingface_sb3",
-		docsUrl: "https://huggingface.co/docs/hub/stable-baselines3",
-		snippets: stableBaselines3,
-	},
-	"ml-agents": {
-		btnLabel: "ml-agents",
-		repoName: "ml-agents",
-		repoUrl: "https://github.com/Unity-Technologies/ml-agents",
-		docsUrl: "https://huggingface.co/docs/hub/ml-agents",
-		snippets: mlAgents,
-	},
-	"unity-sentis": {
-		btnLabel: "unity-sentis",
-		repoName: "unity-sentis",
-		repoUrl: "https://github.com/Unity-Technologies/sentis-samples",
-		snippets: sentis,
-	},
-	pythae: {
-		btnLabel: "pythae",
-		repoName: "pythae",
-		repoUrl: "https://github.com/clementchadebec/benchmark_VAE",
-		snippets: pythae,
-	},
-} as const;

packages/tasks/src/model-libraries.ts

@@ -1,52 +1,383 @@
+import * as snippets from "./model-libraries-snippets";
+import type { ModelData } from "./model-data";
+import type { ElasticBoolQueryFilter } from "./model-libraries-downloads";
+
+/**
+ * Elements configurable by a model library.
+ */
+export interface LibraryUiElement {
+	/**
+	 * Pretty name of the library.
+	 * displayed in tags, and on the main
+	 * call-to-action button on the model page.
+	 */
+	prettyLabel: string;
+	/**
+	 * Repo name of the library's (usually on GitHub) code repo
+	 */
+	repoName: string;
+	/**
+	 * URL to library's (usually on GitHub) code repo
+	 */
+	repoUrl: string;
+	/**
+	 * URL to library's docs
+	 */
+	docsUrl?: string;
+	/**
+	 * Code snippet(s) displayed on model page
+	 */
+	snippets?: (model: ModelData) => string[];
+	/**
+	 * Elastic query used to count this library's model downloads
+	 *
+	 * By default, those files are counted:
+	 * "config.json", "config.yaml", "hyperparams.yaml", "meta.yaml"
+	 */
+	countDownloads?: ElasticBoolQueryFilter;
+	/**
+	 * should we display this library in hf.co/models filter
+	 * (only for popular libraries with > 100 models)
+	 */
+	filter?: boolean;
+}
+
 /**
  * Add your new library here.
  *
  * This is for modeling (= architectures) libraries, not for file formats (like ONNX, etc).
- * File formats live in an enum inside the internal codebase.
+ * (unlike libraries, file formats live in an enum inside the internal codebase.)
+ *
+ * Doc on how to add a library to the Hub:
+ *
+ * https://huggingface.co/docs/hub/models-adding-libraries
+ *
+ * /!\ IMPORTANT
+ *
+ * The key you choose is the tag your models have in their library_name on the Hub.
  */
-export enum ModelLibrary {
-	"adapter-transformers" = "Adapters",
-	"allennlp" = "allenNLP",
-	"asteroid" = "Asteroid",
-	"bertopic" = "BERTopic",
-	"diffusers" = "Diffusers",
-	"doctr" = "docTR",
-	"espnet" = "ESPnet",
-	"fairseq" = "Fairseq",
-	"flair" = "Flair",
-	"keras" = "Keras",
-	"k2" = "K2",
-	"mlx" = "MLX",
-	"nemo" = "NeMo",
-	"open_clip" = "OpenCLIP",
-	"paddlenlp" = "PaddleNLP",
-	"peft" = "PEFT",
-	"pyannote-audio" = "pyannote.audio",
-	"sample-factory" = "Sample Factory",
-	"sentence-transformers" = "Sentence Transformers",
-	"setfit" = "SetFit",
-	"sklearn" = "Scikit-learn",
-	"spacy" = "spaCy",
-	"span-marker" = "SpanMarker",
-	"speechbrain" = "speechbrain",
-	"tensorflowtts" = "TensorFlowTTS",
-	"timm" = "Timm",
-	"fastai" = "fastai",
-	"transformers" = "Transformers",
-	"transformers.js" = "Transformers.js",
-	"stanza" = "Stanza",
-	"fasttext" = "fastText",
-	"stable-baselines3" = "Stable-Baselines3",
-	"ml-agents" = "Unity ML-Agents",
-	"pythae" = "Pythae",
-	"mindspore" = "MindSpore",
-	"unity-sentis" = "Unity Sentis",
-}
 
-export type ModelLibraryKey = keyof typeof ModelLibrary;
+export const MODEL_LIBRARIES_UI_ELEMENTS = {
+	"adapter-transformers": {
+		prettyLabel: "Adapters",
+		repoName: "adapters",
+		repoUrl: "https://github.com/Adapter-Hub/adapters",
+		docsUrl: "https://huggingface.co/docs/hub/adapters",
+		snippets: snippets.adapters,
+		filter: true,
+		countDownloads: {
+			term: { path: "adapter_config.json" },
+		},
+	},
+	allennlp: {
+		prettyLabel: "AllenNLP",
+		repoName: "AllenNLP",
+		repoUrl: "https://github.com/allenai/allennlp",
+		docsUrl: "https://huggingface.co/docs/hub/allennlp",
+		snippets: snippets.allennlp,
+		filter: true,
+	},
+	asteroid: {
+		prettyLabel: "Asteroid",
+		repoName: "Asteroid",
+		repoUrl: "https://github.com/asteroid-team/asteroid",
+		docsUrl: "https://huggingface.co/docs/hub/asteroid",
+		snippets: snippets.asteroid,
+		filter: true,
+		countDownloads: {
+			term: { path: "pytorch_model.bin" },
+		},
+	},
+	bertopic: {
+		prettyLabel: "BERTopic",
+		repoName: "BERTopic",
+		repoUrl: "https://github.com/MaartenGr/BERTopic",
+		snippets: snippets.bertopic,
+		filter: true,
+	},
+	diffusers: {
+		prettyLabel: "Diffusers",
+		repoName: "🤗/diffusers",
+		repoUrl: "https://github.com/huggingface/diffusers",
+		docsUrl: "https://huggingface.co/docs/hub/diffusers",
+		snippets: snippets.diffusers,
+		filter: true,
+		/// diffusers has its own more complex "countDownloads" query
+	},
+	doctr: {
+		prettyLabel: "docTR",
+		repoName: "doctr",
+		repoUrl: "https://github.com/mindee/doctr",
+	},
+	espnet: {
+		prettyLabel: "ESPnet",
+		repoName: "ESPnet",
+		repoUrl: "https://github.com/espnet/espnet",
+		docsUrl: "https://huggingface.co/docs/hub/espnet",
+		snippets: snippets.espnet,
+		filter: true,
+	},
+	fairseq: {
+		prettyLabel: "Fairseq",
+		repoName: "fairseq",
+		repoUrl: "https://github.com/pytorch/fairseq",
+		snippets: snippets.fairseq,
+		filter: true,
+	},
+	fastai: {
+		prettyLabel: "fastai",
+		repoName: "fastai",
+		repoUrl: "https://github.com/fastai/fastai",
+		docsUrl: "https://huggingface.co/docs/hub/fastai",
+		snippets: snippets.fastai,
+		filter: true,
+	},
+	fasttext: {
+		prettyLabel: "fastText",
+		repoName: "fastText",
+		repoUrl: "https://fasttext.cc/",
+		snippets: snippets.fasttext,
+		filter: true,
+	},
+	flair: {
+		prettyLabel: "Flair",
+		repoName: "Flair",
+		repoUrl: "https://github.com/flairNLP/flair",
+		docsUrl: "https://huggingface.co/docs/hub/flair",
+		snippets: snippets.flair,
+		filter: true,
+		countDownloads: {
+			term: { path: "pytorch_model.bin" },
+		},
+	},
+	keras: {
+		prettyLabel: "Keras",
+		repoName: "Keras",
+		repoUrl: "https://github.com/keras-team/keras",
+		docsUrl: "https://huggingface.co/docs/hub/keras",
+		snippets: snippets.keras,
+		filter: true,
+		countDownloads: { term: { path: "saved_model.pb" } },
+	},
+	k2: {
+		prettyLabel: "K2",
+		repoName: "k2",
+		repoUrl: "https://github.com/k2-fsa/k2",
+	},
+	mindspore: {
+		prettyLabel: "MindSpore",
+		repoName: "mindspore",
+		repoUrl: "https://github.com/mindspore-ai/mindspore",
+	},
+	"ml-agents": {
+		prettyLabel: "ml-agents",
+		repoName: "ml-agents",
+		repoUrl: "https://github.com/Unity-Technologies/ml-agents",
+		docsUrl: "https://huggingface.co/docs/hub/ml-agents",
+		snippets: snippets.mlAgents,
+		filter: true,
+		countDownloads: { wildcard: { path: "*.onnx" } },
+	},
+	mlx: {
+		prettyLabel: "MLX",
+		repoName: "MLX",
+		repoUrl: "https://github.com/ml-explore/mlx-examples/tree/main",
+		snippets: snippets.mlx,
+		filter: true,
+	},
+	nemo: {
+		prettyLabel: "NeMo",
+		repoName: "NeMo",
+		repoUrl: "https://github.com/NVIDIA/NeMo",
+		snippets: snippets.nemo,
+		filter: true,
+		countDownloads: { wildcard: { path: "*.nemo" } },
+	},
+	open_clip: {
+		prettyLabel: "OpenCLIP",
+		repoName: "OpenCLIP",
+		repoUrl: "https://github.com/mlfoundations/open_clip",
+		snippets: snippets.open_clip,
+		filter: true,
+		countDownloads: { wildcard: { path: "*pytorch_model.bin" } },
+	},
+	paddlenlp: {
+		prettyLabel: "paddlenlp",
+		repoName: "PaddleNLP",
+		repoUrl: "https://github.com/PaddlePaddle/PaddleNLP",
+		docsUrl: "https://huggingface.co/docs/hub/paddlenlp",
+		snippets: snippets.paddlenlp,
+		filter: true,
+		countDownloads: {
+			term: { path: "model_config.json" },
+		},
+	},
+	peft: {
+		prettyLabel: "PEFT",
+		repoName: "PEFT",
+		repoUrl: "https://github.com/huggingface/peft",
+		snippets: snippets.peft,
+		filter: true,
+		countDownloads: {
+			term: { path: "adapter_config.json" },
+		},
+	},
+	"pyannote-audio": {
+		prettyLabel: "pyannote.audio",
+		repoName: "pyannote-audio",
+		repoUrl: "https://github.com/pyannote/pyannote-audio",
+		snippets: snippets.pyannote_audio,
+		filter: true,
+	},
+	pythae: {
+		prettyLabel: "pythae",
+		repoName: "pythae",
+		repoUrl: "https://github.com/clementchadebec/benchmark_VAE",
+		snippets: snippets.pythae,
+		filter: true,
+	},
+	"sample-factory": {
+		prettyLabel: "sample-factory",
+		repoName: "sample-factory",
+		repoUrl: "https://github.com/alex-petrenko/sample-factory",
+		docsUrl: "https://huggingface.co/docs/hub/sample-factory",
+		snippets: snippets.sampleFactory,
+		filter: true,
+		countDownloads: { term: { path: "cfg.json" } },
+	},
+	"sentence-transformers": {
+		prettyLabel: "sentence-transformers",
+		repoName: "sentence-transformers",
+		repoUrl: "https://github.com/UKPLab/sentence-transformers",
+		docsUrl: "https://huggingface.co/docs/hub/sentence-transformers",
+		snippets: snippets.sentenceTransformers,
+		filter: true,
+	},
+	setfit: {
+		prettyLabel: "setfit",
+		repoName: "setfit",
+		repoUrl: "https://github.com/huggingface/setfit",
+		docsUrl: "https://huggingface.co/docs/hub/setfit",
+		snippets: snippets.setfit,
+		filter: true,
+	},
+	sklearn: {
+		prettyLabel: "Scikit-learn",
+		repoName: "Scikit-learn",
+		repoUrl: "https://github.com/scikit-learn/scikit-learn",
+		snippets: snippets.sklearn,
+		filter: true,
+		countDownloads: {
+			term: { path: "sklearn_model.joblib" },
+		},
+	},
+	spacy: {
+		prettyLabel: "spaCy",
+		repoName: "spaCy",
+		repoUrl: "https://github.com/explosion/spaCy",
+		docsUrl: "https://huggingface.co/docs/hub/spacy",
+		snippets: snippets.spacy,
+		filter: true,
+		countDownloads: {
+			wildcard: { path: "*.whl" },
+		},
+	},
+	"span-marker": {
+		prettyLabel: "SpanMarker",
+		repoName: "SpanMarkerNER",
+		repoUrl: "https://github.com/tomaarsen/SpanMarkerNER",
+		docsUrl: "https://huggingface.co/docs/hub/span_marker",
+		snippets: snippets.span_marker,
+		filter: true,
+	},
+	speechbrain: {
+		prettyLabel: "speechbrain",
+		repoName: "speechbrain",
+		repoUrl: "https://github.com/speechbrain/speechbrain",
+		docsUrl: "https://huggingface.co/docs/hub/speechbrain",
+		snippets: snippets.speechbrain,
+		filter: true,
+		countDownloads: {
+			term: { path: "hyperparams.yaml" },
+		},
+	},
+	"stable-baselines3": {
+		prettyLabel: "stable-baselines3",
+		repoName: "stable-baselines3",
+		repoUrl: "https://github.com/huggingface/huggingface_sb3",
+		docsUrl: "https://huggingface.co/docs/hub/stable-baselines3",
+		snippets: snippets.stableBaselines3,
+		filter: true,
+		countDownloads: {
+			wildcard: { path: "*.zip" },
+		},
+	},
+	stanza: {
+		prettyLabel: "Stanza",
+		repoName: "stanza",
+		repoUrl: "https://github.com/stanfordnlp/stanza",
+		docsUrl: "https://huggingface.co/docs/hub/stanza",
+		snippets: snippets.stanza,
+		filter: true,
+		countDownloads: {
+			term: { path: "models/default.zip" },
+		},
+	},
+	tensorflowtts: {
+		prettyLabel: "TensorFlowTTS",
+		repoName: "TensorFlowTTS",
+		repoUrl: "https://github.com/TensorSpeech/TensorFlowTTS",
+		snippets: snippets.tensorflowtts,
+	},
+	timm: {
+		prettyLabel: "timm",
+		repoName: "pytorch-image-models",
+		repoUrl: "https://github.com/rwightman/pytorch-image-models",
+		docsUrl: "https://huggingface.co/docs/hub/timm",
+		snippets: snippets.timm,
+		filter: true,
+		countDownloads: {
+			terms: { path: ["pytorch_model.bin", "model.safetensors"] },
+		},
+	},
+	transformers: {
+		prettyLabel: "Transformers",
+		repoName: "🤗/transformers",
+		repoUrl: "https://github.com/huggingface/transformers",
+		docsUrl: "https://huggingface.co/docs/hub/transformers",
+		snippets: snippets.transformers,
+		filter: true,
+	},
+	"transformers.js": {
+		prettyLabel: "Transformers.js",
+		repoName: "transformers.js",
+		repoUrl: "https://github.com/xenova/transformers.js",
+		docsUrl: "https://huggingface.co/docs/hub/transformers-js",
+		snippets: snippets.transformersJS,
+		filter: true,
+	},
+	"unity-sentis": {
+		prettyLabel: "unity-sentis",
+		repoName: "unity-sentis",
+		repoUrl: "https://github.com/Unity-Technologies/sentis-samples",
+		snippets: snippets.sentis,
+		filter: true,
+		countDownloads: {
+			wildcard: { path: "*.sentis" },
+		},
+	},
+} satisfies Record<string, LibraryUiElement>;
+
+export type ModelLibraryKey = keyof typeof MODEL_LIBRARIES_UI_ELEMENTS;
 
-export const ALL_MODEL_LIBRARY_KEYS = Object.keys(ModelLibrary) as ModelLibraryKey[];
+export const ALL_MODEL_LIBRARY_KEYS = Object.keys(MODEL_LIBRARIES_UI_ELEMENTS) as ModelLibraryKey[];
 
-export const ALL_DISPLAY_MODEL_LIBRARY_KEYS = ALL_MODEL_LIBRARY_KEYS.filter(
-	(k) => !["doctr", "k2", "mindspore", "tensorflowtts"].includes(k)
-);
+export const ALL_DISPLAY_MODEL_LIBRARY_KEYS = (
+	Object.entries(MODEL_LIBRARIES_UI_ELEMENTS as Record<ModelLibraryKey, LibraryUiElement>) as [
+		ModelLibraryKey,
+		LibraryUiElement,
+	][]
+)
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	.filter(([_, v]) => v.filter)
+	.map(([k]) => k);

packages/tasks/src/pipelines.ts

@@ -225,20 +225,9 @@ export const PIPELINE_DATA = {
 		modality: "nlp",
 		color: "indigo",
 	},
-	conversational: {
-		name: "Conversational",
-		subtasks: [
-			{
-				type: "dialogue-generation",
-				name: "Dialogue Generation",
-			},
-		],
-		modality: "nlp",
-		color: "green",
-	},
 	"feature-extraction": {
 		name: "Feature Extraction",
-		modality: "multimodal",
+		modality: "nlp",
 		color: "red",
 	},
 	"text-generation": {
@@ -248,6 +237,14 @@ export const PIPELINE_DATA = {
 				type: "dialogue-modeling",
 				name: "Dialogue Modeling",
 			},
+			{
+				type: "dialogue-generation",
+				name: "Dialogue Generation",
+			},
+			{
+				type: "conversational",
+				name: "Conversational",
+			},
 			{
 				type: "language-modeling",
 				name: "Language Modeling",
@@ -419,7 +416,7 @@ export const PIPELINE_DATA = {
 	},
 	"text-to-image": {
 		name: "Text-to-Image",
-		modality: "multimodal",
+		modality: "cv",
 		color: "yellow",
 	},
 	"image-to-text": {
@@ -430,7 +427,7 @@ export const PIPELINE_DATA = {
 				name: "Image Captioning",
 			},
 		],
-		modality: "multimodal",
+		modality: "cv",
 		color: "red",
 	},
 	"image-to-image": {
@@ -454,7 +451,7 @@ export const PIPELINE_DATA = {
 	},
 	"image-to-video": {
 		name: "Image-to-Video",
-		modality: "multimodal",
+		modality: "cv",
 		color: "indigo",
 	},
 	"unconditional-image-generation": {
@@ -589,9 +586,15 @@ export const PIPELINE_DATA = {
 	},
 	"text-to-video": {
 		name: "Text-to-Video",
-		modality: "multimodal",
+		modality: "cv",
 		color: "green",
 	},
+	"image-text-to-text": {
+		name: "Image + Text to Text (VLLMs)",
+		modality: "multimodal",
+		color: "red",
+		hideInDatasets: true,
+	},
 	"visual-question-answering": {
 		name: "Visual Question Answering",
 		subtasks: [
@@ -622,7 +625,7 @@ export const PIPELINE_DATA = {
 	},
 	"graph-ml": {
 		name: "Graph Machine Learning",
-		modality: "multimodal",
+		modality: "other",
 		color: "green",
 	},
 	"mask-generation": {
@@ -637,14 +640,19 @@ export const PIPELINE_DATA = {
 	},
 	"text-to-3d": {
 		name: "Text-to-3D",
-		modality: "multimodal",
+		modality: "cv",
 		color: "yellow",
 	},
 	"image-to-3d": {
 		name: "Image-to-3D",
-		modality: "multimodal",
+		modality: "cv",
 		color: "green",
 	},
+	"image-feature-extraction": {
+		name: "Image Feature Extraction",
+		modality: "cv",
+		color: "indigo",
+	},
 	other: {
 		name: "Other",
 		modality: "other",
@@ -656,6 +664,8 @@ export const PIPELINE_DATA = {
 
 export type PipelineType = keyof typeof PIPELINE_DATA;
 
+export type WidgetType = PipelineType | "conversational";
+
 export const PIPELINE_TYPES = Object.keys(PIPELINE_DATA) as PipelineType[];
 
 export const SUBTASK_TYPES = Object.values(PIPELINE_DATA)

packages/tasks/src/snippets/curl.ts

@@ -34,7 +34,6 @@ export const curlSnippets: Partial<Record<PipelineType, (model: ModelData, acces
 	"zero-shot-classification": snippetZeroShotClassification,
 	translation: snippetBasic,
 	summarization: snippetBasic,
-	conversational: snippetBasic,
 	"feature-extraction": snippetBasic,
 	"text-generation": snippetBasic,
 	"text2text-generation": snippetBasic,

packages/tasks/src/snippets/inputs.ts

@@ -9,13 +9,6 @@ const inputsTranslation = () => `"Меня зовут Вольфганг и я
 const inputsSummarization = () =>
 	`"The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct."`;
 
-const inputsConversational = () =>
-	`{
-		"past_user_inputs": ["Which movie is the best ?"],
-		"generated_responses": ["It is Die Hard for sure."],
-		"text": "Can you explain why ?"
-	}`;
-
 const inputsTableQuestionAnswering = () =>
 	`{
 		"query": "How many stars does the transformers repository have?",
@@ -96,7 +89,6 @@ const modelInputSnippets: {
 	"audio-to-audio": inputsAudioToAudio,
 	"audio-classification": inputsAudioClassification,
 	"automatic-speech-recognition": inputsAutomaticSpeechRecognition,
-	conversational: inputsConversational,
 	"document-question-answering": inputsVisualQuestionAnswering,
 	"feature-extraction": inputsFeatureExtraction,
 	"fill-mask": inputsFillMask,

packages/tasks/src/snippets/js.ts

@@ -121,7 +121,6 @@ export const jsSnippets: Partial<Record<PipelineType, (model: ModelData, accessT
 	"zero-shot-classification": snippetZeroShotClassification,
 	translation: snippetBasic,
 	summarization: snippetBasic,
-	conversational: snippetBasic,
 	"feature-extraction": snippetBasic,
 	"text-generation": snippetBasic,
 	"text2text-generation": snippetBasic,

packages/tasks/src/snippets/python.ts

@@ -116,7 +116,6 @@ export const pythonSnippets: Partial<Record<PipelineType, (model: ModelData) =>
 	"zero-shot-classification": snippetZeroShotClassification,
 	translation: snippetBasic,
 	summarization: snippetBasic,
-	conversational: snippetBasic,
 	"feature-extraction": snippetBasic,
 	"text-generation": snippetBasic,
 	"text2text-generation": snippetBasic,

packages/tasks/src/tasks/conversational/about.md

@@ -1,50 +0,0 @@
-## Use Cases
-
-### Chatbot 💬
-
-Chatbots are used to have conversations instead of providing direct contact with a live human. They are used to provide customer service, sales, and can even be used to play games (see [ELIZA](https://en.wikipedia.org/wiki/ELIZA) from 1966 for one of the earliest examples).
-
-## Voice Assistants 🎙️
-
-Conversational response models are used as part of voice assistants to provide appropriate responses to voice based queries.
-
-## Inference
-
-You can infer with Conversational models with the 🤗 Transformers library using the `conversational` pipeline. This pipeline takes a conversation prompt or a list of conversations and generates responses for each prompt. The models that this pipeline can use are models that have been fine-tuned on a multi-turn conversational task (see https://huggingface.co/models?filter=conversational for a list of updated Conversational models).
-
-```python
-from transformers import pipeline, Conversation
-converse = pipeline("conversational")
-
-conversation_1 = Conversation("Going to the movies tonight - any suggestions?")
-conversation_2 = Conversation("What's the last book you have read?")
-converse([conversation_1, conversation_2])
-
-## Output:
-## Conversation 1
-## user >> Going to the movies tonight - any suggestions?
-## bot >> The Big Lebowski ,
-## Conversation 2
-## user >> What's the last book you have read?
-## bot >> The Last Question
-```
-
-You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer with conversational models on Hugging Face Hub.
-
-```javascript
-import { HfInference } from "@huggingface/inference";
-
-const inference = new HfInference(HF_TOKEN);
-await inference.conversational({
-	model: "facebook/blenderbot-400M-distill",
-	inputs: "Going to the movies tonight - any suggestions?",
-});
-```
-
-## Useful Resources
-
-- Learn how ChatGPT and InstructGPT work in this blog: [Illustrating Reinforcement Learning from Human Feedback (RLHF)](https://huggingface.co/blog/rlhf)
-- [Reinforcement Learning from Human Feedback From Zero to ChatGPT](https://www.youtube.com/watch?v=EAd4oQtEJOM)
-- [A guide on Dialog Agents](https://huggingface.co/blog/dialog-agents)
-
-This page was made possible thanks to the efforts of [Viraat Aryabumi](https://huggingface.co/viraat).

packages/tasks/src/tasks/conversational/data.ts

@@ -1,66 +0,0 @@
-import type { TaskDataCustom } from "..";
-
-const taskData: TaskDataCustom = {
-	datasets: [
-		{
-			description:
-				"A dataset of 7k conversations explicitly designed to exhibit multiple conversation modes: displaying personality, having empathy, and demonstrating knowledge.",
-			id: "blended_skill_talk",
-		},
-		{
-			description:
-				"ConvAI is a dataset of human-to-bot conversations labeled for quality. This data can be used to train a metric for evaluating dialogue systems",
-			id: "conv_ai_2",
-		},
-		{
-			description: "EmpatheticDialogues, is a dataset of 25k conversations grounded in emotional situations",
-			id: "empathetic_dialogues",
-		},
-	],
-	demo: {
-		inputs: [
-			{
-				label: "Input",
-				content: "Hey my name is Julien! How are you?",
-				type: "text",
-			},
-		],
-		outputs: [
-			{
-				label: "Answer",
-				content: "Hi Julien! My name is Julia! I am well.",
-				type: "text",
-			},
-		],
-	},
-	metrics: [
-		{
-			description:
-				"BLEU score is calculated by counting the number of shared single or subsequent tokens between the generated sequence and the reference. Subsequent n tokens are called “n-grams”. Unigram refers to a single token while bi-gram refers to token pairs and n-grams refer to n subsequent tokens. The score ranges from 0 to 1, where 1 means the translation perfectly matched and 0 did not match at all",
-			id: "bleu",
-		},
-	],
-	models: [
-		{
-			description: "A faster and smaller model than the famous BERT model.",
-			id: "facebook/blenderbot-400M-distill",
-		},
-		{
-			description:
-				"DialoGPT is a large-scale pretrained dialogue response generation model for multiturn conversations.",
-			id: "microsoft/DialoGPT-large",
-		},
-	],
-	spaces: [
-		{
-			description: "A chatbot based on Blender model.",
-			id: "EXFINITE/BlenderBot-UI",
-		},
-	],
-	summary:
-		"Conversational response modelling is the task of generating conversational text that is relevant, coherent and knowledgable given a prompt. These models have applications in chatbots, and as a part of voice assistants",
-	widgetModels: ["facebook/blenderbot-400M-distill"],
-	youtubeId: "",
-};
-
-export default taskData;

packages/tasks/src/tasks/index.ts

@@ -1,9 +1,9 @@
-import { type PipelineType, PIPELINE_DATA } from "../pipelines";
+import type { PipelineType } from "../pipelines";
+import { PIPELINE_DATA } from "../pipelines";
 
 import audioClassification from "./audio-classification/data";
 import audioToAudio from "./audio-to-audio/data";
 import automaticSpeechRecognition from "./automatic-speech-recognition/data";
-import conversational from "./conversational/data";
 import documentQuestionAnswering from "./document-question-answering/data";
 import featureExtraction from "./feature-extraction/data";
 import fillMask from "./fill-mask/data";
@@ -45,14 +45,15 @@ export const TASKS_MODEL_LIBRARIES: Record<PipelineType, ModelLibraryKey[]> = {
 	"audio-classification": ["speechbrain", "transformers", "transformers.js"],
 	"audio-to-audio": ["asteroid", "speechbrain"],
 	"automatic-speech-recognition": ["espnet", "nemo", "speechbrain", "transformers", "transformers.js"],
-	conversational: ["transformers"],
 	"depth-estimation": ["transformers", "transformers.js"],
 	"document-question-answering": ["transformers", "transformers.js"],
 	"feature-extraction": ["sentence-transformers", "transformers", "transformers.js"],
 	"fill-mask": ["transformers", "transformers.js"],
 	"graph-ml": ["transformers"],
 	"image-classification": ["keras", "timm", "transformers", "transformers.js"],
+	"image-feature-extraction": ["timm", "transformers"],
 	"image-segmentation": ["transformers", "transformers.js"],
+	"image-text-to-text": ["transformers"],
 	"image-to-image": ["diffusers", "transformers", "transformers.js"],
 	"image-to-text": ["transformers", "transformers.js"],
 	"image-to-video": ["diffusers"],
@@ -122,7 +123,6 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"audio-classification": getData("audio-classification", audioClassification),
 	"audio-to-audio": getData("audio-to-audio", audioToAudio),
 	"automatic-speech-recognition": getData("automatic-speech-recognition", automaticSpeechRecognition),
-	conversational: getData("conversational", conversational),
 	"depth-estimation": getData("depth-estimation", depthEstimation),
 	"document-question-answering": getData("document-question-answering", documentQuestionAnswering),
 	"feature-extraction": getData("feature-extraction", featureExtraction),
@@ -130,6 +130,7 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"graph-ml": undefined,
 	"image-classification": getData("image-classification", imageClassification),
 	"image-segmentation": getData("image-segmentation", imageSegmentation),
+	"image-text-to-text": undefined,
 	"image-to-image": getData("image-to-image", imageToImage),
 	"image-to-text": getData("image-to-text", imageToText),
 	"image-to-video": undefined,
@@ -167,6 +168,7 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"zero-shot-object-detection": getData("zero-shot-object-detection", zeroShotObjectDetection),
 	"text-to-3d": getData("text-to-3d", placeholder),
 	"image-to-3d": getData("image-to-3d", placeholder),
+	"image-feature-extraction": getData("image-feature-extraction", placeholder),
 } as const;
 
 export interface ExampleRepo {

packages/tasks/src/tokenizer-data.ts

@@ -0,0 +1,24 @@
+export const SPECIAL_TOKENS_ATTRIBUTES = [
+	"bos_token",
+	"eos_token",
+	"unk_token",
+	"sep_token",
+	"pad_token",
+	"cls_token",
+	"mask_token",
+	// additional_special_tokens (TODO)
+] as const;
+
+/**
+ * Public interface for a tokenizer's special tokens mapping
+ */
+export type SpecialTokensMap = {
+	[key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string;
+};
+/**
+ * Public interface for tokenizer config
+ */
+export interface TokenizerConfig extends SpecialTokensMap {
+	use_default_system_prompt?: boolean;
+	chat_template?: string;
+}

packages/widgets/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/widgets",
 	"packageManager": "pnpm@8.10.5",
-	"version": "0.1.3",
+	"version": "0.1.4",
 	"publishConfig": {
 		"access": "public"
 	},
@@ -45,7 +45,8 @@
 		"static/audioProcessor.js"
 	],
 	"dependencies": {
-		"@huggingface/tasks": "workspace:^"
+		"@huggingface/tasks": "workspace:^",
+		"@huggingface/jinja": "workspace:^"
 	},
 	"peerDependencies": {
 		"svelte": "^3.59.2"

packages/widgets/pnpm-lock.yaml

@@ -5,6 +5,9 @@ settings:
   excludeLinksFromLockfile: false
 
 dependencies:
+  '@huggingface/jinja':
+    specifier: workspace:^
+    version: link:../jinja
   '@huggingface/tasks':
     specifier: workspace:^
     version: link:../tasks

packages/widgets/src/lib/components/Icons/IconImageAndTextToText.svelte

@@ -0,0 +1,28 @@
+<script lang="ts">
+	export let classNames = "";
+</script>
+
+<svg
+	class={classNames}
+	xmlns="http://www.w3.org/2000/svg"
+	xmlns:xlink="http://www.w3.org/1999/xlink"
+	aria-hidden="true"
+	role="img"
+	width="1em"
+	height="1em"
+	preserveAspectRatio="xMidYMid meet"
+	viewBox="0 0 32 32"
+>
+	<path
+		d="M29.707 19.293l-3-3a1 1 0 0 0-1.414 0L16 25.586V30h4.414l9.293-9.293a1 1 0 0 0 0-1.414zM19.586 28H18v-1.586l5-5L24.586 23zM26 21.586L24.414 20L26 18.414L27.586 20z"
+		fill="currentColor"
+	/>
+	<path
+		d="M20 13v-2h-2.142a3.94 3.94 0 0 0-.425-1.019l1.517-1.517l-1.414-1.414l-1.517 1.517A3.944 3.944 0 0 0 15 8.142V6h-2v2.142a3.944 3.944 0 0 0-1.019.425L10.464 7.05L9.05 8.464l1.517 1.517A3.94 3.94 0 0 0 10.142 11H8v2h2.142a3.94 3.94 0 0 0 .425 1.019L9.05 15.536l1.414 1.414l1.517-1.517a3.944 3.944 0 0 0 1.019.425V18h2v-2.142a3.944 3.944 0 0 0 1.019-.425l1.517 1.517l1.414-1.414l-1.517-1.517A3.94 3.94 0 0 0 17.858 13zm-6 1a2 2 0 1 1 2-2a2.002 2.002 0 0 1-2 2z"
+		fill="currentColor"
+	/>
+	<path
+		d="M12 30H6a2.002 2.002 0 0 1-2-2V4a2.002 2.002 0 0 1 2-2h16a2.002 2.002 0 0 1 2 2v10h-2V4H6v24h6z"
+		fill="currentColor"
+	/>
+</svg>

packages/widgets/src/lib/components/Icons/IconImageFeatureExtraction.svelte

@@ -0,0 +1,22 @@
+<script lang="ts">
+	export let classNames = "";
+</script>
+
+<svg
+	class={classNames}
+	xmlns="http://www.w3.org/2000/svg"
+	xmlns:xlink="http://www.w3.org/1999/xlink"
+	aria-hidden="true"
+	fill="currentColor"
+	focusable="false"
+	role="img"
+	width="1em"
+	height="1em"
+	preserveAspectRatio="xMidYMid meet"
+	viewBox="0 0 24 24"
+>
+	<path
+		fill="currentColor"
+		d="M5 21q-.825 0-1.412-.587T3 19V5q0-.825.588-1.412T5 3h5v2H5v14h14v-5.35l2 2V19q0 .825-.587 1.413T19 21H5Zm1-4l3-4l2.25 3l3-4L18 17H6Zm15.55-3.6l-3.1-3.1q-.525.35-1.125.525T16.05 11q-1.85 0-3.15-1.312T11.6 6.5q0-1.875 1.313-3.187T16.1 2q1.875 0 3.188 1.313T20.6 6.5q0 .675-.2 1.3t-.5 1.15L22.95 12l-1.4 1.4ZM16.1 9q1.05 0 1.775-.725T18.6 6.5q0-1.05-.725-1.775T16.1 4q-1.05 0-1.775.725T13.6 6.5q0 1.05.725 1.775T16.1 9Z"
+	/>
+</svg>

packages/widgets/src/lib/components/InferenceWidget/InferenceWidget.svelte

@@ -26,7 +26,7 @@
 	import VisualQuestionAnsweringWidget from "./widgets/VisualQuestionAnsweringWidget/VisualQuestionAnsweringWidget.svelte";
 	import ZeroShotClassificationWidget from "./widgets/ZeroShotClassificationWidget/ZeroShotClassificationWidget.svelte";
 	import ZeroShotImageClassificationWidget from "./widgets/ZeroShotImageClassificationWidget/ZeroShotImageClassificationWidget.svelte";
-	import type { PipelineType } from "@huggingface/tasks";
+	import type { WidgetType } from "@huggingface/tasks";
 	import WidgetInfo from "./shared/WidgetInfo/WidgetInfo.svelte";
 
 	export let apiToken: WidgetProps["apiToken"] = undefined;
@@ -44,7 +44,7 @@
 	// In the future it may be useful / easier to maintain if we created
 	// a single dedicated widget for each pipeline type.
 	const WIDGET_COMPONENTS: {
-		[key in PipelineType]?: typeof SvelteComponent;
+		[key in WidgetType]?: typeof SvelteComponent;
 	} = {
 		"audio-to-audio": AudioToAudioWidget,
 		"audio-classification": AudioClassificationWidget,
@@ -79,9 +79,11 @@
 	};
 
 	$: widgetComponent =
-		model.pipeline_tag && model.pipeline_tag in WIDGET_COMPONENTS
-			? WIDGET_COMPONENTS[model.pipeline_tag as keyof typeof WIDGET_COMPONENTS]
-			: undefined;
+		model.pipeline_tag === "text-generation" && model.tags?.includes("conversational")
+			? (ConversationalWidget as typeof SvelteComponent)
+			: model.pipeline_tag && model.pipeline_tag in WIDGET_COMPONENTS
+			  ? WIDGET_COMPONENTS[model.pipeline_tag as keyof typeof WIDGET_COMPONENTS]
+			  : undefined;
 
 	// prettier-ignore
 	$: widgetProps = ({

packages/widgets/src/lib/components/InferenceWidget/shared/WidgetOutputConvo/WidgetOutputConvo.svelte

@@ -5,9 +5,9 @@
 	import WidgetOutputConvoBubble from "../WidgetOuputConvoBubble/WidgetOutputConvoBubble.svelte";
 
 	export let modelId: string;
-	export let output: Array<{
-		input: string;
-		response: string;
+	export let messages: Array<{
+		role: string;
+		content: string;
 	}>;
 
 	let wrapperEl: HTMLElement;
@@ -25,9 +25,12 @@
 		<strong>{modelId}</strong>.
 	</div>
 	<div class="flex flex-col items-end space-y-4 p-3">
-		{#each output as exchange}
-			<WidgetOutputConvoBubble position="right" text={exchange.input} />
-			<WidgetOutputConvoBubble position="left" text={exchange.response} />
+		{#each messages as message}
+			{#if message.role === "user"}
+				<WidgetOutputConvoBubble position="right" text={message.content} />
+			{:else}
+				<WidgetOutputConvoBubble position="left" text={message.content} />
+			{/if}
 		{/each}
 	</div>
 </div>

packages/widgets/src/lib/components/InferenceWidget/widgets/ConversationalWidget/ConversationalWidget.svelte

@@ -1,6 +1,9 @@
 <script lang="ts">
+	import { onMount } from "svelte";
 	import type { WidgetProps, ExampleRunOpts, InferenceRunOpts } from "../../shared/types.js";
-	import type { WidgetExampleTextInput } from "@huggingface/tasks";
+	import { Template } from "@huggingface/jinja";
+	import type { SpecialTokensMap, TokenizerConfig, WidgetExampleTextInput } from "@huggingface/tasks";
+	import { SPECIAL_TOKENS_ATTRIBUTES } from "@huggingface/tasks";
 
 	import WidgetOutputConvo from "../../shared/WidgetOutputConvo/WidgetOutputConvo.svelte";
 	import WidgetQuickInput from "../../shared/WidgetQuickInput/WidgetQuickInput.svelte";
@@ -19,58 +22,91 @@
 
 	$: isDisabled = $widgetStates?.[model.id]?.isDisabled;
 
-	interface Conversation {
-		generated_responses: string[];
-		past_user_inputs: string[];
+	interface Message {
+		role: string;
+		content: string;
 	}
-	interface Response {
-		conversation: Conversation;
-		generated_text: string;
-	}
-
-	type Output = Array<{
-		input: string;
-		response: string;
-	}>;
 
 	let computeTime = "";
-	let conversation: {
-		generated_responses: string[];
-		past_user_inputs: string[];
-	} = {
-		generated_responses: [],
-		past_user_inputs: [],
-	};
+	let messages: Message[] = [];
 	let error: string = "";
 	let isLoading = false;
 	let modelLoading = {
 		isLoading: false,
 		estimatedTime: 0,
 	};
-	let output: Output = [];
 	let outputJson: string;
 	let text = "";
 
-	async function getOutput({
-		withModelLoading = false,
-		isOnLoadCall = false,
-		exampleOutput = undefined,
-	}: InferenceRunOpts = {}) {
-		const trimmedText = text.trim();
+	let compiledTemplate: Template;
+	let tokenizerConfig: TokenizerConfig;
+
+	// Check config and compile template
+	onMount(() => {
+		const config = model.config;
+		if (config === undefined) {
+			error = "Model config not found";
+			return;
+		}
+
+		if (config.tokenizer === undefined) {
+			error = "Tokenizer config not found";
+			return;
+		}
+		tokenizerConfig = config.tokenizer;
+
+		const chatTemplate = tokenizerConfig.chat_template;
+		if (chatTemplate === undefined) {
+			error = "No chat template found in tokenizer config";
+			return;
+		}
+		try {
+			compiledTemplate = new Template(chatTemplate);
+		} catch (e) {
+			error = `Invalid chat template: "${(e as Error).message}"`;
+			return;
+		}
+	});
 
+	async function getOutput({ withModelLoading = false, isOnLoadCall = false }: InferenceRunOpts = {}) {
+		if (!compiledTemplate) {
+			return;
+		}
+
+		const trimmedText = text.trim();
 		if (!trimmedText) {
 			return;
 		}
 
-		if (shouldUpdateUrl && !conversation.past_user_inputs.length) {
+		if (shouldUpdateUrl && !messages.length) {
 			updateUrl({ text: trimmedText });
 		}
 
+		if (!withModelLoading) {
+			// Add user message to chat
+			messages = [...messages, { role: "user", content: trimmedText }];
+		}
+
+		// Render chat template
+		const special_tokens_map = extractSpecialTokensMap(tokenizerConfig);
+
+		let chatText;
+		try {
+			chatText = compiledTemplate.render({
+				messages,
+				add_generation_prompt: true,
+				...special_tokens_map,
+			});
+		} catch (e) {
+			error = `An error occurred while rendering the chat template: "${(e as Error).message}"`;
+			return;
+		}
+
 		const requestBody = {
-			inputs: {
-				generated_responses: conversation.generated_responses,
-				past_user_inputs: conversation.past_user_inputs,
-				text: trimmedText,
+			inputs: chatText,
+			parameters: {
+				return_full_text: false,
+				max_new_tokens: 100,
 			},
 		};
 		addInferenceParameters(requestBody, model);
@@ -82,7 +118,7 @@
 			model.id,
 			requestBody,
 			apiToken,
-			parseOutput,
+			(body) => parseOutput(body, messages),
 			withModelLoading,
 			includeCredentials,
 			isOnLoadCall
@@ -99,8 +135,7 @@
 			computeTime = res.computeTime;
 			outputJson = res.outputJson;
 			if (res.output) {
-				conversation = res.output.conversation;
-				output = res.output.output;
+				messages = res.output;
 			}
 			// Emptying input value
 			text = "";
@@ -115,34 +150,28 @@
 		}
 	}
 
-	function isValidOutput(arg: any): arg is Response {
-		return (
-			arg && Array.isArray(arg?.conversation?.generated_responses) && Array.isArray(arg?.conversation?.past_user_inputs)
-		);
+	function parseOutput(body: unknown, chat: Message[]): Message[] {
+		if (Array.isArray(body) && body.length) {
+			const text = body[0]?.generated_text ?? "";
+
+			if (!text.length) {
+				throw new Error("Model did not generate a response.");
+			}
+
+			return [...chat, { role: "assistant", content: text }];
+		}
+		throw new TypeError("Invalid output: output must be of type Array & non-empty");
 	}
 
-	function parseOutput(body: unknown): {
-		conversation: Conversation;
-		output: Output;
-	} {
-		if (isValidOutput(body)) {
-			const conversation = body.conversation;
-			const pastUserInputs = conversation.past_user_inputs;
-			const generatedResponses = conversation.generated_responses;
-			const output = pastUserInputs
-				.filter(
-					(x, i) =>
-						x !== null && x !== undefined && generatedResponses[i] !== null && generatedResponses[i] !== undefined
-				)
-				.map((x, i) => ({
-					input: x ?? "",
-					response: generatedResponses[i] ?? "",
-				}));
-			return { conversation, output };
+	function extractSpecialTokensMap(tokenizerConfig: TokenizerConfig): SpecialTokensMap {
+		const specialTokensMap = Object.create(null);
+		for (const key of SPECIAL_TOKENS_ATTRIBUTES) {
+			const value = tokenizerConfig[key];
+			if (typeof value === "string") {
+				specialTokensMap[key] = value;
+			}
 		}
-		throw new TypeError(
-			"Invalid output: output must be of type <conversation: <generated_responses:Array; past_user_inputs:Array>>"
-		);
+		return specialTokensMap;
 	}
 
 	function applyWidgetExample(sample: WidgetExampleTextInput, opts: ExampleRunOpts = {}) {
@@ -165,7 +194,7 @@
 		{applyWidgetExample}
 		validateExample={isTextInput}
 	/>
-	<WidgetOutputConvo modelId={model.id} {output} />
+	<WidgetOutputConvo modelId={model.id} {messages} />
 
 	<WidgetQuickInput
 		bind:value={text}

packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte

@@ -7,6 +7,7 @@
 	import IconConversational from "../Icons/IconConversational.svelte";
 	import IconFeatureExtraction from "../Icons/IconFeatureExtraction.svelte";
 	import IconFillMask from "../Icons/IconFillMask.svelte";
+	import IconImageAndTextToText from "../Icons/IconImageAndTextToText.svelte";
 	import IconImageClassification from "../Icons/IconImageClassification.svelte";
 	import IconImageSegmentation from "../Icons/IconImageSegmentation.svelte";
 	import IconObjectDetection from "../Icons/IconObjectDetection.svelte";
@@ -40,13 +41,14 @@
 	import IconMaskGeneration from "../Icons/IconMaskGeneration.svelte";
 	import IconTextTo3D from "../Icons/IconTextTo3D.svelte";
 	import IconImageTo3D from "../Icons/IconImageTo3D.svelte";
-	import type { PipelineType } from "@huggingface/tasks";
+	import IconImageFeatureExtraction from "../Icons/IconImageFeatureExtraction.svelte";
+	import type { WidgetType } from "@huggingface/tasks";
 
 	export let classNames = "";
 	export let pipeline = "";
 
 	const ICON_COMPONENTS: {
-		[key in PipelineType]?: typeof SvelteComponent;
+		[key in WidgetType]?: typeof SvelteComponent;
 	} = {
 		/// Keep same order as in huggingface_hub/Types.ts
 		/// for easy mapping.
@@ -75,6 +77,7 @@
 		"video-classification": IconVideoClassification,
 		"image-segmentation": IconImageSegmentation,
 		"text-to-image": IconTextToImage,
+		"image-text-to-text": IconImageAndTextToText,
 		"image-to-text": IconImageToText,
 		"image-to-image": IconImageToImage,
 		"image-to-video": IconImageToVideo,
@@ -90,6 +93,7 @@
 		"zero-shot-object-detection": IconZeroShotObjectDetection,
 		"text-to-3d": IconTextTo3D,
 		"image-to-3d": IconImageTo3D,
+		"image-feature-extraction": IconImageFeatureExtraction,
 	};
 
 	$: iconComponent =

packages/widgets/src/routes/+page.svelte

@@ -30,6 +30,23 @@
 	});
 
 	const models: ModelData[] = [
+		{
+			id: "HuggingFaceH4/zephyr-7b-beta",
+			pipeline_tag: "text-generation",
+			tags: ["conversational"],
+			inference: InferenceDisplayability.Yes,
+			config: {
+				tokenizer: {
+					bos_token: "<s>",
+					chat_template:
+						"{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+					eos_token: "</s>",
+					pad_token: "</s>",
+					unk_token: "<unk>",
+					use_default_system_prompt: true,
+				},
+			},
+		},
 		{
 			id: "WizardLM/WizardLM-70B-V1.0",
 			pipeline_tag: "text-generation",
@@ -315,7 +332,7 @@
 		},
 		{
 			id: "facebook/blenderbot-400M-distill",
-			pipeline_tag: "conversational",
+			pipeline_tag: "text2text-generation",
 			inference: InferenceDisplayability.Yes,
 			widgetData: [{ text: "Hey my name is Julien! How are you?" }],
 		},