Sync widgets demo

packages/tasks/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/tasks",
 	"packageManager": "pnpm@8.10.5",
-	"version": "0.0.5",
+	"version": "0.0.6",
 	"description": "List of ML tasks for huggingface.co/tasks",
 	"repository": "https://github.com/huggingface/huggingface.js.git",
 	"publishConfig": {

packages/tasks/src/const.ts

@@ -19,6 +19,7 @@ export const TASKS_MODEL_LIBRARIES: Record<PipelineType, ModelLibraryKey[]> = {
 	"image-to-image": [],
 	"image-to-text": ["transformers.js"],
 	"video-classification": [],
+	"mask-generation": ["transformers"],
 	"multiple-choice": ["transformers"],
 	"object-detection": ["transformers", "transformers.js"],
 	other: [],
@@ -56,4 +57,5 @@ export const TASKS_MODEL_LIBRARIES: Record<PipelineType, ModelLibraryKey[]> = {
 	"voice-activity-detection": [],
 	"zero-shot-classification": ["transformers", "transformers.js"],
 	"zero-shot-image-classification": ["transformers.js"],
+	"zero-shot-object-detection": ["transformers"],
 };

packages/tasks/src/index.ts

@@ -8,6 +8,10 @@ export {
 	type Modality,
 	MODALITIES,
 	MODALITY_LABELS,
+	SUBTASK_TYPES,
+	PIPELINE_TYPES_SET,
 } from "./pipelines";
-export { ModelLibrary } from "./modelLibraries";
+export { ModelLibrary, ALL_DISPLAY_MODEL_LIBRARY_KEYS } from "./modelLibraries";
 export type { ModelLibraryKey } from "./modelLibraries";
+
+export { TAG_NFAA_CONTENT, OTHER_TAGS_SUGGESTIONS, TAG_TEXT_GENERATION_INFERENCE, TAG_CUSTOM_CODE } from "./tags";

packages/tasks/src/modelLibraries.ts

@@ -41,3 +41,7 @@ export enum ModelLibrary {
 }
 
 export type ModelLibraryKey = keyof typeof ModelLibrary;
+
+export const ALL_DISPLAY_MODEL_LIBRARY_KEYS = Object.keys(ModelLibrary).filter(
+	(k) => !["doctr", "k2", "mindspore", "tensorflowtts"].includes(k)
+);

packages/tasks/src/pipelines.ts

@@ -606,6 +606,16 @@ export const PIPELINE_DATA = {
 		modality: "multimodal",
 		color: "green",
 	},
+	"mask-generation": {
+		name: "Mask Generation",
+		modality: "cv",
+		color: "indigo",
+	},
+	"zero-shot-object-detection": {
+		name: "Zero-Shot Object Detection",
+		modality: "cv",
+		color: "yellow",
+	},
 	other: {
 		name: "Other",
 		modality: "other",
@@ -616,4 +626,11 @@ export const PIPELINE_DATA = {
 } satisfies Record<string, PipelineData>;
 
 export type PipelineType = keyof typeof PIPELINE_DATA;
+
 export const PIPELINE_TYPES = Object.keys(PIPELINE_DATA) as PipelineType[];
+
+export const SUBTASK_TYPES = Object.values(PIPELINE_DATA)
+	.flatMap((data) => ("subtasks" in data ? data.subtasks : []))
+	.map((s) => s.type);
+
+export const PIPELINE_TYPES_SET = new Set(PIPELINE_TYPES);

packages/tasks/src/tags.ts

@@ -0,0 +1,15 @@
+export const TAG_NFAA_CONTENT = "not-for-all-audiences";
+export const OTHER_TAGS_SUGGESTIONS = [
+	"chemistry",
+	"biology",
+	"finance",
+	"legal",
+	"music",
+	"art",
+	"code",
+	"climate",
+	"medical",
+	TAG_NFAA_CONTENT,
+];
+export const TAG_TEXT_GENERATION_INFERENCE = "text-generation-inference";
+export const TAG_CUSTOM_CODE = "custom_code";

packages/tasks/src/tasksData.ts

@@ -54,6 +54,7 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"image-segmentation": getData("image-segmentation", imageSegmentation),
 	"image-to-image": getData("image-to-image", imageToImage),
 	"image-to-text": getData("image-to-text", imageToText),
+	"mask-generation": getData("mask-generation", placeholder),
 	"multiple-choice": undefined,
 	"object-detection": getData("object-detection", objectDetection),
 	"video-classification": getData("video-classification", videoClassification),
@@ -84,6 +85,7 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"voice-activity-detection": undefined,
 	"zero-shot-classification": getData("zero-shot-classification", zeroShotClassification),
 	"zero-shot-image-classification": getData("zero-shot-image-classification", zeroShotImageClassification),
+	"zero-shot-object-detection": getData("zero-shot-object-detection", placeholder),
 } as const;
 
 /**

packages/tasks/src/video-classification/about.md

@@ -15,34 +15,14 @@ Models trained in video classification can improve user experience by organizing
 Below you can find code for inferring with a pre-trained video classification model.
 
 ```python
-from transformers import VideoMAEFeatureExtractor, VideoMAEForVideoClassification
-from pytorchvideo.transforms import UniformTemporalSubsample
-from pytorchvideo.data.encoded_video import EncodedVideo
-
-
-# Load the video.
-video = EncodedVideo.from_path("path_to_video.mp4")
-video_data = video.get_clip(start_sec=0, end_sec=4.0)["video"]
-
-# Sub-sample a fixed set of frames and convert them to a NumPy array.
-num_frames = 16
-subsampler = UniformTemporalSubsample(num_frames)
-subsampled_frames = subsampler(video_data)
-video_data_np = subsampled_frames.numpy().transpose(1, 2, 3, 0)
-
-# Preprocess the video frames.
-inputs = feature_extractor(list(video_data_np), return_tensors="pt")
-
-# Run inference
-with torch.no_grad():
-    outputs = model(**inputs)
-    logits = outputs.logits
-
-# Model predicts one of the 400 Kinetics 400 classes
-predicted_label = logits.argmax(-1).item()
-print(model.config.id2label[predicted_label])
-# `eating spaghetti` (if you chose this video:
-# https://hf.co/datasets/nielsr/video-demo/resolve/main/eating_spaghetti.mp4)
+from transformers import pipeline
+
+pipe = pipeline(task = "video-classification", model="nateraw/videomae-base-finetuned-ucf101-subset")
+pipe("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/basketball.avi?download=true")
+
+#[{'score': 0.90, 'label': 'BasketballDunk'},
+# {'score': 0.02, 'label': 'BalanceBeam'},
+# ... ]
 ```
 
 ## Useful Resources

packages/widgets/src/lib/components/Icons/IconMaskGeneration.svelte

@@ -0,0 +1,22 @@
+<script lang="ts">
+	export let classNames = "";
+</script>
+
+<svg
+	class={classNames}
+	xmlns="http://www.w3.org/2000/svg"
+	xmlns:xlink="http://www.w3.org/1999/xlink"
+	aria-hidden="true"
+	focusable="false"
+	role="img"
+	width="1em"
+	height="1em"
+	preserveAspectRatio="xMidYMid meet"
+	viewBox="0 0 12 12"
+	><path
+		fill="currentColor"
+		fill-rule="evenodd"
+		d="M1.84.73h6.63a.7.7 0 0 1 .7.7v4.36h-.7V1.43H1.84v3.54l.9-.9a.7.7 0 0 1 1 0l1.74 1.75a.79.79 0 0 0-.52.47L3.24 4.57l-1.4 1.4v2.08h3.07v.7H1.84a.7.7 0 0 1-.7-.7V1.43a.7.7 0 0 1 .7-.7Zm5.38 4.74.32.32H5.91l.32-.32a.7.7 0 0 1 .99 0Zm-.61-1.43A1.05 1.05 0 1 1 5.45 2.3 1.05 1.05 0 0 1 6.6 4.04Zm-.39-1.16a.35.35 0 1 0-.39.58.35.35 0 0 0 .4-.58Zm3.99 8.43a.65.65 0 0 0 .56-.64v-1.3h-.65.65v1.3a.65.65 0 0 1-.56.64Zm-.09-.64h-1.3 1.3Zm-1.33.68h1.33a.68.68 0 0 0 .68-.68V9.34h-.7v1.3H8.8v.71ZM6.22 8.43v-1.3 1.3Zm1.3-1.3v-.66h-1.3 1.3v.65ZM5.54 8.45h.7v-1.3h1.3v-.72H6.23a.68.68 0 0 0-.68.68v1.34Zm1.98 2.86v-.65h-1.3v-1.3h-.65.65v1.3h1.3v.65ZM5.54 9.34v1.33a.68.68 0 0 0 .68.68h1.33v-.71h-1.3v-1.3h-.71Zm5.23-.91v-1.3a.65.65 0 0 0-.65-.66h-1.3 1.3a.65.65 0 0 1 .65.65v1.3ZM8.8 6.44v.71h1.3v1.3h.71V7.13a.68.68 0 0 0-.68-.68H8.8Z"
+		clip-rule="evenodd"
+	/></svg
+>

packages/widgets/src/lib/components/Icons/IconZeroShotObjectDetection.svelte

@@ -0,0 +1,27 @@
+<script lang="ts">
+	export let classNames = "";
+</script>
+
+<svg
+	class={classNames}
+	xmlns="http://www.w3.org/2000/svg"
+	xmlns:xlink="http://www.w3.org/1999/xlink"
+	aria-hidden="true"
+	focusable="false"
+	role="img"
+	width="1em"
+	height="1em"
+	preserveAspectRatio="xMidYMid meet"
+	viewBox="0 0 12 12"
+	><path
+		fill="currentColor"
+		fill-rule="evenodd"
+		d="M1.84.73h6.63a.7.7 0 0 1 .7.7v4.36h-.7V1.43H1.84v3.54l.9-.9a.7.7 0 0 1 1 0l1.74 1.75a.79.79 0 0 0-.52.47L3.24 4.57l-1.4 1.4v2.08h3.07v.7H1.84a.7.7 0 0 1-.7-.7V1.43a.7.7 0 0 1 .7-.7Zm5.38 4.74.32.32H5.91l.32-.32a.7.7 0 0 1 .99 0Zm-.61-1.43A1.05 1.05 0 1 1 5.45 2.3 1.05 1.05 0 0 1 6.6 4.04Zm-.39-1.16a.35.35 0 1 0-.39.58.35.35 0 0 0 .4-.58Z"
+		clip-rule="evenodd"
+	/><path
+		fill="currentColor"
+		fill-rule="evenodd"
+		d="M7.77 7.07h-1.6v1.42h1.6V7.07Zm-2.2-.6v2.62h2.8V6.47h-2.8ZM8.53 10.17H6.17v.43h2.36v-.43Zm-2.96-.6v1.63h3.55V9.57H5.58ZM10.16 7.07h-.72v1.42h.72V7.07Zm-1.32-.6v2.62h1.92V6.47H8.84Z"
+		clip-rule="evenodd"
+	/></svg
+>

packages/widgets/src/lib/components/InferenceWidget/InferenceWidget.svelte

@@ -27,6 +27,7 @@
 	import ZeroShotClassificationWidget from "./widgets/ZeroShowClassificationWidget/ZeroShotClassificationWidget.svelte";
 	import ZeroShotImageClassificationWidget from "./widgets/ZeroShotImageClassificationWidget/ZeroShotImageClassificationWidget.svelte";
 	import type { PipelineType } from "@huggingface/tasks";
+	import WidgetInfo from "./shared/WidgetInfo/WidgetInfo.svelte";
 
 	export let apiToken: WidgetProps["apiToken"] = undefined;
 	export let callApiOnMount = false;
@@ -97,4 +98,7 @@
 
 {#if widgetComponent}
 	<svelte:component this={widgetComponent} {...widgetProps} />
+{:else}
+	<!-- Still show widget error (such as "pipeline not support", etc.) when there is no widget for a task -->
+	<WidgetInfo {model} />
 {/if}

packages/widgets/src/lib/components/InferenceWidget/shared/WidgetInfo/WidgetInfo.svelte

@@ -5,8 +5,8 @@
 	import IconInfo from "$lib/components/Icons/IconInfo.svelte";
 
 	export let model: WidgetProps["model"];
-	export let computeTime: string;
-	export let error: string;
+	export let computeTime: string = "";
+	export let error: string = "";
 	export let modelLoadInfo: ModelLoadInfo | undefined = undefined;
 	export let modelTooBig = false;
 

packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte

@@ -35,6 +35,8 @@
 	import IconUnconditionalImageGeneration from "../Icons/IconUnconditionalImageGeneration.svelte";
 	import IconDocumentQuestionAnswering from "../Icons/IconDocumentQuestionAnswering.svelte";
 	import IconGraphML from "../Icons/IconGraphML.svelte";
+	import IconZeroShotObjectDetection from "../Icons/IconZeroShotClassification.svelte";
+	import IconMaskGeneration from "../Icons/IconMaskGeneration.svelte";
 	import type { PipelineType } from "@huggingface/tasks";
 
 	export let classNames = "";
@@ -80,6 +82,8 @@
 		"tabular-regression": IconTabularRegression,
 		"text-to-video": IconTextToVideo,
 		"document-question-answering": IconDocumentQuestionAnswering,
+		"mask-generation": IconMaskGeneration,
+		"zero-shot-object-detection": IconZeroShotObjectDetection,
 	};
 
 	$: iconComponent =

packages/widgets/src/routes/+page.svelte

@@ -442,8 +442,8 @@
 	const modelsDisabled: ModelData[] = [
 		{
 			id: "gpt2",
-			pipeline_tag: "text-generation",
-			inference: InferenceDisplayability.CustomCode,
+			pipeline_tag: undefined,
+			inference: InferenceDisplayability.PipelineNotDetected,
 		},
 		{
 			id: "gpt2",