Repository: xenova/transformers.js
File: packages/transformers/src/tokenization_utils.js (JavaScript)

Code before:
                throw new Error(
                    'Unable to decode tensor with `batch size !== 1`. Use `tokenizer.batch_decode(...)` for batched inputs.',
                );
            }
            return tensor.tolist()[0];
        default:
            throw new Error(`Expected tensor to have 1-2 dimensions, got ${dims.length}.`);
    }
}

const SPECIAL_TOKEN_ATTRIBUTES = [
    'bos_token',
    'eos_token',
    'unk_token',
    'sep_token',
    'pad_token',
    'cls_token',
    'mask_token',
    // additional_special_tokens (TODO)
];

/**
 * @typedef {{ type: 'text', text: string, source_lang_code?: string, target_lang_code?: string, [key: string]: any }} TextContent
 * @property {'text'} type The type of content (must be 'text').
 * @property {string} text The text content.
 * @property {string} [source_lang_code] The source language code.
 * @property {string} [target_lang_code] The target language code.
 */

/**
 * @typedef {{ type: 'image', image?: string | import('./utils/image.js').RawImage, [key: string]: any }} ImageContent
 * @property {'image'} type The type of content (must be 'image').
 * @property {string | import('./utils/image.js').RawImage} [image] Optional URL or instance of the image.
 *
 * Note: This works for SmolVLM. Qwen2VL and Idefics3 have different implementations.
 */

/**
 * @typedef {TextContent | ImageContent | { type: string & {}, [key: string]: any }} MessageContent
 * Base type for message content. This is a discriminated union that can be extended with additional content types.
 * Example: `@typedef {TextContent | ImageContent | AudioContent} MessageContent`
 */

/**
 * @typedef {Object} Message
 * @property {'user' | 'assistant' | 'system' | (string & {})} role The role of the message.
 * @property {string | MessageContent[]} content The content of the message. Can be a simple string or an array of content objects.
 */

/**
 *

Review comment (suggestion):
```suggestion
/**
 * @typedef {{ type: 'text', text: string, [key: string]: any }} TextContent
 * @property {'text'} type The type of content (must be 'text').
 * @property {string} text The text content.
 */
```

since `source_lang_code` and `target_lang_code` are very specific to translategemma, I think we can remove them from here. The generic `[key: string]: any` should be sufficient.

Code after:
                throw new Error(
                    'Unable to decode tensor with `batch size !== 1`. Use `tokenizer.batch_decode(...)` for batched inputs.',
                );
            }
            return tensor.tolist()[0];
        default:
            throw new Error(`Expected tensor to have 1-2 dimensions, got ${dims.length}.`);
    }
}

const SPECIAL_TOKEN_ATTRIBUTES = [
    'bos_token',
    'eos_token',
    'unk_token',
    'sep_token',
    'pad_token',
    'cls_token',
    'mask_token',
    // additional_special_tokens (TODO)
];

/**
 * @typedef {{ type: 'text', text: string, [key: string]: any }} TextContent
 * @property {'text'} type The type of content (must be 'text').
 * @property {string} text The text content.
 */

/**
 * @typedef {{ type: 'image', image?: string | import('./utils/image.js').RawImage, [key: string]: any }} ImageContent
 * @property {'image'} type The type of content (must be 'image').
 * @property {string | import('./utils/image.js').RawImage} [image] Optional URL or instance of the image.
 *
 * Note: This works for SmolVLM. Qwen2VL and Idefics3 have different implementations.
 */

/**
 * @typedef {TextContent | ImageContent | { type: string & {}, [key: string]: any }} MessageContent
 * Base type for message content. This is a discriminated union that can be extended with additional content types.
 * Example: `@typedef {TextContent | ImageContent | AudioContent} MessageContent`
 */

/**
 * @typedef {Object} Message
 * @property {'user' | 'assistant' | 'system' | (string & {})} role The role of the message.
 * @property {string | MessageContent[]} content The content of the message. Can be a simple string or an array of content objects.
 */

/**
 *
 * Helper function for padding values of an object, which are each arrays.
 * NOTE: No additional checks are made here for validity of arguments.