Spaces:
Sleeping
Sleeping
machineuser
commited on
Commit
•
32d7cd6
1
Parent(s):
5a13705
Sync widgets demo
Browse files- packages/jinja/package.json +1 -1
- packages/jinja/src/ast.ts +15 -5
- packages/jinja/src/lexer.ts +18 -8
- packages/jinja/src/parser.ts +50 -3
- packages/jinja/src/runtime.ts +217 -126
- packages/jinja/test/e2e.test.js +90 -0
- packages/jinja/test/interpreter.test.js +27 -0
- packages/jinja/test/templates.test.js +484 -16
- packages/widgets/package.json +1 -1
packages/jinja/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
{
|
2 |
"name": "@huggingface/jinja",
|
3 |
"packageManager": "pnpm@8.10.5",
|
4 |
-
"version": "0.2.
|
5 |
"description": "A minimalistic JavaScript implementation of the Jinja templating engine, specifically designed for parsing and rendering ML chat templates.",
|
6 |
"repository": "https://github.com/huggingface/huggingface.js.git",
|
7 |
"publishConfig": {
|
|
|
1 |
{
|
2 |
"name": "@huggingface/jinja",
|
3 |
"packageManager": "pnpm@8.10.5",
|
4 |
+
"version": "0.2.1",
|
5 |
"description": "A minimalistic JavaScript implementation of the Jinja templating engine, specifically designed for parsing and rendering ML chat templates.",
|
6 |
"repository": "https://github.com/huggingface/huggingface.js.git",
|
7 |
"publishConfig": {
|
packages/jinja/src/ast.ts
CHANGED
@@ -120,10 +120,6 @@ export class NumericLiteral extends Literal<number> {
|
|
120 |
*/
|
121 |
export class StringLiteral extends Literal<string> {
|
122 |
override type = "StringLiteral";
|
123 |
-
|
124 |
-
constructor(value: string) {
|
125 |
-
super(value);
|
126 |
-
}
|
127 |
}
|
128 |
|
129 |
/**
|
@@ -133,6 +129,20 @@ export class BooleanLiteral extends Literal<boolean> {
|
|
133 |
override type = "BooleanLiteral";
|
134 |
}
|
135 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
136 |
/**
|
137 |
* An operation with two sides, separated by an operator.
|
138 |
* Note: Either side can be a Complex Expression, with order
|
@@ -159,7 +169,7 @@ export class FilterExpression extends Expression {
|
|
159 |
|
160 |
constructor(
|
161 |
public operand: Expression,
|
162 |
-
public filter: Identifier
|
163 |
) {
|
164 |
super();
|
165 |
}
|
|
|
120 |
*/
|
121 |
export class StringLiteral extends Literal<string> {
|
122 |
override type = "StringLiteral";
|
|
|
|
|
|
|
|
|
123 |
}
|
124 |
|
125 |
/**
|
|
|
129 |
override type = "BooleanLiteral";
|
130 |
}
|
131 |
|
132 |
+
/**
|
133 |
+
* Represents an array literal in the template.
|
134 |
+
*/
|
135 |
+
export class ArrayLiteral extends Literal<Expression[]> {
|
136 |
+
override type = "ArrayLiteral";
|
137 |
+
}
|
138 |
+
|
139 |
+
/**
|
140 |
+
* Represents an object literal in the template.
|
141 |
+
*/
|
142 |
+
export class ObjectLiteral extends Literal<Map<Expression, Expression>> {
|
143 |
+
override type = "ObjectLiteral";
|
144 |
+
}
|
145 |
+
|
146 |
/**
|
147 |
* An operation with two sides, separated by an operator.
|
148 |
* Note: Either side can be a Complex Expression, with order
|
|
|
169 |
|
170 |
constructor(
|
171 |
public operand: Expression,
|
172 |
+
public filter: Identifier | CallExpression
|
173 |
) {
|
174 |
super();
|
175 |
}
|
packages/jinja/src/lexer.ts
CHANGED
@@ -17,6 +17,8 @@ export const TOKEN_TYPES = Object.freeze({
|
|
17 |
CloseExpression: "CloseExpression", // }}
|
18 |
OpenSquareBracket: "OpenSquareBracket", // [
|
19 |
CloseSquareBracket: "CloseSquareBracket", // ]
|
|
|
|
|
20 |
Comma: "Comma", // ,
|
21 |
Dot: "Dot", // .
|
22 |
Colon: "Colon", // :
|
@@ -104,6 +106,8 @@ const ORDERED_MAPPING_TABLE: [string, TokenType][] = [
|
|
104 |
// Single character tokens
|
105 |
["(", TOKEN_TYPES.OpenParen],
|
106 |
[")", TOKEN_TYPES.CloseParen],
|
|
|
|
|
107 |
["[", TOKEN_TYPES.OpenSquareBracket],
|
108 |
["]", TOKEN_TYPES.CloseSquareBracket],
|
109 |
[",", TOKEN_TYPES.Comma],
|
@@ -154,19 +158,25 @@ function preprocess(template: string, options: PreprocessOptions = {}): string {
|
|
154 |
template = template.slice(0, -1);
|
155 |
}
|
156 |
|
157 |
-
|
158 |
-
|
159 |
-
|
160 |
-
|
161 |
-
}
|
162 |
if (options.lstrip_blocks) {
|
163 |
// The lstrip_blocks option can also be set to strip tabs and spaces from the
|
164 |
// beginning of a line to the start of a block. (Nothing will be stripped if
|
165 |
// there are other characters before the start of the block.)
|
166 |
-
template = template.replace(/^[ \t]*{
|
|
|
|
|
|
|
|
|
|
|
|
|
167 |
}
|
168 |
|
169 |
return template
|
|
|
170 |
.replace(/-%}\s*/g, "%}")
|
171 |
.replace(/\s*{%-/g, "{%")
|
172 |
.replace(/-}}\s*/g, "}}")
|
@@ -283,9 +293,9 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token
|
|
283 |
}
|
284 |
}
|
285 |
|
286 |
-
if (char === "'") {
|
287 |
++cursorPosition; // Skip the opening quote
|
288 |
-
const str = consumeWhile((
|
289 |
tokens.push(new Token(str, TOKEN_TYPES.StringLiteral));
|
290 |
++cursorPosition; // Skip the closing quote
|
291 |
continue;
|
|
|
17 |
CloseExpression: "CloseExpression", // }}
|
18 |
OpenSquareBracket: "OpenSquareBracket", // [
|
19 |
CloseSquareBracket: "CloseSquareBracket", // ]
|
20 |
+
OpenCurlyBracket: "OpenCurlyBracket", // {
|
21 |
+
CloseCurlyBracket: "CloseCurlyBracket", // }
|
22 |
Comma: "Comma", // ,
|
23 |
Dot: "Dot", // .
|
24 |
Colon: "Colon", // :
|
|
|
106 |
// Single character tokens
|
107 |
["(", TOKEN_TYPES.OpenParen],
|
108 |
[")", TOKEN_TYPES.CloseParen],
|
109 |
+
["{", TOKEN_TYPES.OpenCurlyBracket],
|
110 |
+
["}", TOKEN_TYPES.CloseCurlyBracket],
|
111 |
["[", TOKEN_TYPES.OpenSquareBracket],
|
112 |
["]", TOKEN_TYPES.CloseSquareBracket],
|
113 |
[",", TOKEN_TYPES.Comma],
|
|
|
158 |
template = template.slice(0, -1);
|
159 |
}
|
160 |
|
161 |
+
// Replace all comments with a placeholder
|
162 |
+
// This ensures that comments don't interfere with the following options
|
163 |
+
template = template.replace(/{#.*?#}/gs, "{##}");
|
164 |
+
|
|
|
165 |
if (options.lstrip_blocks) {
|
166 |
// The lstrip_blocks option can also be set to strip tabs and spaces from the
|
167 |
// beginning of a line to the start of a block. (Nothing will be stripped if
|
168 |
// there are other characters before the start of the block.)
|
169 |
+
template = template.replace(/^[ \t]*({[#%])/gm, "$1");
|
170 |
+
}
|
171 |
+
|
172 |
+
if (options.trim_blocks) {
|
173 |
+
// If an application configures Jinja to trim_blocks, the first newline after
|
174 |
+
// a template tag is removed automatically (like in PHP).
|
175 |
+
template = template.replace(/([#%]})\n/g, "$1");
|
176 |
}
|
177 |
|
178 |
return template
|
179 |
+
.replace(/{##}/g, "") // Remove comments
|
180 |
.replace(/-%}\s*/g, "%}")
|
181 |
.replace(/\s*{%-/g, "{%")
|
182 |
.replace(/-}}\s*/g, "}}")
|
|
|
293 |
}
|
294 |
}
|
295 |
|
296 |
+
if (char === "'" || char === '"') {
|
297 |
++cursorPosition; // Skip the opening quote
|
298 |
+
const str = consumeWhile((c) => c !== char);
|
299 |
tokens.push(new Token(str, TOKEN_TYPES.StringLiteral));
|
300 |
++cursorPosition; // Skip the closing quote
|
301 |
continue;
|
packages/jinja/src/parser.ts
CHANGED
@@ -12,6 +12,8 @@ import {
|
|
12 |
NumericLiteral,
|
13 |
StringLiteral,
|
14 |
BooleanLiteral,
|
|
|
|
|
15 |
BinaryExpression,
|
16 |
FilterExpression,
|
17 |
TestExpression,
|
@@ -197,7 +199,16 @@ export function parse(tokens: Token[]): Program {
|
|
197 |
|
198 |
function parseExpression(): Statement {
|
199 |
// Choose parse function with lowest precedence
|
200 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
201 |
}
|
202 |
|
203 |
function parseLogicalOrExpression(): Statement {
|
@@ -423,11 +434,14 @@ export function parse(tokens: Token[]): Program {
|
|
423 |
while (is(TOKEN_TYPES.Pipe)) {
|
424 |
// Support chaining filters
|
425 |
++current; // consume pipe
|
426 |
-
|
427 |
if (!(filter instanceof Identifier)) {
|
428 |
throw new SyntaxError(`Expected identifier for the filter`);
|
429 |
}
|
430 |
-
|
|
|
|
|
|
|
431 |
}
|
432 |
return operand;
|
433 |
}
|
@@ -457,6 +471,39 @@ export function parse(tokens: Token[]): Program {
|
|
457 |
++current; // consume closing parenthesis
|
458 |
return expression;
|
459 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
460 |
default:
|
461 |
throw new SyntaxError(`Unexpected token: ${token.type}`);
|
462 |
}
|
|
|
12 |
NumericLiteral,
|
13 |
StringLiteral,
|
14 |
BooleanLiteral,
|
15 |
+
ArrayLiteral,
|
16 |
+
ObjectLiteral,
|
17 |
BinaryExpression,
|
18 |
FilterExpression,
|
19 |
TestExpression,
|
|
|
199 |
|
200 |
function parseExpression(): Statement {
|
201 |
// Choose parse function with lowest precedence
|
202 |
+
const a = parseLogicalOrExpression();
|
203 |
+
if (is(TOKEN_TYPES.If)) {
|
204 |
+
// Ternary expression
|
205 |
+
++current; // consume if
|
206 |
+
const predicate = parseLogicalOrExpression();
|
207 |
+
expect(TOKEN_TYPES.Else, "Expected else token");
|
208 |
+
const b = parseLogicalOrExpression();
|
209 |
+
return new If(predicate, [a], [b]);
|
210 |
+
}
|
211 |
+
return a;
|
212 |
}
|
213 |
|
214 |
function parseLogicalOrExpression(): Statement {
|
|
|
434 |
while (is(TOKEN_TYPES.Pipe)) {
|
435 |
// Support chaining filters
|
436 |
++current; // consume pipe
|
437 |
+
let filter = parsePrimaryExpression(); // should be an identifier
|
438 |
if (!(filter instanceof Identifier)) {
|
439 |
throw new SyntaxError(`Expected identifier for the filter`);
|
440 |
}
|
441 |
+
if (is(TOKEN_TYPES.OpenParen)) {
|
442 |
+
filter = parseCallExpression(filter);
|
443 |
+
}
|
444 |
+
operand = new FilterExpression(operand, filter as Identifier | CallExpression);
|
445 |
}
|
446 |
return operand;
|
447 |
}
|
|
|
471 |
++current; // consume closing parenthesis
|
472 |
return expression;
|
473 |
}
|
474 |
+
case TOKEN_TYPES.OpenSquareBracket: {
|
475 |
+
++current; // consume opening square bracket
|
476 |
+
|
477 |
+
const values = [];
|
478 |
+
while (!is(TOKEN_TYPES.CloseSquareBracket)) {
|
479 |
+
values.push(parseExpression());
|
480 |
+
|
481 |
+
if (is(TOKEN_TYPES.Comma)) {
|
482 |
+
++current; // consume comma
|
483 |
+
}
|
484 |
+
}
|
485 |
+
++current; // consume closing square bracket
|
486 |
+
|
487 |
+
return new ArrayLiteral(values);
|
488 |
+
}
|
489 |
+
case TOKEN_TYPES.OpenCurlyBracket: {
|
490 |
+
++current; // consume opening curly bracket
|
491 |
+
|
492 |
+
const values = new Map();
|
493 |
+
while (!is(TOKEN_TYPES.CloseCurlyBracket)) {
|
494 |
+
const key = parseExpression();
|
495 |
+
expect(TOKEN_TYPES.Colon, "Expected colon between key and value in object literal");
|
496 |
+
const value = parseExpression();
|
497 |
+
values.set(key, value);
|
498 |
+
|
499 |
+
if (is(TOKEN_TYPES.Comma)) {
|
500 |
+
++current; // consume comma
|
501 |
+
}
|
502 |
+
}
|
503 |
+
++current; // consume closing curly bracket
|
504 |
+
|
505 |
+
return new ObjectLiteral(values);
|
506 |
+
}
|
507 |
default:
|
508 |
throw new SyntaxError(`Unexpected token: ${token.type}`);
|
509 |
}
|
packages/jinja/src/runtime.ts
CHANGED
@@ -2,6 +2,7 @@ import type {
|
|
2 |
NumericLiteral,
|
3 |
StringLiteral,
|
4 |
BooleanLiteral,
|
|
|
5 |
Statement,
|
6 |
Program,
|
7 |
If,
|
@@ -16,6 +17,7 @@ import type {
|
|
16 |
UnaryExpression,
|
17 |
SliceExpression,
|
18 |
KeywordArgumentExpression,
|
|
|
19 |
} from "./ast";
|
20 |
import { slice, titleCase } from "./utils";
|
21 |
|
@@ -125,6 +127,18 @@ export class ObjectValue extends RuntimeValue<Map<string, AnyRuntimeValue>> {
|
|
125 |
override __bool__(): BooleanValue {
|
126 |
return new BooleanValue(this.value.size > 0);
|
127 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
128 |
}
|
129 |
|
130 |
/**
|
@@ -190,13 +204,61 @@ export class Environment {
|
|
190 |
],
|
191 |
]);
|
192 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
193 |
constructor(public parent?: Environment) {}
|
194 |
|
195 |
/**
|
196 |
* Set the value of a variable in the current environment.
|
197 |
*/
|
198 |
-
|
199 |
-
set(name: string, value: any): AnyRuntimeValue {
|
200 |
return this.declareVariable(name, convertToRuntimeValues(value));
|
201 |
}
|
202 |
|
@@ -215,16 +277,11 @@ export class Environment {
|
|
215 |
// }
|
216 |
|
217 |
/**
|
218 |
-
*
|
|
|
219 |
*/
|
220 |
setVariable(name: string, value: AnyRuntimeValue): AnyRuntimeValue {
|
221 |
-
|
222 |
-
try {
|
223 |
-
env = this.resolve(name);
|
224 |
-
} catch {
|
225 |
-
/* empty */
|
226 |
-
}
|
227 |
-
(env ?? this).variables.set(name, value);
|
228 |
return value;
|
229 |
}
|
230 |
|
@@ -247,7 +304,11 @@ export class Environment {
|
|
247 |
}
|
248 |
|
249 |
lookupVariable(name: string): AnyRuntimeValue {
|
250 |
-
|
|
|
|
|
|
|
|
|
251 |
}
|
252 |
}
|
253 |
|
@@ -318,6 +379,12 @@ export class Interpreter {
|
|
318 |
case "<=":
|
319 |
return new BooleanValue(left.value <= right.value);
|
320 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
321 |
} else if (right instanceof ArrayValue) {
|
322 |
const member = right.value.find((x) => x.value === left.value) !== undefined;
|
323 |
switch (node.operator.value) {
|
@@ -345,6 +412,15 @@ export class Interpreter {
|
|
345 |
}
|
346 |
}
|
347 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
348 |
throw new SyntaxError(`Unknown operator "${node.operator.value}" between ${left.type} and ${right.type}`);
|
349 |
}
|
350 |
|
@@ -365,62 +441,118 @@ export class Interpreter {
|
|
365 |
// return filter.value([operand], environment);
|
366 |
|
367 |
// https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-filters
|
368 |
-
|
369 |
-
|
370 |
-
|
371 |
-
|
372 |
-
|
373 |
-
|
374 |
-
|
375 |
-
|
376 |
-
|
377 |
-
|
378 |
-
|
379 |
-
|
380 |
-
|
381 |
-
|
382 |
-
|
383 |
-
|
384 |
-
|
385 |
-
|
386 |
-
|
387 |
-
|
388 |
-
|
389 |
-
|
390 |
-
|
391 |
-
|
392 |
-
|
393 |
-
|
394 |
-
|
395 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
396 |
}
|
397 |
-
|
398 |
-
|
399 |
-
|
400 |
-
|
401 |
-
|
402 |
-
|
403 |
-
case "lower":
|
404 |
-
return new StringValue(operand.value.toLowerCase());
|
405 |
-
case "title":
|
406 |
-
return new StringValue(titleCase(operand.value));
|
407 |
-
case "capitalize":
|
408 |
-
return new StringValue(operand.value.charAt(0).toUpperCase() + operand.value.slice(1));
|
409 |
-
case "trim":
|
410 |
-
return new StringValue(operand.value.trim());
|
411 |
-
default:
|
412 |
-
throw new Error(`Unknown StringValue filter: ${node.filter.value}`);
|
413 |
}
|
414 |
-
|
415 |
-
|
416 |
-
|
417 |
-
|
418 |
-
|
419 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
420 |
}
|
421 |
}
|
422 |
-
|
423 |
-
throw new Error(`Cannot apply filter "${node.filter.value}" to type: ${operand.type}`);
|
424 |
}
|
425 |
|
426 |
/**
|
@@ -431,65 +563,13 @@ export class Interpreter {
|
|
431 |
// https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-tests
|
432 |
//
|
433 |
// TODO: Add support for non-identifier tests. e.g., divisibleby(number)
|
|
|
434 |
|
435 |
-
const
|
436 |
-
|
437 |
-
|
438 |
-
|
439 |
-
|
440 |
-
case "boolean":
|
441 |
-
return operand.type === "BooleanValue";
|
442 |
-
case "callable":
|
443 |
-
return operand instanceof FunctionValue;
|
444 |
-
case "odd":
|
445 |
-
if (operand.type !== "NumericValue") {
|
446 |
-
throw new Error(`Cannot apply test "odd" to type: ${operand.type}`);
|
447 |
-
}
|
448 |
-
return (operand as NumericValue).value % 2 !== 0;
|
449 |
-
case "even":
|
450 |
-
if (operand.type !== "NumericValue") {
|
451 |
-
throw new Error(`Cannot apply test "even" to type: ${operand.type}`);
|
452 |
-
}
|
453 |
-
return (operand as NumericValue).value % 2 === 0;
|
454 |
-
case "false":
|
455 |
-
return operand.type === "BooleanValue" && !(operand as BooleanValue).value;
|
456 |
-
case "true":
|
457 |
-
return operand.type === "BooleanValue" && (operand as BooleanValue).value;
|
458 |
-
case "number":
|
459 |
-
return operand.type === "NumericValue";
|
460 |
-
case "integer":
|
461 |
-
return operand.type === "NumericValue" && Number.isInteger((operand as NumericValue).value);
|
462 |
-
case "iterable":
|
463 |
-
return operand instanceof ArrayValue || operand instanceof StringValue;
|
464 |
-
case "lower": {
|
465 |
-
const str = (operand as StringValue).value;
|
466 |
-
return operand.type === "StringValue" && str === str.toLowerCase();
|
467 |
-
}
|
468 |
-
case "upper": {
|
469 |
-
const str = (operand as StringValue).value;
|
470 |
-
return operand.type === "StringValue" && str === str.toUpperCase();
|
471 |
-
}
|
472 |
-
case "none":
|
473 |
-
return operand.type === "NullValue";
|
474 |
-
case "defined":
|
475 |
-
return true;
|
476 |
-
case "undefined":
|
477 |
-
return false;
|
478 |
-
}
|
479 |
-
throw new Error(`Unknown test: ${node.test.value}`);
|
480 |
-
} catch (e) {
|
481 |
-
if (node.operand.type === "Identifier") {
|
482 |
-
// Special cases where we want to check if a variable is defined
|
483 |
-
if (node.test.value === "defined") {
|
484 |
-
return false;
|
485 |
-
} else if (node.test.value === "undefined") {
|
486 |
-
return true;
|
487 |
-
}
|
488 |
-
}
|
489 |
-
throw e;
|
490 |
-
}
|
491 |
-
})();
|
492 |
-
|
493 |
return new BooleanValue(node.negate ? !result : result);
|
494 |
}
|
495 |
|
@@ -519,7 +599,7 @@ export class Interpreter {
|
|
519 |
for (const statement of statements) {
|
520 |
const lastEvaluated = this.evaluate(statement, environment);
|
521 |
|
522 |
-
if (lastEvaluated.type !== "NullValue") {
|
523 |
result += lastEvaluated.value;
|
524 |
}
|
525 |
}
|
@@ -623,10 +703,8 @@ export class Interpreter {
|
|
623 |
}
|
624 |
value = object.builtins.get(property.value);
|
625 |
}
|
626 |
-
|
627 |
-
|
628 |
-
}
|
629 |
-
return value;
|
630 |
}
|
631 |
|
632 |
private evaluateSet(node: SetStatement, environment: Environment): NullValue {
|
@@ -719,6 +797,19 @@ export class Interpreter {
|
|
719 |
return new StringValue((statement as StringLiteral).value);
|
720 |
case "BooleanLiteral":
|
721 |
return new BooleanValue((statement as BooleanLiteral).value);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
722 |
case "Identifier":
|
723 |
return this.evaluateIdentifier(statement as Identifier, environment);
|
724 |
case "CallExpression":
|
|
|
2 |
NumericLiteral,
|
3 |
StringLiteral,
|
4 |
BooleanLiteral,
|
5 |
+
ArrayLiteral,
|
6 |
Statement,
|
7 |
Program,
|
8 |
If,
|
|
|
17 |
UnaryExpression,
|
18 |
SliceExpression,
|
19 |
KeywordArgumentExpression,
|
20 |
+
ObjectLiteral,
|
21 |
} from "./ast";
|
22 |
import { slice, titleCase } from "./utils";
|
23 |
|
|
|
127 |
override __bool__(): BooleanValue {
|
128 |
return new BooleanValue(this.value.size > 0);
|
129 |
}
|
130 |
+
|
131 |
+
override builtins: Map<string, AnyRuntimeValue> = new Map<string, AnyRuntimeValue>([
|
132 |
+
[
|
133 |
+
"get",
|
134 |
+
new FunctionValue(([key, defaultValue]) => {
|
135 |
+
if (!(key instanceof StringValue)) {
|
136 |
+
throw new Error(`Object key must be a string: got ${key.type}`);
|
137 |
+
}
|
138 |
+
return this.value.get(key.value) ?? defaultValue ?? new NullValue();
|
139 |
+
}),
|
140 |
+
],
|
141 |
+
]);
|
142 |
}
|
143 |
|
144 |
/**
|
|
|
204 |
],
|
205 |
]);
|
206 |
|
207 |
+
/**
|
208 |
+
* The tests available in this environment.
|
209 |
+
*/
|
210 |
+
tests: Map<string, (...value: AnyRuntimeValue[]) => boolean> = new Map([
|
211 |
+
["boolean", (operand) => operand.type === "BooleanValue"],
|
212 |
+
["callable", (operand) => operand instanceof FunctionValue],
|
213 |
+
[
|
214 |
+
"odd",
|
215 |
+
(operand) => {
|
216 |
+
if (operand.type !== "NumericValue") {
|
217 |
+
throw new Error(`Cannot apply test "odd" to type: ${operand.type}`);
|
218 |
+
}
|
219 |
+
return (operand as NumericValue).value % 2 !== 0;
|
220 |
+
},
|
221 |
+
],
|
222 |
+
[
|
223 |
+
"even",
|
224 |
+
(operand) => {
|
225 |
+
if (operand.type !== "NumericValue") {
|
226 |
+
throw new Error(`Cannot apply test "even" to type: ${operand.type}`);
|
227 |
+
}
|
228 |
+
return (operand as NumericValue).value % 2 === 0;
|
229 |
+
},
|
230 |
+
],
|
231 |
+
["false", (operand) => operand.type === "BooleanValue" && !(operand as BooleanValue).value],
|
232 |
+
["true", (operand) => operand.type === "BooleanValue" && (operand as BooleanValue).value],
|
233 |
+
["number", (operand) => operand.type === "NumericValue"],
|
234 |
+
["integer", (operand) => operand.type === "NumericValue" && Number.isInteger((operand as NumericValue).value)],
|
235 |
+
["iterable", (operand) => operand instanceof ArrayValue || operand instanceof StringValue],
|
236 |
+
[
|
237 |
+
"lower",
|
238 |
+
(operand) => {
|
239 |
+
const str = (operand as StringValue).value;
|
240 |
+
return operand.type === "StringValue" && str === str.toLowerCase();
|
241 |
+
},
|
242 |
+
],
|
243 |
+
[
|
244 |
+
"upper",
|
245 |
+
(operand) => {
|
246 |
+
const str = (operand as StringValue).value;
|
247 |
+
return operand.type === "StringValue" && str === str.toUpperCase();
|
248 |
+
},
|
249 |
+
],
|
250 |
+
["none", (operand) => operand.type === "NullValue"],
|
251 |
+
["defined", (operand) => operand.type !== "UndefinedValue"],
|
252 |
+
["undefined", (operand) => operand.type === "UndefinedValue"],
|
253 |
+
["equalto", (a, b) => a.value === b.value],
|
254 |
+
]);
|
255 |
+
|
256 |
constructor(public parent?: Environment) {}
|
257 |
|
258 |
/**
|
259 |
* Set the value of a variable in the current environment.
|
260 |
*/
|
261 |
+
set(name: string, value: unknown): AnyRuntimeValue {
|
|
|
262 |
return this.declareVariable(name, convertToRuntimeValues(value));
|
263 |
}
|
264 |
|
|
|
277 |
// }
|
278 |
|
279 |
/**
|
280 |
+
* Set variable in the current scope.
|
281 |
+
* See https://jinja.palletsprojects.com/en/3.0.x/templates/#assignments for more information.
|
282 |
*/
|
283 |
setVariable(name: string, value: AnyRuntimeValue): AnyRuntimeValue {
|
284 |
+
this.variables.set(name, value);
|
|
|
|
|
|
|
|
|
|
|
|
|
285 |
return value;
|
286 |
}
|
287 |
|
|
|
304 |
}
|
305 |
|
306 |
lookupVariable(name: string): AnyRuntimeValue {
|
307 |
+
try {
|
308 |
+
return this.resolve(name).variables.get(name) ?? new UndefinedValue();
|
309 |
+
} catch {
|
310 |
+
return new UndefinedValue();
|
311 |
+
}
|
312 |
}
|
313 |
}
|
314 |
|
|
|
379 |
case "<=":
|
380 |
return new BooleanValue(left.value <= right.value);
|
381 |
}
|
382 |
+
} else if (left instanceof ArrayValue && right instanceof ArrayValue) {
|
383 |
+
// Evaluate array operands with binary operator.
|
384 |
+
switch (node.operator.value) {
|
385 |
+
case "+":
|
386 |
+
return new ArrayValue(left.value.concat(right.value));
|
387 |
+
}
|
388 |
} else if (right instanceof ArrayValue) {
|
389 |
const member = right.value.find((x) => x.value === left.value) !== undefined;
|
390 |
switch (node.operator.value) {
|
|
|
412 |
}
|
413 |
}
|
414 |
|
415 |
+
if (left instanceof StringValue && right instanceof ObjectValue) {
|
416 |
+
switch (node.operator.value) {
|
417 |
+
case "in":
|
418 |
+
return new BooleanValue(right.value.has(left.value));
|
419 |
+
case "not in":
|
420 |
+
return new BooleanValue(!right.value.has(left.value));
|
421 |
+
}
|
422 |
+
}
|
423 |
+
|
424 |
throw new SyntaxError(`Unknown operator "${node.operator.value}" between ${left.type} and ${right.type}`);
|
425 |
}
|
426 |
|
|
|
441 |
// return filter.value([operand], environment);
|
442 |
|
443 |
// https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-filters
|
444 |
+
|
445 |
+
if (node.filter.type === "Identifier") {
|
446 |
+
const filter = node.filter as Identifier;
|
447 |
+
|
448 |
+
if (operand instanceof ArrayValue) {
|
449 |
+
switch (filter.value) {
|
450 |
+
case "list":
|
451 |
+
return operand;
|
452 |
+
case "first":
|
453 |
+
return operand.value[0];
|
454 |
+
case "last":
|
455 |
+
return operand.value[operand.value.length - 1];
|
456 |
+
case "length":
|
457 |
+
return new NumericValue(operand.value.length);
|
458 |
+
case "reverse":
|
459 |
+
return new ArrayValue(operand.value.reverse());
|
460 |
+
case "sort":
|
461 |
+
return new ArrayValue(
|
462 |
+
operand.value.sort((a, b) => {
|
463 |
+
if (a.type !== b.type) {
|
464 |
+
throw new Error(`Cannot compare different types: ${a.type} and ${b.type}`);
|
465 |
+
}
|
466 |
+
switch (a.type) {
|
467 |
+
case "NumericValue":
|
468 |
+
return (a as NumericValue).value - (b as NumericValue).value;
|
469 |
+
case "StringValue":
|
470 |
+
return (a as StringValue).value.localeCompare((b as StringValue).value);
|
471 |
+
default:
|
472 |
+
throw new Error(`Cannot compare type: ${a.type}`);
|
473 |
+
}
|
474 |
+
})
|
475 |
+
);
|
476 |
+
default:
|
477 |
+
throw new Error(`Unknown ArrayValue filter: ${filter.value}`);
|
478 |
+
}
|
479 |
+
} else if (operand instanceof StringValue) {
|
480 |
+
switch (filter.value) {
|
481 |
+
case "length":
|
482 |
+
return new NumericValue(operand.value.length);
|
483 |
+
case "upper":
|
484 |
+
return new StringValue(operand.value.toUpperCase());
|
485 |
+
case "lower":
|
486 |
+
return new StringValue(operand.value.toLowerCase());
|
487 |
+
case "title":
|
488 |
+
return new StringValue(titleCase(operand.value));
|
489 |
+
case "capitalize":
|
490 |
+
return new StringValue(operand.value.charAt(0).toUpperCase() + operand.value.slice(1));
|
491 |
+
case "trim":
|
492 |
+
return new StringValue(operand.value.trim());
|
493 |
+
default:
|
494 |
+
throw new Error(`Unknown StringValue filter: ${filter.value}`);
|
495 |
+
}
|
496 |
+
} else if (operand instanceof NumericValue) {
|
497 |
+
switch (filter.value) {
|
498 |
+
case "abs":
|
499 |
+
return new NumericValue(Math.abs(operand.value));
|
500 |
+
default:
|
501 |
+
throw new Error(`Unknown NumericValue filter: ${filter.value}`);
|
502 |
+
}
|
503 |
}
|
504 |
+
throw new Error(`Cannot apply filter "${filter.value}" to type: ${operand.type}`);
|
505 |
+
} else if (node.filter.type === "CallExpression") {
|
506 |
+
const filter = node.filter as CallExpression;
|
507 |
+
|
508 |
+
if (filter.callee.type !== "Identifier") {
|
509 |
+
throw new Error(`Unknown filter: ${filter.callee.type}`);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
510 |
}
|
511 |
+
const filterName = (filter.callee as Identifier).value;
|
512 |
+
|
513 |
+
if (operand instanceof ArrayValue) {
|
514 |
+
switch (filterName) {
|
515 |
+
case "selectattr": {
|
516 |
+
if (operand.value.some((x) => !(x instanceof ObjectValue))) {
|
517 |
+
throw new Error("`selectattr` can only be applied to array of objects");
|
518 |
+
}
|
519 |
+
if (filter.args.some((x) => x.type !== "StringLiteral")) {
|
520 |
+
throw new Error("arguments of `selectattr` must be strings");
|
521 |
+
}
|
522 |
+
|
523 |
+
const [attr, testName, value] = filter.args.map((x) => this.evaluate(x, environment)) as StringValue[];
|
524 |
+
|
525 |
+
let testFunction: (...x: AnyRuntimeValue[]) => boolean;
|
526 |
+
if (testName) {
|
527 |
+
// Get the test function from the environment
|
528 |
+
const test = environment.tests.get(testName.value);
|
529 |
+
if (!test) {
|
530 |
+
throw new Error(`Unknown test: ${testName.value}`);
|
531 |
+
}
|
532 |
+
testFunction = test;
|
533 |
+
} else {
|
534 |
+
// Default to truthiness of first argument
|
535 |
+
testFunction = (...x: AnyRuntimeValue[]) => x[0].__bool__().value;
|
536 |
+
}
|
537 |
+
|
538 |
+
// Filter the array using the test function
|
539 |
+
const filtered = (operand.value as ObjectValue[]).filter((item) => {
|
540 |
+
const a = item.value.get(attr.value);
|
541 |
+
if (a) {
|
542 |
+
return testFunction(a, value);
|
543 |
+
}
|
544 |
+
return false;
|
545 |
+
});
|
546 |
+
|
547 |
+
return new ArrayValue(filtered);
|
548 |
+
}
|
549 |
+
}
|
550 |
+
throw new Error(`Unknown ArrayValue filter: ${filterName}`);
|
551 |
+
} else {
|
552 |
+
throw new Error(`Cannot apply filter "${filterName}" to type: ${operand.type}`);
|
553 |
}
|
554 |
}
|
555 |
+
throw new Error(`Unknown filter: ${node.filter.type}`);
|
|
|
556 |
}
|
557 |
|
558 |
/**
|
|
|
563 |
// https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-tests
|
564 |
//
|
565 |
// TODO: Add support for non-identifier tests. e.g., divisibleby(number)
|
566 |
+
const operand = this.evaluate(node.operand, environment);
|
567 |
|
568 |
+
const test = environment.tests.get(node.test.value);
|
569 |
+
if (!test) {
|
570 |
+
throw new Error(`Unknown test: ${node.test.value}`);
|
571 |
+
}
|
572 |
+
const result = test(operand);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
573 |
return new BooleanValue(node.negate ? !result : result);
|
574 |
}
|
575 |
|
|
|
599 |
for (const statement of statements) {
|
600 |
const lastEvaluated = this.evaluate(statement, environment);
|
601 |
|
602 |
+
if (lastEvaluated.type !== "NullValue" && lastEvaluated.type !== "UndefinedValue") {
|
603 |
result += lastEvaluated.value;
|
604 |
}
|
605 |
}
|
|
|
703 |
}
|
704 |
value = object.builtins.get(property.value);
|
705 |
}
|
706 |
+
|
707 |
+
return value instanceof RuntimeValue ? value : new UndefinedValue();
|
|
|
|
|
708 |
}
|
709 |
|
710 |
private evaluateSet(node: SetStatement, environment: Environment): NullValue {
|
|
|
797 |
return new StringValue((statement as StringLiteral).value);
|
798 |
case "BooleanLiteral":
|
799 |
return new BooleanValue((statement as BooleanLiteral).value);
|
800 |
+
case "ArrayLiteral":
|
801 |
+
return new ArrayValue((statement as ArrayLiteral).value.map((x) => this.evaluate(x, environment)));
|
802 |
+
case "ObjectLiteral": {
|
803 |
+
const mapping = new Map();
|
804 |
+
for (const [key, value] of (statement as ObjectLiteral).value) {
|
805 |
+
const evaluatedKey = this.evaluate(key, environment);
|
806 |
+
if (!(evaluatedKey instanceof StringValue)) {
|
807 |
+
throw new Error(`Object keys must be strings: got ${evaluatedKey.type}`);
|
808 |
+
}
|
809 |
+
mapping.set(evaluatedKey.value, this.evaluate(value, environment));
|
810 |
+
}
|
811 |
+
return new ObjectValue(mapping);
|
812 |
+
}
|
813 |
case "Identifier":
|
814 |
return this.evaluateIdentifier(statement as Identifier, environment);
|
815 |
case "CallExpression":
|
packages/jinja/test/e2e.test.js
CHANGED
@@ -18,6 +18,64 @@ const EXAMPLE_CHAT_WITH_SYSTEM = [
|
|
18 |
...EXAMPLE_CHAT,
|
19 |
];
|
20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
21 |
/**
|
22 |
* Defined in https://github.com/huggingface/transformers
|
23 |
* Keys correspond to `model_type` in the transformers repo.
|
@@ -286,6 +344,38 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({
|
|
286 |
},
|
287 |
target: `<|begin▁of▁sentence|>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\nHello, how are you?\n### Response:\nI'm doing great. How can I help you today?\n<|EOT|>\n### Instruction:\nI'd like to show off how chat templating works!\n`,
|
288 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
289 |
});
|
290 |
|
291 |
describe("End-to-end tests", () => {
|
|
|
18 |
...EXAMPLE_CHAT,
|
19 |
];
|
20 |
|
21 |
+
const EXAMPLE_FUNCTION_CALLING = [
|
22 |
+
{
|
23 |
+
role: "assistant",
|
24 |
+
content: null,
|
25 |
+
tool_calls: [
|
26 |
+
{
|
27 |
+
type: "function",
|
28 |
+
function: {
|
29 |
+
name: "get_current_weather",
|
30 |
+
arguments: '{\n "location": "Hanoi"\n}',
|
31 |
+
},
|
32 |
+
},
|
33 |
+
],
|
34 |
+
},
|
35 |
+
{ role: "user", content: "what's the weather like in Hanoi?" },
|
36 |
+
];
|
37 |
+
|
38 |
+
// Example adapted from https://huggingface.co/fireworks-ai/firefunction-v1
|
39 |
+
const EXAMPLE_FUNCTION_SPEC = [
|
40 |
+
{
|
41 |
+
name: "get_stock_price",
|
42 |
+
description: "Get the current stock price",
|
43 |
+
parameters: {
|
44 |
+
type: "object",
|
45 |
+
properties: {
|
46 |
+
symbol: {
|
47 |
+
type: "string",
|
48 |
+
description: "The stock symbol, e.g. AAPL, GOOG",
|
49 |
+
},
|
50 |
+
},
|
51 |
+
required: ["symbol"],
|
52 |
+
},
|
53 |
+
},
|
54 |
+
{
|
55 |
+
name: "check_word_anagram",
|
56 |
+
description: "Check if two words are anagrams of each other",
|
57 |
+
parameters: {
|
58 |
+
type: "object",
|
59 |
+
properties: {
|
60 |
+
word1: {
|
61 |
+
type: "string",
|
62 |
+
description: "The first word",
|
63 |
+
},
|
64 |
+
word2: {
|
65 |
+
type: "string",
|
66 |
+
description: "The second word",
|
67 |
+
},
|
68 |
+
},
|
69 |
+
required: ["word1", "word2"],
|
70 |
+
},
|
71 |
+
},
|
72 |
+
];
|
73 |
+
const EXAMPLE_FUNCTION_CALLING_WITH_SYSTEM = [
|
74 |
+
{ role: "functions", content: JSON.stringify(EXAMPLE_FUNCTION_SPEC, null, 4) },
|
75 |
+
{ role: "system", content: "You are a helpful assistant with access to functions. Use them if required." },
|
76 |
+
{ role: "user", content: "Hi, can you tell me the current stock price of AAPL?" },
|
77 |
+
];
|
78 |
+
|
79 |
/**
|
80 |
* Defined in https://github.com/huggingface/transformers
|
81 |
* Keys correspond to `model_type` in the transformers repo.
|
|
|
344 |
},
|
345 |
target: `<|begin▁of▁sentence|>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\nHello, how are you?\n### Response:\nI'm doing great. How can I help you today?\n<|EOT|>\n### Instruction:\nI'd like to show off how chat templating works!\n`,
|
346 |
},
|
347 |
+
"meetkai/functionary-medium-v2.2": {
|
348 |
+
chat_template: `{#v2.2#}\n{% for message in messages %}\n{% if message['role'] == 'user' or message['role'] == 'system' %}\n{{ '<|from|>' + message['role'] + '\n<|recipient|>all\n<|content|>' + message['content'] + '\n' }}{% elif message['role'] == 'tool' %}\n{{ '<|from|>' + message['name'] + '\n<|recipient|>all\n<|content|>' + message['content'] + '\n' }}{% else %}\n{% set contain_content='no'%}\n{% if message['content'] is not none %}\n{{ '<|from|>assistant\n<|recipient|>all\n<|content|>' + message['content'] }}{% set contain_content='yes'%}\n{% endif %}\n{% if 'tool_calls' in message and message['tool_calls'] is not none %}\n{% for tool_call in message['tool_calls'] %}\n{% set prompt='<|from|>assistant\n<|recipient|>' + tool_call['function']['name'] + '\n<|content|>' + tool_call['function']['arguments'] %}\n{% if loop.index == 1 and contain_content == "no" %}\n{{ prompt }}{% else %}\n{{ '\n' + prompt}}{% endif %}\n{% endfor %}\n{% endif %}\n{{ '<|stop|>\n' }}{% endif %}\n{% endfor %}\n{% if add_generation_prompt %}{{ '<|from|>assistant\n<|recipient|>' }}{% endif %}`,
|
349 |
+
data: {
|
350 |
+
messages: EXAMPLE_FUNCTION_CALLING,
|
351 |
+
bos_token: "<s>",
|
352 |
+
eos_token: "</s>",
|
353 |
+
add_generation_prompt: false,
|
354 |
+
},
|
355 |
+
target: `<|from|>assistant\n<|recipient|>get_current_weather\n<|content|>{\n "location": "Hanoi"\n}<|stop|>\n<|from|>user\n<|recipient|>all\n<|content|>what's the weather like in Hanoi?\n`,
|
356 |
+
},
|
357 |
+
"fireworks-ai/firefunction-v1": {
|
358 |
+
chat_template: `{%- set message_roles = ['SYSTEM', 'FUNCTIONS', 'USER', 'ASSISTANT', 'TOOL'] -%}\n{%- set ns = namespace(seen_non_system=false, messages=messages, content='', functions=[]) -%}\n{{ bos_token }}\n{#- Basic consistency checks -#}\n{%- if not ns.messages -%}\n {{ raise_exception('No messages') }}\n{%- endif -%}\n{%- if ns.messages[0]['role'] | upper != 'SYSTEM' -%}\n {%- set ns.messages = [{'role': 'SYSTEM', 'content': 'You are a helpful assistant with access to functions. Use them if required.'}] + ns.messages -%}\n{%- endif -%}\n{%- if ns.messages | length < 2 or ns.messages[0]['role'] | upper != 'SYSTEM' or ns.messages[1]['role'] | upper != 'FUNCTIONS' -%}\n {{ raise_exception('Expected either "functions" or ["system", "functions"] as the first messages') }}\n{%- endif -%}\n{%- for message in ns.messages -%}\n {%- set role = message['role'] | upper -%}\n {#- Validation -#}\n {%- if role not in message_roles -%}\n {{ raise_exception('Invalid role ' + message['role'] + '. Only ' + message_roles + ' are supported.') }}\n {%- endif -%}\n {%- set ns.content = message['content'] if message.get('content') else '' -%}\n {#- Move tool calls inside the content -#}\n {%- if 'tool_calls' in message -%}\n {%- for call in message['tool_calls'] -%}\n {%- set ns.content = ns.content + '<functioncall>{"name": "' + call['function']['name'] + '", "arguments": ' + call['function']['arguments'] + '}' -%}\n {%- endfor -%}\n {%- endif -%}\n {%- if role == 'ASSISTANT' and '<functioncall>' not in ns.content -%}\n {%- set ns.content = '<plain>' + ns.content -%}\n {%- endif -%}\n {%- if role == 'ASSISTANT' -%}\n {%- set ns.content = ns.content + eos_token -%}\n {%- endif -%}\n {{ role }}: {{ ns.content }}{{ '\\n\\n' }}\n{%- endfor -%}\nASSISTANT:{{ ' ' }}\n`,
|
359 |
+
data: {
|
360 |
+
messages: EXAMPLE_FUNCTION_CALLING_WITH_SYSTEM,
|
361 |
+
bos_token: "<s>",
|
362 |
+
eos_token: "</s>",
|
363 |
+
add_generation_prompt: false,
|
364 |
+
},
|
365 |
+
target: `<s>SYSTEM: You are a helpful assistant with access to functions. Use them if required.\n\nFUNCTIONS: [\n {\n "name": "get_stock_price",\n "description": "Get the current stock price",\n "parameters": {\n "type": "object",\n "properties": {\n "symbol": {\n "type": "string",\n "description": "The stock symbol, e.g. AAPL, GOOG"\n }\n },\n "required": [\n "symbol"\n ]\n }\n },\n {\n "name": "check_word_anagram",\n "description": "Check if two words are anagrams of each other",\n "parameters": {\n "type": "object",\n "properties": {\n "word1": {\n "type": "string",\n "description": "The first word"\n },\n "word2": {\n "type": "string",\n "description": "The second word"\n }\n },\n "required": [\n "word1",\n "word2"\n ]\n }\n }\n]\n\nSYSTEM: You are a helpful assistant with access to functions. Use them if required.\n\nUSER: Hi, can you tell me the current stock price of AAPL?\n\nASSISTANT: `,
|
366 |
+
},
|
367 |
+
"maywell/PiVoT-MoE": {
|
368 |
+
chat_template: `{{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }}{% for message in messages %}{% if message['role'] == 'system' %}{{ message['content']|trim }}{% elif message['role'] == 'user' %}### Instruction: {{ message['content']|trim }}{% elif message['role'] == 'assistant' %}### Response: {{ message['content']|trim }}{% elif message['role'] == 'user_context' %}### Input: {{ message['content']|trim }}{% endif %}{% if not loop.last %}\n{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}### Response:{% endif %}`,
|
369 |
+
data: {
|
370 |
+
messages: EXAMPLE_CHAT_WITH_SYSTEM,
|
371 |
+
bos_token: "<s>",
|
372 |
+
eos_token: "</s>",
|
373 |
+
add_generation_prompt: false,
|
374 |
+
},
|
375 |
+
// NOTE: There is a bug in the model's chat template which causes the system prompt
|
376 |
+
// to be repeated twice. We replicate this behaviour here.
|
377 |
+
target: `You are a friendly chatbot who always responds in the style of a pirateYou are a friendly chatbot who always responds in the style of a pirate### Instruction: Hello, how are you?### Response: I'm doing great. How can I help you today?### Instruction: I'd like to show off how chat templating works!`,
|
378 |
+
},
|
379 |
});
|
380 |
|
381 |
describe("End-to-end tests", () => {
|
packages/jinja/test/interpreter.test.js
CHANGED
@@ -12,6 +12,7 @@ describe("Test interpreter options", () => {
|
|
12 |
const EXAMPLE_FOR_TEMPLATE_2 = `{% for item in seq -%}\n {{ item }}\n{% endfor %}`;
|
13 |
const EXAMPLE_FOR_TEMPLATE_3 = `{% for item in seq %}\n {{ item }}\n{%- endfor %}`;
|
14 |
const EXAMPLE_FOR_TEMPLATE_4 = `{% for item in seq -%}\n {{ item }}\n{%- endfor %}`;
|
|
|
15 |
|
16 |
const seq = [1, 2, 3, 4, 5, 6, 7, 8, 9];
|
17 |
|
@@ -98,6 +99,32 @@ describe("Test interpreter options", () => {
|
|
98 |
options: {},
|
99 |
target: `123456789`,
|
100 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
101 |
];
|
102 |
|
103 |
for (const test of TESTS) {
|
|
|
12 |
const EXAMPLE_FOR_TEMPLATE_2 = `{% for item in seq -%}\n {{ item }}\n{% endfor %}`;
|
13 |
const EXAMPLE_FOR_TEMPLATE_3 = `{% for item in seq %}\n {{ item }}\n{%- endfor %}`;
|
14 |
const EXAMPLE_FOR_TEMPLATE_4 = `{% for item in seq -%}\n {{ item }}\n{%- endfor %}`;
|
15 |
+
const EXAMPLE_COMMENT_TEMPLATE = ` {# comment #}\n {# {% if true %} {% endif %} #}\n`;
|
16 |
|
17 |
const seq = [1, 2, 3, 4, 5, 6, 7, 8, 9];
|
18 |
|
|
|
99 |
options: {},
|
100 |
target: `123456789`,
|
101 |
},
|
102 |
+
|
103 |
+
// Comment tests
|
104 |
+
{
|
105 |
+
template: EXAMPLE_COMMENT_TEMPLATE,
|
106 |
+
data: {},
|
107 |
+
options: {},
|
108 |
+
target: ` \n `,
|
109 |
+
},
|
110 |
+
{
|
111 |
+
template: EXAMPLE_COMMENT_TEMPLATE,
|
112 |
+
data: {},
|
113 |
+
options: { lstrip_blocks: true },
|
114 |
+
target: `\n`,
|
115 |
+
},
|
116 |
+
{
|
117 |
+
template: EXAMPLE_COMMENT_TEMPLATE,
|
118 |
+
data: {},
|
119 |
+
options: { trim_blocks: true },
|
120 |
+
target: ` `,
|
121 |
+
},
|
122 |
+
{
|
123 |
+
template: EXAMPLE_COMMENT_TEMPLATE,
|
124 |
+
data: {},
|
125 |
+
options: { lstrip_blocks: true, trim_blocks: true },
|
126 |
+
target: ``,
|
127 |
+
},
|
128 |
];
|
129 |
|
130 |
for (const test of TESTS) {
|
packages/jinja/test/templates.test.js
CHANGED
@@ -39,6 +39,8 @@ const TEST_STRINGS = {
|
|
39 |
|
40 |
// Strings
|
41 |
STRINGS: `{{ 'Bye' }}{{ bos_token + '[INST] ' }}`,
|
|
|
|
|
42 |
|
43 |
// Function calls
|
44 |
FUNCTIONS: `{{ func() }}{{ func(apple) }}{{ func(x, 'test', 2, false) }}`,
|
@@ -72,6 +74,8 @@ const TEST_STRINGS = {
|
|
72 |
FILTER_OPERATOR: `{{ arr | length }}{{ 1 + arr | length }}{{ 2 + arr | sort | length }}{{ (arr | sort)[0] }}`,
|
73 |
FILTER_OPERATOR_2: `|{{ 'abc' | length }}|{{ 'aBcD' | upper }}|{{ 'aBcD' | lower }}|{{ 'test test' | capitalize}}|{{ 'test test' | title }}|{{ ' a b ' | trim }}|{{ ' A B ' | trim | lower | length }}|`,
|
74 |
FILTER_OPERATOR_3: `|{{ -1 | abs }}|{{ 1 | abs }}|`,
|
|
|
|
|
75 |
|
76 |
// Logical operators between non-Booleans
|
77 |
BOOLEAN_NUMERICAL: `|{{ 1 and 2 }}|{{ 1 and 0 }}|{{ 0 and 1 }}|{{ 0 and 0 }}|{{ 1 or 2 }}|{{ 1 or 0 }}|{{ 0 or 1 }}|{{ 0 or 0 }}|{{ not 1 }}|{{ not 0 }}|`,
|
@@ -95,6 +99,30 @@ const TEST_STRINGS = {
|
|
95 |
NAMESPACE: `{% set ns = namespace() %}{% set ns.foo = 'bar' %}{{ ns.foo }}`,
|
96 |
NAMESPACE_1: `{% set ns = namespace(default=false) %}{{ ns.default }}`,
|
97 |
NAMESPACE_2: `{% set ns = namespace(default=false, number=1+1) %}|{{ ns.default }}|{{ ns.number }}|`,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98 |
};
|
99 |
|
100 |
const TEST_PARSED = {
|
@@ -613,6 +641,62 @@ const TEST_PARSED = {
|
|
613 |
{ value: "[INST] ", type: "StringLiteral" },
|
614 |
{ value: "}}", type: "CloseExpression" },
|
615 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
616 |
|
617 |
// Function calls
|
618 |
FUNCTIONS: [
|
@@ -1231,6 +1315,34 @@ const TEST_PARSED = {
|
|
1231 |
{ value: "}}", type: "CloseExpression" },
|
1232 |
{ value: "|", type: "Text" },
|
1233 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1234 |
|
1235 |
// Logical operators between non-Booleans
|
1236 |
BOOLEAN_NUMERICAL: [
|
@@ -1777,6 +1889,303 @@ const TEST_PARSED = {
|
|
1777 |
{ value: "}}", type: "CloseExpression" },
|
1778 |
{ value: "|", type: "Text" },
|
1779 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1780 |
};
|
1781 |
|
1782 |
const TEST_CONTEXT = {
|
@@ -1825,6 +2234,8 @@ const TEST_CONTEXT = {
|
|
1825 |
STRINGS: {
|
1826 |
bos_token: "<s>",
|
1827 |
},
|
|
|
|
|
1828 |
|
1829 |
// Function calls
|
1830 |
FUNCTIONS: {
|
@@ -1887,6 +2298,12 @@ const TEST_CONTEXT = {
|
|
1887 |
},
|
1888 |
FILTER_OPERATOR_2: {},
|
1889 |
FILTER_OPERATOR_3: {},
|
|
|
|
|
|
|
|
|
|
|
|
|
1890 |
|
1891 |
// Logical operators between non-Booleans
|
1892 |
BOOLEAN_NUMERICAL: {},
|
@@ -1914,6 +2331,40 @@ const TEST_CONTEXT = {
|
|
1914 |
NAMESPACE: {},
|
1915 |
NAMESPACE_1: {},
|
1916 |
NAMESPACE_2: {},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1917 |
};
|
1918 |
|
1919 |
const EXPECTED_OUTPUTS = {
|
@@ -1951,6 +2402,8 @@ const EXPECTED_OUTPUTS = {
|
|
1951 |
|
1952 |
// Strings
|
1953 |
STRINGS: "Bye<s>[INST] ",
|
|
|
|
|
1954 |
|
1955 |
// Function calls
|
1956 |
FUNCTIONS: "014",
|
@@ -1984,6 +2437,8 @@ const EXPECTED_OUTPUTS = {
|
|
1984 |
FILTER_OPERATOR: `3451`,
|
1985 |
FILTER_OPERATOR_2: `|3|ABCD|abcd|Test test|Test Test|a b|4|`,
|
1986 |
FILTER_OPERATOR_3: `|1|1|`,
|
|
|
|
|
1987 |
|
1988 |
// Logical operators between non-Booleans
|
1989 |
BOOLEAN_NUMERICAL: `|2|0|0|0|1|1|1|0|false|true|`,
|
@@ -2007,6 +2462,30 @@ const EXPECTED_OUTPUTS = {
|
|
2007 |
NAMESPACE: `bar`,
|
2008 |
NAMESPACE_1: `false`,
|
2009 |
NAMESPACE_2: `|false|2|`,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010 |
};
|
2011 |
|
2012 |
describe("Templates", () => {
|
@@ -2080,6 +2559,11 @@ describe("Error checking", () => {
|
|
2080 |
const text = "{{ invalid ! invalid }}";
|
2081 |
expect(() => tokenize(text)).toThrowError();
|
2082 |
});
|
|
|
|
|
|
|
|
|
|
|
2083 |
});
|
2084 |
|
2085 |
describe("Parsing errors", () => {
|
@@ -2127,22 +2611,6 @@ describe("Error checking", () => {
|
|
2127 |
});
|
2128 |
|
2129 |
describe("Runtime errors", () => {
|
2130 |
-
it("Undefined variable", () => {
|
2131 |
-
const env = new Environment();
|
2132 |
-
const interpreter = new Interpreter(env);
|
2133 |
-
const tokens = tokenize("{{ undefined_variable }}");
|
2134 |
-
const ast = parse(tokens);
|
2135 |
-
expect(() => interpreter.run(ast)).toThrowError();
|
2136 |
-
});
|
2137 |
-
|
2138 |
-
it("Undefined attribute access", () => {
|
2139 |
-
const env = new Environment();
|
2140 |
-
const interpreter = new Interpreter(env);
|
2141 |
-
const tokens = tokenize("{{ object.undefined_attribute }}");
|
2142 |
-
const ast = parse(tokens);
|
2143 |
-
expect(() => interpreter.run(ast)).toThrowError();
|
2144 |
-
});
|
2145 |
-
|
2146 |
it("Undefined function call", () => {
|
2147 |
const env = new Environment();
|
2148 |
const interpreter = new Interpreter(env);
|
|
|
39 |
|
40 |
// Strings
|
41 |
STRINGS: `{{ 'Bye' }}{{ bos_token + '[INST] ' }}`,
|
42 |
+
STRINGS_1: `|{{ "test" }}|{{ "a" + 'b' + "c" }}|{{ '"' + "'" }}|{{ '\\'' }}|{{ "\\"" }}|`,
|
43 |
+
STRINGS_2: `|{{ "" | length }}|{{ "a" | length }}|{{ '' | length }}|{{ 'a' | length }}|`,
|
44 |
|
45 |
// Function calls
|
46 |
FUNCTIONS: `{{ func() }}{{ func(apple) }}{{ func(x, 'test', 2, false) }}`,
|
|
|
74 |
FILTER_OPERATOR: `{{ arr | length }}{{ 1 + arr | length }}{{ 2 + arr | sort | length }}{{ (arr | sort)[0] }}`,
|
75 |
FILTER_OPERATOR_2: `|{{ 'abc' | length }}|{{ 'aBcD' | upper }}|{{ 'aBcD' | lower }}|{{ 'test test' | capitalize}}|{{ 'test test' | title }}|{{ ' a b ' | trim }}|{{ ' A B ' | trim | lower | length }}|`,
|
76 |
FILTER_OPERATOR_3: `|{{ -1 | abs }}|{{ 1 | abs }}|`,
|
77 |
+
FILTER_OPERATOR_4: `{{ items | selectattr('key') | length }}`,
|
78 |
+
FILTER_OPERATOR_5: `{{ messages | selectattr('role', 'equalto', 'system') | length }}`,
|
79 |
|
80 |
// Logical operators between non-Booleans
|
81 |
BOOLEAN_NUMERICAL: `|{{ 1 and 2 }}|{{ 1 and 0 }}|{{ 0 and 1 }}|{{ 0 and 0 }}|{{ 1 or 2 }}|{{ 1 or 0 }}|{{ 0 or 1 }}|{{ 0 or 0 }}|{{ not 1 }}|{{ not 0 }}|`,
|
|
|
99 |
NAMESPACE: `{% set ns = namespace() %}{% set ns.foo = 'bar' %}{{ ns.foo }}`,
|
100 |
NAMESPACE_1: `{% set ns = namespace(default=false) %}{{ ns.default }}`,
|
101 |
NAMESPACE_2: `{% set ns = namespace(default=false, number=1+1) %}|{{ ns.default }}|{{ ns.number }}|`,
|
102 |
+
|
103 |
+
// Object operators
|
104 |
+
OBJECT_OPERATORS: `|{{ 'known' in obj }}|{{ 'known' not in obj }}|{{ 'unknown' in obj }}|{{ 'unknown' not in obj }}|`,
|
105 |
+
OBJECT_OPERATORS_1: `|{{ obj.get('known') }}|{{ obj.get('unknown') is none }}|{{ obj.get('unknown') is defined }}|`,
|
106 |
+
|
107 |
+
// Scope
|
108 |
+
SCOPE: `{% set ns = namespace(found=false) %}{% for num in nums %}{% if num == 1 %}{{ 'found=' }}{% set ns.found = true %}{% endif %}{% endfor %}{{ ns.found }}`,
|
109 |
+
SCOPE_1: `{% set found = false %}{% for num in nums %}{% if num == 1 %}{{ 'found=' }}{% set found = true %}{% endif %}{% endfor %}{{ found }}`,
|
110 |
+
|
111 |
+
// Undefined
|
112 |
+
UNDEFINED_VARIABLES: `{{ undefined_variable }}`,
|
113 |
+
UNDEFINED_ACCESS: `{{ object.undefined_attribute }}`,
|
114 |
+
|
115 |
+
// Ternary operator
|
116 |
+
TERNARY_OPERATOR: `|{{ 'a' if true else 'b' }}|{{ 'a' if false else 'b' }}|{{ 'a' if 1 + 1 == 2 else 'b' }}|{{ 'a' if 1 + 1 == 3 or 1 * 2 == 3 else 'b' }}|`,
|
117 |
+
|
118 |
+
// Array literals
|
119 |
+
ARRAY_LITERALS: `{{ [1, true, 'hello', [1, 2, 3, 4], var] | length }}`,
|
120 |
+
|
121 |
+
// Object literals
|
122 |
+
OBJECT_LITERALS: `{{ { 'key': 'value', key: 'value2', "key3": [1, {'foo': 'bar'} ] }['key'] }}`,
|
123 |
+
|
124 |
+
// Array operators
|
125 |
+
ARRAY_OPERATORS: `{{ ([1, 2, 3] + [4, 5, 6]) | length }}`,
|
126 |
};
|
127 |
|
128 |
const TEST_PARSED = {
|
|
|
641 |
{ value: "[INST] ", type: "StringLiteral" },
|
642 |
{ value: "}}", type: "CloseExpression" },
|
643 |
],
|
644 |
+
STRINGS_1: [
|
645 |
+
{ value: "|", type: "Text" },
|
646 |
+
{ value: "{{", type: "OpenExpression" },
|
647 |
+
{ value: "test", type: "StringLiteral" },
|
648 |
+
{ value: "}}", type: "CloseExpression" },
|
649 |
+
{ value: "|", type: "Text" },
|
650 |
+
{ value: "{{", type: "OpenExpression" },
|
651 |
+
{ value: "a", type: "StringLiteral" },
|
652 |
+
{ value: "+", type: "AdditiveBinaryOperator" },
|
653 |
+
{ value: "b", type: "StringLiteral" },
|
654 |
+
{ value: "+", type: "AdditiveBinaryOperator" },
|
655 |
+
{ value: "c", type: "StringLiteral" },
|
656 |
+
{ value: "}}", type: "CloseExpression" },
|
657 |
+
{ value: "|", type: "Text" },
|
658 |
+
{ value: "{{", type: "OpenExpression" },
|
659 |
+
{ value: '"', type: "StringLiteral" },
|
660 |
+
{ value: "+", type: "AdditiveBinaryOperator" },
|
661 |
+
{ value: "'", type: "StringLiteral" },
|
662 |
+
{ value: "}}", type: "CloseExpression" },
|
663 |
+
{ value: "|", type: "Text" },
|
664 |
+
{ value: "{{", type: "OpenExpression" },
|
665 |
+
{ value: "'", type: "StringLiteral" },
|
666 |
+
{ value: "}}", type: "CloseExpression" },
|
667 |
+
{ value: "|", type: "Text" },
|
668 |
+
{ value: "{{", type: "OpenExpression" },
|
669 |
+
{ value: '"', type: "StringLiteral" },
|
670 |
+
{ value: "}}", type: "CloseExpression" },
|
671 |
+
{ value: "|", type: "Text" },
|
672 |
+
],
|
673 |
+
STRINGS_2: [
|
674 |
+
{ value: "|", type: "Text" },
|
675 |
+
{ value: "{{", type: "OpenExpression" },
|
676 |
+
{ value: "", type: "StringLiteral" },
|
677 |
+
{ value: "|", type: "Pipe" },
|
678 |
+
{ value: "length", type: "Identifier" },
|
679 |
+
{ value: "}}", type: "CloseExpression" },
|
680 |
+
{ value: "|", type: "Text" },
|
681 |
+
{ value: "{{", type: "OpenExpression" },
|
682 |
+
{ value: "a", type: "StringLiteral" },
|
683 |
+
{ value: "|", type: "Pipe" },
|
684 |
+
{ value: "length", type: "Identifier" },
|
685 |
+
{ value: "}}", type: "CloseExpression" },
|
686 |
+
{ value: "|", type: "Text" },
|
687 |
+
{ value: "{{", type: "OpenExpression" },
|
688 |
+
{ value: "", type: "StringLiteral" },
|
689 |
+
{ value: "|", type: "Pipe" },
|
690 |
+
{ value: "length", type: "Identifier" },
|
691 |
+
{ value: "}}", type: "CloseExpression" },
|
692 |
+
{ value: "|", type: "Text" },
|
693 |
+
{ value: "{{", type: "OpenExpression" },
|
694 |
+
{ value: "a", type: "StringLiteral" },
|
695 |
+
{ value: "|", type: "Pipe" },
|
696 |
+
{ value: "length", type: "Identifier" },
|
697 |
+
{ value: "}}", type: "CloseExpression" },
|
698 |
+
{ value: "|", type: "Text" },
|
699 |
+
],
|
700 |
|
701 |
// Function calls
|
702 |
FUNCTIONS: [
|
|
|
1315 |
{ value: "}}", type: "CloseExpression" },
|
1316 |
{ value: "|", type: "Text" },
|
1317 |
],
|
1318 |
+
FILTER_OPERATOR_4: [
|
1319 |
+
{ value: "{{", type: "OpenExpression" },
|
1320 |
+
{ value: "items", type: "Identifier" },
|
1321 |
+
{ value: "|", type: "Pipe" },
|
1322 |
+
{ value: "selectattr", type: "Identifier" },
|
1323 |
+
{ value: "(", type: "OpenParen" },
|
1324 |
+
{ value: "key", type: "StringLiteral" },
|
1325 |
+
{ value: ")", type: "CloseParen" },
|
1326 |
+
{ value: "|", type: "Pipe" },
|
1327 |
+
{ value: "length", type: "Identifier" },
|
1328 |
+
{ value: "}}", type: "CloseExpression" },
|
1329 |
+
],
|
1330 |
+
FILTER_OPERATOR_5: [
|
1331 |
+
{ value: "{{", type: "OpenExpression" },
|
1332 |
+
{ value: "messages", type: "Identifier" },
|
1333 |
+
{ value: "|", type: "Pipe" },
|
1334 |
+
{ value: "selectattr", type: "Identifier" },
|
1335 |
+
{ value: "(", type: "OpenParen" },
|
1336 |
+
{ value: "role", type: "StringLiteral" },
|
1337 |
+
{ value: ",", type: "Comma" },
|
1338 |
+
{ value: "equalto", type: "StringLiteral" },
|
1339 |
+
{ value: ",", type: "Comma" },
|
1340 |
+
{ value: "system", type: "StringLiteral" },
|
1341 |
+
{ value: ")", type: "CloseParen" },
|
1342 |
+
{ value: "|", type: "Pipe" },
|
1343 |
+
{ value: "length", type: "Identifier" },
|
1344 |
+
{ value: "}}", type: "CloseExpression" },
|
1345 |
+
],
|
1346 |
|
1347 |
// Logical operators between non-Booleans
|
1348 |
BOOLEAN_NUMERICAL: [
|
|
|
1889 |
{ value: "}}", type: "CloseExpression" },
|
1890 |
{ value: "|", type: "Text" },
|
1891 |
],
|
1892 |
+
|
1893 |
+
// Object operators
|
1894 |
+
OBJECT_OPERATORS: [
|
1895 |
+
{ value: "|", type: "Text" },
|
1896 |
+
{ value: "{{", type: "OpenExpression" },
|
1897 |
+
{ value: "known", type: "StringLiteral" },
|
1898 |
+
{ value: "in", type: "In" },
|
1899 |
+
{ value: "obj", type: "Identifier" },
|
1900 |
+
{ value: "}}", type: "CloseExpression" },
|
1901 |
+
{ value: "|", type: "Text" },
|
1902 |
+
{ value: "{{", type: "OpenExpression" },
|
1903 |
+
{ value: "known", type: "StringLiteral" },
|
1904 |
+
{ value: "not in", type: "NotIn" },
|
1905 |
+
{ value: "obj", type: "Identifier" },
|
1906 |
+
{ value: "}}", type: "CloseExpression" },
|
1907 |
+
{ value: "|", type: "Text" },
|
1908 |
+
{ value: "{{", type: "OpenExpression" },
|
1909 |
+
{ value: "unknown", type: "StringLiteral" },
|
1910 |
+
{ value: "in", type: "In" },
|
1911 |
+
{ value: "obj", type: "Identifier" },
|
1912 |
+
{ value: "}}", type: "CloseExpression" },
|
1913 |
+
{ value: "|", type: "Text" },
|
1914 |
+
{ value: "{{", type: "OpenExpression" },
|
1915 |
+
{ value: "unknown", type: "StringLiteral" },
|
1916 |
+
{ value: "not in", type: "NotIn" },
|
1917 |
+
{ value: "obj", type: "Identifier" },
|
1918 |
+
{ value: "}}", type: "CloseExpression" },
|
1919 |
+
{ value: "|", type: "Text" },
|
1920 |
+
],
|
1921 |
+
OBJECT_OPERATORS_1: [
|
1922 |
+
{ value: "|", type: "Text" },
|
1923 |
+
{ value: "{{", type: "OpenExpression" },
|
1924 |
+
{ value: "obj", type: "Identifier" },
|
1925 |
+
{ value: ".", type: "Dot" },
|
1926 |
+
{ value: "get", type: "Identifier" },
|
1927 |
+
{ value: "(", type: "OpenParen" },
|
1928 |
+
{ value: "known", type: "StringLiteral" },
|
1929 |
+
{ value: ")", type: "CloseParen" },
|
1930 |
+
{ value: "}}", type: "CloseExpression" },
|
1931 |
+
{ value: "|", type: "Text" },
|
1932 |
+
{ value: "{{", type: "OpenExpression" },
|
1933 |
+
{ value: "obj", type: "Identifier" },
|
1934 |
+
{ value: ".", type: "Dot" },
|
1935 |
+
{ value: "get", type: "Identifier" },
|
1936 |
+
{ value: "(", type: "OpenParen" },
|
1937 |
+
{ value: "unknown", type: "StringLiteral" },
|
1938 |
+
{ value: ")", type: "CloseParen" },
|
1939 |
+
{ value: "is", type: "Is" },
|
1940 |
+
{ value: "none", type: "Identifier" },
|
1941 |
+
{ value: "}}", type: "CloseExpression" },
|
1942 |
+
{ value: "|", type: "Text" },
|
1943 |
+
{ value: "{{", type: "OpenExpression" },
|
1944 |
+
{ value: "obj", type: "Identifier" },
|
1945 |
+
{ value: ".", type: "Dot" },
|
1946 |
+
{ value: "get", type: "Identifier" },
|
1947 |
+
{ value: "(", type: "OpenParen" },
|
1948 |
+
{ value: "unknown", type: "StringLiteral" },
|
1949 |
+
{ value: ")", type: "CloseParen" },
|
1950 |
+
{ value: "is", type: "Is" },
|
1951 |
+
{ value: "defined", type: "Identifier" },
|
1952 |
+
{ value: "}}", type: "CloseExpression" },
|
1953 |
+
{ value: "|", type: "Text" },
|
1954 |
+
],
|
1955 |
+
|
1956 |
+
// Scope
|
1957 |
+
SCOPE: [
|
1958 |
+
{ value: "{%", type: "OpenStatement" },
|
1959 |
+
{ value: "set", type: "Set" },
|
1960 |
+
{ value: "ns", type: "Identifier" },
|
1961 |
+
{ value: "=", type: "Equals" },
|
1962 |
+
{ value: "namespace", type: "Identifier" },
|
1963 |
+
{ value: "(", type: "OpenParen" },
|
1964 |
+
{ value: "found", type: "Identifier" },
|
1965 |
+
{ value: "=", type: "Equals" },
|
1966 |
+
{ value: "false", type: "BooleanLiteral" },
|
1967 |
+
{ value: ")", type: "CloseParen" },
|
1968 |
+
{ value: "%}", type: "CloseStatement" },
|
1969 |
+
{ value: "{%", type: "OpenStatement" },
|
1970 |
+
{ value: "for", type: "For" },
|
1971 |
+
{ value: "num", type: "Identifier" },
|
1972 |
+
{ value: "in", type: "In" },
|
1973 |
+
{ value: "nums", type: "Identifier" },
|
1974 |
+
{ value: "%}", type: "CloseStatement" },
|
1975 |
+
{ value: "{%", type: "OpenStatement" },
|
1976 |
+
{ value: "if", type: "If" },
|
1977 |
+
{ value: "num", type: "Identifier" },
|
1978 |
+
{ value: "==", type: "ComparisonBinaryOperator" },
|
1979 |
+
{ value: "1", type: "NumericLiteral" },
|
1980 |
+
{ value: "%}", type: "CloseStatement" },
|
1981 |
+
{ value: "{{", type: "OpenExpression" },
|
1982 |
+
{ value: "found=", type: "StringLiteral" },
|
1983 |
+
{ value: "}}", type: "CloseExpression" },
|
1984 |
+
{ value: "{%", type: "OpenStatement" },
|
1985 |
+
{ value: "set", type: "Set" },
|
1986 |
+
{ value: "ns", type: "Identifier" },
|
1987 |
+
{ value: ".", type: "Dot" },
|
1988 |
+
{ value: "found", type: "Identifier" },
|
1989 |
+
{ value: "=", type: "Equals" },
|
1990 |
+
{ value: "true", type: "BooleanLiteral" },
|
1991 |
+
{ value: "%}", type: "CloseStatement" },
|
1992 |
+
{ value: "{%", type: "OpenStatement" },
|
1993 |
+
{ value: "endif", type: "EndIf" },
|
1994 |
+
{ value: "%}", type: "CloseStatement" },
|
1995 |
+
{ value: "{%", type: "OpenStatement" },
|
1996 |
+
{ value: "endfor", type: "EndFor" },
|
1997 |
+
{ value: "%}", type: "CloseStatement" },
|
1998 |
+
{ value: "{{", type: "OpenExpression" },
|
1999 |
+
{ value: "ns", type: "Identifier" },
|
2000 |
+
{ value: ".", type: "Dot" },
|
2001 |
+
{ value: "found", type: "Identifier" },
|
2002 |
+
{ value: "}}", type: "CloseExpression" },
|
2003 |
+
],
|
2004 |
+
SCOPE_1: [
|
2005 |
+
{ value: "{%", type: "OpenStatement" },
|
2006 |
+
{ value: "set", type: "Set" },
|
2007 |
+
{ value: "found", type: "Identifier" },
|
2008 |
+
{ value: "=", type: "Equals" },
|
2009 |
+
{ value: "false", type: "BooleanLiteral" },
|
2010 |
+
{ value: "%}", type: "CloseStatement" },
|
2011 |
+
{ value: "{%", type: "OpenStatement" },
|
2012 |
+
{ value: "for", type: "For" },
|
2013 |
+
{ value: "num", type: "Identifier" },
|
2014 |
+
{ value: "in", type: "In" },
|
2015 |
+
{ value: "nums", type: "Identifier" },
|
2016 |
+
{ value: "%}", type: "CloseStatement" },
|
2017 |
+
{ value: "{%", type: "OpenStatement" },
|
2018 |
+
{ value: "if", type: "If" },
|
2019 |
+
{ value: "num", type: "Identifier" },
|
2020 |
+
{ value: "==", type: "ComparisonBinaryOperator" },
|
2021 |
+
{ value: "1", type: "NumericLiteral" },
|
2022 |
+
{ value: "%}", type: "CloseStatement" },
|
2023 |
+
{ value: "{{", type: "OpenExpression" },
|
2024 |
+
{ value: "found=", type: "StringLiteral" },
|
2025 |
+
{ value: "}}", type: "CloseExpression" },
|
2026 |
+
{ value: "{%", type: "OpenStatement" },
|
2027 |
+
{ value: "set", type: "Set" },
|
2028 |
+
{ value: "found", type: "Identifier" },
|
2029 |
+
{ value: "=", type: "Equals" },
|
2030 |
+
{ value: "true", type: "BooleanLiteral" },
|
2031 |
+
{ value: "%}", type: "CloseStatement" },
|
2032 |
+
{ value: "{%", type: "OpenStatement" },
|
2033 |
+
{ value: "endif", type: "EndIf" },
|
2034 |
+
{ value: "%}", type: "CloseStatement" },
|
2035 |
+
{ value: "{%", type: "OpenStatement" },
|
2036 |
+
{ value: "endfor", type: "EndFor" },
|
2037 |
+
{ value: "%}", type: "CloseStatement" },
|
2038 |
+
{ value: "{{", type: "OpenExpression" },
|
2039 |
+
{ value: "found", type: "Identifier" },
|
2040 |
+
{ value: "}}", type: "CloseExpression" },
|
2041 |
+
],
|
2042 |
+
|
2043 |
+
// Undefined
|
2044 |
+
UNDEFINED_VARIABLES: [
|
2045 |
+
{ value: "{{", type: "OpenExpression" },
|
2046 |
+
{ value: "undefined_variable", type: "Identifier" },
|
2047 |
+
{ value: "}}", type: "CloseExpression" },
|
2048 |
+
],
|
2049 |
+
UNDEFINED_ACCESS: [
|
2050 |
+
{ value: "{{", type: "OpenExpression" },
|
2051 |
+
{ value: "object", type: "Identifier" },
|
2052 |
+
{ value: ".", type: "Dot" },
|
2053 |
+
{ value: "undefined_attribute", type: "Identifier" },
|
2054 |
+
{ value: "}}", type: "CloseExpression" },
|
2055 |
+
],
|
2056 |
+
|
2057 |
+
// Ternary operator
|
2058 |
+
TERNARY_OPERATOR: [
|
2059 |
+
{ value: "|", type: "Text" },
|
2060 |
+
{ value: "{{", type: "OpenExpression" },
|
2061 |
+
{ value: "a", type: "StringLiteral" },
|
2062 |
+
{ value: "if", type: "If" },
|
2063 |
+
{ value: "true", type: "BooleanLiteral" },
|
2064 |
+
{ value: "else", type: "Else" },
|
2065 |
+
{ value: "b", type: "StringLiteral" },
|
2066 |
+
{ value: "}}", type: "CloseExpression" },
|
2067 |
+
{ value: "|", type: "Text" },
|
2068 |
+
{ value: "{{", type: "OpenExpression" },
|
2069 |
+
{ value: "a", type: "StringLiteral" },
|
2070 |
+
{ value: "if", type: "If" },
|
2071 |
+
{ value: "false", type: "BooleanLiteral" },
|
2072 |
+
{ value: "else", type: "Else" },
|
2073 |
+
{ value: "b", type: "StringLiteral" },
|
2074 |
+
{ value: "}}", type: "CloseExpression" },
|
2075 |
+
{ value: "|", type: "Text" },
|
2076 |
+
{ value: "{{", type: "OpenExpression" },
|
2077 |
+
{ value: "a", type: "StringLiteral" },
|
2078 |
+
{ value: "if", type: "If" },
|
2079 |
+
{ value: "1", type: "NumericLiteral" },
|
2080 |
+
{ value: "+", type: "AdditiveBinaryOperator" },
|
2081 |
+
{ value: "1", type: "NumericLiteral" },
|
2082 |
+
{ value: "==", type: "ComparisonBinaryOperator" },
|
2083 |
+
{ value: "2", type: "NumericLiteral" },
|
2084 |
+
{ value: "else", type: "Else" },
|
2085 |
+
{ value: "b", type: "StringLiteral" },
|
2086 |
+
{ value: "}}", type: "CloseExpression" },
|
2087 |
+
{ value: "|", type: "Text" },
|
2088 |
+
{ value: "{{", type: "OpenExpression" },
|
2089 |
+
{ value: "a", type: "StringLiteral" },
|
2090 |
+
{ value: "if", type: "If" },
|
2091 |
+
{ value: "1", type: "NumericLiteral" },
|
2092 |
+
{ value: "+", type: "AdditiveBinaryOperator" },
|
2093 |
+
{ value: "1", type: "NumericLiteral" },
|
2094 |
+
{ value: "==", type: "ComparisonBinaryOperator" },
|
2095 |
+
{ value: "3", type: "NumericLiteral" },
|
2096 |
+
{ value: "or", type: "Or" },
|
2097 |
+
{ value: "1", type: "NumericLiteral" },
|
2098 |
+
{ value: "*", type: "MultiplicativeBinaryOperator" },
|
2099 |
+
{ value: "2", type: "NumericLiteral" },
|
2100 |
+
{ value: "==", type: "ComparisonBinaryOperator" },
|
2101 |
+
{ value: "3", type: "NumericLiteral" },
|
2102 |
+
{ value: "else", type: "Else" },
|
2103 |
+
{ value: "b", type: "StringLiteral" },
|
2104 |
+
{ value: "}}", type: "CloseExpression" },
|
2105 |
+
{ value: "|", type: "Text" },
|
2106 |
+
],
|
2107 |
+
|
2108 |
+
// Array literals
|
2109 |
+
ARRAY_LITERALS: [
|
2110 |
+
{ value: "{{", type: "OpenExpression" },
|
2111 |
+
{ value: "[", type: "OpenSquareBracket" },
|
2112 |
+
{ value: "1", type: "NumericLiteral" },
|
2113 |
+
{ value: ",", type: "Comma" },
|
2114 |
+
{ value: "true", type: "BooleanLiteral" },
|
2115 |
+
{ value: ",", type: "Comma" },
|
2116 |
+
{ value: "hello", type: "StringLiteral" },
|
2117 |
+
{ value: ",", type: "Comma" },
|
2118 |
+
{ value: "[", type: "OpenSquareBracket" },
|
2119 |
+
{ value: "1", type: "NumericLiteral" },
|
2120 |
+
{ value: ",", type: "Comma" },
|
2121 |
+
{ value: "2", type: "NumericLiteral" },
|
2122 |
+
{ value: ",", type: "Comma" },
|
2123 |
+
{ value: "3", type: "NumericLiteral" },
|
2124 |
+
{ value: ",", type: "Comma" },
|
2125 |
+
{ value: "4", type: "NumericLiteral" },
|
2126 |
+
{ value: "]", type: "CloseSquareBracket" },
|
2127 |
+
{ value: ",", type: "Comma" },
|
2128 |
+
{ value: "var", type: "Identifier" },
|
2129 |
+
{ value: "]", type: "CloseSquareBracket" },
|
2130 |
+
{ value: "|", type: "Pipe" },
|
2131 |
+
{ value: "length", type: "Identifier" },
|
2132 |
+
{ value: "}}", type: "CloseExpression" },
|
2133 |
+
],
|
2134 |
+
|
2135 |
+
// Object literals
|
2136 |
+
OBJECT_LITERALS: [
|
2137 |
+
{ value: "{{", type: "OpenExpression" },
|
2138 |
+
{ value: "{", type: "OpenCurlyBracket" },
|
2139 |
+
{ value: "key", type: "StringLiteral" },
|
2140 |
+
{ value: ":", type: "Colon" },
|
2141 |
+
{ value: "value", type: "StringLiteral" },
|
2142 |
+
{ value: ",", type: "Comma" },
|
2143 |
+
{ value: "key", type: "Identifier" },
|
2144 |
+
{ value: ":", type: "Colon" },
|
2145 |
+
{ value: "value2", type: "StringLiteral" },
|
2146 |
+
{ value: ",", type: "Comma" },
|
2147 |
+
{ value: "key3", type: "StringLiteral" },
|
2148 |
+
{ value: ":", type: "Colon" },
|
2149 |
+
{ value: "[", type: "OpenSquareBracket" },
|
2150 |
+
{ value: "1", type: "NumericLiteral" },
|
2151 |
+
{ value: ",", type: "Comma" },
|
2152 |
+
{ value: "{", type: "OpenCurlyBracket" },
|
2153 |
+
{ value: "foo", type: "StringLiteral" },
|
2154 |
+
{ value: ":", type: "Colon" },
|
2155 |
+
{ value: "bar", type: "StringLiteral" },
|
2156 |
+
{ value: "}", type: "CloseCurlyBracket" },
|
2157 |
+
{ value: "]", type: "CloseSquareBracket" },
|
2158 |
+
{ value: "}", type: "CloseCurlyBracket" },
|
2159 |
+
{ value: "[", type: "OpenSquareBracket" },
|
2160 |
+
{ value: "key", type: "StringLiteral" },
|
2161 |
+
{ value: "]", type: "CloseSquareBracket" },
|
2162 |
+
{ value: "}}", type: "CloseExpression" },
|
2163 |
+
],
|
2164 |
+
|
2165 |
+
// Array operators
|
2166 |
+
ARRAY_OPERATORS: [
|
2167 |
+
{ value: "{{", type: "OpenExpression" },
|
2168 |
+
{ value: "(", type: "OpenParen" },
|
2169 |
+
{ value: "[", type: "OpenSquareBracket" },
|
2170 |
+
{ value: "1", type: "NumericLiteral" },
|
2171 |
+
{ value: ",", type: "Comma" },
|
2172 |
+
{ value: "2", type: "NumericLiteral" },
|
2173 |
+
{ value: ",", type: "Comma" },
|
2174 |
+
{ value: "3", type: "NumericLiteral" },
|
2175 |
+
{ value: "]", type: "CloseSquareBracket" },
|
2176 |
+
{ value: "+", type: "AdditiveBinaryOperator" },
|
2177 |
+
{ value: "[", type: "OpenSquareBracket" },
|
2178 |
+
{ value: "4", type: "NumericLiteral" },
|
2179 |
+
{ value: ",", type: "Comma" },
|
2180 |
+
{ value: "5", type: "NumericLiteral" },
|
2181 |
+
{ value: ",", type: "Comma" },
|
2182 |
+
{ value: "6", type: "NumericLiteral" },
|
2183 |
+
{ value: "]", type: "CloseSquareBracket" },
|
2184 |
+
{ value: ")", type: "CloseParen" },
|
2185 |
+
{ value: "|", type: "Pipe" },
|
2186 |
+
{ value: "length", type: "Identifier" },
|
2187 |
+
{ value: "}}", type: "CloseExpression" },
|
2188 |
+
],
|
2189 |
};
|
2190 |
|
2191 |
const TEST_CONTEXT = {
|
|
|
2234 |
STRINGS: {
|
2235 |
bos_token: "<s>",
|
2236 |
},
|
2237 |
+
STRINGS_1: {},
|
2238 |
+
STRINGS_2: {},
|
2239 |
|
2240 |
// Function calls
|
2241 |
FUNCTIONS: {
|
|
|
2298 |
},
|
2299 |
FILTER_OPERATOR_2: {},
|
2300 |
FILTER_OPERATOR_3: {},
|
2301 |
+
FILTER_OPERATOR_4: {
|
2302 |
+
items: [{ key: "a" }, { key: 0 }, { key: 1 }, {}, { key: false }],
|
2303 |
+
},
|
2304 |
+
FILTER_OPERATOR_5: {
|
2305 |
+
messages: [{ role: "system" }, { role: "user" }, { role: "assistant" }],
|
2306 |
+
},
|
2307 |
|
2308 |
// Logical operators between non-Booleans
|
2309 |
BOOLEAN_NUMERICAL: {},
|
|
|
2331 |
NAMESPACE: {},
|
2332 |
NAMESPACE_1: {},
|
2333 |
NAMESPACE_2: {},
|
2334 |
+
|
2335 |
+
// Object operators
|
2336 |
+
OBJECT_OPERATORS: {
|
2337 |
+
obj: {
|
2338 |
+
known: true,
|
2339 |
+
},
|
2340 |
+
},
|
2341 |
+
OBJECT_OPERATORS_1: {
|
2342 |
+
obj: {
|
2343 |
+
known: true,
|
2344 |
+
},
|
2345 |
+
},
|
2346 |
+
|
2347 |
+
// Scope
|
2348 |
+
SCOPE: { nums: [1, 2, 3] },
|
2349 |
+
SCOPE_1: { nums: [1, 2, 3] },
|
2350 |
+
|
2351 |
+
// Undefined
|
2352 |
+
UNDEFINED_VARIABLES: {},
|
2353 |
+
UNDEFINED_ACCESS: { object: {} },
|
2354 |
+
|
2355 |
+
// Ternary operator
|
2356 |
+
TERNARY_OPERATOR: {},
|
2357 |
+
|
2358 |
+
// Array literals
|
2359 |
+
ARRAY_LITERALS: { var: true },
|
2360 |
+
|
2361 |
+
// Object literals
|
2362 |
+
OBJECT_LITERALS: {
|
2363 |
+
key: "key2",
|
2364 |
+
},
|
2365 |
+
|
2366 |
+
// Array operators
|
2367 |
+
ARRAY_OPERATORS: {},
|
2368 |
};
|
2369 |
|
2370 |
const EXPECTED_OUTPUTS = {
|
|
|
2402 |
|
2403 |
// Strings
|
2404 |
STRINGS: "Bye<s>[INST] ",
|
2405 |
+
STRINGS_1: `|test|abc|"'|'|"|`,
|
2406 |
+
STRINGS_2: `|0|1|0|1|`,
|
2407 |
|
2408 |
// Function calls
|
2409 |
FUNCTIONS: "014",
|
|
|
2437 |
FILTER_OPERATOR: `3451`,
|
2438 |
FILTER_OPERATOR_2: `|3|ABCD|abcd|Test test|Test Test|a b|4|`,
|
2439 |
FILTER_OPERATOR_3: `|1|1|`,
|
2440 |
+
FILTER_OPERATOR_4: `2`,
|
2441 |
+
FILTER_OPERATOR_5: `1`,
|
2442 |
|
2443 |
// Logical operators between non-Booleans
|
2444 |
BOOLEAN_NUMERICAL: `|2|0|0|0|1|1|1|0|false|true|`,
|
|
|
2462 |
NAMESPACE: `bar`,
|
2463 |
NAMESPACE_1: `false`,
|
2464 |
NAMESPACE_2: `|false|2|`,
|
2465 |
+
|
2466 |
+
// Object operators
|
2467 |
+
OBJECT_OPERATORS: `|true|false|false|true|`,
|
2468 |
+
OBJECT_OPERATORS_1: `|true|true|true|`,
|
2469 |
+
|
2470 |
+
// Scope
|
2471 |
+
SCOPE: `found=true`,
|
2472 |
+
SCOPE_1: `found=false`,
|
2473 |
+
|
2474 |
+
// Undefined
|
2475 |
+
UNDEFINED_VARIABLES: ``,
|
2476 |
+
UNDEFINED_ACCESS: ``,
|
2477 |
+
|
2478 |
+
// Ternary operator
|
2479 |
+
TERNARY_OPERATOR: `|a|b|a|b|`,
|
2480 |
+
|
2481 |
+
// Array literals
|
2482 |
+
ARRAY_LITERALS: `5`,
|
2483 |
+
|
2484 |
+
// Object literals
|
2485 |
+
OBJECT_LITERALS: `value`,
|
2486 |
+
|
2487 |
+
// Array operators
|
2488 |
+
ARRAY_OPERATORS: `6`,
|
2489 |
};
|
2490 |
|
2491 |
describe("Templates", () => {
|
|
|
2559 |
const text = "{{ invalid ! invalid }}";
|
2560 |
expect(() => tokenize(text)).toThrowError();
|
2561 |
});
|
2562 |
+
|
2563 |
+
it("Invalid quote character", () => {
|
2564 |
+
const text = "{{ \u2018text\u2019 }}";
|
2565 |
+
expect(() => tokenize(text)).toThrowError();
|
2566 |
+
});
|
2567 |
});
|
2568 |
|
2569 |
describe("Parsing errors", () => {
|
|
|
2611 |
});
|
2612 |
|
2613 |
describe("Runtime errors", () => {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2614 |
it("Undefined function call", () => {
|
2615 |
const env = new Environment();
|
2616 |
const interpreter = new Interpreter(env);
|
packages/widgets/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
{
|
2 |
"name": "@huggingface/widgets",
|
3 |
"packageManager": "pnpm@8.10.5",
|
4 |
-
"version": "0.2.
|
5 |
"publishConfig": {
|
6 |
"access": "public"
|
7 |
},
|
|
|
1 |
{
|
2 |
"name": "@huggingface/widgets",
|
3 |
"packageManager": "pnpm@8.10.5",
|
4 |
+
"version": "0.2.4",
|
5 |
"publishConfig": {
|
6 |
"access": "public"
|
7 |
},
|