vision-agent / lib /messageUtils.ts
wuyiqunLu
feat: parse new msg format (#57)
63d0776 unverified
raw
history blame
No virus
8.37 kB
import { MessageBase } from './types';
import { CLEANED_SEPARATOR } from './constants';
const PAIRS: Record<string, string> = {
'┍': 'β”‘',
'┝': 'β”₯',
'β”œ': '─',
'β”•': 'β”™',
};
const MIDDLE_STARTER = '┝';
const MIDDLE_SEPARATOR = 'β”Ώ';
const ANSWERS_PREFIX = 'answers';
const INPUT_PREFIX = 'input';
export const generateAnswersImageMarkdown = (index: number, url: string) => {
return `![${ANSWERS_PREFIX}-${index}](${url})`;
};
export const generateInputImageMarkdown = (url: string, index = 0) => {
if (url.toLowerCase().endsWith('.mp4')) {
const prefix = 'input-video';
return `![${INPUT_PREFIX}-${index}](<${url}>)`;
} else {
const prefix = 'input';
return `![${INPUT_PREFIX}-${index}](<${url}>)`;
}
};
export const cleanInputMessage = (content: string) => {
return content
.replace(/!\[input-.*?\)/g, '')
.replace(/<video[^>]*>.*?<\/video>/g, '');
};
export const cleanAnswerMessage = (content: string) => {
return content.replace(/!\[answers.*?\.png\)/g, '');
};
const generateJSONArrayMarkdown = (
message: string,
payload: Array<Record<string, string | boolean>>,
) => {
if (payload.length === 0) return '';
const keys = Object.keys(payload[0]);
message += '\n';
message += '| ' + keys.join(' | ') + ' |' + '\n';
message += new Array(keys.length + 1).fill('|').join(' :- ') + '\n';
payload.forEach((obj: any) => {
message +=
'| ' +
keys
.map(key => {
if (key === 'documentation') {
const doc = `\`\`\`\n${obj[key]}\n\`\`\`\n`;
return `<button data-details=${JSON.stringify(encodeURI(doc))}>Show</button>`;
} else {
return obj[key];
}
})
.join(' | ') +
' |' +
'\n';
});
message += '\n';
return message;
};
const generateStringArrayMarkdown = (
message: string,
header: string,
payload: Array<string>,
) => {
message += '\n';
message += '| ' + header + ' |' + '\n';
message += '| ' + ':-' + ' |' + '\n';
payload.forEach((tool: string) => {
message += '| ' + tool + ' |' + '\n';
});
message += '\n';
return message;
};
const generateCodeExecutionMarkdown = (
message: string,
payload: {
code: string;
test: string;
result?: string;
},
) => {
let Details = 'Code: \n';
Details += `\`\`\`python\n${payload.code}\n\`\`\`\n`;
Details += 'Test: \n';
Details += `\`\`\`python\n${payload.test}\n\`\`\`\n`;
if (payload.result) {
Details += 'Execution result: \n';
Details += `\`\`\`python\n${payload.result}\n\`\`\`\n`;
}
message += `<button data-details=${JSON.stringify(encodeURI(Details))}>View details</button> \n`;
return message;
};
const generateFinalCodeMarkdown = (
message: string,
payload: {
code: string;
test: string;
result: string;
},
) => {
message += 'Final Code: \n';
message += `\`\`\`python\n${payload.code}\n\`\`\`\n`;
message += 'Final test: \n';
message += `\`\`\`python\n${payload.test}\n\`\`\`\n`;
message += `Final result: \n`;
message += `\`\`\`\n${payload.result}\n\`\`\`\n`;
return message;
};
const generateFinalResultMarkdown = (
message: string,
payload: {
success: boolean;
reach_max_retries: boolean;
},
) => {
if (payload.reach_max_retries) {
message += 'Reach max debug retries!\n';
} else if (payload.success) {
message += 'Success!';
}
message += '\n';
return message;
};
type PlansBody =
| {
type: 'plans';
status: 'started';
}
| {
type: 'plans';
status: 'completed';
payload: Array<Record<string, string>>;
};
type ToolsBody =
| {
type: 'tools';
status: 'started';
}
| {
type: 'tools';
status: 'completed';
payload: Array<Record<string, string>>;
};
type CodeBody =
| {
type: 'code';
status: 'started';
}
| {
type: 'code';
status: 'running';
payload: {
code: string;
test: string;
};
}
| {
type: 'code';
status: 'completed' | 'failed';
payload: {
code: string;
test: string;
result: string;
};
};
type FinalCodeBody = {
type: 'final_code';
status: 'completed' | 'failed';
payload: {
code: string;
test: string;
result: string;
};
};
// this will return if self_reflection flag is true
type ReflectionBody =
| {
type: 'self_reflection';
status: 'started';
}
| {
type: 'self_reflection';
status: 'completed' | 'failed';
payload: { feedback: string; success: boolean };
};
type MessageBody =
| PlansBody
| ToolsBody
| CodeBody
| ReflectionBody
| FinalCodeBody;
const getMessageTitle = (json: MessageBody) => {
switch (json.type) {
case 'plans':
if (json.status === 'started') {
return '🎬 Start generating plans...\n';
} else {
return 'βœ… Going to run the following plan(s) in sequence:\n';
}
case 'tools':
if (json.status === 'started') {
return '🎬 Start retrieving tools...\n';
} else {
return 'βœ… Tools retrieved:\n';
}
case 'code':
if (json.status === 'started') {
return '🎬 Start generating code...\n';
} else if (json.status === 'running') {
return '🎬 Code generated, start execution... ';
} else if (json.status === 'completed') {
return 'βœ… Code executed successfully. ';
} else {
return '❌ Code execution failed. ';
}
case 'self_reflection':
if (json.status === 'started') {
return '🎬 Start self reflection...\n';
} else if (json.status === 'completed') {
return 'βœ… Self reflection completed: \n';
} else {
return '❌ Self reflection failed: \n';
}
case 'final_code':
if (json.status === 'completed') {
return 'βœ… The vision agent has concluded the chat, the last execution is successful. \n';
} else {
return '❌ he vision agent has concluded the chat, the last execution is failed. \n';
}
default:
throw 'Not supported type';
}
};
const parseLine = (json: MessageBody) => {
const title = getMessageTitle(json);
if (json.status === 'started') {
return title;
}
switch (json.type) {
case 'plans':
return generateJSONArrayMarkdown(title, json.payload);
case 'tools':
return generateJSONArrayMarkdown(title, json.payload);
case 'code':
return generateCodeExecutionMarkdown(title, json.payload);
case 'self_reflection':
return generateJSONArrayMarkdown(title, [json.payload]);
case 'final_code':
return generateFinalCodeMarkdown(title, json.payload);
default:
throw 'Not supported type';
}
};
export const getCleanedUpMessages = ({
content,
role,
}: Pick<MessageBase, 'role' | 'content'>) => {
if (role === 'user') {
return {
logs: content,
};
}
if (content.split(CLEANED_SEPARATOR).length === 2) {
return {
logs: content.split(CLEANED_SEPARATOR)[0],
content: content.split(CLEANED_SEPARATOR)[1],
};
}
const [logs = '', answer = ''] = content.split('<ANSWER>');
const lines = logs.split('\n');
let formattedLogs = '';
const jsons: MessageBody[] = [];
for (let line of lines) {
if (!line.trim()) {
continue;
}
try {
const json = JSON.parse(line) as MessageBody;
if (
jsons.length > 0 &&
json.type === jsons[jsons.length - 1].type &&
json.status !== 'started'
) {
jsons[jsons.length - 1] = json;
} else {
jsons.push(json);
}
} catch (e) {
console.error((e as Error).message);
console.error(line);
}
}
jsons.forEach(json => (formattedLogs += parseLine(json)));
const [answerText, imagesStr = ''] = answer.split('<VIZ>');
const [imagesArrayStr, ...rest] = imagesStr.split('</VIZ>');
const images = imagesArrayStr
.split('</IMG>')
.map(str => str.replace('<IMG>', ''))
.slice(0, -1);
return {
logs: formattedLogs,
content:
answerText.replace('</</ANSWER>', '').replace('</ANSWER>', '') +
'\n\n' +
images
.map((_, index) => generateAnswersImageMarkdown(index, '/loading.gif'))
.join('') +
rest.join(''),
images: images,
};
};