vision-agent / lib /messageUtils.ts
wuyiqunLu
feat: save raw message to db (#63)
3ebf44a unverified
raw
history blame
No virus
7.45 kB
import { MessageBase } from './types';
import { CLEANED_SEPARATOR } from './constants';
const PAIRS: Record<string, string> = {
'┍': 'β”‘',
'┝': 'β”₯',
'β”œ': '─',
'β”•': 'β”™',
};
const MIDDLE_STARTER = '┝';
const MIDDLE_SEPARATOR = 'β”Ώ';
const ANSWERS_PREFIX = 'answers';
const INPUT_PREFIX = 'input';
export const generateAnswersImageMarkdown = (index: number, url: string) => {
return `![${ANSWERS_PREFIX}-${index}](${url})`;
};
export const generateInputImageMarkdown = (url: string, index = 0) => {
if (url.toLowerCase().endsWith('.mp4')) {
const prefix = 'input-video';
return `![${INPUT_PREFIX}-${index}](<${url}>)`;
} else {
const prefix = 'input';
return `![${INPUT_PREFIX}-${index}](<${url}>)`;
}
};
export const cleanInputMessage = (content: string) => {
return content
.replace(/!\[input-.*?\)/g, '')
.replace(/<video[^>]*>.*?<\/video>/g, '');
};
export const cleanAnswerMessage = (content: string) => {
return content.replace(/!\[answers.*?\.png\)/g, '');
};
const generateJSONArrayMarkdown = (
message: string,
payload: Array<Record<string, string | boolean>>,
) => {
if (payload.length === 0) return '';
const keys = Object.keys(payload[0]);
message += '\n';
message += '| ' + keys.join(' | ') + ' |' + '\n';
message += new Array(keys.length + 1).fill('|').join(' :- ') + '\n';
payload.forEach((obj: any) => {
message +=
'| ' +
keys
.map(key => {
if (key === 'documentation') {
const doc = `\`\`\`\n${obj[key]}\n\`\`\`\n`;
return `<button data-details=${JSON.stringify(encodeURI(doc))}>Show</button>`;
} else {
return obj[key];
}
})
.join(' | ') +
' |' +
'\n';
});
message += '\n';
return message;
};
const generateCodeExecutionMarkdown = (
message: string,
payload: {
code: string;
test: string;
result?: string;
},
) => {
let Details = 'Code: \n';
Details += `\`\`\`python\n${payload.code}\n\`\`\`\n`;
Details += 'Test: \n';
Details += `\`\`\`python\n${payload.test}\n\`\`\`\n`;
if (payload.result) {
Details += 'Execution result: \n';
Details += `\`\`\`python\n${payload.result}\n\`\`\`\n`;
}
message += `<button data-details=${JSON.stringify(encodeURI(Details))}>View details</button> \n`;
return message;
};
const generateFinalCodeMarkdown = (
message: string,
payload: FinalCodeBody['payload'],
) => {
message += 'Final Code: \n';
message += `\`\`\`python\n${payload.code}\n\`\`\`\n`;
message += 'Final test: \n';
message += `\`\`\`python\n${payload.test}\n\`\`\`\n`;
message += `Final result: \n`;
message += `\`\`\`\n${payload.result}\n\`\`\`\n`;
return message;
};
const generateFinalResultMarkdown = (
message: string,
payload: {
success: boolean;
reach_max_retries: boolean;
},
) => {
if (payload.reach_max_retries) {
message += 'Reach max debug retries!\n';
} else if (payload.success) {
message += 'Success!';
}
message += '\n';
return message;
};
type PlansBody =
| {
type: 'plans';
status: 'started';
}
| {
type: 'plans';
status: 'completed';
payload: Array<Record<string, string>>;
};
type ToolsBody =
| {
type: 'tools';
status: 'started';
}
| {
type: 'tools';
status: 'completed';
payload: Array<Record<string, string>>;
};
type CodeBody =
| {
type: 'code';
status: 'started';
}
| {
type: 'code';
status: 'running';
payload: {
code: string;
test: string;
};
}
| {
type: 'code';
status: 'completed' | 'failed';
payload: {
code: string;
test: string;
result: string;
};
};
type FinalCodeBody = {
type: 'final_code';
status: 'completed' | 'failed';
payload: {
code: string;
test: string;
result: {
logs: {
stderr: string[];
stdout: string[];
};
results: Array<{
png: string;
text: string;
is_main_result: boolean;
}>;
};
};
};
// this will return if self_reflection flag is true
type ReflectionBody =
| {
type: 'self_reflection';
status: 'started';
}
| {
type: 'self_reflection';
status: 'completed' | 'failed';
payload: { feedback: string; success: boolean };
};
type MessageBody =
| PlansBody
| ToolsBody
| CodeBody
| ReflectionBody
| FinalCodeBody;
const getMessageTitle = (json: MessageBody) => {
switch (json.type) {
case 'plans':
if (json.status === 'started') {
return '🎬 Start generating plans...\n';
} else {
return 'βœ… Going to run the following plan(s) in sequence:\n';
}
case 'tools':
if (json.status === 'started') {
return '🎬 Start retrieving tools...\n';
} else {
return 'βœ… Tools retrieved:\n';
}
case 'code':
if (json.status === 'started') {
return '🎬 Start generating code...\n';
} else if (json.status === 'running') {
return '🎬 Code generated, start execution... ';
} else if (json.status === 'completed') {
return 'βœ… Code executed successfully. ';
} else {
return '❌ Code execution failed. ';
}
case 'self_reflection':
if (json.status === 'started') {
return '🎬 Start self reflection...\n';
} else if (json.status === 'completed') {
return 'βœ… Self reflection completed: \n';
} else {
return '❌ Self reflection failed: \n';
}
case 'final_code':
if (json.status === 'completed') {
return 'βœ… The vision agent has concluded the chat, the last execution is successful. \n';
} else {
return '❌ The vision agent has concluded the chat, the last execution is failed. \n';
}
default:
throw 'Not supported type';
}
};
const parseLine = (json: MessageBody) => {
const title = getMessageTitle(json);
if (json.status === 'started') {
return title;
}
switch (json.type) {
case 'plans':
return generateJSONArrayMarkdown(title, json.payload);
case 'tools':
return generateJSONArrayMarkdown(title, json.payload);
case 'code':
return generateCodeExecutionMarkdown(title, json.payload);
case 'self_reflection':
return generateJSONArrayMarkdown(title, [json.payload]);
case 'final_code':
return generateFinalCodeMarkdown(title, json.payload);
default:
throw 'Not supported type';
}
};
export const getCleanedUpMessages = ({
content,
role,
}: Pick<MessageBase, 'role' | 'content'>) => {
if (role === 'user') {
return {
content,
};
}
const lines = content.split('\n');
let formattedLogs = '';
const jsons: MessageBody[] = [];
for (let line of lines) {
if (!line.trim()) {
continue;
}
try {
const json = JSON.parse(line) as MessageBody;
if (
jsons.length > 0 &&
json.type === jsons[jsons.length - 1].type &&
json.status !== 'started'
) {
jsons[jsons.length - 1] = json;
} else {
jsons.push(json);
}
} catch (e) {
console.error((e as Error).message);
console.error(line);
}
}
jsons.forEach(json => (formattedLogs += parseLine(json)));
return {
content: formattedLogs,
};
};