Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <meta name="author" content="Chirag Patnaik"> | |
| <meta name="version" content="2.0.0"> | |
| <meta name="description" content="Private AI chatbot running Gemma entirely in your browser. No server, no API keys, no data leaves your device."> | |
| <title>LocalMind β Private AI Chat in Your Browser</title> | |
| <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'%3E%3Ctext y='.9em' font-size='90'%3E%F0%9F%A7%A0%3C/text%3E%3C/svg%3E"> | |
| <!-- KaTeX for inline math ($β¦$) and display math ($$β¦$$) in assistant | |
| responses. Loaded from jsdelivr with defer to avoid blocking parse. | |
| TODO: add SRI hashes alongside the transformers@4 SRI work. --> | |
| <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.11/dist/katex.min.css" crossorigin="anonymous"> | |
| <script defer src="https://cdn.jsdelivr.net/npm/katex@0.16.11/dist/katex.min.js" crossorigin="anonymous"></script> | |
| <style> | |
| /* Rangrez Β· westafrica-01 Β· BAOBAB β Senegalese baobab bark, savanna pale dawn */ | |
| :root { | |
| --gray-900: #1a1408; /* INK */ | |
| --gray-800: #2a2014; | |
| --gray-700: #4a3818; | |
| --gray-600: #7a5a28; | |
| --gray-400: #b89a68; | |
| --gray-200: #ddd6c0; /* line */ | |
| --gray-100: #f6f3ec; /* BODY */ | |
| --gray-50: #ece8de; /* row */ | |
| --white: #fdfbf4; /* PANEL */ | |
| --indigo-500: #08184a; /* ACT β savanna near-black navy */ | |
| --indigo-600: #050f30; | |
| --indigo-100: #d8dceb; | |
| --indigo-focus-ring: rgba(8, 24, 74, 0.20); | |
| --success-bg: #dceadb; | |
| --success-border: #2a8a3a; | |
| --success-text: #0a3014; | |
| --loading-bg: #fde6c4; | |
| --loading-border: #c08018; | |
| --loading-text: #744210; | |
| --error-bg: #f8d4dc; | |
| --error-border: #b81848; /* BRAND β hot rose */ | |
| --error-text: #6a0a30; | |
| } | |
| * { box-sizing: border-box; margin: 0; padding: 0; } | |
| body { | |
| font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; | |
| background: var(--gray-100); | |
| color: var(--gray-800); | |
| line-height: 1.6; | |
| padding: 24px 16px 60px; | |
| } | |
| .app { | |
| max-width: 780px; | |
| margin: 0 auto; | |
| display: flex; | |
| flex-direction: column; | |
| height: calc(100vh - 108px); | |
| position: relative; | |
| overflow: hidden; | |
| } | |
| /* Header */ | |
| .header { | |
| display: flex; | |
| justify-content: space-between; | |
| align-items: center; | |
| margin-bottom: 16px; | |
| flex-shrink: 0; | |
| } | |
| .header h1 { | |
| font-size: 1.5rem; | |
| font-weight: 700; | |
| color: var(--gray-900); | |
| } | |
| .header-right { | |
| display: flex; | |
| align-items: center; | |
| gap: 10px; | |
| } | |
| /* Help button */ | |
| .help-wrap { position: relative; } | |
| .help-btn { | |
| width: 26px; | |
| height: 26px; | |
| border-radius: 50%; | |
| border: 1.5px solid var(--gray-400); | |
| background: var(--white); | |
| color: var(--gray-600); | |
| font-size: 0.8rem; | |
| font-weight: 600; | |
| cursor: pointer; | |
| display: flex; | |
| align-items: center; | |
| justify-content: center; | |
| transition: border-color 0.15s, color 0.15s; | |
| } | |
| .help-btn:hover { | |
| border-color: var(--indigo-500); | |
| color: var(--indigo-500); | |
| } | |
| .help-popover { | |
| display: none; | |
| position: absolute; | |
| top: calc(100% + 10px); | |
| right: 0; | |
| width: 340px; | |
| background: var(--white); | |
| border-radius: 12px; | |
| box-shadow: 0 4px 24px rgba(0,0,0,0.12); | |
| font-size: 0.82rem; | |
| color: var(--gray-700); | |
| line-height: 1.55; | |
| z-index: 100; | |
| overflow: hidden; | |
| } | |
| .help-popover::before { | |
| content: ''; | |
| position: absolute; | |
| top: -6px; | |
| right: 10px; | |
| width: 12px; | |
| height: 12px; | |
| background: var(--white); | |
| transform: rotate(45deg); | |
| box-shadow: -2px -2px 4px rgba(0,0,0,0.04); | |
| z-index: 1; | |
| } | |
| .help-popover.open { display: block; } | |
| .help-tabs { | |
| display: flex; | |
| border-bottom: 1px solid var(--gray-200); | |
| background: var(--gray-50); | |
| } | |
| .help-tab { | |
| flex: 1; | |
| padding: 10px 6px; | |
| font-size: 0.72rem; | |
| font-weight: 600; | |
| color: var(--gray-500); | |
| text-align: center; | |
| cursor: pointer; | |
| border: none; | |
| background: none; | |
| border-bottom: 2px solid transparent; | |
| transition: color 0.15s, border-color 0.15s; | |
| } | |
| .help-tab:hover { color: var(--gray-700); } | |
| .help-tab.active { | |
| color: var(--indigo-600); | |
| border-bottom-color: var(--indigo-500); | |
| } | |
| .help-tab-content { | |
| display: none; | |
| padding: 14px 16px; | |
| max-height: 320px; | |
| overflow-y: auto; | |
| } | |
| .help-tab-content.active { display: block; } | |
| .help-tab-content h4 { | |
| font-size: 0.82rem; | |
| font-weight: 600; | |
| color: var(--gray-900); | |
| margin: 10px 0 4px; | |
| } | |
| .help-tab-content h4:first-child { margin-top: 0; } | |
| .help-tab-content p { margin-bottom: 8px; } | |
| .help-tab-content ul { | |
| padding-left: 18px; | |
| margin-bottom: 8px; | |
| } | |
| .help-tab-content li { margin-bottom: 4px; } | |
| .help-tab-content .help-note { | |
| font-size: 0.75rem; | |
| color: var(--gray-400); | |
| border-top: 1px solid var(--gray-200); | |
| padding-top: 8px; | |
| margin-top: 8px; | |
| } | |
| .try-prompt { | |
| display: block; | |
| background: var(--gray-50); | |
| border: 1px solid var(--gray-200); | |
| border-radius: 8px; | |
| padding: 7px 10px; | |
| margin-bottom: 6px; | |
| font-size: 0.78rem; | |
| color: var(--gray-700); | |
| font-style: italic; | |
| cursor: pointer; | |
| transition: background 0.15s, border-color 0.15s; | |
| } | |
| .try-prompt:hover { | |
| background: rgba(102, 126, 234, 0.06); | |
| border-color: var(--indigo-300); | |
| } | |
| /* Status badge */ | |
| .status-badge { | |
| display: flex; | |
| align-items: center; | |
| gap: 8px; | |
| padding: 4px 12px; | |
| border-radius: 20px; | |
| font-size: 0.75rem; | |
| font-weight: 500; | |
| transition: background 0.4s, border-color 0.4s, color 0.4s; | |
| } | |
| .status-badge.loading { | |
| background: var(--loading-bg); | |
| border: 1px solid var(--loading-border); | |
| color: var(--loading-text); | |
| } | |
| .status-badge.ready { | |
| background: var(--success-bg); | |
| border: 1px solid var(--success-border); | |
| color: var(--success-text); | |
| } | |
| .status-badge.error { | |
| background: var(--error-bg); | |
| border: 1px solid var(--error-border); | |
| color: var(--error-text); | |
| } | |
| .spinner { | |
| width: 10px; | |
| height: 10px; | |
| border: 2px solid currentColor; | |
| border-top-color: transparent; | |
| border-radius: 50%; | |
| animation: spin 0.75s linear infinite; | |
| } | |
| @keyframes spin { to { transform: rotate(360deg); } } | |
| /* Card */ | |
| .card { | |
| background: var(--white); | |
| border-radius: 16px; | |
| box-shadow: 0 2px 12px rgba(0,0,0,0.08); | |
| overflow: hidden; | |
| display: flex; | |
| flex-direction: column; | |
| flex: 1; | |
| min-height: 0; | |
| } | |
| /* Chat area */ | |
| .chat-area { | |
| flex: 1; | |
| overflow-y: auto; | |
| padding: 20px; | |
| display: flex; | |
| flex-direction: column; | |
| gap: 16px; | |
| min-height: 0; | |
| } | |
| .chat-area::-webkit-scrollbar { width: 6px; } | |
| .chat-area::-webkit-scrollbar-track { background: transparent; } | |
| .chat-area::-webkit-scrollbar-thumb { background: var(--gray-200); border-radius: 3px; } | |
| /* Welcome message */ | |
| .welcome { | |
| text-align: center; | |
| color: var(--gray-400); | |
| font-size: 0.88rem; | |
| margin: auto 0; | |
| padding: 40px 20px; | |
| } | |
| .welcome-icon { | |
| font-size: 2.5rem; | |
| margin-bottom: 12px; | |
| } | |
| .welcome h2 { | |
| font-size: 1.1rem; | |
| font-weight: 600; | |
| color: var(--gray-600); | |
| margin-bottom: 6px; | |
| } | |
| /* Messages */ | |
| .msg { | |
| display: flex; | |
| flex-direction: column; | |
| max-width: 85%; | |
| animation: fadeIn 0.2s ease; | |
| } | |
| @keyframes fadeIn { from { opacity: 0; transform: translateY(6px); } to { opacity: 1; transform: translateY(0); } } | |
| .msg.user { align-self: flex-end; } | |
| .msg.assistant { align-self: flex-start; } | |
| .msg-bubble { | |
| padding: 10px 14px; | |
| border-radius: 14px; | |
| font-size: 0.9rem; | |
| line-height: 1.55; | |
| word-break: break-word; | |
| } | |
| .msg.user .msg-bubble { | |
| background: var(--indigo-500); | |
| color: var(--white); | |
| border-bottom-right-radius: 4px; | |
| } | |
| .msg.assistant .msg-bubble { | |
| background: var(--gray-100); | |
| color: var(--gray-800); | |
| border-bottom-left-radius: 4px; | |
| } | |
| /* User message attachment thumbnails */ | |
| .msg-attachments { | |
| display: flex; | |
| flex-wrap: wrap; | |
| gap: 6px; | |
| margin-bottom: 6px; | |
| } | |
| .msg-attachments img { | |
| width: 80px; | |
| height: 80px; | |
| object-fit: cover; | |
| border-radius: 8px; | |
| border: 2px solid rgba(255,255,255,0.3); | |
| } | |
| .msg-attachments .audio-tag { | |
| display: flex; | |
| align-items: center; | |
| gap: 4px; | |
| background: rgba(255,255,255,0.15); | |
| border-radius: 8px; | |
| padding: 6px 10px; | |
| font-size: 0.75rem; | |
| color: rgba(255,255,255,0.9); | |
| } | |
| /* Markdown in assistant messages */ | |
| .msg.assistant .msg-bubble code { | |
| background: rgba(0,0,0,0.06); | |
| padding: 1px 5px; | |
| border-radius: 4px; | |
| font-size: 0.82rem; | |
| font-family: 'SF Mono', SFMono-Regular, Consolas, 'Liberation Mono', Menlo, monospace; | |
| } | |
| .msg.assistant .msg-bubble pre { | |
| background: var(--gray-900); | |
| color: var(--gray-200); | |
| padding: 12px 14px; | |
| border-radius: 8px; | |
| overflow-x: auto; | |
| margin: 8px 0; | |
| font-size: 0.8rem; | |
| line-height: 1.5; | |
| } | |
| .msg.assistant .msg-bubble pre code { | |
| background: none; | |
| padding: 0; | |
| color: inherit; | |
| font-size: inherit; | |
| } | |
| .msg.assistant .msg-bubble pre { | |
| position: relative; | |
| } | |
| .code-download-btn { | |
| position: absolute; | |
| top: 6px; | |
| right: 6px; | |
| background: rgba(255,255,255,0.8); | |
| border: 1px solid var(--gray-200); | |
| border-radius: 4px; | |
| width: 24px; | |
| height: 24px; | |
| font-size: 0.75rem; | |
| cursor: pointer; | |
| display: flex; | |
| align-items: center; | |
| justify-content: center; | |
| opacity: 0; | |
| transition: opacity 0.15s; | |
| } | |
| .msg.assistant .msg-bubble pre:hover .code-download-btn { opacity: 1; } | |
| .save-md-btn { | |
| display: inline-block; | |
| margin-top: 6px; | |
| padding: 2px 8px; | |
| font-size: 0.68rem; | |
| color: var(--gray-400); | |
| border: 1px solid var(--gray-200); | |
| border-radius: 4px; | |
| background: none; | |
| cursor: pointer; | |
| transition: color 0.15s, border-color 0.15s; | |
| } | |
| .save-md-btn:hover { | |
| color: var(--gray-600); | |
| border-color: var(--gray-400); | |
| } | |
| /* Collapsible tool-trace block (step labels + tool-call blocks) that | |
| sits above the bubble when expanded. Default collapsed; toggle | |
| button sits next to Save as MD. */ | |
| .tool-trace-container { margin: 0 0 8px 0; } | |
| .tool-trace-container.collapsed { display: none; } | |
| .msg.assistant .msg-bubble ol, | |
| .msg.assistant .msg-bubble ul { | |
| padding-left: 20px; | |
| margin: 6px 0; | |
| } | |
| .msg.assistant .msg-bubble li { margin-bottom: 2px; } | |
| .msg.assistant .msg-bubble strong { font-weight: 600; } | |
| .msg.assistant .msg-bubble em { font-style: italic; } | |
| .msg.assistant .msg-bubble p { margin-bottom: 8px; } | |
| .msg.assistant .msg-bubble p:last-child { margin-bottom: 0; } | |
| /* Thinking block */ | |
| .thinking-block { | |
| background: rgba(102, 126, 234, 0.06); | |
| border-left: 3px solid var(--indigo-500); | |
| padding: 8px 12px; | |
| border-radius: 0 8px 8px 0; | |
| margin-bottom: 8px; | |
| font-size: 0.82rem; | |
| color: var(--gray-600); | |
| font-style: italic; | |
| } | |
| .thinking-toggle { | |
| cursor: pointer; | |
| font-size: 0.75rem; | |
| color: var(--indigo-500); | |
| font-weight: 500; | |
| display: flex; | |
| align-items: center; | |
| gap: 4px; | |
| margin-bottom: 4px; | |
| font-style: normal; | |
| } | |
| .thinking-content { | |
| display: none; | |
| white-space: pre-wrap; | |
| } | |
| .thinking-content.open { display: block; } | |
| /* Tool call block */ | |
| .tool-call-block { | |
| background: rgba(245, 158, 11, 0.06); | |
| border-left: 3px solid var(--loading-border); | |
| padding: 8px 12px; | |
| border-radius: 0 8px 8px 0; | |
| margin: 8px 0; | |
| font-size: 0.82rem; | |
| color: var(--gray-600); | |
| } | |
| .tool-call-toggle { | |
| cursor: pointer; | |
| font-size: 0.75rem; | |
| color: var(--loading-text); | |
| font-weight: 500; | |
| display: flex; | |
| align-items: center; | |
| gap: 4px; | |
| margin-bottom: 4px; | |
| } | |
| .tool-call-toggle strong { | |
| font-family: 'SF Mono', 'Cascadia Code', 'Consolas', monospace; | |
| } | |
| .tool-call-result { | |
| display: none; | |
| white-space: pre-wrap; | |
| font-size: 0.78rem; | |
| margin-top: 4px; | |
| } | |
| .tool-call-result.open { display: block; } | |
| /* Planner blocks (multi-step planning) β mirror tool-call-block but indigo */ | |
| .planner-block { | |
| background: rgba(99, 102, 241, 0.06); | |
| border-left: 3px solid var(--indigo-500); | |
| padding: 8px 12px; | |
| border-radius: 0 8px 8px 0; | |
| margin: 8px 0; | |
| font-size: 0.82rem; | |
| color: var(--gray-600); | |
| } | |
| .planner-toggle { | |
| cursor: pointer; | |
| font-size: 0.75rem; | |
| color: var(--indigo-500); | |
| font-weight: 500; | |
| display: flex; | |
| align-items: center; | |
| gap: 4px; | |
| } | |
| .planner-body { | |
| display: none; | |
| margin-top: 6px; | |
| white-space: pre-wrap; | |
| font-size: 0.78rem; | |
| color: var(--gray-600); | |
| } | |
| .planner-body.open { display: block; } | |
| .planner-status { | |
| font-size: 0.78rem; | |
| color: var(--gray-500); | |
| font-style: italic; | |
| padding: 6px 0; | |
| } | |
| /* Mermaid + KaTeX */ | |
| .mermaid { | |
| background: var(--white); | |
| border: 1px solid var(--gray-200); | |
| border-radius: 6px; | |
| padding: 12px; | |
| margin: 8px 0; | |
| overflow-x: auto; | |
| text-align: center; | |
| } | |
| .mermaid-error, .katex-error { | |
| color: var(--error-text); | |
| background: rgba(185, 28, 28, 0.08); | |
| padding: 2px 6px; | |
| border-radius: 4px; | |
| font-family: monospace; | |
| font-size: 0.78rem; | |
| } | |
| /* Artifact preview (sandboxed iframe for ```html, ```svg, ```artifact) */ | |
| .artifact-preview { | |
| margin: 8px 0; | |
| border: 1px solid var(--gray-200); | |
| border-radius: 6px; | |
| overflow: hidden; | |
| background: var(--white); | |
| } | |
| .artifact-header { | |
| display: flex; | |
| align-items: center; | |
| justify-content: space-between; | |
| padding: 6px 10px; | |
| background: var(--gray-50); | |
| border-bottom: 1px solid var(--gray-200); | |
| font-size: 0.72rem; | |
| color: var(--gray-600); | |
| } | |
| .artifact-header strong { | |
| font-family: 'SF Mono', 'Cascadia Code', 'Consolas', monospace; | |
| color: var(--indigo-500); | |
| } | |
| .artifact-frame { | |
| width: 100%; | |
| min-height: 200px; | |
| border: 0; | |
| display: block; | |
| background: var(--white); | |
| } | |
| /* Message context menu (right-click / long-press) */ | |
| .msg-context-menu { | |
| position: fixed; | |
| display: none; | |
| background: var(--white); | |
| border: 1px solid var(--gray-200); | |
| border-radius: 6px; | |
| box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); | |
| padding: 4px 0; | |
| font-size: 0.82rem; | |
| z-index: 1000; | |
| min-width: 160px; | |
| } | |
| .msg-context-menu.visible { display: block; } | |
| .msg-context-menu button { | |
| display: block; | |
| width: 100%; | |
| text-align: left; | |
| background: none; | |
| border: none; | |
| padding: 8px 12px; | |
| cursor: pointer; | |
| color: var(--gray-700); | |
| font-size: inherit; | |
| } | |
| .msg-context-menu button:hover { background: var(--gray-50); } | |
| .tool-call-result pre { | |
| margin: 0; | |
| padding: 6px 8px; | |
| background: rgba(245, 158, 11, 0.04); | |
| border-radius: 4px; | |
| font-size: 0.78rem; | |
| overflow-x: auto; | |
| } | |
| /* Segmentation overlay */ | |
| .segmentation-result { | |
| position: relative; | |
| display: inline-block; | |
| margin: 8px 0; | |
| max-width: 100%; | |
| } | |
| .segmentation-result img { | |
| max-width: 100%; | |
| border-radius: 8px; | |
| display: block; | |
| } | |
| .segmentation-result canvas { | |
| position: absolute; | |
| top: 0; | |
| left: 0; | |
| width: 100%; | |
| height: 100%; | |
| border-radius: 8px; | |
| pointer-events: none; | |
| } | |
| .segmentation-download { | |
| display: inline-block; | |
| margin-top: 4px; | |
| font-size: 0.72rem; | |
| color: var(--indigo-500); | |
| cursor: pointer; | |
| border: none; | |
| background: none; | |
| padding: 2px 0; | |
| } | |
| .segmentation-download:hover { | |
| text-decoration: underline; | |
| } | |
| /* Typing indicator */ | |
| .typing-dot { | |
| display: inline-block; | |
| width: 6px; | |
| height: 6px; | |
| border-radius: 50%; | |
| background: var(--gray-400); | |
| animation: blink 1.4s infinite; | |
| margin-right: 3px; | |
| } | |
| .typing-dot:nth-child(2) { animation-delay: 0.2s; } | |
| .typing-dot:nth-child(3) { animation-delay: 0.4s; margin-right: 0; } | |
| @keyframes blink { | |
| 0%, 60%, 100% { opacity: 0.3; } | |
| 30% { opacity: 1; } | |
| } | |
| /* Attachment bar */ | |
| .attachment-bar { | |
| display: none; | |
| padding: 8px 16px 4px; | |
| border-top: 1px solid var(--gray-200); | |
| background: var(--gray-50); | |
| flex-shrink: 0; | |
| overflow-x: auto; | |
| gap: 8px; | |
| } | |
| .attachment-bar.visible { | |
| display: flex; | |
| } | |
| .attachment-bar::-webkit-scrollbar { height: 4px; } | |
| .attachment-bar::-webkit-scrollbar-thumb { background: var(--gray-200); border-radius: 2px; } | |
| .attachment-chip { | |
| position: relative; | |
| flex-shrink: 0; | |
| width: 56px; | |
| height: 56px; | |
| border-radius: 10px; | |
| overflow: hidden; | |
| border: 1.5px solid var(--gray-200); | |
| background: var(--gray-100); | |
| display: flex; | |
| align-items: center; | |
| justify-content: center; | |
| } | |
| .attachment-chip.video-chip { | |
| width: 100px; | |
| height: 64px; | |
| } | |
| .attachment-chip.audio-chip { | |
| width: 180px; | |
| height: 48px; | |
| padding: 4px 6px; | |
| gap: 4px; | |
| } | |
| .attachment-chip img { | |
| width: 100%; | |
| height: 100%; | |
| object-fit: cover; | |
| } | |
| .attachment-chip video { | |
| width: 100%; | |
| height: 100%; | |
| object-fit: cover; | |
| pointer-events: none; | |
| } | |
| .attachment-chip .video-badge { | |
| position: absolute; | |
| bottom: 3px; | |
| left: 3px; | |
| background: rgba(0,0,0,0.6); | |
| color: var(--white); | |
| font-size: 0.5rem; | |
| font-weight: 600; | |
| padding: 1px 4px; | |
| border-radius: 3px; | |
| line-height: 1.4; | |
| } | |
| .attachment-chip audio { | |
| width: 100%; | |
| height: 28px; | |
| } | |
| .attachment-chip .audio-label { | |
| font-size: 0.55rem; | |
| color: var(--gray-600); | |
| overflow: hidden; | |
| text-overflow: ellipsis; | |
| white-space: nowrap; | |
| width: 100%; | |
| text-align: center; | |
| line-height: 1.2; | |
| } | |
| .attachment-chip .audio-icon { | |
| font-size: 1.2rem; | |
| color: var(--gray-600); | |
| } | |
| .attachment-chip .audio-name { | |
| position: absolute; | |
| bottom: 2px; | |
| left: 2px; | |
| right: 2px; | |
| font-size: 0.55rem; | |
| color: var(--gray-600); | |
| text-align: center; | |
| overflow: hidden; | |
| text-overflow: ellipsis; | |
| white-space: nowrap; | |
| background: rgba(255,255,255,0.85); | |
| border-radius: 3px; | |
| padding: 0 2px; | |
| } | |
| .attachment-chip .remove-btn { | |
| position: absolute; | |
| top: 2px; | |
| right: 2px; | |
| width: 18px; | |
| height: 18px; | |
| border-radius: 50%; | |
| background: rgba(0,0,0,0.55); | |
| color: var(--white); | |
| border: none; | |
| font-size: 0.65rem; | |
| cursor: pointer; | |
| display: flex; | |
| align-items: center; | |
| justify-content: center; | |
| line-height: 1; | |
| opacity: 0; | |
| transition: opacity 0.15s; | |
| } | |
| .attachment-chip:hover .remove-btn { opacity: 1; } | |
| .attachment-chip .experimental-badge { | |
| position: absolute; | |
| top: 2px; | |
| left: 2px; | |
| background: var(--loading-border); | |
| color: var(--white); | |
| font-size: 0.5rem; | |
| font-weight: 600; | |
| padding: 0 3px; | |
| border-radius: 3px; | |
| line-height: 1.4; | |
| } | |
| /* Input bar */ | |
| .input-bar { | |
| display: flex; | |
| align-items: flex-end; | |
| gap: 8px; | |
| padding: 12px 16px; | |
| border-top: 1px solid var(--gray-200); | |
| background: var(--gray-50); | |
| flex-shrink: 0; | |
| } | |
| .input-bar textarea { | |
| flex: 1; | |
| border: 1.5px solid var(--gray-200); | |
| border-radius: 12px; | |
| padding: 10px 14px; | |
| font-size: 0.9rem; | |
| font-family: inherit; | |
| line-height: 1.4; | |
| resize: none; | |
| outline: none; | |
| min-height: 42px; | |
| max-height: 140px; | |
| transition: border-color 0.15s; | |
| background: var(--white); | |
| } | |
| .input-bar textarea:focus { | |
| border-color: var(--indigo-500); | |
| box-shadow: 0 0 0 3px var(--indigo-focus-ring); | |
| } | |
| .input-bar textarea:disabled { | |
| background: var(--gray-100); | |
| color: var(--gray-400); | |
| cursor: not-allowed; | |
| } | |
| .input-bar textarea::placeholder { color: var(--gray-400); } | |
| /* Input affordance buttons */ | |
| .input-affordances { | |
| display: none; | |
| align-items: flex-end; | |
| gap: 4px; | |
| flex-shrink: 0; | |
| } | |
| .input-affordances.visible { display: flex; } | |
| .afford-btn { | |
| width: 36px; | |
| height: 36px; | |
| border-radius: 10px; | |
| border: 1.5px solid var(--gray-200); | |
| background: var(--white); | |
| color: var(--gray-600); | |
| font-size: 1rem; | |
| cursor: pointer; | |
| display: flex; | |
| align-items: center; | |
| justify-content: center; | |
| transition: border-color 0.15s, color 0.15s, background 0.15s; | |
| flex-shrink: 0; | |
| } | |
| .afford-btn:hover:not(:disabled) { | |
| border-color: var(--indigo-500); | |
| color: var(--indigo-500); | |
| } | |
| .afford-btn:disabled { opacity: 0.3; cursor: not-allowed; } | |
| .afford-btn.recording { | |
| background: var(--error-bg); | |
| border-color: var(--error-text); | |
| color: var(--error-text); | |
| animation: pulse-rec 1.2s ease-in-out infinite; | |
| } | |
| @keyframes pulse-rec { | |
| 0%, 100% { box-shadow: 0 0 0 0 rgba(229, 62, 62, 0.3); } | |
| 50% { box-shadow: 0 0 0 6px rgba(229, 62, 62, 0); } | |
| } | |
| .send-btn { | |
| width: 42px; | |
| height: 42px; | |
| border-radius: 12px; | |
| border: none; | |
| background: var(--indigo-500); | |
| color: var(--white); | |
| font-size: 1.1rem; | |
| cursor: pointer; | |
| display: flex; | |
| align-items: center; | |
| justify-content: center; | |
| transition: background 0.15s, opacity 0.15s; | |
| flex-shrink: 0; | |
| } | |
| .send-btn:hover:not(:disabled) { background: var(--indigo-600); } | |
| .send-btn:disabled { opacity: 0.4; cursor: not-allowed; } | |
| .send-btn.stop { background: var(--error-text); } | |
| .send-btn.stop:hover:not(:disabled) { background: var(--error-text); } | |
| /* Action buttons bar */ | |
| .actions-bar { | |
| display: flex; | |
| align-items: center; | |
| gap: 6px; | |
| padding: 6px 16px; | |
| border-top: 1px solid var(--gray-200); | |
| background: var(--gray-50); | |
| flex-shrink: 0; | |
| } | |
| .btn-icon { | |
| background: none; | |
| border: none; | |
| border-radius: 6px; | |
| font-size: 0.75rem; | |
| color: var(--gray-600); | |
| padding: 4px 8px; | |
| cursor: pointer; | |
| transition: background 0.15s, color 0.15s; | |
| } | |
| .btn-icon:hover:not(:disabled) { | |
| background: var(--gray-200); | |
| color: var(--gray-800); | |
| } | |
| .btn-icon:disabled { opacity: 0.4; cursor: not-allowed; } | |
| .btn-icon.folder-open { | |
| background: var(--indigo-100); | |
| color: var(--indigo-600); | |
| } | |
| /* Model selector */ | |
| .model-select { | |
| appearance: none; | |
| -webkit-appearance: none; | |
| background: var(--white); | |
| border: 1.5px solid var(--gray-200); | |
| border-radius: 8px; | |
| padding: 3px 24px 3px 8px; | |
| font-size: 0.72rem; | |
| font-family: inherit; | |
| color: var(--gray-700); | |
| cursor: pointer; | |
| outline: none; | |
| transition: border-color 0.15s; | |
| background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='10' height='6'%3E%3Cpath d='M0 0l5 6 5-6z' fill='%23718096'/%3E%3C/svg%3E"); | |
| background-repeat: no-repeat; | |
| background-position: right 8px center; | |
| } | |
| .model-select:hover { border-color: var(--gray-400); } | |
| .model-select:focus { border-color: var(--indigo-500); box-shadow: 0 0 0 3px var(--indigo-focus-ring); } | |
| .model-select:disabled { opacity: 0.4; cursor: not-allowed; } | |
| /* Progress section */ | |
| .progress-section { | |
| padding: 12px 16px; | |
| border-top: 1px solid var(--gray-200); | |
| background: var(--gray-50); | |
| flex-shrink: 0; | |
| display: none; | |
| } | |
| .progress-section.visible { display: block; } | |
| .progress-bar { | |
| height: 6px; | |
| background: var(--gray-200); | |
| border-radius: 8px; | |
| overflow: hidden; | |
| } | |
| .progress-fill { | |
| height: 100%; | |
| background: linear-gradient(90deg, var(--indigo-500), #b81848); | |
| border-radius: 8px; | |
| transition: width 0.3s ease; | |
| width: 0%; | |
| } | |
| .progress-text-row { | |
| display: flex; | |
| align-items: center; | |
| justify-content: space-between; | |
| gap: 8px; | |
| margin-top: 6px; | |
| } | |
| .progress-text { | |
| font-size: 0.72rem; | |
| color: var(--gray-600); | |
| flex: 1; | |
| } | |
| .progress-cancel-btn { | |
| padding: 2px 10px; | |
| font-size: 0.7rem; | |
| background: transparent; | |
| border: 1px solid var(--gray-200); | |
| color: var(--gray-600); | |
| border-radius: 4px; | |
| cursor: pointer; | |
| flex-shrink: 0; | |
| transition: border-color 0.15s, color 0.15s; | |
| } | |
| .progress-cancel-btn:hover { | |
| border-color: var(--gray-400); | |
| color: var(--gray-800); | |
| } | |
| /* Settings panel */ | |
| .settings-panel { | |
| display: none; | |
| padding: 12px 16px; | |
| border-top: 1px solid var(--gray-200); | |
| background: var(--gray-50); | |
| flex-shrink: 1; | |
| max-height: 50vh; | |
| overflow-y: auto; | |
| } | |
| .settings-panel.open { display: block; } | |
| /* API chip next to the model selector β only visible when the | |
| window.localmind API is enabled in Settings. Flashes on each call. */ | |
| .api-chip { | |
| display: none; | |
| align-items: center; | |
| gap: 4px; | |
| padding: 2px 8px; | |
| margin-right: 6px; | |
| font-size: 0.7rem; | |
| font-weight: 600; | |
| letter-spacing: 0.04em; | |
| color: var(--indigo-600); | |
| background: var(--indigo-100); | |
| border: 1px solid var(--indigo-500); | |
| border-radius: 10px; | |
| cursor: pointer; | |
| transition: background-color 0.2s, color 0.2s; | |
| } | |
| .api-chip.on { display: inline-flex; } | |
| .api-chip.flash { | |
| background: var(--indigo-500); | |
| color: var(--white); | |
| } | |
| .api-chip::before { | |
| content: ''; | |
| width: 6px; | |
| height: 6px; | |
| border-radius: 50%; | |
| background: var(--indigo-500); | |
| } | |
| .api-chip.flash::before { background: var(--white); } | |
| /* Activity log modal β reuses the share-modal backdrop pattern */ | |
| .api-log-modal { | |
| display: none; | |
| position: fixed; | |
| inset: 0; | |
| z-index: 200; | |
| background: rgba(26, 32, 44, 0.55); | |
| align-items: center; | |
| justify-content: center; | |
| padding: 24px; | |
| } | |
| .api-log-modal.open { display: flex; } | |
| .api-log-modal-content { | |
| background: var(--white); | |
| border-radius: 12px; | |
| max-width: 720px; | |
| width: 100%; | |
| max-height: 80vh; | |
| display: flex; | |
| flex-direction: column; | |
| overflow: hidden; | |
| box-shadow: 0 20px 40px rgba(0,0,0,0.2); | |
| } | |
| .api-log-header { | |
| padding: 14px 18px; | |
| border-bottom: 1px solid var(--gray-200); | |
| display: flex; | |
| align-items: center; | |
| justify-content: space-between; | |
| } | |
| .api-log-header strong { font-size: 0.95rem; } | |
| .api-log-list { | |
| overflow-y: auto; | |
| padding: 8px 0; | |
| font-size: 0.75rem; | |
| font-family: ui-monospace, SFMono-Regular, Menlo, monospace; | |
| } | |
| .api-log-empty { | |
| padding: 30px 18px; | |
| text-align: center; | |
| color: var(--gray-500); | |
| } | |
| .api-log-row { | |
| display: grid; | |
| grid-template-columns: 90px 1fr 80px 70px 70px; | |
| gap: 12px; | |
| padding: 6px 18px; | |
| border-bottom: 1px solid var(--gray-100); | |
| } | |
| .api-log-row:hover { background: var(--gray-50); } | |
| .api-log-row .t { color: var(--gray-500); } | |
| .api-log-row .m { color: var(--gray-800); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } | |
| .api-log-row .dur { color: var(--gray-600); text-align: right; } | |
| .api-log-row .ok { color: var(--success-border); } | |
| .api-log-row .err { color: var(--error-text); } | |
| .api-log-row .busy { color: var(--loading-text); } | |
| .settings-panel label { | |
| display: block; | |
| font-size: 0.75rem; | |
| font-weight: 500; | |
| color: var(--gray-600); | |
| margin-bottom: 4px; | |
| } | |
| .settings-panel textarea { | |
| width: 100%; | |
| border: 1.5px solid var(--gray-200); | |
| border-radius: 8px; | |
| padding: 8px 10px; | |
| font-size: 0.8rem; | |
| font-family: inherit; | |
| line-height: 1.4; | |
| resize: vertical; | |
| outline: none; | |
| min-height: 60px; | |
| max-height: 120px; | |
| background: var(--white); | |
| } | |
| .settings-panel textarea:focus { | |
| border-color: var(--indigo-500); | |
| box-shadow: 0 0 0 3px var(--indigo-focus-ring); | |
| } | |
| .toggle-row { | |
| display: flex; | |
| align-items: center; | |
| gap: 8px; | |
| margin-top: 10px; | |
| font-size: 0.8rem; | |
| color: var(--gray-700); | |
| cursor: pointer; | |
| } | |
| .toggle-row input[type="checkbox"] { | |
| width: 16px; | |
| height: 16px; | |
| accent-color: var(--indigo-500); | |
| cursor: pointer; | |
| } | |
| .toggle-row.hidden { display: none; } | |
| .settings-divider { | |
| border-top: 1px solid var(--gray-200); | |
| margin-top: 12px; | |
| padding-top: 12px; | |
| } | |
| .settings-select, .settings-input { | |
| width: 100%; | |
| padding: 8px 10px; | |
| border: 1px solid var(--gray-200); | |
| border-radius: 8px; | |
| font-size: 0.82rem; | |
| background: var(--white); | |
| margin-top: 4px; | |
| margin-bottom: 8px; | |
| outline: none; | |
| } | |
| .settings-select:focus, .settings-input:focus { | |
| border-color: var(--indigo-500); | |
| box-shadow: 0 0 0 3px var(--indigo-focus-ring); | |
| } | |
| /* Dual send buttons */ | |
| .send-group { | |
| display: flex; | |
| gap: 4px; | |
| } | |
| .search-send-btn { | |
| width: 40px; | |
| height: 40px; | |
| border-radius: 50%; | |
| border: 1.5px solid var(--indigo-400); | |
| background: var(--white); | |
| color: var(--indigo-600); | |
| font-size: 1rem; | |
| cursor: pointer; | |
| display: flex; | |
| align-items: center; | |
| justify-content: center; | |
| transition: background 0.15s, transform 0.1s; | |
| flex-shrink: 0; | |
| } | |
| .search-send-btn:hover:not(:disabled) { | |
| background: rgba(102, 126, 234, 0.08); | |
| } | |
| .search-send-btn:disabled { | |
| opacity: 0.4; | |
| cursor: default; | |
| } | |
| /* Source badges on messages */ | |
| .msg-source-badge { | |
| display: inline-block; | |
| font-size: 0.65rem; | |
| padding: 2px 6px; | |
| border-radius: 4px; | |
| margin-bottom: 4px; | |
| font-weight: 500; | |
| } | |
| .msg-source-badge.on-device { | |
| background: var(--gray-100); | |
| color: var(--gray-500); | |
| } | |
| .msg-source-badge.web-enriched { | |
| background: rgba(16, 185, 129, 0.1); | |
| color: var(--success-border); | |
| } | |
| .source-links { | |
| margin-top: 8px; | |
| padding-top: 6px; | |
| border-top: 1px solid var(--gray-200); | |
| font-size: 0.72rem; | |
| } | |
| .source-links a { | |
| color: var(--indigo-600); | |
| text-decoration: none; | |
| display: block; | |
| margin-bottom: 2px; | |
| overflow: hidden; | |
| text-overflow: ellipsis; | |
| white-space: nowrap; | |
| } | |
| .source-links a:hover { text-decoration: underline; } | |
| /* Memory inspector panel */ | |
| .memory-panel { | |
| display: none; | |
| padding: 12px 16px; | |
| border-top: 1px solid var(--gray-200); | |
| max-height: 400px; | |
| overflow-y: auto; | |
| } | |
| .memory-panel.open { display: block; } | |
| .memory-header { | |
| display: flex; | |
| justify-content: space-between; | |
| align-items: center; | |
| margin-bottom: 8px; | |
| font-size: 0.82rem; | |
| color: var(--gray-900); | |
| } | |
| .memory-count { | |
| font-size: 0.72rem; | |
| color: var(--gray-400); | |
| } | |
| .memory-search-row { | |
| display: flex; | |
| gap: 6px; | |
| margin-bottom: 6px; | |
| } | |
| .memory-search-input { | |
| flex: 1; | |
| padding: 6px 10px; | |
| border: 1px solid var(--gray-200); | |
| border-radius: 8px; | |
| font-size: 0.78rem; | |
| background: var(--white); | |
| outline: none; | |
| } | |
| .memory-search-input:focus { | |
| border-color: var(--indigo-500); | |
| } | |
| .memory-clear-btn { | |
| font-size: 0.7rem; | |
| color: var(--gray-500); | |
| } | |
| /* Category filter pills */ | |
| .memory-cat-pills { | |
| display: flex; | |
| flex-wrap: wrap; | |
| gap: 4px; | |
| margin-bottom: 8px; | |
| } | |
| .memory-cat-pill { | |
| padding: 2px 8px; | |
| border-radius: 20px; | |
| border: 1px solid var(--gray-200); | |
| background: var(--white); | |
| font-size: 0.7rem; | |
| color: var(--gray-600); | |
| cursor: pointer; | |
| white-space: nowrap; | |
| transition: background 0.12s, border-color 0.12s, color 0.12s; | |
| } | |
| .memory-cat-pill:hover { border-color: var(--gray-400); } | |
| .memory-cat-pill.active { | |
| background: var(--indigo-100); | |
| border-color: var(--indigo-500); | |
| color: var(--indigo-600); | |
| font-weight: 600; | |
| } | |
| /* Source group (for document categories) */ | |
| .memory-source-group { | |
| border: 1px solid var(--gray-200); | |
| border-radius: 10px; | |
| overflow: hidden; | |
| margin-bottom: 6px; | |
| } | |
| .memory-source-header { | |
| display: flex; | |
| align-items: center; | |
| gap: 6px; | |
| padding: 6px 10px; | |
| background: var(--gray-100); | |
| font-size: 0.75rem; | |
| } | |
| .memory-source-name { | |
| flex: 1; | |
| font-weight: 600; | |
| color: var(--gray-700); | |
| overflow: hidden; | |
| text-overflow: ellipsis; | |
| white-space: nowrap; | |
| } | |
| .memory-source-count { | |
| color: var(--gray-400); | |
| flex-shrink: 0; | |
| } | |
| .memory-source-del { | |
| font-size: 0.68rem; | |
| color: var(--gray-400); | |
| background: none; | |
| border: 1px solid var(--gray-200); | |
| border-radius: 4px; | |
| padding: 1px 6px; | |
| cursor: pointer; | |
| flex-shrink: 0; | |
| } | |
| .memory-source-del:hover { color: var(--error-border); border-color: var(--error-border); } | |
| .memory-source-group .memory-item { | |
| border-radius: 0; | |
| border-top: 1px solid var(--gray-200); | |
| } | |
| .memory-list { | |
| display: flex; | |
| flex-direction: column; | |
| gap: 6px; | |
| } | |
| .memory-item { | |
| display: flex; | |
| align-items: flex-start; | |
| gap: 8px; | |
| padding: 8px 10px; | |
| background: var(--gray-50); | |
| border-radius: 8px; | |
| font-size: 0.78rem; | |
| color: var(--gray-700); | |
| line-height: 1.45; | |
| } | |
| .memory-item-text { flex: 1; min-width: 0; } | |
| .memory-item-meta { | |
| display: flex; | |
| align-items: center; | |
| flex-wrap: wrap; | |
| gap: 4px; | |
| font-size: 0.68rem; | |
| color: var(--gray-400); | |
| margin-top: 2px; | |
| } | |
| .memory-item-del { | |
| cursor: pointer; | |
| color: var(--gray-400); | |
| font-size: 0.8rem; | |
| padding: 0 4px; | |
| border: none; | |
| background: none; | |
| line-height: 1; | |
| flex-shrink: 0; | |
| } | |
| .memory-item-del:hover { color: var(--error-border); } | |
| /* Category badges */ | |
| .mem-cat { | |
| display: inline-block; | |
| padding: 0 5px; | |
| border-radius: 4px; | |
| font-size: 0.65rem; | |
| font-weight: 600; | |
| text-transform: uppercase; | |
| letter-spacing: 0.02em; | |
| } | |
| .mem-cat-fact { background: #dbeafe; color: #1d4ed8; } | |
| .mem-cat-preference { background: #ede9fe; color: #6d28d9; } | |
| .mem-cat-finding { background: #d1fae5; color: #065f46; } | |
| .mem-cat-document { background: #ffedd5; color: #c2410c; } | |
| .mem-cat-document_summary { background: #fef3c7; color: #92400e; } | |
| .mem-cat-conversation { background: var(--gray-200); color: var(--gray-700); } | |
| .memory-empty { | |
| text-align: center; | |
| color: var(--gray-400); | |
| font-size: 0.78rem; | |
| padding: 16px 0; | |
| } | |
| /* Audit view */ | |
| .audit-section { margin-bottom: 10px; } | |
| .audit-section-header { | |
| display: flex; | |
| align-items: center; | |
| gap: 6px; | |
| padding: 5px 0; | |
| font-size: 0.75rem; | |
| font-weight: 600; | |
| color: var(--gray-700); | |
| border-bottom: 1px solid var(--gray-200); | |
| margin-bottom: 6px; | |
| } | |
| .audit-section-header span { flex: 1; } | |
| .audit-clean { | |
| text-align: center; | |
| background: #d1fae5; | |
| color: #065f46; | |
| border-radius: 8px; | |
| padding: 14px; | |
| font-size: 0.82rem; | |
| font-weight: 500; | |
| } | |
| .audit-warn { | |
| font-size: 0.7rem; | |
| color: var(--gray-500); | |
| background: var(--loading-bg); | |
| border: 1px solid var(--loading-border); | |
| border-radius: 6px; | |
| padding: 6px 10px; | |
| margin-bottom: 8px; | |
| } | |
| .history-item { | |
| display: flex; | |
| align-items: center; | |
| gap: 8px; | |
| padding: 8px 10px; | |
| background: var(--gray-50); | |
| border-radius: 8px; | |
| font-size: 0.78rem; | |
| color: var(--gray-700); | |
| cursor: pointer; | |
| transition: background 0.15s; | |
| } | |
| .history-item:hover { background: rgba(102, 126, 234, 0.06); } | |
| .history-item-text { flex: 1; overflow: hidden; } | |
| .history-item-title { | |
| white-space: nowrap; | |
| overflow: hidden; | |
| text-overflow: ellipsis; | |
| font-weight: 500; | |
| } | |
| .history-item-meta { | |
| font-size: 0.68rem; | |
| color: var(--gray-400); | |
| margin-top: 1px; | |
| } | |
| .history-item-del { | |
| cursor: pointer; | |
| color: var(--gray-400); | |
| font-size: 0.8rem; | |
| padding: 0 4px; | |
| border: none; | |
| background: none; | |
| line-height: 1; | |
| flex-shrink: 0; | |
| } | |
| .history-item-del:hover { color: var(--error-border); } | |
| /* History sidebar */ | |
| .history-backdrop { | |
| display: none; | |
| position: absolute; | |
| inset: 0; | |
| background: rgba(0,0,0,0.2); | |
| z-index: 60; | |
| border-radius: 16px; | |
| } | |
| .history-backdrop.open { display: block; } | |
| .history-sidebar { | |
| position: absolute; | |
| top: 0; | |
| left: 0; | |
| bottom: 0; | |
| width: 280px; | |
| background: var(--white); | |
| border-right: 1px solid var(--gray-200); | |
| border-radius: 16px 0 0 16px; | |
| z-index: 70; | |
| display: flex; | |
| flex-direction: column; | |
| transform: translateX(-100%); | |
| transition: transform 0.25s ease; | |
| overflow: hidden; | |
| } | |
| .history-sidebar.open { transform: translateX(0); } | |
| .history-sidebar-header { | |
| display: flex; | |
| justify-content: space-between; | |
| align-items: center; | |
| padding: 14px 16px 10px; | |
| border-bottom: 1px solid var(--gray-200); | |
| font-size: 0.85rem; | |
| color: var(--gray-900); | |
| flex-shrink: 0; | |
| } | |
| .history-sidebar-list { | |
| flex: 1; | |
| overflow-y: auto; | |
| padding: 8px; | |
| display: flex; | |
| flex-direction: column; | |
| gap: 4px; | |
| } | |
| .btn-danger-text { color: var(--gray-400); } | |
| .btn-danger-text:hover { color: var(--error-border); } | |
| /* Camera overlay */ | |
| .camera-overlay { | |
| display: none; | |
| position: fixed; | |
| inset: 0; | |
| z-index: 200; | |
| background: rgba(0,0,0,0.85); | |
| flex-direction: column; | |
| align-items: center; | |
| justify-content: center; | |
| gap: 16px; | |
| } | |
| .camera-overlay.open { display: flex; } | |
| .camera-overlay video { | |
| max-width: 90vw; | |
| max-height: 60vh; | |
| border-radius: 12px; | |
| background: #000; | |
| } | |
| .camera-overlay .cam-controls { | |
| display: flex; | |
| gap: 16px; | |
| } | |
| .camera-overlay button { | |
| padding: 10px 24px; | |
| border-radius: 24px; | |
| border: none; | |
| font-size: 0.9rem; | |
| font-weight: 500; | |
| cursor: pointer; | |
| transition: opacity 0.15s; | |
| } | |
| .camera-overlay button:hover { opacity: 0.85; } | |
| .cam-capture-btn { | |
| background: var(--indigo-500); | |
| color: var(--white); | |
| } | |
| .cam-cancel-btn { | |
| background: var(--gray-200); | |
| color: var(--gray-800); | |
| } | |
| /* Batch panel */ | |
| .batch-panel { | |
| display: none; | |
| padding: 12px 16px; | |
| border-top: 1px solid var(--gray-200); | |
| background: var(--gray-50); | |
| flex-shrink: 0; | |
| } | |
| .batch-panel.open { display: block; } | |
| .batch-textarea { | |
| width: 100%; | |
| min-height: 96px; | |
| max-height: 200px; | |
| border: 1.5px solid var(--gray-200); | |
| border-radius: 8px; | |
| padding: 8px 10px; | |
| font-size: 0.78rem; | |
| font-family: inherit; | |
| color: var(--gray-800); | |
| background: var(--white); | |
| resize: vertical; | |
| outline: none; | |
| line-height: 1.5; | |
| margin: 8px 0; | |
| } | |
| .batch-textarea:focus { border-color: var(--indigo-500); } | |
| .batch-controls { | |
| display: flex; | |
| align-items: center; | |
| gap: 8px; | |
| flex-wrap: wrap; | |
| } | |
| .batch-chain-label { | |
| display: flex; | |
| align-items: center; | |
| gap: 5px; | |
| font-size: 0.75rem; | |
| color: var(--gray-600); | |
| cursor: pointer; | |
| flex: 1; | |
| } | |
| .batch-progress { | |
| font-size: 0.72rem; | |
| color: var(--gray-500); | |
| min-width: 80px; | |
| text-align: right; | |
| } | |
| .btn-run { | |
| padding: 4px 14px; | |
| border-radius: 7px; | |
| border: none; | |
| background: var(--indigo-500); | |
| color: white; | |
| font-size: 0.78rem; | |
| font-weight: 600; | |
| cursor: pointer; | |
| transition: background 0.15s; | |
| } | |
| .btn-run:hover:not(:disabled) { background: var(--indigo-600); } | |
| .btn-run:disabled { opacity: 0.45; cursor: not-allowed; } | |
| /* Share modal */ | |
| .share-backdrop { | |
| position: fixed; inset: 0; | |
| background: rgba(0,0,0,0.45); | |
| z-index: 600; | |
| display: none; | |
| align-items: center; | |
| justify-content: center; | |
| } | |
| .share-backdrop.open { display: flex; } | |
| .share-modal { | |
| background: var(--white); | |
| border-radius: 14px; | |
| padding: 24px; | |
| width: min(480px, 90vw); | |
| box-shadow: 0 12px 40px rgba(0,0,0,0.18); | |
| display: flex; | |
| flex-direction: column; | |
| gap: 14px; | |
| } | |
| .share-modal h3 { | |
| font-size: 1rem; | |
| font-weight: 700; | |
| color: var(--gray-900); | |
| margin: 0; | |
| } | |
| .share-modal label { | |
| font-size: 0.82rem; | |
| color: var(--gray-600); | |
| display: flex; | |
| align-items: center; | |
| gap: 8px; | |
| cursor: pointer; | |
| } | |
| .share-modal input[type="text"], | |
| .share-modal input[type="password"] { | |
| width: 100%; | |
| border: 1.5px solid var(--gray-200); | |
| border-radius: 8px; | |
| padding: 8px 10px; | |
| font-size: 0.82rem; | |
| color: var(--gray-800); | |
| background: var(--gray-50); | |
| outline: none; | |
| transition: border-color 0.15s; | |
| font-family: inherit; | |
| } | |
| .share-modal input:focus { border-color: var(--indigo-500); } | |
| .share-url-row { | |
| display: flex; | |
| gap: 6px; | |
| } | |
| .share-url-row input { flex: 1; font-family: monospace; font-size: 0.75rem; } | |
| .share-modal-footer { | |
| display: flex; | |
| justify-content: flex-end; | |
| gap: 8px; | |
| } | |
| .share-modal-footer button { | |
| padding: 7px 16px; | |
| border-radius: 8px; | |
| font-size: 0.82rem; | |
| font-weight: 600; | |
| cursor: pointer; | |
| border: 1.5px solid var(--gray-200); | |
| background: var(--white); | |
| color: var(--gray-700); | |
| transition: background 0.15s, border-color 0.15s; | |
| } | |
| .share-modal-footer button:hover { background: var(--gray-100); border-color: var(--gray-400); } | |
| .share-modal-footer .btn-primary { | |
| background: var(--indigo-500); | |
| color: white; | |
| border-color: var(--indigo-500); | |
| } | |
| .share-modal-footer .btn-primary:hover { background: var(--indigo-600); border-color: var(--indigo-600); } | |
| .share-modal-footer .btn-primary:disabled { opacity: 0.5; cursor: not-allowed; } | |
| .share-meta { font-size: 0.78rem; color: var(--gray-600); } | |
| /* Import banner */ | |
| .import-banner { | |
| position: fixed; | |
| top: 0; left: 0; right: 0; | |
| background: var(--indigo-500); | |
| color: white; | |
| padding: 12px 20px; | |
| display: none; | |
| align-items: center; | |
| gap: 12px; | |
| z-index: 700; | |
| font-size: 0.85rem; | |
| flex-wrap: wrap; | |
| } | |
| .import-banner.open { display: flex; } | |
| .import-banner strong { flex: 1; min-width: 180px; } | |
| .import-banner input[type="password"] { | |
| border: none; | |
| border-radius: 6px; | |
| padding: 5px 10px; | |
| font-size: 0.82rem; | |
| width: 160px; | |
| } | |
| .import-banner button { | |
| background: var(--white); | |
| color: var(--indigo-600); | |
| border: none; | |
| border-radius: 6px; | |
| padding: 5px 14px; | |
| font-size: 0.82rem; | |
| font-weight: 600; | |
| cursor: pointer; | |
| } | |
| .import-banner .dismiss-btn { | |
| background: transparent; | |
| color: rgba(255,255,255,0.8); | |
| font-weight: 400; | |
| padding: 5px 8px; | |
| } | |
| /* Drag-drop overlay */ | |
| .drag-overlay { | |
| display: none; | |
| position: absolute; | |
| inset: 0; | |
| z-index: 50; | |
| background: rgba(102, 126, 234, 0.08); | |
| border: 2px dashed var(--indigo-500); | |
| border-radius: 16px; | |
| align-items: center; | |
| justify-content: center; | |
| font-size: 0.9rem; | |
| color: var(--indigo-500); | |
| font-weight: 500; | |
| pointer-events: none; | |
| } | |
| .drag-overlay.visible { display: flex; } | |
| /* WebGPU error */ | |
| .webgpu-error { | |
| text-align: center; | |
| padding: 40px 20px; | |
| color: var(--error-text); | |
| } | |
| .webgpu-error h2 { | |
| font-size: 1.1rem; | |
| margin-bottom: 8px; | |
| } | |
| .webgpu-error p { | |
| font-size: 0.88rem; | |
| color: var(--gray-600); | |
| max-width: 400px; | |
| margin: 0 auto; | |
| } | |
| /* Footer */ | |
| .footer-nav { | |
| position: fixed; | |
| bottom: 12px; | |
| left: 50%; | |
| transform: translateX(-50%); | |
| display: flex; | |
| gap: 24px; | |
| background: rgba(255,255,255,0.92); | |
| padding: 6px 18px; | |
| border-radius: 20px; | |
| box-shadow: 0 1px 6px rgba(0,0,0,0.06); | |
| font-size: 0.72rem; | |
| white-space: nowrap; | |
| } | |
| .footer-nav a { | |
| color: #0366d6; | |
| text-decoration: none; | |
| transition: opacity 0.2s; | |
| } | |
| .footer-nav a:hover { opacity: 0.7; } | |
| /* Mobile */ | |
| @media (max-width: 600px) { | |
| body { padding: 12px 8px 56px; } | |
| .app { height: calc(100vh - 80px); } | |
| .header h1 { font-size: 1.2rem; } | |
| .msg { max-width: 92%; } | |
| .chat-area { padding: 14px 12px; } | |
| .help-popover { width: calc(100vw - 32px); right: -8px; max-width: 340px; } | |
| .afford-btn { width: 32px; height: 32px; font-size: 0.9rem; } | |
| .attachment-chip { width: 48px; height: 48px; } | |
| .attachment-chip.video-chip { width: 80px; height: 52px; } | |
| .attachment-chip.audio-chip { width: 150px; height: 44px; } | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="app"> | |
| <!-- Header --> | |
| <div class="header"> | |
| <h1>LocalMind</h1> | |
| <div class="header-right"> | |
| <div class="help-wrap"> | |
| <button class="help-btn" id="helpBtn" aria-label="Help">?</button> | |
| <div class="help-popover" id="helpPopover"> | |
| <div class="help-tabs"> | |
| <button class="help-tab active" data-tab="about">About</button> | |
| <button class="help-tab" data-tab="models">Models</button> | |
| <button class="help-tab" data-tab="features">Features</button> | |
| <button class="help-tab" data-tab="api">JS API</button> | |
| <button class="help-tab" data-tab="try">Things to Try</button> | |
| </div> | |
| <div class="help-tab-content active" data-tab="about"> | |
| <p>A private AI research agent running <strong>Gemma</strong> entirely in your browser via WebGPU. Tool calling, persistent memory, and web search — all on-device.</p> | |
| <p>Only your search queries touch the network (and only when you choose to). All reasoning stays on your device.</p> | |
| <p>Models are cached after first download — future visits load instantly.</p> | |
| <p><strong>Requirements:</strong> Chrome 113+, Edge 113+, or Firefox 130+ with WebGPU.</p> | |
| <div class="help-note">v2.0.0 · Powered by Transformers.js + Google Gemma (Apache 2.0).</div> | |
| </div> | |
| <div class="help-tab-content" data-tab="models"> | |
| <h4>Available Models</h4> | |
| <ul> | |
| <li><strong>Ternary Bonsai 1.7B</strong> (~470 MB, default) — text + agent (tool calling). Smallest download with tool calling. 1.58-bit ternary weights, Qwen3 backbone, Apache-2.0.</li> | |
| <li><strong>Ternary Bonsai 4B</strong> (~1.1 GB) — same capabilities, better quality.</li> | |
| <li><strong>Ternary Bonsai 8B</strong> (~2.2 GB) — best Bonsai quality, 65K context.</li> | |
| <li><strong>Qwen3 4B</strong> (~2.8 GB) — text + agent (tool calling). Standard Qwen3-4B at q4f16, Apache-2.0, 32K context.</li> | |
| <li><strong>Gemma 3 1B</strong> (~760 MB) — text-only, no tool calling. Fallback option.</li> | |
| <li><strong>Gemma 4 E2B</strong> (~1.5 GB) — multimodal (image + audio) + agent.</li> | |
| <li><strong>Gemma 4 E4B</strong> (~4.9 GB) — multimodal + agent, best quality.</li> | |
| </ul> | |
| <h4>Agent Tools (Ternary Bonsai + Qwen3 + Gemma 4)</h4> | |
| <ul> | |
| <li><strong>calculate</strong> — math, percentages, conversions</li> | |
| <li><strong>get_current_time</strong> — date/time with timezone</li> | |
| <li><strong>store_memory</strong> — save facts to persistent memory</li> | |
| <li><strong>search_memory</strong> — recall from stored memories</li> | |
| <li><strong>web_search</strong> — search the web (requires API key)</li> | |
| <li><strong>fetch_page</strong> — read a web page’s content</li> | |
| <li><strong>set_reminder</strong> — browser notification after N minutes</li> | |
| <li><strong>list_memories</strong> — show what’s stored in memory</li> | |
| <li><strong>delete_memory</strong> — forget specific memories</li> | |
| <li><strong>segment_image</strong> — segment objects in attached images (SAM)</li> | |
| </ul> | |
| <h4>Image Segmentation (SAM)</h4> | |
| <p>Gemma 4 can call SAM (Segment Anything Model) to segment objects in attached images. Choose your SAM model in Settings — loaded on first use.</p> | |
| <ul> | |
| <li><strong>SlimSAM 50</strong> (~10 MB) — fastest, good enough for most tasks</li> | |
| <li><strong>SlimSAM 77</strong> (~14 MB) — default, better accuracy</li> | |
| <li><strong>SAM ViT-Base</strong> (~350 MB) — full quality, slower download</li> | |
| <li><strong>SAM 3</strong> (latest) — newest architecture</li> | |
| </ul> | |
| <h4>Things to try</h4> | |
| <ul> | |
| <li>“Segment the main object in this image”</li> | |
| <li>“Outline the person on the left”</li> | |
| <li>“Isolate the background”</li> | |
| <li>“How many distinct objects are in this image?”</li> | |
| </ul> | |
| <p>Translation works directly — Gemma 4 speaks 140+ languages natively, no tool needed.</p> | |
| <p>Gemma 3 1B works as a simple chatbot — no agent tools. Prefer Ternary Bonsai 1.7B if you want tool calling at a similar size.</p> | |
| </div> | |
| <div class="help-tab-content" data-tab="features"> | |
| <h4>Document Upload</h4> | |
| <ul> | |
| <li><strong>Text</strong> — .txt, .md, .json, .csv</li> | |
| <li><strong>PDF</strong> — extracted via PDF.js, auto-summarized</li> | |
| <li><strong>DOCX</strong> — extracted via mammoth.js, auto-summarized</li> | |
| </ul> | |
| <p>Documents are chunked, embedded, and stored as searchable knowledge. A summary is generated on upload.</p> | |
| <h4>Multimodal (Gemma 4)</h4> | |
| <ul> | |
| <li><strong>📎 Attach</strong> — images, audio, MP4 video, or documents</li> | |
| <li><strong>📷 Camera</strong> / <strong>🎤 Mic</strong> / <strong>Paste</strong> / <strong>Drag & drop</strong></li> | |
| </ul> | |
| <h4>Document Upload</h4> | |
| <ul> | |
| <li><strong>Text</strong> — .txt, .md, .json, .csv</li> | |
| <li><strong>PDF</strong> — extracted via PDF.js, auto-summarized</li> | |
| <li><strong>DOCX</strong> — extracted via mammoth.js, auto-summarized</li> | |
| <li><strong>Folder</strong> — open a local folder to ingest all .md/.txt/.pdf files at once; re-open to sync only changed files (incremental)</li> | |
| </ul> | |
| <h4>Conversations</h4> | |
| <ul> | |
| <li><strong>New Chat</strong> — archives to History + starts fresh</li> | |
| <li><strong>Clear</strong> — deletes without saving</li> | |
| <li><strong>History</strong> — sidebar, click to resume any past chat</li> | |
| <li><strong>Share</strong> — generate an encrypted or plain link to share any conversation; recipient opens the URL to load it</li> | |
| </ul> | |
| <h4>Memory browser</h4> | |
| <ul> | |
| <li><strong>Category pills</strong> — filter by fact / preference / finding / document / conversation with live counts</li> | |
| <li><strong>Source grouping</strong> — document chunks grouped by file; bulk “Delete all” per source</li> | |
| <li><strong>Audit</strong> — flags stale (>60 days), near-duplicates (cosine sim ≥0.92), and outliers (low avg similarity to category); bulk or per-item delete; auto-reruns after each deletion</li> | |
| </ul> | |
| <h4>Output & Export</h4> | |
| <ul> | |
| <li><strong>Save as MD</strong> — download any response as Markdown (or write directly to open folder if one is active)</li> | |
| <li><strong>Code download</strong> — hover code blocks for download button</li> | |
| <li><strong>Export / Import</strong> — in Memory panel, full data as JSON</li> | |
| <li><strong>Auto-backup</strong> — toggle in Settings to download on New Chat</li> | |
| </ul> | |
| <h4>Batch Prompts</h4> | |
| <ul> | |
| <li>Enter one prompt per line in the Batch panel — they run sequentially through the full agent loop</li> | |
| <li><strong>{{previous}}</strong> — explicit placeholder substituted with the previous response text</li> | |
| <li><strong>Auto-inject</strong> — checkbox (on by default) appends the previous response as context even without a placeholder; disabled for any prompt that already contains {{previous}}</li> | |
| <li><strong>Stop</strong> — halts after the current generation finishes; progress shown live</li> | |
| </ul> | |
| <h4>Other</h4> | |
| <ul> | |
| <li><strong>Web Search</strong> — Settings → provider + API key → 🌐 button</li> | |
| <li><strong>Thinking Mode</strong> — see chain-of-thought (collapses when done)</li> | |
| <li><strong>Multi-step planning</strong> (experimental, Gemma 4 only) — Settings → tick the toggle. Each message is planned into 2–5 steps, each step executed (with tools), then synthesised. Plan + per-step outputs render as collapsible blocks below the answer. Slower (3×+ model calls) but handles research-style queries better.</li> | |
| <li><strong>Branch from here</strong> — right-click (or long-press) a user message → "Branch from here". Archives the current conversation, then forks a new one containing messages up to that point. Continue the new branch from that question.</li> | |
| <li><strong>Custom tools</strong> (agent-capable models) — Settings → Custom tools. Paste a tool definition as JSON (<code>name</code>, <code>description</code>, <code>parameters</code>, <code>endpoint</code>). On a tool call the model's args are <code>POST</code>ed to your endpoint as a JSON body; the response is fed back to the model. CORS must allow this origin.</li> | |
| <li><strong>MCP servers</strong> (agent-capable models) — Settings → MCP servers. Paste a Streamable HTTP MCP endpoint URL (plus optional bearer). LocalMind opens a JSON-RPC 2.0 session, discovers tools via <code>tools/list</code>, and registers each with an <code>mcp_</code> prefix so the agent loop can use them alongside built-ins.</li> | |
| <li><strong>Math & diagrams</strong> — inline <code>$\int x^2 dx$</code> and display <code>$$\\sum_{i=1}^n i$$</code> math render via KaTeX; <code>```mermaid</code> blocks render as SVG via lazy-loaded Mermaid.</li> | |
| <li><strong>Artifact preview</strong> — <code>```html</code> / <code>```svg</code> / <code>```artifact</code> code blocks get a live sandboxed iframe below the code (<code>sandbox="allow-scripts"</code>, no same-origin). Safe to run model-generated UI inline.</li> | |
| <li><strong>Voice to text</strong> — π£ button left of the input records mic audio, decodes to 16 kHz mono PCM on-device, and runs Whisper-base via WebGPU to transcribe into the input. ~80 MB first-use download; nothing leaves the device.</li> | |
| <li><strong>Python code tool</strong> (agent-capable models) — model can call <code>run_python</code> to execute Python in a sandboxed Pyodide worker. numpy / pandas / matplotlib auto-install on import. ~10 MB first-use download.</li> | |
| <li><strong>Cache management</strong> — view/clear cached models in Settings</li> | |
| <li><strong>Custom models</strong> — Settings → paste a Hugging Face ONNX repo id (causal LMs only). The validator probes the HF API, picks the best available quantisation, estimates real load size, and hard-blocks anything that exceeds the device’s WebGPU buffer limit or the 6 GB ceiling.</li> | |
| <li><strong>Response badges</strong> — On-device / Agent / Web-enriched</li> | |
| </ul> | |
| </div> | |
| <div class="help-tab-content" data-tab="api"> | |
| <h4>JavaScript API <span style="font-weight:400;color:var(--gray-500)">(experimental)</span></h4> | |
| <p>Settings → tick <strong>Expose <code>window.localmind</code></strong>. Same-tab only — cross-origin scripts cannot reach it. The object is frozen and non-writable; disable the toggle to detach.</p> | |
| <h4>Surface (v1.0)</h4> | |
| <ul> | |
| <li><strong>version</strong> · <strong>ready</strong> · <strong>model</strong> — live state getters</li> | |
| <li><strong>listModels()</strong> — full registry incl. custom models with <code>loaded</code> flag</li> | |
| <li><strong>load(idOrKey)</strong> — loads a model (short key or HF id); resolves when ready</li> | |
| <li><strong>chat.completions.create({ messages, max_tokens, temperature, top_p, model })</strong> — non-streaming, returns OpenAI-shaped <code>chat.completion</code></li> | |
| <li><strong>chat.completions.create({ …, stream: true })</strong> — async iterator yielding <code>chat.completion.chunk</code> objects</li> | |
| </ul> | |
| <h4>Not exposed</h4> | |
| <ul> | |
| <li>Tools / tool calling</li> | |
| <li>Memory read/write</li> | |
| <li>File system, web search, search API keys, user profile</li> | |
| <li>Multimodal input</li> | |
| </ul> | |
| <h4>Activity log</h4> | |
| <p>Every API call is logged in-memory (last 50). Click the <code>• API</code> chip in the toolbar or <strong>Settings → View activity log</strong>. Each call shows method, prompt length, tokens generated, duration, and outcome (ok / err / busy).</p> | |
| <h4>Demo</h4> | |
| <p>Open <code>demo.html</code> in the same folder. It iframes LocalMind, auto-flips the toggle, waits for the model, and runs both a non-streaming and a streaming completion against <code>iframe.contentWindow.localmind</code>.</p> | |
| <div class="help-note">Experimental — the shape may change before a stable v1.1.</div> | |
| </div> | |
| <div class="help-tab-content" data-tab="try"> | |
| <h4>Math & Conversions</h4> | |
| <span class="try-prompt" data-prompt="What is 15% of 2450?">What is 15% of 2450?</span> | |
| <span class="try-prompt" data-prompt="Convert 72 Fahrenheit to Celsius">Convert 72 Fahrenheit to Celsius</span> | |
| <span class="try-prompt" data-prompt="If I invest $10,000 at 7% annual return, how much after 5 years with compound interest?">Compound interest: $10K at 7% for 5 years?</span> | |
| <h4>Time & Reminders</h4> | |
| <span class="try-prompt" data-prompt="What time is it in Tokyo?">What time is it in Tokyo?</span> | |
| <span class="try-prompt" data-prompt="Remind me in 5 minutes to check the oven">Remind me in 5 minutes to check the oven</span> | |
| <h4>Memory</h4> | |
| <span class="try-prompt" data-prompt="Remember that I'm a software engineer working on a React project called Dashboard Pro">Remember: I'm a software engineer on Dashboard Pro</span> | |
| <span class="try-prompt" data-prompt="What do you know about me and my projects?">What do you know about me?</span> | |
| <span class="try-prompt" data-prompt="Forget everything about my preferences">Forget my preferences</span> | |
| <h4>Translation</h4> | |
| <span class="try-prompt" data-prompt="Translate 'Good morning, how are you?' to Japanese, French, and Hindi">Translate "Good morning" to Japanese, French, Hindi</span> | |
| <span class="try-prompt" data-prompt="How do you say 'Where is the nearest train station?' in Spanish and German?">Train station directions in Spanish & German</span> | |
| <h4>Writing & Analysis</h4> | |
| <span class="try-prompt" data-prompt="Write a professional email declining a meeting invitation politely">Write a polite meeting decline email</span> | |
| <span class="try-prompt" data-prompt="Summarize the pros and cons of microservices vs monolithic architecture">Microservices vs monolith: pros and cons</span> | |
| <span class="try-prompt" data-prompt="Explain the concept of WebGPU to a non-technical person in 3 sentences">Explain WebGPU in 3 simple sentences</span> | |
| <h4>Documents (attach a PDF, DOCX, or text file)</h4> | |
| <span class="try-prompt" data-prompt="Summarize the key points from the document I just uploaded">Summarize the uploaded document</span> | |
| <span class="try-prompt" data-prompt="What are the main conclusions or recommendations in my document?">Main conclusions from my document</span> | |
| <h4>Multimodal (attach an image first)</h4> | |
| <span class="try-prompt" data-prompt="Describe this image in detail">Describe this image in detail</span> | |
| <span class="try-prompt" data-prompt="What text can you see in this image? Transcribe it.">Transcribe text from this image</span> | |
| <h4>Web Research (requires API key)</h4> | |
| <span class="try-prompt" data-prompt="What are the top tech news stories today?">Top tech news today</span> | |
| <span class="try-prompt" data-prompt="Search for the latest WebGPU browser support status and summarize">Latest WebGPU browser support status</span> | |
| <span class="try-prompt" data-prompt="Find recent articles about AI running locally in the browser and give me a summary with sources">AI in the browser: recent articles with sources</span> | |
| <h4>Coding</h4> | |
| <span class="try-prompt" data-prompt="Write a Python function that finds all prime numbers up to n using the Sieve of Eratosthenes">Sieve of Eratosthenes in Python</span> | |
| <span class="try-prompt" data-prompt="Explain the difference between async/await and Promises in JavaScript with examples">async/await vs Promises explained</span> | |
| <h4>Math & Diagrams</h4> | |
| <span class="try-prompt" data-prompt="Derive the quadratic formula step by step, using inline and display LaTeX math.">Derive the quadratic formula with LaTeX</span> | |
| <span class="try-prompt" data-prompt="Show the Pythagorean identity sin^2 x + cos^2 x = 1 and sketch a short proof using LaTeX display math.">Pythagorean identity with display math</span> | |
| <span class="try-prompt" data-prompt="Draw a Mermaid flowchart for a login flow: user submits credentials, server validates, issues a JWT, client stores it, requests authorized resources.">Mermaid: login + JWT flow</span> | |
| <span class="try-prompt" data-prompt="Draw a Mermaid sequence diagram for the classic producer-consumer pattern with a bounded buffer.">Mermaid: producer-consumer sequence</span> | |
| <h4>Live HTML / SVG Artifacts</h4> | |
| <span class="try-prompt" data-prompt="Output only a full standalone HTML document in a ```html block: a click counter with +, -, and reset buttons. Style it nicely.">Interactive counter (HTML artifact)</span> | |
| <span class="try-prompt" data-prompt="Output only a complete HTML document in a ```html block: a Pomodoro timer with 25-minute countdown, start, pause, reset.">Pomodoro timer (HTML artifact)</span> | |
| <span class="try-prompt" data-prompt="Output only an SVG in a ```svg block: a sunrise scene with gradient sky, sun, and three mountain silhouettes.">Sunrise scene (SVG artifact)</span> | |
| <h4>Python (Gemma 4, run_python tool)</h4> | |
| <span class="try-prompt" data-prompt="Use run_python to compute the first 20 Fibonacci numbers and print them as a comma-separated list.">First 20 Fibonacci via Python</span> | |
| <span class="try-prompt" data-prompt="Use run_python with pandas to generate a synthetic dataframe of 1000 rows with columns age (18-80), income (30k-200k, roughly log-normal), and compute the correlation.">Pandas synthetic data + correlation</span> | |
| <span class="try-prompt" data-prompt="Use run_python to solve this system of equations with numpy: 2x + 3y - z = 5, x - y + 2z = 3, 3x + y + z = 10">Solve a 3x3 linear system</span> | |
| <h4>Multi-step planning (Gemma 4, Settings toggle)</h4> | |
| <span class="try-prompt" data-prompt="Compare cold brew, pour-over, and French press coffee: typical brew time, caffeine content, flavor profile, and equipment cost. Give a recommendation for someone new to specialty coffee.">Compare 3 coffee brewing methods</span> | |
| <span class="try-prompt" data-prompt="Research the history of the Silk Road in 3 phases (ancient origins, medieval peak, modern revival), then summarize the key goods traded at each phase.">Silk Road in 3 phases</span> | |
| <h4>MCP tools (after adding a server in Settings)</h4> | |
| <span class="try-prompt" data-prompt="List every MCP tool you have available and what each one does.">List all MCP tools I have</span> | |
| <span class="try-prompt" data-prompt="Use mcp_fetch_url (or whatever fetch-like MCP tool is available) to grab https://example.com and summarise the response.">Use an MCP fetch tool</span> | |
| <h4>Voice to text (no prompt needed)</h4> | |
| <p style="font-size:0.78rem;color:var(--gray-500);margin:4px 0 8px">Click the π£ button to the left of the input, speak a sentence, click again to stop. Whisper runs on-device. Try saying: <em>“Write a short summary of the Great Pyramid of Giza in three sentences.”</em></p> | |
| <div class="help-note"> | |
| <strong>Tips.</strong> | |
| <ul style="margin:4px 0 0 16px;padding:0;font-size:0.72rem;line-height:1.5"> | |
| <li>For artifact and Mermaid prompts, start with <em>“Output only this…”</em> or <em>“Copy this exactly…”</em> — smaller models tend to wrap code blocks in prose otherwise.</li> | |
| <li>If a <code>```mermaid</code> block renders as plain code, the model stripped the language label. Re-send with <em>“Preserve the language tag after the backticks”</em>.</li> | |
| <li>Click any prompt above to paste it. Web search needs a provider in Settings. Multimodal needs a Gemma 4 model + an attached image. Artifact, math, and Mermaid rendering work on any model; <code>run_python</code>, MCP tools, and multi-step planning require an agent-capable (Gemma 4) model.</li> | |
| </ul> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="status-badge loading" id="statusBadge"> | |
| <div class="spinner" id="statusSpinner"></div> | |
| <span id="statusText">Initializing...</span> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- Main card --> | |
| <div class="card" style="position:relative"> | |
| <!-- Drag-drop overlay --> | |
| <div class="drag-overlay" id="dragOverlay">Drop image, audio, or video here</div> | |
| <!-- Chat messages --> | |
| <div class="chat-area" id="chatArea"> | |
| <div class="welcome" id="welcomeMsg"> | |
| <div class="welcome-icon">🧠</div> | |
| <h2>Private AI, right in your browser</h2> | |
| <p>Models run entirely on your device via WebGPU. Nothing is sent to any server.</p> | |
| <p style="font-size:0.78rem;color:var(--gray-400);margin-top:4px">Pick a Ternary Bonsai or Qwen3 model for tool calling and memory; pick a Gemma 4 model for multimodal (image + audio) input.<br>Web search requires an API key (Tavily, Brave) or a self-hosted SearXNG instance.</p> | |
| </div> | |
| </div> | |
| <!-- Progress bar (model loading) --> | |
| <div class="progress-section" id="progressSection"> | |
| <div class="progress-bar"> | |
| <div class="progress-fill" id="progressFill"></div> | |
| </div> | |
| <div class="progress-text-row"> | |
| <div class="progress-text" id="progressText">Preparing to download model...</div> | |
| <button class="progress-cancel-btn" id="progressCancelBtn" title="Cancel download (partial chunks are preserved for resume)">Cancel</button> | |
| </div> | |
| </div> | |
| <!-- Progress bar (SAM loading) --> | |
| <div class="progress-section" id="samProgressSection"> | |
| <div class="progress-bar"> | |
| <div class="progress-fill" id="samProgressFill" style="background:linear-gradient(90deg,#059669,#10b981)"></div> | |
| </div> | |
| <div class="progress-text" id="samProgressText">Loading SAM model...</div> | |
| </div> | |
| <!-- Settings (system prompt + thinking toggle) --> | |
| <div class="settings-panel" id="settingsPanel"> | |
| <label for="systemPrompt">System prompt (optional)</label> | |
| <textarea id="systemPrompt" placeholder="e.g. You are a helpful coding assistant."></textarea> | |
| <label class="toggle-row hidden" id="thinkingRow"> | |
| <input type="checkbox" id="thinkingToggle"> | |
| Show reasoning (thinking mode) | |
| </label> | |
| <label class="toggle-row hidden" id="plannerRow"> | |
| <input type="checkbox" id="plannerToggle"> | |
| Multi-step planning <span style="font-weight:400;color:var(--gray-500)">(experimental — slower but handles complex research tasks better)</span> | |
| </label> | |
| <div class="settings-divider" id="searchSettingsSection" style="display:none"> | |
| <label for="searchProvider">Web search provider</label> | |
| <select id="searchProvider" class="settings-select"> | |
| <option value="none">None (offline only)</option> | |
| <option value="tavily">Tavily (free tier, no card)</option> | |
| <option value="brave">Brave Search (privacy-first)</option> | |
| <option value="searxng">SearXNG (self-hosted)</option> | |
| </select> | |
| <div id="apiKeyRow" style="display:none"> | |
| <label for="searchApiKey">API key</label> | |
| <input type="password" id="searchApiKey" class="settings-input" placeholder="Paste your API key"> | |
| </div> | |
| <div id="searxngUrlRow" style="display:none"> | |
| <label for="searxngUrl">SearXNG instance URL</label> | |
| <input type="text" id="searxngUrl" class="settings-input" placeholder="https://searx.example.com"> | |
| </div> | |
| </div> | |
| <div class="settings-divider" id="samSettingsSection" style="display:none"> | |
| <label for="samModelSelect">Segmentation model (SAM)</label> | |
| <select id="samModelSelect" class="settings-select"> | |
| <option value="Xenova/slimsam-50-uniform">SlimSAM 50 · ~10 MB (fastest)</option> | |
| <option value="Xenova/slimsam-77-uniform" selected>SlimSAM 77 · ~14 MB (default)</option> | |
| <option value="Xenova/sam-vit-base">SAM ViT-Base · ~350 MB</option> | |
| <option value="onnx-community/sam3-tracker-ONNX">SAM 3 · Latest</option> | |
| </select> | |
| <div style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5"> | |
| Loaded on first use. Gemma calls this when you ask it to segment, outline, or identify objects in attached images. | |
| </div> | |
| </div> | |
| <label class="toggle-row" style="border-top:1px solid var(--gray-200);padding-top:10px;margin-top:10px"> | |
| <input type="checkbox" id="autoBackupToggle"> | |
| Auto-download backup on New Chat | |
| </label> | |
| <div class="settings-divider"> | |
| <label>Model cache</label> | |
| <div id="cacheInfo" style="font-size:0.75rem;color:var(--gray-500);margin-top:4px">Loading...</div> | |
| <label class="toggle-row" style="padding-top:8px"> | |
| <input type="checkbox" id="resumableDownloadsToggle"> | |
| Resumable downloads <span style="font-weight:400;color:var(--gray-500)">(5 MB chunks cached in IndexedDB; interrupted downloads resume from the last chunk)</span> | |
| </label> | |
| </div> | |
| <div class="settings-divider"> | |
| <label for="voiceLanguageSelect">Voice to text language <span style="font-weight:400;color:var(--gray-500)">(Whisper — leaving as English improves accuracy on short clips)</span></label> | |
| <select id="voiceLanguageSelect" class="settings-select"> | |
| <option value="en">English</option> | |
| <option value="es">Spanish</option> | |
| <option value="fr">French</option> | |
| <option value="de">German</option> | |
| <option value="it">Italian</option> | |
| <option value="pt">Portuguese</option> | |
| <option value="ru">Russian</option> | |
| <option value="ja">Japanese</option> | |
| <option value="zh">Chinese</option> | |
| <option value="hi">Hindi</option> | |
| <option value="ar">Arabic</option> | |
| <option value="ko">Korean</option> | |
| </select> | |
| </div> | |
| <div class="settings-divider"> | |
| <label>Custom models <span style="font-weight:400;color:var(--gray-500)">(causal LMs with ONNX exports on Hugging Face)</span></label> | |
| <div style="display:flex;gap:6px;margin-top:4px"> | |
| <input type="text" id="customModelInput" class="settings-input" placeholder="owner/repo e.g. onnx-community/Llama-3.2-1B-Instruct" style="flex:1"> | |
| <button class="btn-icon" id="customModelAddBtn">Add</button> | |
| </div> | |
| <div id="customModelStatus" style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5;min-height:1em"></div> | |
| <div id="customModelList" style="margin-top:6px"></div> | |
| <div style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5"> | |
| Must be a Hugging Face repo with ONNX files under <code>onnx/</code>. Multimodal | |
| custom models are not yet supported. Added models appear in the model selector. | |
| </div> | |
| </div> | |
| <div class="settings-divider"> | |
| <label>Custom tools <span style="font-weight:400;color:var(--gray-500)">(agent-capable models; HTTP POST to your endpoint)</span></label> | |
| <textarea id="customToolInput" class="settings-input" rows="6" placeholder='{ | |
| "name": "my_tool", | |
| "description": "What this tool does", | |
| "parameters": { "type": "object", "properties": { "query": { "type": "string" } }, "required": ["query"] }, | |
| "endpoint": "https://example.com/tool" | |
| }' style="margin-top:4px;font-family:monospace;font-size:0.72rem"></textarea> | |
| <div style="display:flex;gap:6px;margin-top:4px"> | |
| <button class="btn-icon" id="customToolAddBtn">Add tool</button> | |
| </div> | |
| <div id="customToolStatus" style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5;min-height:1em"></div> | |
| <div id="customToolList" style="margin-top:6px"></div> | |
| <div style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5"> | |
| On a tool call, LocalMind does <code>POST <endpoint></code> with the model-generated | |
| args as JSON body; the response JSON is fed back to the model. The endpoint must send | |
| CORS headers for this origin. Name must be <code>[a-zA-Z_][a-zA-Z0-9_]*</code> and not | |
| collide with a built-in. | |
| </div> | |
| </div> | |
| <div class="settings-divider"> | |
| <label>MCP servers <span style="font-weight:400;color:var(--gray-500)">(Streamable HTTP transport; tools discovered via JSON-RPC <code>tools/list</code>)</span></label> | |
| <div style="display:flex;gap:6px;margin-top:4px"> | |
| <input type="text" id="mcpUrlInput" class="settings-input" placeholder="https://mcp.example.com" style="flex:1"> | |
| <input type="password" id="mcpAuthInput" class="settings-input" placeholder="Bearer (optional)" style="flex:0 0 40%"> | |
| <button class="btn-icon" id="mcpAddBtn">Add server</button> | |
| </div> | |
| <div id="mcpStatus" style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5;min-height:1em"></div> | |
| <div id="mcpList" style="margin-top:6px"></div> | |
| <div style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5"> | |
| Tools discovered from an MCP server are registered with the prefix <code>mcp_</code> | |
| to avoid collisions with built-ins. The server must allow CORS for this origin and | |
| accept JSON-RPC 2.0 requests at the given URL. Connections re-established on page load. | |
| </div> | |
| </div> | |
| <div class="settings-divider"> | |
| <label>JavaScript API <span style="font-weight:400;color:var(--gray-500)">(experimental)</span></label> | |
| <label class="toggle-row" style="padding-top:4px"> | |
| <input type="checkbox" id="apiEnabledToggle"> | |
| Expose <code>window.localmind</code> to scripts in this page | |
| </label> | |
| <div style="font-size:0.7rem;color:var(--gray-500);margin-top:4px;line-height:1.5"> | |
| Lets other JavaScript on this page call the loaded model via an OpenAI-shaped | |
| method. Same-tab only β cross-origin scripts cannot reach it. No tool calling, | |
| memory access, or web search is exposed. | |
| </div> | |
| <button class="btn-icon" id="apiLogBtn" style="margin-top:8px">View activity log</button> | |
| </div> | |
| </div> | |
| <!-- Attachment bar --> | |
| <div class="attachment-bar" id="attachmentBar"></div> | |
| <!-- Memory inspector panel --> | |
| <div class="memory-panel" id="memoryPanel"> | |
| <div class="memory-header"> | |
| <strong>Memory</strong> | |
| <span class="memory-count" id="memoryCount">0 memories</span> | |
| </div> | |
| <div class="memory-search-row"> | |
| <input type="text" id="memorySearch" placeholder="Search memories..." class="memory-search-input"> | |
| <button class="btn-icon memory-clear-btn" id="memoryClearAll" title="Clear all memories">Clear All</button> | |
| </div> | |
| <div class="memory-cat-pills" id="memoryCatPills"></div> | |
| <div class="memory-list" id="memoryList"></div> | |
| <div class="memory-search-row" style="margin-top:8px;border-top:1px solid var(--gray-200);padding-top:8px"> | |
| <button class="btn-icon" id="memoryExport" title="Export all data">Export</button> | |
| <button class="btn-icon" id="memoryImport" title="Import data">Import</button> | |
| <button class="btn-icon" id="memoryAuditBtn" title="Audit memory for stale, duplicate, and outlier chunks">Audit</button> | |
| <input type="file" id="importFileInput" accept=".json" style="display:none"> | |
| </div> | |
| </div> | |
| <!-- Batch panel --> | |
| <div class="batch-panel" id="batchPanel"> | |
| <div class="memory-header"> | |
| <strong>Batch Prompts</strong> | |
| <span class="memory-count" id="batchCount">0 prompts</span> | |
| </div> | |
| <textarea class="batch-textarea" id="batchTextarea" placeholder="One prompt per line. Use {{previous}} to insert the previous response into the next prompt. Example: Summarise the history of WebGPU Now translate this to French: {{previous}}"></textarea> | |
| <div class="batch-controls"> | |
| <label class="batch-chain-label"> | |
| <input type="checkbox" id="batchChainToggle" checked> | |
| Auto-inject {{previous}} | |
| </label> | |
| <span class="batch-progress" id="batchProgress"></span> | |
| <button class="btn-icon" id="batchStopBtn" disabled>Stop</button> | |
| <button class="btn-run" id="batchRunBtn" disabled>Run</button> | |
| </div> | |
| </div> | |
| <!-- Actions bar --> | |
| <div class="actions-bar"> | |
| <button class="btn-icon" id="newChatBtn" disabled title="Archive and start new chat">New Chat</button> | |
| <button class="btn-icon btn-danger-text" id="clearBtn" disabled title="Delete current chat">Clear</button> | |
| <button class="btn-icon" id="settingsBtn" title="System prompt settings">Settings</button> | |
| <button class="btn-icon" id="memoryBtn" title="View stored memories">Memory</button> | |
| <button class="btn-icon" id="folderBtn" title="Open folder β ingest all .md/.txt/.pdf files into memory">Folder</button> | |
| <button class="btn-icon" id="historyBtn" title="Past conversations">History</button> | |
| <button class="btn-icon" id="shareBtn" title="Share conversation (encrypted link)">Share</button> | |
| <button class="btn-icon" id="batchBtn" title="Batch prompts β run a list of questions sequentially">Batch</button> | |
| <span style="flex:1"></span> | |
| <span class="api-chip" id="apiChip" title="window.localmind API is enabled β click to view activity log">API</span> | |
| <select class="model-select" id="modelSelect" title="Choose model"> | |
| <option value="bonsai-ternary-1.7b" selected>Ternary Bonsai 1.7B · Agent (~470 MB)</option> | |
| <option value="bonsai-ternary-4b">Ternary Bonsai 4B · Agent (~1.1 GB)</option> | |
| <option value="bonsai-ternary-8b">Ternary Bonsai 8B · Agent (~2.2 GB)</option> | |
| <option value="qwen3-4b">Qwen3 4B · Agent (~2.8 GB)</option> | |
| <option value="gemma3-1b">Gemma 3 1B · Fast (~760 MB)</option> | |
| <option value="gemma4-e2b">Gemma 4 E2B · Multimodal (~1.5 GB)</option> | |
| <option value="gemma4-e4b">Gemma 4 E4B · Multimodal (~4.9 GB)</option> | |
| </select> | |
| </div> | |
| <!-- Input --> | |
| <div class="input-bar"> | |
| <div class="input-affordances" id="inputAffordances"> | |
| <button class="afford-btn" id="attachBtn" title="Attach file" aria-label="Attach file">📎</button> | |
| <button class="afford-btn" id="cameraBtn" title="Camera" aria-label="Take photo">📷</button> | |
| <button class="afford-btn" id="micBtn" title="Record audio" aria-label="Record audio">🎤</button> | |
| </div> | |
| <button class="afford-btn" id="voiceBtn" title="Voice to text (Whisper, on-device)" aria-label="Voice to text">🗣</button> | |
| <textarea id="chatInput" rows="1" placeholder="Type a message..." disabled></textarea> | |
| <div class="send-group"> | |
| <button class="send-btn" id="sendBtn" disabled aria-label="Send" title="Send (offline)">▶</button> | |
| <button class="search-send-btn" id="searchSendBtn" disabled aria-label="Search and Send" title="Search web + Send (makes an external API call)" style="display:none">🌐</button> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- History sidebar --> | |
| <div class="history-backdrop" id="historyBackdrop"></div> | |
| <div class="history-sidebar" id="historySidebar"> | |
| <div class="history-sidebar-header"> | |
| <strong>History</strong> | |
| <span class="memory-count" id="historyCount">0 conversations</span> | |
| </div> | |
| <div class="history-sidebar-list" id="historyList"></div> | |
| </div> | |
| </div> | |
| <!-- Import banner (shown when page loads with a share link) --> | |
| <div class="import-banner" id="importBanner"> | |
| <strong id="importBannerMsg">Shared conversation detected</strong> | |
| <input type="password" id="importPassphrase" placeholder="Passphrase" style="display:none"> | |
| <button id="importConfirmBtn">Load conversation</button> | |
| <button class="dismiss-btn" id="importDismissBtn">Dismiss</button> | |
| </div> | |
| <!-- Message context menu (right-click / long-press on a message) --> | |
| <div class="msg-context-menu" id="msgContextMenu"> | |
| <button id="msgBranchBtn">Branch from here</button> | |
| </div> | |
| <!-- API activity log modal --> | |
| <div class="api-log-modal" id="apiLogModal"> | |
| <div class="api-log-modal-content"> | |
| <div class="api-log-header"> | |
| <strong>API activity log</strong> | |
| <div> | |
| <button class="btn-icon" id="apiLogClearBtn">Clear</button> | |
| <button class="btn-icon" id="apiLogCloseBtn">Close</button> | |
| </div> | |
| </div> | |
| <div class="api-log-list" id="apiLogList"> | |
| <div class="api-log-empty">No API calls yet.</div> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- Share modal --> | |
| <div class="share-backdrop" id="shareBackdrop"> | |
| <div class="share-modal"> | |
| <h3>Share Conversation</h3> | |
| <p class="share-meta" id="shareMetaText"></p> | |
| <label> | |
| <input type="checkbox" id="shareUsePassphrase"> | |
| Encrypt with passphrase (AES-256-GCM) | |
| </label> | |
| <input type="password" id="sharePassphrase" placeholder="Passphrase" style="display:none" autocomplete="new-password"> | |
| <div class="share-url-row"> | |
| <input type="text" id="shareUrlInput" readonly placeholder="Click Generate to create linkβ¦"> | |
| </div> | |
| <div class="share-modal-footer"> | |
| <button id="shareCopyBtn" disabled>Copy Link</button> | |
| <button class="btn-primary" id="shareGenerateBtn">Generate</button> | |
| <button id="shareCloseBtn">Close</button> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- Camera overlay --> | |
| <div class="camera-overlay" id="cameraOverlay"> | |
| <video id="cameraPreview" autoplay playsinline></video> | |
| <div class="cam-controls"> | |
| <button class="cam-capture-btn" id="camCaptureBtn">Capture</button> | |
| <button class="cam-cancel-btn" id="camCancelBtn">Cancel</button> | |
| </div> | |
| </div> | |
| <!-- Hidden file input --> | |
| <input type="file" id="fileInput" style="display:none" multiple accept="image/*,audio/*,video/mp4,.txt,.md,.json,.csv,.pdf,.docx"> | |
| <!-- Footer --> | |
| <div class="footer-nav"> | |
| <a href="https://naklitechie.github.io/" target="_blank" rel="noopener">⬅ More Projects</a> | |
| <a href="https://github.com/NakliTechie/LocalMind" target="_blank" rel="noopener">⌨ Source</a> | |
| </div> | |
| <script type="module"> | |
| // ββ WebGPU check ββββββββββββββββββββββββββββββββββββββββββββ | |
| const hasWebGPU = !!navigator.gpu; | |
| if (!hasWebGPU) { | |
| document.getElementById('chatArea').innerHTML = ` | |
| <div class="webgpu-error"> | |
| <div style="font-size:2rem;margin-bottom:12px">⚠</div> | |
| <h2>WebGPU Not Available</h2> | |
| <p>LocalMind requires WebGPU to run the AI model in your browser. | |
| Please use Chrome 113+, Edge 113+, or Firefox 130+ on a device with a compatible GPU.</p> | |
| </div>`; | |
| document.getElementById('statusBadge').className = 'status-badge error'; | |
| document.getElementById('statusSpinner').style.display = 'none'; | |
| document.getElementById('statusText').textContent = 'Not supported'; | |
| throw new Error('WebGPU not available'); | |
| } | |
| // ββ Model registry ββββββββββββββββββββββββββββββββββββββββββ | |
| const MODELS = { | |
| 'bonsai-ternary-1.7b': { | |
| id: 'onnx-community/Ternary-Bonsai-1.7B-ONNX', | |
| label: 'Ternary Bonsai 1.7B', | |
| dtype: 'q2f16', | |
| size: '~470 MB', | |
| type: 'causal', | |
| multimodal: false, | |
| agentCapable: true, | |
| contextSize: 32768, | |
| // Qwen3 recommended sampling for chat/agent. min_p not exposed by | |
| // Transformers.js generate(), top_p covers it adequately. | |
| genConfig: { temperature: 0.7, top_k: 20, top_p: 0.8, max_new_tokens: 2048, repetition_penalty: 1.05 }, | |
| }, | |
| 'bonsai-ternary-4b': { | |
| id: 'onnx-community/Ternary-Bonsai-4B-ONNX', | |
| label: 'Ternary Bonsai 4B', | |
| dtype: 'q2f16', | |
| size: '~1.1 GB', | |
| type: 'causal', | |
| multimodal: false, | |
| agentCapable: true, | |
| contextSize: 32768, | |
| genConfig: { temperature: 0.7, top_k: 20, top_p: 0.8, max_new_tokens: 2048, repetition_penalty: 1.05 }, | |
| }, | |
| 'bonsai-ternary-8b': { | |
| id: 'onnx-community/Ternary-Bonsai-8B-ONNX', | |
| label: 'Ternary Bonsai 8B', | |
| dtype: 'q2f16', | |
| size: '~2.2 GB', | |
| type: 'causal', | |
| multimodal: false, | |
| agentCapable: true, | |
| contextSize: 65536, | |
| genConfig: { temperature: 0.7, top_k: 20, top_p: 0.8, max_new_tokens: 2048, repetition_penalty: 1.05 }, | |
| }, | |
| 'qwen3-4b': { | |
| id: 'onnx-community/Qwen3-4B-ONNX', | |
| label: 'Qwen3 4B', | |
| dtype: 'q4f16', | |
| size: '~2.8 GB', | |
| type: 'causal', | |
| multimodal: false, | |
| agentCapable: true, | |
| contextSize: 32768, | |
| // Qwen3 recommended sampling for non-thinking mode (temperature 0.7, | |
| // top_p 0.8, top_k 20). Apache-2.0, native tool calling. | |
| genConfig: { temperature: 0.7, top_k: 20, top_p: 0.8, max_new_tokens: 2048, repetition_penalty: 1.05 }, | |
| }, | |
| 'gemma3-1b': { | |
| id: 'onnx-community/gemma-3-1b-it-ONNX-GQA', | |
| label: 'Gemma 3 1B', | |
| dtype: 'q4f16', | |
| size: '~760 MB', | |
| type: 'causal', // runtime loads this as a text-only causal model | |
| multimodal: false, | |
| agentCapable: false, | |
| contextSize: 4096, | |
| genConfig: { temperature: 0.7, top_k: 50, top_p: 0.95, max_new_tokens: 2048 }, | |
| }, | |
| 'gemma4-e2b': { | |
| id: 'onnx-community/gemma-4-E2B-it-ONNX', | |
| label: 'Gemma 4 E2B', | |
| dtype: 'q4f16', | |
| size: '~1.5 GB', | |
| type: 'multimodal', // runtime loads this via the multimodal processor path | |
| multimodal: true, | |
| agentCapable: true, | |
| contextSize: 8192, | |
| genConfig: { temperature: 0.7, top_k: 40, top_p: 0.95, max_new_tokens: 2048, repetition_penalty: 1.1 }, | |
| }, | |
| 'gemma4-e4b': { | |
| id: 'onnx-community/gemma-4-E4B-it-ONNX', | |
| label: 'Gemma 4 E4B', | |
| dtype: 'q4f16', | |
| size: '~4.9 GB', | |
| type: 'multimodal', // runtime loads this via the multimodal processor path | |
| multimodal: true, | |
| agentCapable: true, | |
| contextSize: 12288, | |
| genConfig: { temperature: 0.7, top_k: 40, top_p: 0.95, max_new_tokens: 2048, repetition_penalty: 1.1 }, | |
| }, | |
| }; | |
| // ββ State βββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| let messages = []; | |
| let generating = false; | |
| let msgIdCounter = 0; | |
| let currentAssistantEl = null; | |
| let currentAssistantText = ''; | |
| let activeModelKey = null; | |
| let worker = null; | |
| let modelReady = false; | |
| let attachments = []; // [{type:'image'|'audio', blob:Blob, thumb:string, name:string, experimental?:bool}] | |
| let mediaRecorder = null; | |
| let cameraStream = null; | |
| let generateResolve = null; // Promise resolve for agentic loop | |
| let dirHandle = null; // FileSystemDirectoryHandle for open folder | |
| let loadResolvers = null; // { resolve, reject } for window.localmind.load() awaiters | |
| let currentInferenceContext = null; // { onToken } when a streaming API caller wants per-token notifications | |
| // Most recent file the worker reported activity on β used so the | |
| // 'progress_total' event (aggregate across all files, no `file` field) | |
| // can still show *which* file is currently streaming bytes. | |
| let currentDownloadFile = ''; | |
| // ββ JavaScript API state ββββββββββββββββββββββββββββββββββββ | |
| // Feature-flag persisted in localStorage. The real window.localmind | |
| // object is attached in a later step; this step only builds the | |
| // toggle, chip, and activity log infrastructure. | |
| let apiEnabled = false; | |
| try { apiEnabled = localStorage.getItem('lm_api_enabled') === '1'; } catch {} | |
| // Multi-step planning: when on + agent-capable model, sendMessage routes | |
| // through runPlannerAgent (plan β per-step execute β synthesize) instead | |
| // of the single-pass agent loop. Default off; Gemma 4 at ~4.5B produces | |
| // inconsistent plans, so this is labelled experimental. | |
| let plannerEnabled = false; | |
| try { plannerEnabled = localStorage.getItem('lm_planner_enabled') === '1'; } catch {} | |
| const API_LOG_MAX = 50; | |
| const apiLog = []; // newest first: [{ ts, method, promptLen, tokens, durationMs, outcome }] | |
| // Restore from sessionStorage | |
| try { | |
| const saved = sessionStorage.getItem('localmind_chat'); | |
| if (saved) messages = JSON.parse(saved); | |
| } catch {} | |
| // ββ Agent: Tool Registry ββββββββββββββββββββββββββββββββββ | |
| const TOOL_REGISTRY = { | |
| calculate: { | |
| description: 'Evaluate a mathematical expression. Use for arithmetic, unit conversions, percentages.', | |
| parameters: { | |
| type: 'object', | |
| properties: { expression: { type: 'string', description: 'Math expression to evaluate, e.g. "2 * 3 + 1"' } }, | |
| required: ['expression'], | |
| }, | |
| execute(args) { | |
| try { | |
| const expr = String(args.expression).replace(/[^0-9+\-*/.()% ]/g, ''); | |
| const result = Function('"use strict"; return (' + expr + ')')(); | |
| return { result: Number(result) }; | |
| } catch (e) { return { error: e.message }; } | |
| }, | |
| }, | |
| get_current_time: { | |
| description: 'Get the current date and time.', | |
| parameters: { | |
| type: 'object', | |
| properties: { timezone: { type: 'string', description: 'IANA timezone, e.g. "America/New_York". Defaults to local.' } }, | |
| required: [], | |
| }, | |
| execute(args) { | |
| try { | |
| const opts = { dateStyle: 'full', timeStyle: 'long' }; | |
| if (args.timezone) opts.timeZone = args.timezone; | |
| const formatted = new Intl.DateTimeFormat('en-US', opts).format(new Date()); | |
| return { datetime: formatted, iso: new Date().toISOString() }; | |
| } catch (e) { return { error: e.message }; } | |
| }, | |
| }, | |
| store_memory: { | |
| description: 'Store an important fact, preference, or finding for later recall. Memories persist across sessions.', | |
| parameters: { | |
| type: 'object', | |
| properties: { | |
| content: { type: 'string', description: 'The information to remember' }, | |
| category: { type: 'string', enum: ['fact', 'preference', 'finding', 'task'], description: 'Category of memory' }, | |
| }, | |
| required: ['content'], | |
| }, | |
| async execute(args) { | |
| try { | |
| const count = await embedAndStore(args.content, args.category || 'fact', 'assistant'); | |
| return { stored: true, chunks: count }; | |
| } catch (e) { return { error: e.message }; } | |
| }, | |
| }, | |
| search_memory: { | |
| description: 'Search your stored memories, past conversations, and documents. Use when the user asks about something you may have been told before.', | |
| parameters: { | |
| type: 'object', | |
| properties: { query: { type: 'string', description: 'What to search for in memory' } }, | |
| required: ['query'], | |
| }, | |
| async execute(args) { | |
| try { | |
| const results = await searchMemory(args.query, 5); | |
| if (results.length === 0) return { found: false, message: 'No relevant memories found.' }; | |
| return { | |
| found: true, | |
| count: results.length, | |
| memories: results.map(r => ({ | |
| content: r.text, | |
| category: r.category, | |
| score: Math.round(r.score * 100) / 100, | |
| source: r.source, | |
| })), | |
| }; | |
| } catch (e) { return { error: e.message }; } | |
| }, | |
| }, | |
| set_reminder: { | |
| description: 'Set a reminder that will notify the user after a specified number of minutes. Uses browser notifications.', | |
| parameters: { | |
| type: 'object', | |
| properties: { | |
| message: { type: 'string', description: 'The reminder message to show' }, | |
| minutes: { type: 'number', description: 'Minutes from now (1-120)' }, | |
| }, | |
| required: ['message', 'minutes'], | |
| }, | |
| execute(args) { | |
| const mins = Math.max(1, Math.min(120, Number(args.minutes) || 5)); | |
| const msg = args.message || 'Reminder'; | |
| // Request notification permission if needed | |
| if ('Notification' in window && Notification.permission === 'default') { | |
| Notification.requestPermission(); | |
| } | |
| setTimeout(() => { | |
| // Browser notification | |
| if ('Notification' in window && Notification.permission === 'granted') { | |
| new Notification('LocalMind Reminder', { body: msg, icon: 'data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100"><text y=".9em" font-size="90">π§ </text></svg>' }); | |
| } | |
| // Also show a toast in case notifications are blocked | |
| showToast('Reminder: ' + msg); | |
| }, mins * 60 * 1000); | |
| const time = new Date(Date.now() + mins * 60 * 1000); | |
| return { set: true, message: msg, fires_at: time.toLocaleTimeString(), minutes: mins }; | |
| }, | |
| }, | |
| list_memories: { | |
| description: 'List all stored memories grouped by category with counts. Use to see what you remember about the user.', | |
| parameters: { type: 'object', properties: {}, required: [] }, | |
| async execute() { | |
| try { | |
| const chunks = await getAllChunks(); | |
| if (chunks.length === 0) return { total: 0, message: 'No memories stored.' }; | |
| const byCategory = {}; | |
| for (const c of chunks) { | |
| byCategory[c.category] = (byCategory[c.category] || 0) + 1; | |
| } | |
| const recent = chunks.sort((a, b) => b.timestamp - a.timestamp).slice(0, 5); | |
| return { | |
| total: chunks.length, | |
| categories: byCategory, | |
| recent: recent.map(r => ({ text: r.text.slice(0, 80), category: r.category, source: r.source })), | |
| }; | |
| } catch (e) { return { error: e.message }; } | |
| }, | |
| }, | |
| delete_memory: { | |
| description: 'Delete memories matching a search query. Use when the user asks you to forget something.', | |
| parameters: { | |
| type: 'object', | |
| properties: { query: { type: 'string', description: 'What to forget β will delete matching memories' } }, | |
| required: ['query'], | |
| }, | |
| async execute(args) { | |
| try { | |
| const results = await searchMemory(args.query, 10); | |
| if (results.length === 0) return { deleted: 0, message: 'No matching memories found.' }; | |
| // Delete matches above 0.5 similarity (confident matches only) | |
| const toDelete = results.filter(r => r.score >= 0.5); | |
| for (const r of toDelete) await deleteChunk(r.id); | |
| return { deleted: toDelete.length, message: `Deleted ${toDelete.length} matching memor${toDelete.length === 1 ? 'y' : 'ies'}.` }; | |
| } catch (e) { return { error: e.message }; } | |
| }, | |
| }, | |
| segment_image: { | |
| description: 'Segment objects in the attached image using SAM. YOU MUST estimate the point coordinates yourself by looking at the image β do NOT ask the user for coordinates. Coordinates are [x, y] in normalized [0,1] range: (0,0)=top-left, (1,1)=bottom-right, (0.5,0.5)=center. Example: to segment a dog in the center-left of the image, use points [[0.3, 0.5]] with labels [1]. Always call this tool immediately when the user asks to segment, outline, isolate, or identify objects β pick your best estimate for the object center.', | |
| parameters: { | |
| type: 'object', | |
| properties: { | |
| points: { | |
| type: 'array', | |
| description: 'Array of [x, y] pairs in [0,1] range. YOU choose these by estimating where the target object is in the image. Place at the center of each object to segment.', | |
| items: { type: 'array', items: { type: 'number' }, minItems: 2, maxItems: 2 }, | |
| }, | |
| labels: { | |
| type: 'array', | |
| description: 'Same length as points. 1 = foreground (the object to segment), 0 = background (area to exclude).', | |
| items: { type: 'integer', enum: [0, 1] }, | |
| }, | |
| }, | |
| required: ['points', 'labels'], | |
| }, | |
| requiresImage: true, | |
| async execute(args, context) { | |
| if (!context || !context.imageBlob) { | |
| return { error: 'No image attached. Ask the user to attach an image first.' }; | |
| } | |
| const { points, labels } = args; | |
| if (!Array.isArray(points) || !Array.isArray(labels) || points.length === 0) { | |
| return { error: 'points and labels must be non-empty arrays of equal length.' }; | |
| } | |
| if (points.length !== labels.length) { | |
| return { error: `points (${points.length}) and labels (${labels.length}) must have the same length.` }; | |
| } | |
| try { | |
| // Get image dimensions to scale normalized coords to pixels | |
| const bmp = await createImageBitmap(context.imageBlob); | |
| const imgW = bmp.width; | |
| const imgH = bmp.height; | |
| bmp.close(); | |
| showToast('Loading SAM model...'); | |
| // Run SAM once per foreground point to get separate masks per object. | |
| // SAM returns 3 alternative masks per prompt β we pick the best one each time. | |
| const objectMasks = []; | |
| const objectScores = []; | |
| for (let i = 0; i < points.length; i++) { | |
| if (labels[i] !== 1) continue; // skip background points | |
| const px = Math.round(Math.max(0, Math.min(1, points[i][0])) * imgW); | |
| const py = Math.round(Math.max(0, Math.min(1, points[i][1])) * imgH); | |
| const { masks, scores: sc } = await LocalMind.runtime.segmentImage(context.imageBlob, [[px, py]], [1]); | |
| // Pick best-scoring mask from the 3 alternatives | |
| let bestIdx = 0, bestScore = -1; | |
| for (let j = 0; j < masks.length; j++) { | |
| if ((sc[j] || 0) > bestScore) { bestScore = sc[j] || 0; bestIdx = j; } | |
| } | |
| objectMasks.push(masks[bestIdx]); | |
| objectScores.push(bestScore); | |
| } | |
| if (objectMasks.length === 0) { | |
| return { error: 'No foreground points provided (all labels were 0).' }; | |
| } | |
| // Build JSON-safe result for the model | |
| const maskSummaries = objectMasks.map((m, i) => { | |
| const totalPixels = m.width * m.height; | |
| const fgPixels = m.data.reduce((sum, v) => sum + (v > 128 ? 1 : 0), 0); | |
| return { | |
| object: i + 1, | |
| score: Math.round(objectScores[i] * 1000) / 1000, | |
| area_percent: Math.round((fgPixels / totalPixels) * 10000) / 100, | |
| }; | |
| }); | |
| const result = { | |
| success: true, | |
| image_width: imgW, | |
| image_height: imgH, | |
| objects_found: objectMasks.length, | |
| objects: maskSummaries, | |
| _rawMasks: objectMasks, | |
| _imageBlob: context.imageBlob, | |
| _scores: objectScores, | |
| }; | |
| return result; | |
| } catch (e) { | |
| return { error: 'Segmentation failed: ' + e.message }; | |
| } | |
| }, | |
| }, | |
| }; | |
| // ββ Web: Search Provider Adapters βββββββββββββββββββββββββ | |
| async function parseApiError(res, provider) { | |
| const status = res.status; | |
| let detail = ''; | |
| try { | |
| const body = await res.text(); | |
| try { | |
| const json = JSON.parse(body); | |
| detail = json.message || json.error?.message || json.detail || json.error || ''; | |
| } catch { detail = body.slice(0, 200); } | |
| } catch {} | |
| // Human-readable messages for common HTTP errors | |
| if (status === 401 || status === 403) return `${provider}: Invalid or expired API key. Check your key in Settings.`; | |
| if (status === 429) return `${provider}: Rate limit or quota exceeded. ${detail || 'Try again later or check your plan.'}`; | |
| if (status === 402) return `${provider}: Payment required. Your free tier may be exhausted.`; | |
| if (status === 503 || status === 502) return `${provider}: Service temporarily unavailable. Try again in a moment.`; | |
| return `${provider} error ${status}: ${detail || res.statusText}`; | |
| } | |
| const SearchProviders = { | |
| async brave(query, apiKey) { | |
| const res = await fetch(`https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(query)}&count=5`, { | |
| headers: { 'Accept': 'application/json', 'Accept-Encoding': 'gzip', 'X-Subscription-Token': apiKey }, | |
| }); | |
| if (!res.ok) throw new Error(await parseApiError(res, 'Brave Search')); | |
| const data = await res.json(); | |
| return (data.web?.results || []).slice(0, 5).map(r => ({ | |
| title: r.title, url: r.url, snippet: r.description || '', content: null, age: r.age || null, | |
| })); | |
| }, | |
| async tavily(query, apiKey) { | |
| const res = await fetch('https://api.tavily.com/search', { | |
| method: 'POST', | |
| headers: { 'Content-Type': 'application/json' }, | |
| body: JSON.stringify({ api_key: apiKey, query, search_depth: 'basic', include_raw_content: false, max_results: 5 }), | |
| }); | |
| if (!res.ok) throw new Error(await parseApiError(res, 'Tavily')); | |
| const data = await res.json(); | |
| return (data.results || []).map(r => ({ | |
| title: r.title, url: r.url, snippet: r.content || '', content: r.raw_content || null, age: null, | |
| })); | |
| }, | |
| async searxng(query, instanceUrl) { | |
| const base = instanceUrl.replace(/\/+$/, ''); | |
| const res = await fetch(`${base}/search?q=${encodeURIComponent(query)}&format=json&categories=general`); | |
| if (!res.ok) throw new Error(await parseApiError(res, 'SearXNG')); | |
| const data = await res.json(); | |
| return (data.results || []).slice(0, 5).map(r => ({ | |
| title: r.title, url: r.url, snippet: r.content || '', content: null, age: null, | |
| })); | |
| }, | |
| }; | |
| function getSearchConfig() { | |
| const provider = localStorage.getItem('lm_search_provider') || 'none'; | |
| const apiKey = localStorage.getItem('lm_search_key') || ''; | |
| const searxngUrl = localStorage.getItem('lm_searxng_url') || ''; | |
| return { provider, apiKey, searxngUrl }; | |
| } | |
| function isSearchConfigured() { | |
| const { provider, apiKey, searxngUrl } = getSearchConfig(); | |
| if (provider === 'none') return false; | |
| if (provider === 'searxng') return !!searxngUrl; | |
| return !!apiKey; | |
| } | |
| async function executeWebSearch(query) { | |
| const { provider, apiKey, searxngUrl } = getSearchConfig(); | |
| if (provider === 'brave') return SearchProviders.brave(query, apiKey); | |
| if (provider === 'tavily') return SearchProviders.tavily(query, apiKey); | |
| if (provider === 'searxng') return SearchProviders.searxng(query, searxngUrl); | |
| throw new Error('No search provider configured'); | |
| } | |
| // ββ Web: Page Fetcher βββββββββββββββββββββββββββββββββββββββ | |
| let readabilityLoaded = false; | |
| async function ensureReadability() { | |
| if (readabilityLoaded) return; | |
| return new Promise((resolve, reject) => { | |
| const script = document.createElement('script'); | |
| script.src = 'https://cdn.jsdelivr.net/npm/@mozilla/readability@0.5.0/Readability.min.js'; | |
| // SRI: if the CDN is compromised or the file is tampered with, the | |
| // browser will refuse to execute the script and fire onerror. | |
| script.integrity = 'sha384-DHTbAPJmhf9shXtIK080V86CoLVziMLp/Gdn1EVZCKVyvIMiENtuUVSfckMLbkYO'; | |
| script.crossOrigin = 'anonymous'; | |
| script.onload = () => { readabilityLoaded = true; resolve(); }; | |
| script.onerror = () => reject(new Error('Failed to load or verify Readability.js')); | |
| document.head.appendChild(script); | |
| }); | |
| } | |
| async function fetchAndExtract(url, userQuery) { | |
| // Try direct fetch, fallback to CORS proxy | |
| let html; | |
| try { | |
| const res = await fetch(url, { signal: AbortSignal.timeout(8000) }); | |
| html = await res.text(); | |
| } catch { | |
| try { | |
| const res = await fetch(`https://api.allorigins.win/raw?url=${encodeURIComponent(url)}`, { signal: AbortSignal.timeout(10000) }); | |
| html = await res.text(); | |
| } catch (e) { | |
| return { error: `Could not fetch page: ${e.message}`, url }; | |
| } | |
| } | |
| // Extract with Readability | |
| try { | |
| await ensureReadability(); | |
| const doc = new DOMParser().parseFromString(html, 'text/html'); | |
| const reader = new Readability(doc); | |
| const article = reader.parse(); | |
| let text = article ? article.textContent : doc.body?.textContent || ''; | |
| text = text.replace(/\s+/g, ' ').trim(); | |
| // Semantic pre-filter if embedding is ready | |
| if (LocalMind.runtime.embeddingsReady && userQuery && text.length > 1500) { | |
| try { | |
| const paragraphs = text.match(/.{200,1000}[.!?\n]|.{200,1000}$/g) || [text.slice(0, 3000)]; | |
| const paraVecs = await LocalMind.runtime.embed(paragraphs); | |
| const queryVec = await LocalMind.runtime.embed(userQuery); | |
| const scored = paragraphs.map((p, i) => ({ text: p, score: cosineSimilarity(queryVec, paraVecs[i]) })); | |
| scored.sort((a, b) => b.score - a.score); | |
| text = scored.slice(0, 5).map(s => s.text).join('\n\n'); | |
| } catch { | |
| text = text.slice(0, 3500); | |
| } | |
| } else { | |
| text = text.slice(0, 3500); | |
| } | |
| return { title: article?.title || '', content: text, url }; | |
| } catch (e) { | |
| return { content: html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').slice(0, 3500), url }; | |
| } | |
| } | |
| // ββ Web: Register web_search + fetch_page tools βββββββββββββ | |
| // Added dynamically so they only appear in the tool schema when configured | |
| TOOL_REGISTRY.web_search = { | |
| description: 'Search the web for current information. Use when you need facts, news, or data you don\'t know.', | |
| parameters: { | |
| type: 'object', | |
| properties: { query: { type: 'string', description: 'Search query, 3-8 words' } }, | |
| required: ['query'], | |
| }, | |
| requiresWeb: true, | |
| async execute(args) { | |
| try { | |
| const results = await executeWebSearch(args.query); | |
| // Cache results in RAG for future offline use | |
| if (LocalMind.runtime.embeddingsReady && results.length > 0) { | |
| const snippetText = results.map(r => `${r.title}: ${r.snippet}`).join('\n'); | |
| embedAndStore(snippetText, 'finding', 'web-search').catch(() => {}); | |
| } | |
| return { provider: getSearchConfig().provider, query: args.query, results }; | |
| } catch (e) { return { error: e.message }; } | |
| }, | |
| }; | |
| TOOL_REGISTRY.fetch_page = { | |
| description: 'Fetch and read content from a URL. Use after web_search to get details from a promising result.', | |
| parameters: { | |
| type: 'object', | |
| properties: { url: { type: 'string', description: 'Full URL to fetch' } }, | |
| required: ['url'], | |
| }, | |
| requiresWeb: true, | |
| async execute(args, context) { | |
| try { | |
| const result = await fetchAndExtract(args.url, context?.userQuery || ''); | |
| // Cache fetched content in RAG | |
| if (LocalMind.runtime.embeddingsReady && result.content) { | |
| embedAndStore(result.content, 'finding', args.url).catch(() => {}); | |
| } | |
| return result; | |
| } catch (e) { return { error: e.message }; } | |
| }, | |
| }; | |
| // ββ Agent: tool-call helpers βββββββββββββββββββββββββββββββββ | |
| // The system-prompt scaffolding, output parser, and tool-result | |
| // formatting now live inside the runtime adapter (search for | |
| // _adapterBuildToolPrompt / _adapterParseToolCalls / | |
| // _adapterFormatToolResultMessage). The agent loop in | |
| // sendMessage() consumes tool_call events from runtime.chat() | |
| // instead of parsing model output itself. | |
| // Singleton Pyodide worker β instantiated lazily the first time the | |
| // run_python tool is called. The blob worker imports Pyodide ESM and | |
| // holds the interpreter across calls so subsequent runs reuse the | |
| // already-loaded runtime and installed packages. | |
| let __pyodideWorker = null; | |
| let __pyodideLoadingToastShown = false; | |
| function runPython(userCode) { | |
| if (!__pyodideWorker) { | |
| __pyodideWorker = createPyodideWorker(); | |
| __pyodideWorker.addEventListener('message', (e) => { | |
| if (e.data.type === 'loading' && !__pyodideLoadingToastShown) { | |
| showToast('Loading Python runtime (first use)\u2026'); | |
| __pyodideLoadingToastShown = true; | |
| } else if (e.data.type === 'loaded') { | |
| showToast('Python runtime ready'); | |
| } | |
| }); | |
| } | |
| return new Promise((resolve) => { | |
| const id = Math.random().toString(36).slice(2); | |
| const onMsg = (e) => { | |
| if (e.data.id !== id) return; | |
| if (e.data.type === 'result') { | |
| __pyodideWorker.removeEventListener('message', onMsg); | |
| resolve(e.data); | |
| } | |
| }; | |
| __pyodideWorker.addEventListener('message', onMsg); | |
| __pyodideWorker.postMessage({ type: 'run', code: userCode, id }); | |
| }); | |
| } | |
| TOOL_REGISTRY.run_python = { | |
| description: 'Execute Python code in a sandboxed in-browser runtime (Pyodide). Imported packages (numpy, pandas, matplotlib, etc.) auto-install on first use. Returns stdout, stderr, and the last-expression result as strings. Good for arithmetic, data manipulation, CSV parsing, quick simulations. First call downloads ~10 MB of Python runtime.', | |
| parameters: { | |
| type: 'object', | |
| properties: { | |
| code: { type: 'string', description: 'Python source to execute. The last expression (if any) is returned in `result`.' }, | |
| }, | |
| required: ['code'], | |
| }, | |
| async execute(args) { | |
| const code = typeof args.code === 'string' ? args.code : ''; | |
| if (!code.trim()) return { error: 'code parameter is required' }; | |
| try { | |
| const out = await runPython(code); | |
| if (out.error) return { error: out.error }; | |
| const compact = { stdout: out.stdout || '', stderr: out.stderr || '' }; | |
| if (out.result) compact.result = out.result; | |
| return compact; | |
| } catch (e) { | |
| return { error: 'Python execution failed: ' + (e.message || e) }; | |
| } | |
| }, | |
| }; | |
| // Helper used only by the agent loop to build the OpenAI-shaped | |
| // tools array from TOOL_REGISTRY. Stays outside the adapter β the | |
| // adapter receives an explicit tools[] array per call and never | |
| // sees TOOL_REGISTRY directly. | |
| function buildToolsForRuntime() { | |
| const webConfigured = isSearchConfigured(); | |
| const caps = LocalMind.runtime.capabilities(); | |
| const hasImage = !!(caps && caps.image); | |
| return Object.entries(TOOL_REGISTRY) | |
| .filter(([_, t]) => !t.requiresWeb || webConfigured) | |
| .filter(([_, t]) => !t.requiresImage || hasImage) | |
| .map(([name, t]) => ({ | |
| name, | |
| description: t.description, | |
| parameters: t.parameters, | |
| })); | |
| } | |
| // ββ Agent: Context Budget Helper ββββββββββββββββββββββββββββ | |
| function approxTokens(text) { | |
| if (!text) return 0; | |
| return Math.ceil(String(text).length / 3.5); | |
| } | |
| // ββ Agent: Sliding Window Conversation Builder ββββββββββββ | |
| let conversationSummary = ''; | |
| function buildContextMessages(allMessages, systemPrompt, modelKey) { | |
| const m = MODELS[modelKey]; | |
| const maxCtx = m ? m.contextSize : 4096; | |
| // Reserve tokens: system ~500, response ~2048, headroom ~300 | |
| const budgetForHistory = maxCtx - 2048 - 300; | |
| const out = []; | |
| if (systemPrompt) out.push({ role: 'system', content: systemPrompt }); | |
| // Calculate system prompt cost | |
| let used = approxTokens(systemPrompt); | |
| // Always include all messages if they fit | |
| let historyTokens = 0; | |
| for (const msg of allMessages) { | |
| historyTokens += approxTokens(typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content)); | |
| } | |
| if (used + historyTokens <= budgetForHistory) { | |
| // Everything fits β send it all | |
| for (const msg of allMessages) { | |
| out.push({ role: msg.role === 'assistant' ? 'assistant' : 'user', content: msg.content }); | |
| } | |
| return out; | |
| } | |
| // Doesn't fit β sliding window: keep last 3 turn-pairs verbatim, summarize the rest | |
| // A "turn pair" = one user + one assistant message | |
| const pairs = []; | |
| let i = 0; | |
| while (i < allMessages.length) { | |
| if (allMessages[i].role === 'user') { | |
| const user = allMessages[i]; | |
| const asst = (i + 1 < allMessages.length && allMessages[i + 1].role === 'assistant') ? allMessages[i + 1] : null; | |
| pairs.push({ user, assistant: asst }); | |
| i += asst ? 2 : 1; | |
| } else { | |
| // Orphan assistant message | |
| pairs.push({ user: null, assistant: allMessages[i] }); | |
| i++; | |
| } | |
| } | |
| const keepPairs = Math.min(3, pairs.length); | |
| const recentPairs = pairs.slice(-keepPairs); | |
| // If we have a conversation summary from older turns, prepend it | |
| if (conversationSummary && pairs.length > keepPairs) { | |
| out.push({ role: 'user', content: '[Previous conversation summary]\n' + conversationSummary }); | |
| out.push({ role: 'assistant', content: 'Understood, I have context from our earlier conversation.' }); | |
| } | |
| // Add recent turns | |
| for (const pair of recentPairs) { | |
| if (pair.user) out.push({ role: 'user', content: pair.user.content }); | |
| if (pair.assistant) out.push({ role: 'assistant', content: pair.assistant.content }); | |
| } | |
| return out; | |
| } | |
| // ββ RAG: Embedding Worker (MiniLM, WASM/CPU) βββββββββββββ | |
| // Lives entirely inside LocalMind.runtime now. Use: | |
| // await LocalMind.runtime.embed(text) -> Float32Array | |
| // await LocalMind.runtime.embed(textArray) -> Float32Array[] | |
| // LocalMind.runtime.embeddingsReady -> boolean (gate) | |
| // The worker is loaded lazily on the first embed() call. | |
| // ββ RAG: IndexedDB Vector Store βββββββββββββββββββββββββββββ | |
| const DB_NAME = 'localmind_rag'; | |
| const DB_VERSION = 2; | |
| function openRAGDB() { | |
| return new Promise((resolve, reject) => { | |
| const req = indexedDB.open(DB_NAME, DB_VERSION); | |
| req.onupgradeneeded = (e) => { | |
| const db = e.target.result; | |
| if (!db.objectStoreNames.contains('chunks')) { | |
| const store = db.createObjectStore('chunks', { keyPath: 'id' }); | |
| store.createIndex('category', 'category', { unique: false }); | |
| store.createIndex('source', 'source', { unique: false }); | |
| } | |
| if (!db.objectStoreNames.contains('profile')) { | |
| db.createObjectStore('profile', { keyPath: 'key' }); | |
| } | |
| if (!db.objectStoreNames.contains('conversations')) { | |
| const convStore = db.createObjectStore('conversations', { keyPath: 'id' }); | |
| convStore.createIndex('updated', 'updated', { unique: false }); | |
| } | |
| }; | |
| req.onsuccess = () => resolve(req.result); | |
| req.onerror = () => reject(req.error); | |
| }); | |
| } | |
| async function storeChunks(chunks) { | |
| const db = await openRAGDB(); | |
| const tx = db.transaction('chunks', 'readwrite'); | |
| const store = tx.objectStore('chunks'); | |
| for (const chunk of chunks) store.put(chunk); | |
| return new Promise((resolve, reject) => { | |
| tx.oncomplete = resolve; | |
| tx.onerror = () => reject(tx.error); | |
| }); | |
| } | |
| async function getAllChunks() { | |
| const db = await openRAGDB(); | |
| return new Promise((resolve, reject) => { | |
| const tx = db.transaction('chunks', 'readonly'); | |
| const req = tx.objectStore('chunks').getAll(); | |
| req.onsuccess = () => resolve(req.result); | |
| req.onerror = () => reject(req.error); | |
| }); | |
| } | |
| async function deleteChunk(id) { | |
| const db = await openRAGDB(); | |
| const tx = db.transaction('chunks', 'readwrite'); | |
| tx.objectStore('chunks').delete(id); | |
| return new Promise((resolve, reject) => { | |
| tx.oncomplete = resolve; | |
| tx.onerror = () => reject(tx.error); | |
| }); | |
| } | |
| async function clearAllChunks() { | |
| const db = await openRAGDB(); | |
| const tx = db.transaction('chunks', 'readwrite'); | |
| tx.objectStore('chunks').clear(); | |
| return new Promise((resolve, reject) => { | |
| tx.oncomplete = resolve; | |
| tx.onerror = () => reject(tx.error); | |
| }); | |
| } | |
| // ββ Conversation History (IndexedDB) ββββββββββββββββββββββ | |
| let activeConversationId = null; | |
| function newConversationId() { return Date.now().toString(36) + '-' + Math.random().toString(36).slice(2, 6); } | |
| async function saveConversation(msgs, id) { | |
| if (!msgs || msgs.length === 0) return; | |
| const db = await openRAGDB(); | |
| const tx = db.transaction('conversations', 'readwrite'); | |
| const now = Date.now(); | |
| // Generate title from first user message | |
| const firstUser = msgs.find(m => m.role === 'user'); | |
| let title = 'Untitled'; | |
| if (firstUser) { | |
| const text = typeof firstUser.content === 'string' ? firstUser.content : firstUser.content.filter(c => c.type === 'text').map(c => c.text).join(' '); | |
| title = text.slice(0, 60) + (text.length > 60 ? '...' : ''); | |
| } | |
| tx.objectStore('conversations').put({ | |
| id: id || activeConversationId || newConversationId(), | |
| title, | |
| messages: msgs.map(m => { | |
| if (m.role === 'assistant') return m; | |
| if (typeof m.content === 'string') return m; | |
| const text = m.content.filter(c => c.type === 'text').map(c => c.text).join(' '); | |
| return { role: m.role, content: text }; | |
| }), | |
| modelKey: activeModelKey, | |
| messageCount: msgs.length, | |
| created: now, | |
| updated: now, | |
| }); | |
| return new Promise((resolve, reject) => { tx.oncomplete = resolve; tx.onerror = () => reject(tx.error); }); | |
| } | |
| async function getAllConversations() { | |
| const db = await openRAGDB(); | |
| return new Promise((resolve, reject) => { | |
| const tx = db.transaction('conversations', 'readonly'); | |
| const req = tx.objectStore('conversations').getAll(); | |
| req.onsuccess = () => resolve(req.result.sort((a, b) => b.updated - a.updated)); | |
| req.onerror = () => reject(req.error); | |
| }); | |
| } | |
| async function getConversation(id) { | |
| const db = await openRAGDB(); | |
| return new Promise((resolve, reject) => { | |
| const tx = db.transaction('conversations', 'readonly'); | |
| const req = tx.objectStore('conversations').get(id); | |
| req.onsuccess = () => resolve(req.result); | |
| req.onerror = () => reject(req.error); | |
| }); | |
| } | |
| async function deleteConversation(id) { | |
| const db = await openRAGDB(); | |
| const tx = db.transaction('conversations', 'readwrite'); | |
| tx.objectStore('conversations').delete(id); | |
| return new Promise((resolve, reject) => { tx.oncomplete = resolve; tx.onerror = () => reject(tx.error); }); | |
| } | |
| async function exportAllData() { | |
| const [chunks, convs, profile] = await Promise.all([getAllChunks(), getAllConversations(), getProfile()]); | |
| return { version: 1, exported: new Date().toISOString(), memories: chunks, conversations: convs, profile }; | |
| } | |
| async function importData(data) { | |
| if (!data || data.version !== 1) throw new Error('Invalid export format'); | |
| if (data.memories?.length) await storeChunks(data.memories); | |
| if (data.profile) await saveProfile(data.profile); | |
| if (data.conversations?.length) { | |
| const db = await openRAGDB(); | |
| const tx = db.transaction('conversations', 'readwrite'); | |
| const store = tx.objectStore('conversations'); | |
| for (const c of data.conversations) store.put(c); | |
| await new Promise((resolve, reject) => { tx.oncomplete = resolve; tx.onerror = () => reject(tx.error); }); | |
| } | |
| } | |
| function cosineSimilarity(a, b) { | |
| let dot = 0, normA = 0, normB = 0; | |
| for (let i = 0; i < a.length; i++) { | |
| dot += a[i] * b[i]; | |
| normA += a[i] * a[i]; | |
| normB += b[i] * b[i]; | |
| } | |
| return dot / (Math.sqrt(normA) * Math.sqrt(normB) + 1e-8); | |
| } | |
| async function searchByVector(queryVec, topK = 5, threshold = 0.3) { | |
| const chunks = await getAllChunks(); | |
| const scored = chunks.map(c => ({ | |
| ...c, | |
| score: cosineSimilarity(queryVec, c.embedding), | |
| })).filter(c => c.score >= threshold); | |
| scored.sort((a, b) => b.score - a.score); | |
| return scored.slice(0, topK); | |
| } | |
| // ββ RAG: User Profile (Working Memory) ββββββββββββββββββββββ | |
| async function getProfile() { | |
| const db = await openRAGDB(); | |
| return new Promise((resolve, reject) => { | |
| const tx = db.transaction('profile', 'readonly'); | |
| const req = tx.objectStore('profile').get('user'); | |
| req.onsuccess = () => resolve(req.result || { key: 'user', name: '', preferences: [], facts: [] }); | |
| req.onerror = () => reject(req.error); | |
| }); | |
| } | |
| async function saveProfile(profile) { | |
| const db = await openRAGDB(); | |
| const tx = db.transaction('profile', 'readwrite'); | |
| tx.objectStore('profile').put({ ...profile, key: 'user' }); | |
| return new Promise((resolve, reject) => { | |
| tx.oncomplete = resolve; | |
| tx.onerror = () => reject(tx.error); | |
| }); | |
| } | |
| // ββ RAG: Chunk Pipeline βββββββββββββββββββββββββββββββββββββ | |
| function chunkText(text, maxChars = 900, overlapChars = 200) { | |
| // Split at sentence boundaries, then group into chunks | |
| const sentences = text.match(/[^.!?\n]+[.!?\n]+|[^.!?\n]+$/g) || [text]; | |
| const chunks = []; | |
| let current = ''; | |
| for (const sentence of sentences) { | |
| if (current.length + sentence.length > maxChars && current.length > 0) { | |
| chunks.push(current.trim()); | |
| // Keep overlap from end of current chunk | |
| const words = current.split(/\s+/); | |
| const overlapWords = []; | |
| let len = 0; | |
| for (let i = words.length - 1; i >= 0 && len < overlapChars; i--) { | |
| overlapWords.unshift(words[i]); | |
| len += words[i].length + 1; | |
| } | |
| current = overlapWords.join(' ') + ' ' + sentence; | |
| } else { | |
| current += (current ? ' ' : '') + sentence; | |
| } | |
| } | |
| if (current.trim()) chunks.push(current.trim()); | |
| return chunks; | |
| } | |
| async function embedAndStore(text, category, source) { | |
| const textChunks = chunkText(text); | |
| const vectors = await LocalMind.runtime.embed(textChunks); | |
| const now = Date.now(); | |
| const chunks = textChunks.map((t, i) => ({ | |
| id: `${now}-${i}-${Math.random().toString(36).slice(2, 6)}`, | |
| text: t, | |
| embedding: vectors[i], | |
| category: category || 'fact', | |
| source: source || 'user', | |
| timestamp: now, | |
| })); | |
| await storeChunks(chunks); | |
| return chunks.length; | |
| } | |
| async function searchMemory(query, topK = 5) { | |
| const vec = await LocalMind.runtime.embed(query); | |
| return searchByVector(vec, topK); | |
| } | |
| // ββ DOM refs ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| const chatArea = document.getElementById('chatArea'); | |
| const chatInput = document.getElementById('chatInput'); | |
| const sendBtn = document.getElementById('sendBtn'); | |
| const clearBtn = document.getElementById('clearBtn'); | |
| const settingsBtn = document.getElementById('settingsBtn'); | |
| const settingsPanel = document.getElementById('settingsPanel'); | |
| const systemPrompt = document.getElementById('systemPrompt'); | |
| const statusBadge = document.getElementById('statusBadge'); | |
| const statusSpinner = document.getElementById('statusSpinner'); | |
| const statusText = document.getElementById('statusText'); | |
| const progressSection = document.getElementById('progressSection'); | |
| const progressFill = document.getElementById('progressFill'); | |
| const progressText = document.getElementById('progressText'); | |
| const welcomeMsg = document.getElementById('welcomeMsg'); | |
| const helpBtn = document.getElementById('helpBtn'); | |
| const helpPopover = document.getElementById('helpPopover'); | |
| const modelSelect = document.getElementById('modelSelect'); | |
| const attachmentBar = document.getElementById('attachmentBar'); | |
| const inputAffordances = document.getElementById('inputAffordances'); | |
| const attachBtn = document.getElementById('attachBtn'); | |
| const cameraBtn = document.getElementById('cameraBtn'); | |
| const micBtn = document.getElementById('micBtn'); | |
| const fileInput = document.getElementById('fileInput'); | |
| const thinkingToggle = document.getElementById('thinkingToggle'); | |
| const thinkingRow = document.getElementById('thinkingRow'); | |
| const plannerRow = document.getElementById('plannerRow'); | |
| const plannerToggle = document.getElementById('plannerToggle'); | |
| plannerToggle.checked = plannerEnabled; | |
| plannerToggle.addEventListener('change', () => { | |
| plannerEnabled = plannerToggle.checked; | |
| try { localStorage.setItem('lm_planner_enabled', plannerEnabled ? '1' : '0'); } catch {} | |
| }); | |
| const cameraOverlay = document.getElementById('cameraOverlay'); | |
| const cameraPreview = document.getElementById('cameraPreview'); | |
| const camCaptureBtn = document.getElementById('camCaptureBtn'); | |
| const camCancelBtn = document.getElementById('camCancelBtn'); | |
| const dragOverlay = document.getElementById('dragOverlay'); | |
| // ββ Help popover ββββββββββββββββββββββββββββββββββββββββββββ | |
| helpBtn.addEventListener('click', e => { | |
| e.stopPropagation(); | |
| helpPopover.classList.toggle('open'); | |
| }); | |
| document.addEventListener('click', () => helpPopover.classList.remove('open')); | |
| // Help popover: tab switching | |
| helpPopover.addEventListener('click', e => { | |
| e.stopPropagation(); | |
| const tab = e.target.closest('.help-tab'); | |
| if (tab) { | |
| helpPopover.querySelectorAll('.help-tab').forEach(t => t.classList.remove('active')); | |
| helpPopover.querySelectorAll('.help-tab-content').forEach(c => c.classList.remove('active')); | |
| tab.classList.add('active'); | |
| helpPopover.querySelector(`.help-tab-content[data-tab="${tab.dataset.tab}"]`).classList.add('active'); | |
| return; | |
| } | |
| // Click-to-paste prompt | |
| const prompt = e.target.closest('.try-prompt'); | |
| if (prompt && prompt.dataset.prompt) { | |
| chatInput.value = prompt.dataset.prompt; | |
| chatInput.dispatchEvent(new Event('input')); | |
| helpPopover.classList.remove('open'); | |
| chatInput.focus(); | |
| } | |
| }); | |
| // ββ Settings panel ββββββββββββββββββββββββββββββββββββββββββ | |
| settingsBtn.addEventListener('click', () => { | |
| settingsPanel.classList.toggle('open'); | |
| memoryPanel.classList.remove('open'); | |
| closeHistorySidebar(); | |
| }); | |
| // ββ Model cache management ββββββββββββββββββββββββββββββββ | |
| const cacheInfo = document.getElementById('cacheInfo'); | |
| async function refreshCacheInfo() { | |
| try { | |
| const keys = await caches.keys(); | |
| const modelCaches = keys.filter(k => k.includes('transformers')); | |
| if (modelCaches.length === 0) { | |
| cacheInfo.innerHTML = 'No cached models'; | |
| return; | |
| } | |
| let totalSize = 0; | |
| const entries = []; | |
| for (const key of modelCaches) { | |
| const cache = await caches.open(key); | |
| const requests = await cache.keys(); | |
| let size = 0; | |
| for (const req of requests) { | |
| try { | |
| const res = await cache.match(req); | |
| if (res) { | |
| const blob = await res.blob(); | |
| size += blob.size; | |
| } | |
| } catch {} | |
| } | |
| totalSize += size; | |
| entries.push({ key, size, count: requests.length }); | |
| } | |
| const sizeMB = (totalSize / 1024 / 1024).toFixed(0); | |
| let html = `<strong>${sizeMB} MB</strong> cached (${entries.reduce((a, e) => a + e.count, 0)} files)<br>`; | |
| html += `<button class="btn-icon btn-danger-text" style="font-size:0.7rem;margin-top:4px" id="clearCacheBtn">Clear model cache</button>`; | |
| cacheInfo.innerHTML = html; | |
| document.getElementById('clearCacheBtn')?.addEventListener('click', async () => { | |
| if (!confirm('Delete all cached models? They will re-download on next use.')) return; | |
| for (const key of modelCaches) await caches.delete(key); | |
| refreshCacheInfo(); | |
| showToast('Model cache cleared'); | |
| }); | |
| } catch (e) { | |
| cacheInfo.innerHTML = 'Cache info unavailable'; | |
| } | |
| } | |
| // Refresh cache info when settings panel opens | |
| settingsBtn.addEventListener('click', () => { if (settingsPanel.classList.contains('open')) refreshCacheInfo(); }); | |
| refreshCacheInfo(); | |
| // Resumable downloads toggle (default on; set to '0' to disable) | |
| const resumableToggle = document.getElementById('resumableDownloadsToggle'); | |
| resumableToggle.checked = localStorage.getItem('lm_resumable_downloads') !== '0'; | |
| resumableToggle.addEventListener('change', () => { | |
| try { localStorage.setItem('lm_resumable_downloads', resumableToggle.checked ? '1' : '0'); } catch {} | |
| showToast(resumableToggle.checked ? 'Resumable downloads on' : 'Resumable downloads off β takes effect on next model load'); | |
| }); | |
| // Voice-to-text language. Whisper's auto-detect is unreliable on short | |
| // clips and often degenerates to filler tokens ("and", "the"); forcing | |
| // an explicit language avoids that. Default English. | |
| const voiceLanguageSelect = document.getElementById('voiceLanguageSelect'); | |
| voiceLanguageSelect.value = localStorage.getItem('lm_voice_language') || 'en'; | |
| voiceLanguageSelect.addEventListener('change', () => { | |
| try { localStorage.setItem('lm_voice_language', voiceLanguageSelect.value); } catch {} | |
| }); | |
| // ββ JavaScript API: chip, toggle, activity log ββββββββββββββ | |
| const apiChip = document.getElementById('apiChip'); | |
| const apiEnabledToggle = document.getElementById('apiEnabledToggle'); | |
| const apiLogBtn = document.getElementById('apiLogBtn'); | |
| const apiLogModal = document.getElementById('apiLogModal'); | |
| const apiLogList = document.getElementById('apiLogList'); | |
| const apiLogCloseBtn = document.getElementById('apiLogCloseBtn'); | |
| const apiLogClearBtn = document.getElementById('apiLogClearBtn'); | |
| function setApiEnabled(on) { | |
| apiEnabled = !!on; | |
| try { localStorage.setItem('lm_api_enabled', apiEnabled ? '1' : '0'); } catch {} | |
| apiChip.classList.toggle('on', apiEnabled); | |
| apiEnabledToggle.checked = apiEnabled; | |
| if (apiEnabled) { | |
| attachLocalmindAPI(); | |
| // One-time console notice the first time it's enabled in a session | |
| if (!window.__lm_api_notice_shown) { | |
| window.__lm_api_notice_shown = true; | |
| console.info('%cLocalMind API enabled', 'color:#5a67d8;font-weight:600', 'β window.localmind is experimental and unstable. Activity is logged and visible via Settings β View activity log.'); | |
| } | |
| } else { | |
| detachLocalmindAPI(); | |
| } | |
| } | |
| function logApiCall(entry) { | |
| // Entry shape: { method, promptLen, tokens, durationMs, outcome, error } | |
| apiLog.unshift({ ts: Date.now(), ...entry }); | |
| if (apiLog.length > API_LOG_MAX) apiLog.length = API_LOG_MAX; | |
| flashApiChip(); | |
| // If the modal is open, refresh it live | |
| if (apiLogModal.classList.contains('open')) renderApiLog(); | |
| } | |
| function flashApiChip() { | |
| if (!apiEnabled) return; | |
| apiChip.classList.add('flash'); | |
| setTimeout(() => apiChip.classList.remove('flash'), 400); | |
| } | |
| function renderApiLog() { | |
| if (apiLog.length === 0) { | |
| apiLogList.innerHTML = '<div class="api-log-empty">No API calls yet.</div>'; | |
| return; | |
| } | |
| const frag = document.createDocumentFragment(); | |
| for (const e of apiLog) { | |
| const row = document.createElement('div'); | |
| row.className = 'api-log-row'; | |
| const time = new Date(e.ts).toLocaleTimeString(); | |
| const outcomeClass = e.outcome === 'ok' ? 'ok' : (e.outcome === 'busy' ? 'busy' : 'err'); | |
| // All user-facing fields are escaped before being written as text. | |
| row.innerHTML = | |
| '<span class="t"></span>' + | |
| '<span class="m"></span>' + | |
| '<span class="dur"></span>' + | |
| '<span class="dur"></span>' + | |
| '<span class="' + outcomeClass + '"></span>'; | |
| row.children[0].textContent = time; | |
| row.children[1].textContent = e.method + (e.promptLen != null ? ' (' + e.promptLen + ' ch)' : ''); | |
| row.children[2].textContent = e.tokens != null ? e.tokens + ' tok' : ''; | |
| row.children[3].textContent = e.durationMs != null ? e.durationMs + ' ms' : ''; | |
| row.children[4].textContent = e.outcome + (e.error ? ': ' + e.error.slice(0, 40) : ''); | |
| frag.appendChild(row); | |
| } | |
| apiLogList.innerHTML = ''; | |
| apiLogList.appendChild(frag); | |
| } | |
| // Resolve a user-supplied model identifier to a short MODELS key. | |
| // Accepts either the short key ("gemma3-1b") or the full HF id | |
| // ("onnx-community/gemma-3-1b-it-ONNX-GQA"). Returns null if unknown. | |
| function resolveModelKey(idOrKey) { | |
| if (MODELS[idOrKey]) return idOrKey; | |
| for (const [k, m] of Object.entries(MODELS)) { | |
| if (m.id === idOrKey) return k; | |
| } | |
| return null; | |
| } | |
| // Promise wrapper around loadModel(). loadModel itself is fire-and-forget | |
| // (success/failure is signalled later via worker messages); we install | |
| // resolvers right after calling it. The order matters: loadModel() is | |
| // synchronous, so no worker message can arrive between the two lines. | |
| function loadModelViaApi(idOrKey) { | |
| const key = resolveModelKey(idOrKey); | |
| if (!key) return Promise.reject(new Error('Unknown model: ' + idOrKey)); | |
| // Already loaded β fast path, no worker churn | |
| if (key === activeModelKey && modelReady) return Promise.resolve(); | |
| return new Promise((resolve, reject) => { | |
| loadModel(key); | |
| loadResolvers = { resolve, reject }; | |
| }); | |
| } | |
| // Tiny push/pull async iterator for streaming responses. Producers call | |
| // push() with each chunk and finish() when done (optionally with an | |
| // error). Consumers iterate via `for await`. Pulls outpace pushes by | |
| // queueing pending promises; pushes outpace pulls by buffering. | |
| // onReturn fires exactly once if the consumer exits the loop early | |
| // (break, return, throw); producers use it to cancel upstream work. | |
| function makeAsyncIterator(onReturn) { | |
| const buffer = []; | |
| const pending = []; | |
| let done = false; | |
| let error = null; | |
| const it = { | |
| push(item) { | |
| if (done) return; | |
| if (pending.length) pending.shift().resolve({ value: item, done: false }); | |
| else buffer.push(item); | |
| }, | |
| finish(err) { | |
| if (done) return; | |
| done = true; | |
| error = err || null; | |
| while (pending.length) { | |
| const p = pending.shift(); | |
| if (error) p.reject(error); | |
| else p.resolve({ value: undefined, done: true }); | |
| } | |
| }, | |
| next() { | |
| if (buffer.length) return Promise.resolve({ value: buffer.shift(), done: false }); | |
| if (done) return error ? Promise.reject(error) : Promise.resolve({ value: undefined, done: true }); | |
| return new Promise((resolve, reject) => pending.push({ resolve, reject })); | |
| }, | |
| return() { | |
| const wasDone = done; | |
| this.finish(); | |
| if (!wasDone && typeof onReturn === 'function') { | |
| try { onReturn(); } catch {} | |
| } | |
| return Promise.resolve({ value: undefined, done: true }); | |
| }, | |
| [Symbol.asyncIterator]() { return this; }, | |
| }; | |
| return it; | |
| } | |
| // Validate and normalise the messages array passed to chat.completions.create. | |
| // Accepts the OpenAI shape: [{ role, content }]. Roles limited to | |
| // system/user/assistant. Content must be a string in v1 (no multimodal). | |
| function validateChatMessages(messages) { | |
| if (!Array.isArray(messages) || messages.length === 0) { | |
| throw new Error('messages must be a non-empty array'); | |
| } | |
| const VALID_ROLES = new Set(['system', 'user', 'assistant']); | |
| const out = []; | |
| for (let i = 0; i < messages.length; i++) { | |
| const m = messages[i]; | |
| if (!m || typeof m !== 'object') throw new Error('messages[' + i + '] must be an object'); | |
| if (!VALID_ROLES.has(m.role)) throw new Error('messages[' + i + '].role must be one of system|user|assistant'); | |
| if (typeof m.content !== 'string') throw new Error('messages[' + i + '].content must be a string (multimodal not supported in v1)'); | |
| out.push({ role: m.role, content: m.content }); | |
| } | |
| return out; | |
| } | |
| // Build an OpenAI-shaped chat.completion.chunk | |
| function makeStreamChunk(id, created, modelId, delta, finishReason) { | |
| return { | |
| id, | |
| object: 'chat.completion.chunk', | |
| created, | |
| model: modelId, | |
| choices: [ | |
| { index: 0, delta, finish_reason: finishReason }, | |
| ], | |
| }; | |
| } | |
| // Streaming variant of chat.completions.create. Returns an async iterator | |
| // immediately (well, the create() function awaits this and returns it), | |
| // and pushes chunks as the worker emits tokens. The runInference call is | |
| // fired without await β completion is signalled by its .then. Errors are | |
| // propagated to the iterator's next() call. | |
| function makeStreamingCompletion(chatMessages, genConfig, modelEntry, startTs) { | |
| const id = 'lmcc-' + Math.random().toString(36).slice(2, 12); | |
| const created = Math.floor(Date.now() / 1000); | |
| const modelId = modelEntry.id; | |
| const promptChars = chatMessages.reduce((s, m) => s + m.content.length, 0); | |
| // If the consumer breaks out of `for await` before generation ends, | |
| // tell the worker to stop so the next API call isn't queued behind | |
| // an abandoned generation that keeps running to max_tokens. | |
| const iter = makeAsyncIterator(() => { | |
| try { | |
| const w = LocalMind.runtime._worker; | |
| if (w) w.postMessage({ type: 'stop' }); | |
| } catch {} | |
| }); | |
| let tokenCount = 0; | |
| runInference({ | |
| chatMessages, | |
| attData: null, | |
| enableThinking: false, | |
| genConfig, | |
| setup: () => { | |
| // Detach the chat-UI bindings β same as the non-streaming path | |
| currentAssistantEl = null; | |
| currentAssistantText = ''; | |
| // First chunk is role-only, matching OpenAI's convention | |
| iter.push(makeStreamChunk(id, created, modelId, { role: 'assistant' }, null)); | |
| // Wire each subsequent token to a content delta | |
| currentInferenceContext = { | |
| onToken: (token) => { | |
| tokenCount++; | |
| iter.push(makeStreamChunk(id, created, modelId, { content: token }, null)); | |
| }, | |
| }; | |
| }, | |
| }).then( | |
| () => { | |
| // Final chunk: empty delta + stop | |
| iter.push(makeStreamChunk(id, created, modelId, {}, 'stop')); | |
| iter.finish(); | |
| logApiCall({ | |
| method: 'chat.completions.create (stream)', | |
| promptLen: promptChars, | |
| tokens: tokenCount, | |
| outcome: 'ok', | |
| durationMs: Date.now() - startTs, | |
| }); | |
| }, | |
| (err) => { | |
| iter.finish(err); | |
| logApiCall({ | |
| method: 'chat.completions.create (stream)', | |
| promptLen: promptChars, | |
| outcome: 'err', | |
| error: err.message, | |
| durationMs: Date.now() - startTs, | |
| }); | |
| } | |
| ); | |
| return iter; | |
| } | |
| // chat.completions.create β non-streaming or streaming depending on | |
| // params.stream. Mirrors the OpenAI SDK shape closely enough that | |
| // simple clients can drop us in by swapping the base. | |
| // Tools, multimodal, and response_format are intentionally rejected | |
| // in v1 β see ROADMAP.md and the SECURITY notes in the README. | |
| async function chatCompletionsCreate(params) { | |
| const start = Date.now(); | |
| params = params || {}; | |
| // Hard-rejects (logged as err) | |
| const reject = (msg) => { | |
| const e = new Error(msg); | |
| logApiCall({ method: 'chat.completions.create', outcome: 'err', error: msg, durationMs: Date.now() - start }); | |
| throw e; | |
| }; | |
| if (!modelReady || !activeModelKey) reject('no model loaded β call window.localmind.load() first'); | |
| if (params.tools) reject('tools not supported in v1'); | |
| if (params.tool_choice) reject('tool_choice not supported in v1'); | |
| if (params.response_format) reject('response_format not supported in v1'); | |
| if (params.n != null && params.n !== 1) reject('only n=1 is supported'); | |
| // Optional model param: if present, must match loaded model (by id or key) | |
| if (params.model) { | |
| const requested = resolveModelKey(params.model); | |
| if (!requested) reject('unknown model: ' + params.model); | |
| if (requested !== activeModelKey) reject('model mismatch: requested ' + params.model + ' but ' + MODELS[activeModelKey].id + ' is loaded β call load() first'); | |
| } | |
| let chatMessages; | |
| try { chatMessages = validateChatMessages(params.messages); } | |
| catch (e) { reject(e.message); } | |
| const m = MODELS[activeModelKey]; | |
| const baseGen = m.genConfig; | |
| // Clamp/sanitise generation parameters | |
| const temperature = (typeof params.temperature === 'number') ? Math.max(0, Math.min(2, params.temperature)) : baseGen.temperature; | |
| const top_p = (typeof params.top_p === 'number') ? Math.max(0, Math.min(1, params.top_p)) : baseGen.top_p; | |
| const reqMaxTokens = (typeof params.max_tokens === 'number' && params.max_tokens > 0) ? Math.floor(params.max_tokens) : baseGen.max_new_tokens; | |
| // Hard cap: never let a caller exceed the model's default ceiling | |
| const max_new_tokens = Math.min(reqMaxTokens, baseGen.max_new_tokens); | |
| const genConfig = { | |
| ...baseGen, | |
| temperature, | |
| top_p, | |
| max_new_tokens, | |
| }; | |
| // Streaming branch: return an async iterator that yields OpenAI-shaped | |
| // chat.completion.chunk objects as tokens arrive. | |
| if (params.stream) { | |
| return makeStreamingCompletion(chatMessages, genConfig, m, start); | |
| } | |
| // Run through the queue with a setup callback that detaches the chat | |
| // UI bindings inside the critical section, so concurrent chat sends | |
| // can't trample our generation. | |
| let responseText; | |
| try { | |
| responseText = await runInference({ | |
| chatMessages, | |
| attData: null, | |
| enableThinking: false, | |
| genConfig, | |
| setup: () => { | |
| currentAssistantEl = null; | |
| currentAssistantText = ''; | |
| }, | |
| }); | |
| } catch (e) { | |
| reject(e.message); | |
| } | |
| // Approximate token counts: prompt by char/4 heuristic, completion by | |
| // tokenizer streaming events isn't surfaced to main thread, so use the | |
| // same heuristic. Marked as approximate in the README. | |
| const promptChars = chatMessages.reduce((s, m) => s + m.content.length, 0); | |
| const promptTokensApprox = Math.max(1, Math.round(promptChars / 4)); | |
| const completionTokensApprox = Math.max(1, Math.round((responseText || '').length / 4)); | |
| const id = 'lmcc-' + Math.random().toString(36).slice(2, 12); | |
| const created = Math.floor(Date.now() / 1000); | |
| const out = { | |
| id, | |
| object: 'chat.completion', | |
| created, | |
| model: m.id, | |
| choices: [ | |
| { | |
| index: 0, | |
| message: { role: 'assistant', content: responseText || '' }, | |
| finish_reason: 'stop', | |
| }, | |
| ], | |
| usage: { | |
| prompt_tokens: promptTokensApprox, | |
| completion_tokens: completionTokensApprox, | |
| total_tokens: promptTokensApprox + completionTokensApprox, | |
| // Marker so callers can tell these aren't exact | |
| _approximate: true, | |
| }, | |
| }; | |
| logApiCall({ | |
| method: 'chat.completions.create', | |
| promptLen: promptChars, | |
| tokens: completionTokensApprox, | |
| outcome: 'ok', | |
| durationMs: Date.now() - start, | |
| }); | |
| return out; | |
| } | |
| // window.localmind v1 β read-only inference surface. | |
| // Live properties (`ready`, `model`) use getters so they reflect current | |
| // state without needing a re-attach on every model change. | |
| const localmindAPI = Object.freeze({ | |
| version: '1.0', | |
| get ready() { return modelReady; }, | |
| get model() { return (activeModelKey && modelReady) ? MODELS[activeModelKey].id : null; }, | |
| listModels() { | |
| return Object.entries(MODELS).map(([key, m]) => ({ | |
| id: m.id, | |
| key, | |
| label: m.label, | |
| size: m.size, | |
| multimodal: !!m.multimodal, | |
| contextSize: m.contextSize, | |
| loaded: key === activeModelKey && modelReady, | |
| })); | |
| }, | |
| load(idOrKey) { | |
| const start = Date.now(); | |
| return loadModelViaApi(idOrKey).then( | |
| () => { | |
| logApiCall({ method: 'load', outcome: 'ok', durationMs: Date.now() - start }); | |
| }, | |
| (err) => { | |
| logApiCall({ method: 'load', outcome: 'err', error: err.message, durationMs: Date.now() - start }); | |
| throw err; | |
| } | |
| ); | |
| }, | |
| chat: Object.freeze({ | |
| completions: Object.freeze({ | |
| create: chatCompletionsCreate, | |
| }), | |
| }), | |
| }); | |
| function attachLocalmindAPI() { | |
| try { | |
| Object.defineProperty(window, 'localmind', { | |
| value: localmindAPI, | |
| writable: false, | |
| configurable: true, | |
| enumerable: false, | |
| }); | |
| } catch (e) { | |
| console.error('Could not attach window.localmind:', e); | |
| } | |
| } | |
| function detachLocalmindAPI() { | |
| try { delete window.localmind; } catch {} | |
| } | |
| apiEnabledToggle.addEventListener('change', (e) => setApiEnabled(e.target.checked)); | |
| apiLogBtn.addEventListener('click', () => { | |
| renderApiLog(); | |
| apiLogModal.classList.add('open'); | |
| }); | |
| apiChip.addEventListener('click', () => { | |
| renderApiLog(); | |
| apiLogModal.classList.add('open'); | |
| }); | |
| apiLogCloseBtn.addEventListener('click', () => apiLogModal.classList.remove('open')); | |
| apiLogModal.addEventListener('click', (e) => { if (e.target === apiLogModal) apiLogModal.classList.remove('open'); }); | |
| apiLogClearBtn.addEventListener('click', () => { apiLog.length = 0; renderApiLog(); }); | |
| // Apply the persisted flag on load | |
| setApiEnabled(apiEnabled); | |
| // ββ Custom models (paste a HF ONNX model id) ββββββββββββββββ | |
| const customModelInput = document.getElementById('customModelInput'); | |
| const customModelAddBtn = document.getElementById('customModelAddBtn'); | |
| const customModelStatus = document.getElementById('customModelStatus'); | |
| const customModelList = document.getElementById('customModelList'); | |
| function setCustomStatus(msg, kind) { | |
| customModelStatus.textContent = msg || ''; | |
| customModelStatus.style.color = kind === 'err' ? 'var(--error-text)' : (kind === 'ok' ? '#2f855a' : 'var(--gray-500)'); | |
| } | |
| // Query the WebGPU adapter once for its hard buffer limits. Cached so | |
| // we don't pay the cost on every validation. | |
| let __webgpuLimits = null; | |
| async function getWebGPULimits() { | |
| if (__webgpuLimits) return __webgpuLimits; | |
| try { | |
| if (!navigator.gpu) return (__webgpuLimits = { maxBufferSize: null, ok: false }); | |
| const adapter = await navigator.gpu.requestAdapter(); | |
| if (!adapter) return (__webgpuLimits = { maxBufferSize: null, ok: false }); | |
| __webgpuLimits = { | |
| maxBufferSize: adapter.limits.maxBufferSize || null, | |
| maxStorageBufferBindingSize: adapter.limits.maxStorageBufferBindingSize || null, | |
| ok: true, | |
| }; | |
| } catch { | |
| __webgpuLimits = { maxBufferSize: null, ok: false }; | |
| } | |
| return __webgpuLimits; | |
| } | |
| function formatBytes(n) { | |
| if (!n || n <= 0) return 'unknown'; | |
| if (n >= 1024 * 1024 * 1024) return (n / 1024 / 1024 / 1024).toFixed(1) + ' GB'; | |
| if (n >= 1024 * 1024) return Math.round(n / 1024 / 1024) + ' MB'; | |
| return Math.round(n / 1024) + ' KB'; | |
| } | |
| // Inspect a list of HF siblings and group ONNX files by quantization. | |
| // Returns { quants: { q4f16: {files, bytes}, q4: {...}, ... }, all: [...] }. | |
| function classifyOnnxFiles(siblings) { | |
| const out = { quants: {}, all: [] }; | |
| // Detect dtype suffix from filename. Map several common conventions to | |
| // the dtype string Transformers.js expects. | |
| const detectDtype = (name) => { | |
| const f = name.toLowerCase(); | |
| if (/_q4f16(\.onnx|_data\b)/.test(f)) return 'q4f16'; | |
| if (/_q4(\.onnx|_data\b)/.test(f) && !/q4f16/.test(f)) return 'q4'; | |
| if (/_int8(\.onnx|_data\b)/.test(f)) return 'int8'; | |
| if (/_uint8(\.onnx|_data\b)/.test(f)) return 'uint8'; | |
| if (/_fp16(\.onnx|_data\b)/.test(f)) return 'fp16'; | |
| if (/_quantized(\.onnx|_data\b)/.test(f)) return 'quantized'; | |
| if (/_bnb4(\.onnx|_data\b)/.test(f)) return 'bnb4'; | |
| if (/_q8(\.onnx|_data\b)/.test(f)) return 'q8'; | |
| if (/_q4f8(\.onnx|_data\b)/.test(f)) return 'q4f8'; | |
| // Bare model.onnx (no suffix) β fp32 | |
| if (/(^|\/)model\.onnx(_data)?$/.test(f)) return 'fp32'; | |
| return null; | |
| }; | |
| for (const s of siblings) { | |
| if (!s || typeof s.rfilename !== 'string') continue; | |
| if (!/\.onnx(_data)?$/.test(s.rfilename)) continue; | |
| out.all.push(s); | |
| const dtype = detectDtype(s.rfilename); | |
| if (!dtype) continue; | |
| if (!out.quants[dtype]) out.quants[dtype] = { files: [], bytes: 0 }; | |
| out.quants[dtype].files.push(s); | |
| if (typeof s.size === 'number') out.quants[dtype].bytes += s.size; | |
| } | |
| return out; | |
| } | |
| // Validate a user-supplied HF repo id and probe the HF API to confirm | |
| // it exists, has usable ONNX files, picks a workable quantization, and | |
| // checks the chosen size against the device's WebGPU buffer limits. | |
| // Returns { entry, warnings: [] } on success, throws on hard failure. | |
| async function probeHuggingFaceModel(id) { | |
| if (!/^[\w.-]+\/[\w.-]+$/.test(id)) { | |
| throw new Error('invalid id β use owner/repo'); | |
| } | |
| // 1. HF model info. ?blobs=true populates each sibling's .size. | |
| let info; | |
| try { | |
| const res = await fetch('https://huggingface.co/api/models/' + id + '?blobs=true', { signal: AbortSignal.timeout(10000) }); | |
| if (!res.ok) throw new Error('HTTP ' + res.status); | |
| info = await res.json(); | |
| } catch (e) { | |
| throw new Error('model not found or network error: ' + e.message); | |
| } | |
| // 2. Find ONNX files anywhere in the repo (not just under onnx/). | |
| const siblings = Array.isArray(info.siblings) ? info.siblings : []; | |
| const classified = classifyOnnxFiles(siblings); | |
| if (classified.all.length === 0) { | |
| throw new Error('no ONNX files found in this repo. Hugging Face has many ONNX exports under the onnx-community/ org and from Xenova β try one of those.'); | |
| } | |
| // 3. Pick a quantization. Order = preference for in-browser inference: | |
| // smallest-and-fastest first, largest last. | |
| const PREFERENCE = ['q4f16', 'q4', 'int8', 'q8', 'uint8', 'bnb4', 'q4f8', 'fp16', 'quantized', 'fp32']; | |
| let chosenDtype = null; | |
| for (const dt of PREFERENCE) { | |
| if (classified.quants[dt] && classified.quants[dt].files.length > 0) { chosenDtype = dt; break; } | |
| } | |
| if (!chosenDtype) { | |
| // ONNX files exist but none match a known dtype convention. Let the | |
| // user know exactly what we found so they can investigate. | |
| const sample = classified.all.slice(0, 4).map(s => s.rfilename).join(', '); | |
| throw new Error('ONNX files found but none match a known quantization (looked for q4f16/q4/int8/fp16/quantized/fp32). Files: ' + sample); | |
| } | |
| // Estimate size as MAX(.onnx) + MAX(.onnx_data) across files in the | |
| // chosen dtype group. Many repos publish multiple variants of the | |
| // same model (model_, decoder_model_, decoder_model_merged_, ...) and | |
| // Transformers.js loads exactly ONE β summing them all overcounts by | |
| // 3-4Γ. The largest single .onnx is the actual loaded weights file; | |
| // .onnx_data (if any) is its companion external-weights file. | |
| let chosenBytes = 0; | |
| { | |
| let maxOnnx = 0, maxData = 0; | |
| for (const f of classified.quants[chosenDtype].files) { | |
| if (typeof f.size !== 'number') continue; | |
| if (/\.onnx_data$/.test(f.rfilename)) { | |
| if (f.size > maxData) maxData = f.size; | |
| } else if (/\.onnx$/.test(f.rfilename)) { | |
| if (f.size > maxOnnx) maxOnnx = f.size; | |
| } | |
| } | |
| chosenBytes = maxOnnx + maxData; | |
| } | |
| const warnings = []; | |
| // 4. Size check vs WebGPU limits. | |
| const limits = await getWebGPULimits(); | |
| if (limits.ok && limits.maxBufferSize && chosenBytes > 0) { | |
| // The single biggest weight tensor must fit in maxBufferSize. We | |
| // approximate by assuming the largest file is one buffer's worth. | |
| // If even the smallest quant exceeds the limit, hard-block. | |
| const largestFile = Math.max(0, ...classified.quants[chosenDtype].files.map(f => f.size || 0)); | |
| if (largestFile > limits.maxBufferSize) { | |
| throw new Error('largest weight file (' + formatBytes(largestFile) + ') exceeds this device\'s WebGPU buffer limit (' + formatBytes(limits.maxBufferSize) + '). Try a smaller model or a device with more GPU memory.'); | |
| } | |
| // If the total exceeds 4Γ the buffer limit, it almost certainly | |
| // won't fit overall β block it. (Models are usually split across | |
| // many buffers, so 4Γ headroom is generous.) | |
| if (chosenBytes > limits.maxBufferSize * 4) { | |
| throw new Error('model is ' + formatBytes(chosenBytes) + ' but this device\'s WebGPU buffer limit is ' + formatBytes(limits.maxBufferSize) + ' β too large to load.'); | |
| } | |
| // Warn if chosen size is over half the buffer limit β likely tight | |
| if (chosenBytes > limits.maxBufferSize) { | |
| warnings.push('size (' + formatBytes(chosenBytes) + ') exceeds the per-buffer WebGPU limit (' + formatBytes(limits.maxBufferSize) + '); load may fail'); | |
| } | |
| } else if (!limits.ok) { | |
| warnings.push('could not query WebGPU limits β load may fail if model exceeds device memory'); | |
| } | |
| // 5. Absolute soft cap: anything bigger than 6 GB is impractical in | |
| // a browser tab regardless of GPU. | |
| if (chosenBytes > 6 * 1024 * 1024 * 1024) { | |
| throw new Error('model is ' + formatBytes(chosenBytes) + ' β too large for in-browser inference. The hard ceiling for LocalMind is 6 GB.'); | |
| } | |
| if (chosenBytes > 2 * 1024 * 1024 * 1024) { | |
| warnings.push('large model (' + formatBytes(chosenBytes) + ') β first download will take a while and may not fit on lower-end GPUs'); | |
| } | |
| // 6. config.json β architecture hint + context size | |
| let config = null; | |
| try { | |
| const res = await fetch('https://huggingface.co/' + id + '/resolve/main/config.json', { signal: AbortSignal.timeout(10000) }); | |
| if (res.ok) config = await res.json(); | |
| } catch {} | |
| // 7. Reject multimodal in v1 (the runtime adapter currently | |
| // only knows how to load the multimodal class for Gemma 4) | |
| const modelType = config && config.model_type ? String(config.model_type) : ''; | |
| if (config && (config.vision_config || config.audio_config)) { | |
| throw new Error('multimodal custom models are not supported in v1 (model_type=' + modelType + ')'); | |
| } | |
| // 8. Build the MODELS entry | |
| const repoShort = id.split('/').pop(); | |
| const sizeLabel = chosenBytes ? '~' + formatBytes(chosenBytes) : 'size unknown'; | |
| const entry = { | |
| id, | |
| label: repoShort + ' (custom)', | |
| dtype: chosenDtype, | |
| size: sizeLabel, | |
| type: 'causal', | |
| multimodal: false, | |
| agentCapable: false, | |
| contextSize: (config && (config.max_position_embeddings || config.max_seq_len)) || 4096, | |
| genConfig: { temperature: 0.7, top_k: 50, top_p: 0.95, max_new_tokens: 1024 }, | |
| custom: true, | |
| }; | |
| return { entry, warnings }; | |
| } | |
| function loadCustomModelsFromStorage() { | |
| try { | |
| const raw = localStorage.getItem('lm_custom_models'); | |
| if (!raw) return []; | |
| const arr = JSON.parse(raw); | |
| if (!Array.isArray(arr)) return []; | |
| return arr.filter(m => m && typeof m.id === 'string'); | |
| } catch { return []; } | |
| } | |
| function saveCustomModelsToStorage() { | |
| const custom = Object.values(MODELS).filter(m => m.custom); | |
| try { localStorage.setItem('lm_custom_models', JSON.stringify(custom)); } catch {} | |
| } | |
| function addCustomModelToRegistry(entry) { | |
| // The HF id is the registry key for custom models β it's unique, stable, | |
| // and matches what the worker will pass to from_pretrained. | |
| MODELS[entry.id] = entry; | |
| // Append to the model selector if not already there | |
| if (!Array.from(modelSelect.options).some(o => o.value === entry.id)) { | |
| const opt = document.createElement('option'); | |
| opt.value = entry.id; | |
| opt.textContent = entry.label + ' Β· ' + entry.size; | |
| modelSelect.appendChild(opt); | |
| } | |
| } | |
| function removeCustomModelFromRegistry(id) { | |
| if (!MODELS[id] || !MODELS[id].custom) return; | |
| // If this is the active model, we can't remove it β the worker is tied to it | |
| if (id === activeModelKey) { | |
| setCustomStatus('cannot remove β this model is currently loaded. Switch to another first.', 'err'); | |
| return; | |
| } | |
| delete MODELS[id]; | |
| const opt = Array.from(modelSelect.options).find(o => o.value === id); | |
| if (opt) opt.remove(); | |
| saveCustomModelsToStorage(); | |
| renderCustomModelList(); | |
| } | |
| function renderCustomModelList() { | |
| customModelList.innerHTML = ''; | |
| const custom = Object.values(MODELS).filter(m => m.custom); | |
| if (custom.length === 0) { | |
| customModelList.textContent = ''; | |
| return; | |
| } | |
| for (const m of custom) { | |
| const row = document.createElement('div'); | |
| row.style.cssText = 'display:flex;align-items:center;justify-content:space-between;gap:8px;padding:4px 6px;margin-top:4px;background:var(--gray-50);border:1px solid var(--gray-200);border-radius:4px;font-size:0.72rem'; | |
| const left = document.createElement('div'); | |
| left.style.cssText = 'flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap'; | |
| left.textContent = m.id + ' Β· ' + m.size; | |
| const del = document.createElement('button'); | |
| del.className = 'btn-icon'; | |
| del.textContent = 'Remove'; | |
| del.style.cssText = 'flex-shrink:0;padding:2px 8px;font-size:0.7rem'; | |
| del.addEventListener('click', () => removeCustomModelFromRegistry(m.id)); | |
| row.appendChild(left); | |
| row.appendChild(del); | |
| customModelList.appendChild(row); | |
| } | |
| } | |
| async function handleAddCustomModel() { | |
| const id = customModelInput.value.trim(); | |
| if (!id) { setCustomStatus('enter a model id', 'err'); return; } | |
| if (MODELS[id]) { setCustomStatus('already added', 'err'); return; } | |
| customModelAddBtn.disabled = true; | |
| setCustomStatus('Probing Hugging Faceβ¦'); | |
| try { | |
| const { entry, warnings } = await probeHuggingFaceModel(id); | |
| addCustomModelToRegistry(entry); | |
| saveCustomModelsToStorage(); | |
| renderCustomModelList(); | |
| customModelInput.value = ''; | |
| let msg = 'Added (' + entry.dtype + ', ' + entry.size + '). Select it from the model dropdown to load.'; | |
| if (warnings && warnings.length) { | |
| msg += '\nWarning: ' + warnings.join('; '); | |
| // Use a softer colour so the user notices but knows it's not a fatal error | |
| customModelStatus.textContent = msg; | |
| customModelStatus.style.color = '#975a16'; | |
| customModelStatus.style.whiteSpace = 'pre-wrap'; | |
| } else { | |
| setCustomStatus(msg, 'ok'); | |
| } | |
| } catch (e) { | |
| setCustomStatus('Failed: ' + e.message, 'err'); | |
| customModelStatus.style.whiteSpace = 'pre-wrap'; | |
| } finally { | |
| customModelAddBtn.disabled = false; | |
| } | |
| } | |
| customModelAddBtn.addEventListener('click', handleAddCustomModel); | |
| customModelInput.addEventListener('keydown', (e) => { | |
| if (e.key === 'Enter') { e.preventDefault(); handleAddCustomModel(); } | |
| }); | |
| // Restore any previously added custom models BEFORE the initial loadModel | |
| // runs β otherwise a saved active selection pointing at a custom model | |
| // would silently fall back to the default. | |
| for (const m of loadCustomModelsFromStorage()) { | |
| try { addCustomModelToRegistry(m); } catch {} | |
| } | |
| renderCustomModelList(); | |
| // ββ Custom tools (user-defined, HTTP POST) ββββββββββββββββ | |
| const customToolInput = document.getElementById('customToolInput'); | |
| const customToolAddBtn = document.getElementById('customToolAddBtn'); | |
| const customToolStatus = document.getElementById('customToolStatus'); | |
| const customToolList = document.getElementById('customToolList'); | |
| function setCustomToolStatus(msg, kind) { | |
| customToolStatus.textContent = msg || ''; | |
| customToolStatus.style.color = kind === 'err' ? 'var(--red-600, #dc2626)' : 'var(--gray-500)'; | |
| } | |
| function validateCustomToolDef(def) { | |
| if (!def || typeof def !== 'object') return 'must be a JSON object'; | |
| if (typeof def.name !== 'string' || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(def.name)) { | |
| return 'name must match /^[a-zA-Z_][a-zA-Z0-9_]*$/'; | |
| } | |
| if (TOOL_REGISTRY[def.name] && !TOOL_REGISTRY[def.name]._custom) { | |
| return 'name collides with a built-in tool'; | |
| } | |
| if (typeof def.description !== 'string' || !def.description.trim()) return 'description required'; | |
| if (typeof def.endpoint !== 'string' || !/^https?:\/\//i.test(def.endpoint)) return 'endpoint must be http(s) URL'; | |
| if (!def.parameters || typeof def.parameters !== 'object') return 'parameters required (JSON schema object)'; | |
| return null; | |
| } | |
| function makeCustomToolExecute(endpoint, toolName) { | |
| return async function execute(args) { | |
| try { | |
| const res = await fetch(endpoint, { | |
| method: 'POST', | |
| headers: { 'content-type': 'application/json' }, | |
| body: JSON.stringify(args || {}), | |
| }); | |
| const ct = res.headers.get('content-type') || ''; | |
| const body = ct.includes('application/json') ? await res.json() : await res.text(); | |
| if (!res.ok) return { error: 'Tool "' + toolName + '" returned ' + res.status, body }; | |
| return body; | |
| } catch (e) { | |
| return { error: 'Tool "' + toolName + '" fetch failed: ' + (e.message || e) }; | |
| } | |
| }; | |
| } | |
| function registerCustomTool(def) { | |
| TOOL_REGISTRY[def.name] = { | |
| description: def.description, | |
| parameters: def.parameters, | |
| execute: makeCustomToolExecute(def.endpoint, def.name), | |
| _custom: true, | |
| _endpoint: def.endpoint, | |
| }; | |
| } | |
| function loadCustomToolsFromStorage() { | |
| try { | |
| const raw = localStorage.getItem('lm_custom_tools'); | |
| if (!raw) return []; | |
| const arr = JSON.parse(raw); | |
| return Array.isArray(arr) ? arr : []; | |
| } catch { return []; } | |
| } | |
| function saveCustomToolsToStorage() { | |
| const arr = Object.entries(TOOL_REGISTRY) | |
| .filter(([, t]) => t._custom) | |
| .map(([name, t]) => ({ name, description: t.description, parameters: t.parameters, endpoint: t._endpoint })); | |
| try { localStorage.setItem('lm_custom_tools', JSON.stringify(arr)); } catch {} | |
| } | |
| function renderCustomToolList() { | |
| customToolList.innerHTML = ''; | |
| const entries = Object.entries(TOOL_REGISTRY).filter(([, t]) => t._custom); | |
| if (entries.length === 0) return; | |
| for (const [name, t] of entries) { | |
| const row = document.createElement('div'); | |
| row.style.cssText = 'display:flex;align-items:center;justify-content:space-between;gap:8px;padding:4px 6px;margin-top:4px;background:var(--gray-50);border:1px solid var(--gray-200);border-radius:4px;font-size:0.72rem'; | |
| const left = document.createElement('div'); | |
| left.style.cssText = 'flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap'; | |
| left.textContent = name + ' \u2192 ' + t._endpoint; | |
| const del = document.createElement('button'); | |
| del.className = 'btn-icon'; | |
| del.textContent = 'Remove'; | |
| del.style.cssText = 'flex-shrink:0;padding:2px 8px;font-size:0.7rem'; | |
| del.addEventListener('click', () => { | |
| delete TOOL_REGISTRY[name]; | |
| saveCustomToolsToStorage(); | |
| renderCustomToolList(); | |
| }); | |
| row.appendChild(left); | |
| row.appendChild(del); | |
| customToolList.appendChild(row); | |
| } | |
| } | |
| customToolAddBtn.addEventListener('click', () => { | |
| let def; | |
| try { def = JSON.parse(customToolInput.value); } | |
| catch (e) { setCustomToolStatus('Invalid JSON: ' + e.message, 'err'); return; } | |
| const err = validateCustomToolDef(def); | |
| if (err) { setCustomToolStatus(err, 'err'); return; } | |
| registerCustomTool(def); | |
| saveCustomToolsToStorage(); | |
| renderCustomToolList(); | |
| customToolInput.value = ''; | |
| setCustomToolStatus('Added "' + def.name + '". Agent-capable models will see it on next message.'); | |
| }); | |
| for (const def of loadCustomToolsFromStorage()) { | |
| const err = validateCustomToolDef(def); | |
| if (!err) registerCustomTool(def); | |
| } | |
| renderCustomToolList(); | |
| // ββ MCP servers (Streamable HTTP JSON-RPC 2.0) βββββββββββββ | |
| // Discover tools on page load from each registered server and add them | |
| // to TOOL_REGISTRY with an mcp_ prefix. Tool execution POSTs a | |
| // tools/call JSON-RPC request back to the same URL. | |
| const mcpUrlInput = document.getElementById('mcpUrlInput'); | |
| const mcpAuthInput = document.getElementById('mcpAuthInput'); | |
| const mcpAddBtn = document.getElementById('mcpAddBtn'); | |
| const mcpStatus = document.getElementById('mcpStatus'); | |
| const mcpListDiv = document.getElementById('mcpList'); | |
| // Keep server connection state here so removeMcpServer can wipe only | |
| // the tools that belong to that server. | |
| const __mcpServers = []; // [{ url, auth, toolNames: [] }] | |
| function mcpSetStatus(msg, kind) { | |
| mcpStatus.textContent = msg || ''; | |
| mcpStatus.style.color = kind === 'err' ? '#dc2626' : 'var(--gray-500)'; | |
| } | |
| async function mcpRpc(url, auth, method, params) { | |
| const body = { jsonrpc: '2.0', id: Math.floor(Math.random() * 1e9), method }; | |
| if (params !== undefined) body.params = params; | |
| const headers = { 'content-type': 'application/json', 'accept': 'application/json, text/event-stream' }; | |
| if (auth) headers['authorization'] = 'Bearer ' + auth; | |
| const res = await fetch(url, { method: 'POST', headers, body: JSON.stringify(body) }); | |
| if (!res.ok) throw new Error('HTTP ' + res.status + ' ' + res.statusText); | |
| const ct = res.headers.get('content-type') || ''; | |
| // MVP: only parse JSON responses. SSE streams aren't supported yet β | |
| // callers that need them should pick a stateless MCP server. | |
| const text = await res.text(); | |
| if (ct.includes('text/event-stream')) { | |
| // Pull the first `data: β¦` line's JSON β sufficient for discrete replies | |
| const line = text.split(/\r?\n/).find((l) => l.startsWith('data:')); | |
| if (!line) throw new Error('MCP SSE response had no data frame'); | |
| const j = JSON.parse(line.slice(5).trim()); | |
| if (j.error) throw new Error('MCP error: ' + (j.error.message || JSON.stringify(j.error))); | |
| return j.result; | |
| } | |
| const j = JSON.parse(text); | |
| if (j.error) throw new Error('MCP error: ' + (j.error.message || JSON.stringify(j.error))); | |
| return j.result; | |
| } | |
| function mcpMakeExecute(url, auth, remoteName) { | |
| return async function execute(args) { | |
| try { | |
| const result = await mcpRpc(url, auth, 'tools/call', { name: remoteName, arguments: args || {} }); | |
| // MCP tool result shape: { content: [{ type: 'text', text: 'β¦' }], isError? } | |
| if (result && Array.isArray(result.content)) { | |
| const textParts = result.content.filter((c) => c && c.type === 'text').map((c) => c.text).join('\n'); | |
| return result.isError ? { error: textParts || 'tool reported error' } : (textParts || result.content); | |
| } | |
| return result; | |
| } catch (e) { | |
| return { error: 'MCP call failed: ' + (e.message || e) }; | |
| } | |
| }; | |
| } | |
| async function mcpConnectServer(url, auth) { | |
| // Initialize handshake + list tools, then register each. | |
| await mcpRpc(url, auth, 'initialize', { | |
| protocolVersion: '2025-03-26', | |
| capabilities: {}, | |
| clientInfo: { name: 'LocalMind', version: '2.0' }, | |
| }); | |
| const list = await mcpRpc(url, auth, 'tools/list'); | |
| const tools = (list && list.tools) || []; | |
| const registered = []; | |
| for (const t of tools) { | |
| if (!t || !t.name) continue; | |
| // Sanitise the MCP tool name into a valid identifier, prefix with mcp_. | |
| const clean = String(t.name).replace(/[^a-zA-Z0-9_]/g, '_'); | |
| const local = 'mcp_' + clean; | |
| if (TOOL_REGISTRY[local] && !TOOL_REGISTRY[local]._mcp) continue; // skip collisions | |
| TOOL_REGISTRY[local] = { | |
| description: t.description || ('MCP tool from ' + url), | |
| parameters: t.inputSchema || { type: 'object', properties: {}, required: [] }, | |
| execute: mcpMakeExecute(url, auth, t.name), | |
| _mcp: true, | |
| _mcpUrl: url, | |
| }; | |
| registered.push(local); | |
| } | |
| return registered; | |
| } | |
| function mcpLoadFromStorage() { | |
| try { | |
| const raw = localStorage.getItem('lm_mcp_servers'); | |
| if (!raw) return []; | |
| const arr = JSON.parse(raw); | |
| return Array.isArray(arr) ? arr : []; | |
| } catch { return []; } | |
| } | |
| function mcpSaveToStorage() { | |
| const list = __mcpServers.map((s) => ({ url: s.url, auth: s.auth || '' })); | |
| try { localStorage.setItem('lm_mcp_servers', JSON.stringify(list)); } catch {} | |
| } | |
| function renderMcpList() { | |
| mcpListDiv.innerHTML = ''; | |
| if (__mcpServers.length === 0) return; | |
| for (const server of __mcpServers) { | |
| const row = document.createElement('div'); | |
| row.style.cssText = 'display:flex;align-items:center;justify-content:space-between;gap:8px;padding:4px 6px;margin-top:4px;background:var(--gray-50);border:1px solid var(--gray-200);border-radius:4px;font-size:0.72rem'; | |
| const left = document.createElement('div'); | |
| left.style.cssText = 'flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap'; | |
| left.textContent = server.url + ' \u2014 ' + server.toolNames.length + ' tool(s)'; | |
| const del = document.createElement('button'); | |
| del.className = 'btn-icon'; | |
| del.textContent = 'Remove'; | |
| del.style.cssText = 'flex-shrink:0;padding:2px 8px;font-size:0.7rem'; | |
| del.addEventListener('click', () => removeMcpServer(server.url)); | |
| row.appendChild(left); | |
| row.appendChild(del); | |
| mcpListDiv.appendChild(row); | |
| } | |
| } | |
| async function addMcpServer(url, auth) { | |
| if (__mcpServers.some((s) => s.url === url)) { mcpSetStatus('already added', 'err'); return; } | |
| mcpSetStatus('Connecting to ' + url + '\u2026'); | |
| mcpAddBtn.disabled = true; | |
| try { | |
| const names = await mcpConnectServer(url, auth); | |
| __mcpServers.push({ url, auth, toolNames: names }); | |
| mcpSaveToStorage(); | |
| renderMcpList(); | |
| mcpSetStatus('Connected. Discovered ' + names.length + ' tool(s).'); | |
| mcpUrlInput.value = ''; | |
| mcpAuthInput.value = ''; | |
| } catch (e) { | |
| mcpSetStatus('Failed: ' + (e.message || e), 'err'); | |
| } finally { | |
| mcpAddBtn.disabled = false; | |
| } | |
| } | |
| function removeMcpServer(url) { | |
| const idx = __mcpServers.findIndex((s) => s.url === url); | |
| if (idx < 0) return; | |
| for (const name of __mcpServers[idx].toolNames) { | |
| if (TOOL_REGISTRY[name] && TOOL_REGISTRY[name]._mcp) delete TOOL_REGISTRY[name]; | |
| } | |
| __mcpServers.splice(idx, 1); | |
| mcpSaveToStorage(); | |
| renderMcpList(); | |
| } | |
| mcpAddBtn.addEventListener('click', () => { | |
| const url = mcpUrlInput.value.trim(); | |
| if (!url || !/^https?:\/\//i.test(url)) { mcpSetStatus('URL must be http(s)://', 'err'); return; } | |
| addMcpServer(url, mcpAuthInput.value.trim()); | |
| }); | |
| // Auto-reconnect saved servers on page load. Silent on failures β | |
| // users will see the stale count in the list and can retry manually. | |
| (async () => { | |
| for (const s of mcpLoadFromStorage()) { | |
| try { | |
| const names = await mcpConnectServer(s.url, s.auth); | |
| __mcpServers.push({ url: s.url, auth: s.auth, toolNames: names }); | |
| } catch { | |
| __mcpServers.push({ url: s.url, auth: s.auth, toolNames: [] }); | |
| } | |
| } | |
| renderMcpList(); | |
| })(); | |
| // ββ Web search settings βββββββββββββββββββββββββββββββββββ | |
| const searchProvider = document.getElementById('searchProvider'); | |
| const searchApiKey = document.getElementById('searchApiKey'); | |
| const searxngUrl = document.getElementById('searxngUrl'); | |
| const apiKeyRow = document.getElementById('apiKeyRow'); | |
| const searxngUrlRow = document.getElementById('searxngUrlRow'); | |
| const searchSettingsSection = document.getElementById('searchSettingsSection'); | |
| const searchSendBtn = document.getElementById('searchSendBtn'); | |
| let webEnrichedMode = false; | |
| // Restore saved settings | |
| searchProvider.value = localStorage.getItem('lm_search_provider') || 'none'; | |
| searchApiKey.value = localStorage.getItem('lm_search_key') || ''; | |
| searxngUrl.value = localStorage.getItem('lm_searxng_url') || ''; | |
| function updateSearchUI() { | |
| const p = searchProvider.value; | |
| apiKeyRow.style.display = (p === 'brave' || p === 'tavily') ? '' : 'none'; | |
| searxngUrlRow.style.display = p === 'searxng' ? '' : 'none'; | |
| const m = MODELS[activeModelKey]; | |
| const isAgent = m && m.agentCapable; | |
| const configured = isSearchConfigured(); | |
| searchSendBtn.style.display = (configured && isAgent) ? '' : 'none'; | |
| } | |
| searchProvider.addEventListener('change', () => { | |
| localStorage.setItem('lm_search_provider', searchProvider.value); | |
| updateSearchUI(); | |
| }); | |
| searchApiKey.addEventListener('input', () => localStorage.setItem('lm_search_key', searchApiKey.value)); | |
| searxngUrl.addEventListener('input', () => localStorage.setItem('lm_searxng_url', searxngUrl.value)); | |
| // SAM model selector | |
| const samModelSelect = document.getElementById('samModelSelect'); | |
| const samSettingsSection = document.getElementById('samSettingsSection'); | |
| const savedSamModel = localStorage.getItem('lm_sam_model'); | |
| if (savedSamModel) samModelSelect.value = savedSamModel; | |
| samModelSelect.addEventListener('change', () => { | |
| localStorage.setItem('lm_sam_model', samModelSelect.value); | |
| }); | |
| // Search+Send button | |
| searchSendBtn.addEventListener('click', () => { | |
| if (generating) return; | |
| webEnrichedMode = true; | |
| sendMessage(); | |
| }); | |
| updateSearchUI(); | |
| // ββ Memory inspector ββββββββββββββββββββββββββββββββββββββββ | |
| const memoryBtn = document.getElementById('memoryBtn'); | |
| const memoryPanel = document.getElementById('memoryPanel'); | |
| const memoryList = document.getElementById('memoryList'); | |
| const memoryCount = document.getElementById('memoryCount'); | |
| const memorySearch = document.getElementById('memorySearch'); | |
| const memoryClearAll = document.getElementById('memoryClearAll'); | |
| const memoryCatPills = document.getElementById('memoryCatPills'); | |
| let memoryCatFilter = 'all'; | |
| memoryBtn.addEventListener('click', () => { | |
| memoryPanel.classList.toggle('open'); | |
| settingsPanel.classList.remove('open'); | |
| closeHistorySidebar(); | |
| if (memoryPanel.classList.contains('open')) { | |
| refreshMemoryPanel(); | |
| } else if (auditMode) { | |
| // Reset audit state when panel is closed | |
| auditMode = false; | |
| memoryAuditBtn.textContent = 'Audit'; | |
| memoryCatPills.style.display = ''; | |
| memorySearch.closest('.memory-search-row').style.display = ''; | |
| } | |
| }); | |
| function relTime(ts) { | |
| const m = Math.floor((Date.now() - ts) / 60000); | |
| if (m < 1) return 'just now'; | |
| if (m < 60) return `${m}m ago`; | |
| const h = Math.floor(m / 60); | |
| if (h < 24) return `${h}h ago`; | |
| const d = Math.floor(h / 24); | |
| if (d < 30) return `${d}d ago`; | |
| return new Date(ts).toLocaleDateString(); | |
| } | |
| function srcBasename(src) { | |
| if (!src) return 'unknown'; | |
| return src.includes('/') ? src.split('/').pop() : src; | |
| } | |
| function catBadgeHtml(cat) { | |
| return `<span class="mem-cat mem-cat-${escapeHtml(cat)}">${escapeHtml(cat)}</span>`; | |
| } | |
| function makeChunkItem(chunk, showCat) { | |
| const item = document.createElement('div'); | |
| item.className = 'memory-item'; | |
| const preview = chunk.text.length > 150 ? chunk.text.slice(0, 150) + 'β¦' : chunk.text; | |
| const src = srcBasename(chunk.source); | |
| item.innerHTML = ` | |
| <div class="memory-item-text"> | |
| ${escapeHtml(preview)} | |
| <div class="memory-item-meta"> | |
| ${showCat ? catBadgeHtml(chunk.category) : ''} | |
| <span title="${escapeHtml(chunk.source || '')}">${escapeHtml(src)}</span> | |
| · ${relTime(chunk.timestamp)} | |
| </div> | |
| </div> | |
| <button class="memory-item-del" title="Delete">×</button> | |
| `; | |
| item.querySelector('.memory-item-del').addEventListener('click', async () => { | |
| await deleteChunk(chunk.id); | |
| refreshMemoryPanel(); | |
| }); | |
| return item; | |
| } | |
| async function refreshMemoryPanel() { | |
| const textQ = memorySearch.value.trim().toLowerCase(); | |
| const catF = memoryCatFilter; | |
| try { | |
| const all = await getAllChunks(); | |
| memoryCount.textContent = `${all.length} chunk${all.length === 1 ? '' : 's'}`; | |
| // Build category counts for pills | |
| const catCounts = {}; | |
| for (const c of all) catCounts[c.category] = (catCounts[c.category] || 0) + 1; | |
| // Render pills | |
| const CAT_ORDER = ['all', 'fact', 'preference', 'finding', 'document', 'document_summary', 'conversation']; | |
| const cats = ['all', ...CAT_ORDER.slice(1).filter(c => catCounts[c])]; | |
| memoryCatPills.innerHTML = ''; | |
| for (const cat of cats) { | |
| const pill = document.createElement('button'); | |
| pill.className = 'memory-cat-pill' + (cat === catF ? ' active' : ''); | |
| const label = cat === 'document_summary' ? 'doc summary' : cat; | |
| const count = cat === 'all' ? all.length : (catCounts[cat] || 0); | |
| pill.textContent = `${label} (${count})`; | |
| pill.addEventListener('click', () => { | |
| memoryCatFilter = cat; | |
| refreshMemoryPanel(); | |
| }); | |
| memoryCatPills.appendChild(pill); | |
| } | |
| // Filter | |
| let display = catF === 'all' ? all : all.filter(c => c.category === catF); | |
| if (textQ) { | |
| display = display.filter(c => | |
| c.text.toLowerCase().includes(textQ) || | |
| c.category.includes(textQ) || | |
| (c.source && c.source.toLowerCase().includes(textQ)) | |
| ); | |
| } | |
| memoryList.innerHTML = ''; | |
| if (display.length === 0) { | |
| memoryList.innerHTML = '<div class="memory-empty">No memories found</div>'; | |
| return; | |
| } | |
| const isDocCat = catF === 'document' || catF === 'document_summary'; | |
| if (isDocCat) { | |
| // Group by source | |
| const groups = {}; | |
| for (const c of display) { | |
| const src = c.source || 'unknown'; | |
| if (!groups[src]) groups[src] = []; | |
| groups[src].push(c); | |
| } | |
| for (const [src, items] of Object.entries(groups).sort()) { | |
| const basename = srcBasename(src); | |
| const group = document.createElement('div'); | |
| group.className = 'memory-source-group'; | |
| const header = document.createElement('div'); | |
| header.className = 'memory-source-header'; | |
| header.innerHTML = ` | |
| <span class="memory-source-name" title="${escapeHtml(src)}">${escapeHtml(basename)}</span> | |
| <span class="memory-source-count">${items.length} chunk${items.length === 1 ? '' : 's'}</span> | |
| <button class="memory-source-del">Delete all</button> | |
| `; | |
| header.querySelector('.memory-source-del').addEventListener('click', async () => { | |
| if (!confirm(`Delete all ${items.length} chunk(s) from "${basename}"?`)) return; | |
| for (const c of items) await deleteChunk(c.id); | |
| refreshMemoryPanel(); | |
| }); | |
| group.appendChild(header); | |
| for (const chunk of items.sort((a, b) => b.timestamp - a.timestamp)) { | |
| group.appendChild(makeChunkItem(chunk, false)); | |
| } | |
| memoryList.appendChild(group); | |
| } | |
| } else { | |
| // Flat list sorted newest-first | |
| const sorted = display.sort((a, b) => b.timestamp - a.timestamp); | |
| const showCat = catF === 'all'; | |
| const LIMIT = 200; | |
| for (const chunk of sorted.slice(0, LIMIT)) { | |
| memoryList.appendChild(makeChunkItem(chunk, showCat)); | |
| } | |
| if (sorted.length > LIMIT) { | |
| memoryList.insertAdjacentHTML('beforeend', | |
| `<div class="memory-empty">${sorted.length - LIMIT} more β use a category filter or search to narrow</div>`); | |
| } | |
| } | |
| } catch (e) { | |
| memoryList.innerHTML = `<div class="memory-empty">Error loading memories</div>`; | |
| console.error('Memory panel error:', e); | |
| } | |
| } | |
| memorySearch.addEventListener('input', () => refreshMemoryPanel()); | |
| memoryClearAll.addEventListener('click', async () => { | |
| if (!confirm('Delete all stored memories? This cannot be undone.')) return; | |
| await clearAllChunks(); | |
| refreshMemoryPanel(); | |
| }); | |
| // ββ Memory audit ββββββββββββββββββββββββββββββββββββββββββββ | |
| const memoryAuditBtn = document.getElementById('memoryAuditBtn'); | |
| let auditMode = false; | |
| memoryAuditBtn.addEventListener('click', async () => { | |
| if (auditMode) { | |
| auditMode = false; | |
| memoryAuditBtn.textContent = 'Audit'; | |
| memoryCatPills.style.display = ''; | |
| memorySearch.closest('.memory-search-row').style.display = ''; | |
| refreshMemoryPanel(); | |
| return; | |
| } | |
| auditMode = true; | |
| memoryAuditBtn.textContent = 'β Back'; | |
| memoryCatPills.style.display = 'none'; | |
| memorySearch.closest('.memory-search-row').style.display = 'none'; | |
| memoryList.innerHTML = '<div class="memory-empty">Running auditβ¦</div>'; | |
| try { | |
| const results = await runMemoryAudit(); | |
| renderAuditResults(results); | |
| } catch (e) { | |
| memoryList.innerHTML = `<div class="memory-empty">Audit failed: ${escapeHtml(e.message)}</div>`; | |
| console.error('Audit error:', e); | |
| } | |
| }); | |
| async function runMemoryAudit() { | |
| const STALE_DAYS = 60; | |
| const DUPE_THRESHOLD = 0.92; | |
| const OUTLIER_MAX_SIM = 0.20; | |
| const MIN_CAT_SIZE = 5; | |
| const CAP = 600; | |
| const all = await getAllChunks(); | |
| const withEmb = all.filter(c => c.embedding && c.embedding.length > 0); | |
| const capped = withEmb.length > CAP; | |
| const work = capped ? withEmb.slice(0, CAP) : withEmb; | |
| // Stale | |
| const stale = work.filter(c => Date.now() - c.timestamp > STALE_DAYS * 86400000); | |
| // Group by category for pairwise ops | |
| const byCat = {}; | |
| for (const c of work) { | |
| (byCat[c.category] = byCat[c.category] || []).push(c); | |
| } | |
| const dupeIds = new Set(); | |
| const dupePairs = []; // [[kept, flagged], ...] | |
| const outliers = []; | |
| for (const members of Object.values(byCat)) { | |
| const n = members.length; | |
| const avgSims = new Float32Array(n); | |
| for (let i = 0; i < n; i++) { | |
| let sum = 0; | |
| for (let j = 0; j < n; j++) { | |
| if (i === j) continue; | |
| const sim = cosineSimilarity(members[i].embedding, members[j].embedding); | |
| sum += sim; | |
| if (sim >= DUPE_THRESHOLD && i < j | |
| && !dupeIds.has(members[i].id) | |
| && !dupeIds.has(members[j].id)) { | |
| dupePairs.push([members[i], members[j]]); | |
| dupeIds.add(members[j].id); | |
| } | |
| } | |
| avgSims[i] = n > 1 ? sum / (n - 1) : 1; | |
| } | |
| if (n >= MIN_CAT_SIZE) { | |
| for (let i = 0; i < n; i++) { | |
| if (avgSims[i] < OUTLIER_MAX_SIM && !dupeIds.has(members[i].id)) { | |
| outliers.push(members[i]); | |
| } | |
| } | |
| } | |
| } | |
| return { total: work.length, capped, stale, dupePairs, dupeIds, outliers }; | |
| } | |
| function renderAuditResults({ total, capped, stale, dupePairs, outliers }) { | |
| memoryList.innerHTML = ''; | |
| if (capped) { | |
| memoryList.insertAdjacentHTML('beforeend', | |
| `<div class="audit-warn">Only the first 600 chunks were analysed. Use category filters to audit subsets of larger stores.</div>`); | |
| } | |
| const flagCount = stale.length + dupePairs.length + outliers.length; | |
| if (flagCount === 0) { | |
| memoryList.insertAdjacentHTML('beforeend', | |
| `<div class="audit-clean">β Memory looks clean β no stale, duplicate, or outlier chunks found across ${total} chunk(s).</div>`); | |
| return; | |
| } | |
| if (stale.length > 0) { | |
| memoryList.appendChild(makeAuditSection( | |
| `${stale.length} stale β older than 60 days`, stale, | |
| async () => { for (const c of stale) await deleteChunk(c.id); rerunAudit(); } | |
| )); | |
| } | |
| if (dupePairs.length > 0) { | |
| const flagged = dupePairs.map(([, b]) => b); | |
| memoryList.appendChild(makeAuditSection( | |
| `${dupePairs.length} near-duplicate(s) β keeping one of each pair`, flagged, | |
| async () => { for (const c of flagged) await deleteChunk(c.id); rerunAudit(); } | |
| )); | |
| } | |
| if (outliers.length > 0) { | |
| memoryList.appendChild(makeAuditSection( | |
| `${outliers.length} outlier(s) β low similarity to their category`, outliers, | |
| async () => { for (const c of outliers) await deleteChunk(c.id); rerunAudit(); } | |
| )); | |
| } | |
| } | |
| async function rerunAudit() { | |
| memoryList.innerHTML = '<div class="memory-empty">Re-runningβ¦</div>'; | |
| try { | |
| renderAuditResults(await runMemoryAudit()); | |
| } catch (e) { | |
| memoryList.innerHTML = `<div class="memory-empty">Audit failed: ${escapeHtml(e.message)}</div>`; | |
| } | |
| } | |
| function makeAuditSection(label, chunks, onDeleteAll) { | |
| const section = document.createElement('div'); | |
| section.className = 'audit-section'; | |
| const header = document.createElement('div'); | |
| header.className = 'audit-section-header'; | |
| const delBtn = document.createElement('button'); | |
| delBtn.className = 'memory-source-del'; | |
| delBtn.textContent = 'Delete all'; | |
| delBtn.addEventListener('click', async () => { | |
| if (!confirm(`Delete ${chunks.length} item(s)?`)) return; | |
| await onDeleteAll(); | |
| }); | |
| header.innerHTML = `<span>${escapeHtml(label)}</span>`; | |
| header.appendChild(delBtn); | |
| section.appendChild(header); | |
| const PREVIEW = 20; | |
| for (const chunk of chunks.slice(0, PREVIEW)) { | |
| const item = makeChunkItem(chunk, true); | |
| // Override the delete handler so it reruns audit afterward | |
| item.querySelector('.memory-item-del').replaceWith((() => { | |
| const btn = document.createElement('button'); | |
| btn.className = 'memory-item-del'; | |
| btn.title = 'Delete'; | |
| btn.textContent = 'Γ'; | |
| btn.addEventListener('click', async () => { | |
| await deleteChunk(chunk.id); | |
| rerunAudit(); | |
| }); | |
| return btn; | |
| })()); | |
| section.appendChild(item); | |
| } | |
| if (chunks.length > PREVIEW) { | |
| section.insertAdjacentHTML('beforeend', | |
| `<div class="memory-empty">β¦and ${chunks.length - PREVIEW} more</div>`); | |
| } | |
| return section; | |
| } | |
| // ββ Export / Import βββββββββββββββββββββββββββββββββββββββββ | |
| const memoryExport = document.getElementById('memoryExport'); | |
| const memoryImport = document.getElementById('memoryImport'); | |
| const importFileInput = document.getElementById('importFileInput'); | |
| memoryExport.addEventListener('click', async () => { | |
| try { | |
| const data = await exportAllData(); | |
| const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' }); | |
| const url = URL.createObjectURL(blob); | |
| const a = document.createElement('a'); | |
| a.href = url; a.download = `localmind-export-${new Date().toISOString().slice(0, 10)}.json`; | |
| a.click(); | |
| URL.revokeObjectURL(url); | |
| showToast('Data exported'); | |
| } catch (e) { showToast('Export failed: ' + e.message); } | |
| }); | |
| memoryImport.addEventListener('click', () => importFileInput.click()); | |
| importFileInput.addEventListener('change', async () => { | |
| const file = importFileInput.files[0]; | |
| if (!file) return; | |
| try { | |
| const text = await file.text(); | |
| const data = JSON.parse(text); | |
| await importData(data); | |
| showToast(`Imported ${data.memories?.length || 0} memories, ${data.conversations?.length || 0} conversations`); | |
| refreshMemoryPanel(); | |
| } catch (e) { showToast('Import failed: ' + e.message); } | |
| importFileInput.value = ''; | |
| }); | |
| // ββ History sidebar ββββββββββββββββββββββββββββββββββββββββ | |
| const historyBtn = document.getElementById('historyBtn'); | |
| const historySidebar = document.getElementById('historySidebar'); | |
| const historyBackdrop = document.getElementById('historyBackdrop'); | |
| const historyList = document.getElementById('historyList'); | |
| const historyCount = document.getElementById('historyCount'); | |
| const newChatBtn = document.getElementById('newChatBtn'); | |
| function openHistorySidebar() { | |
| historySidebar.classList.add('open'); | |
| historyBackdrop.classList.add('open'); | |
| memoryPanel.classList.remove('open'); | |
| settingsPanel.classList.remove('open'); | |
| refreshHistoryPanel(); | |
| } | |
| function closeHistorySidebar() { | |
| historySidebar.classList.remove('open'); | |
| historyBackdrop.classList.remove('open'); | |
| } | |
| historyBtn.addEventListener('click', () => { | |
| if (historySidebar.classList.contains('open')) closeHistorySidebar(); | |
| else openHistorySidebar(); | |
| }); | |
| historyBackdrop.addEventListener('click', closeHistorySidebar); | |
| async function refreshHistoryPanel() { | |
| try { | |
| const convs = await getAllConversations(); | |
| historyCount.textContent = `${convs.length} conversation${convs.length === 1 ? '' : 's'}`; | |
| historyList.innerHTML = ''; | |
| if (convs.length === 0) { | |
| historyList.innerHTML = '<div class="memory-empty">No saved conversations</div>'; | |
| return; | |
| } | |
| for (const conv of convs.slice(0, 50)) { | |
| const item = document.createElement('div'); | |
| item.className = 'history-item'; | |
| const date = new Date(conv.updated).toLocaleDateString(); | |
| const model = MODELS[conv.modelKey]?.label || conv.modelKey || ''; | |
| item.innerHTML = ` | |
| <div class="history-item-text"> | |
| <div class="history-item-title">${escapeHtml(conv.title)}</div> | |
| <div class="history-item-meta">${conv.messageCount} messages · ${model} · ${date}</div> | |
| </div> | |
| <button class="history-item-del" title="Delete">×</button> | |
| `; | |
| item.addEventListener('click', (e) => { | |
| if (e.target.closest('.history-item-del')) return; | |
| resumeConversation(conv); | |
| }); | |
| item.querySelector('.history-item-del').addEventListener('click', async (e) => { | |
| e.stopPropagation(); | |
| await deleteConversation(conv.id); | |
| refreshHistoryPanel(); | |
| }); | |
| historyList.appendChild(item); | |
| } | |
| } catch (e) { | |
| historyList.innerHTML = '<div class="memory-empty">Error loading history</div>'; | |
| } | |
| } | |
| function resetChatUI() { | |
| messages = []; | |
| conversationSummary = ''; | |
| activeConversationId = newConversationId(); | |
| saveChat(); | |
| clearAttachments(); | |
| chatArea.innerHTML = ''; | |
| chatArea.appendChild(welcomeMsg); | |
| welcomeMsg.style.display = ''; | |
| } | |
| function resumeConversation(conv) { | |
| messages = conv.messages || []; | |
| activeConversationId = conv.id; | |
| conversationSummary = ''; | |
| saveChat(); | |
| chatArea.innerHTML = ''; | |
| chatArea.appendChild(welcomeMsg); | |
| welcomeMsg.style.display = 'none'; | |
| renderRestoredMessages(); | |
| closeHistorySidebar(); | |
| } | |
| // ββ Branch from message ββββββββββββββββββββββββββββββββββββ | |
| // Right-click / long-press on a user message β archive the current | |
| // conversation, then slice messages[0..index] into a new conversation | |
| // and switch to it. The original conversation keeps its full history. | |
| async function branchFromMessage(index) { | |
| if (generating) { showToast('Finish the current message first'); return; } | |
| if (!Number.isFinite(index) || index < 0 || index >= messages.length) return; | |
| // Archive the current conversation first so the branch doesn't | |
| // lose messages after the branch point. If no activeConversationId | |
| // exists yet (fresh session, user hasn't hit New Chat), mint one | |
| // now so the original isn't orphaned. | |
| if (messages.length >= 2) { | |
| if (!activeConversationId) activeConversationId = newConversationId(); | |
| try { await saveConversation(messages, activeConversationId); } catch {} | |
| } | |
| const branched = messages.slice(0, index + 1); | |
| const newId = newConversationId(); | |
| messages = branched; | |
| activeConversationId = newId; | |
| conversationSummary = ''; | |
| saveChat(); | |
| try { await saveConversation(messages, newId); } catch {} | |
| chatArea.innerHTML = ''; | |
| chatArea.appendChild(welcomeMsg); | |
| welcomeMsg.style.display = messages.length ? 'none' : ''; | |
| renderRestoredMessages(); | |
| showToast('Branched to new conversation'); | |
| } | |
| const msgContextMenu = document.getElementById('msgContextMenu'); | |
| const msgBranchBtn = document.getElementById('msgBranchBtn'); | |
| let msgContextTargetIndex = null; | |
| function showMsgContextMenu(x, y, index) { | |
| msgContextTargetIndex = index; | |
| // Position near the cursor, clamping to viewport | |
| const menuW = 170, menuH = 40; | |
| const left = Math.min(x, window.innerWidth - menuW - 8); | |
| const top = Math.min(y, window.innerHeight - menuH - 8); | |
| msgContextMenu.style.left = left + 'px'; | |
| msgContextMenu.style.top = top + 'px'; | |
| msgContextMenu.classList.add('visible'); | |
| } | |
| function hideMsgContextMenu() { | |
| msgContextMenu.classList.remove('visible'); | |
| msgContextTargetIndex = null; | |
| } | |
| chatArea.addEventListener('contextmenu', (e) => { | |
| const msgEl = e.target.closest('.msg.user'); | |
| if (!msgEl) return; | |
| const idxStr = msgEl.dataset.msgIndex; | |
| if (idxStr == null) return; | |
| e.preventDefault(); | |
| showMsgContextMenu(e.clientX, e.clientY, parseInt(idxStr, 10)); | |
| }); | |
| msgBranchBtn.addEventListener('click', () => { | |
| const idx = msgContextTargetIndex; | |
| hideMsgContextMenu(); | |
| if (idx != null) branchFromMessage(idx); | |
| }); | |
| document.addEventListener('click', (e) => { | |
| if (!msgContextMenu.contains(e.target)) hideMsgContextMenu(); | |
| }); | |
| document.addEventListener('scroll', hideMsgContextMenu, true); | |
| window.addEventListener('resize', hideMsgContextMenu); | |
| // New Chat: archive current β start fresh | |
| // Auto-backup toggle | |
| const autoBackupToggle = document.getElementById('autoBackupToggle'); | |
| autoBackupToggle.checked = localStorage.getItem('lm_auto_backup') === 'true'; | |
| autoBackupToggle.addEventListener('change', () => localStorage.setItem('lm_auto_backup', autoBackupToggle.checked)); | |
| newChatBtn.addEventListener('click', async () => { | |
| if (generating) return; | |
| if (messages.length >= 2) { | |
| try { await saveConversation(messages, activeConversationId); } catch (e) { console.warn('Failed to archive:', e); } | |
| // Post-session summarization | |
| if (LocalMind.runtime.embeddingsReady) { | |
| try { | |
| const textParts = messages.map(m => { | |
| const role = m.role === 'user' ? 'User' : 'Assistant'; | |
| const content = typeof m.content === 'string' ? m.content : m.content.filter(c => c.type === 'text').map(c => c.text).join(' '); | |
| return `${role}: ${content}`; | |
| }).filter(s => s.length > 10); | |
| const summary = textParts.slice(-10).join('\n').slice(0, 2000); | |
| await embedAndStore(summary, 'conversation', 'session-' + new Date().toISOString().slice(0, 10)); | |
| } catch {} | |
| } | |
| // Auto-backup if enabled | |
| if (autoBackupToggle.checked) { | |
| try { | |
| const data = await exportAllData(); | |
| const blob = new Blob([JSON.stringify(data)], { type: 'application/json' }); | |
| const url = URL.createObjectURL(blob); | |
| const a = document.createElement('a'); | |
| a.href = url; a.download = `localmind-backup-${new Date().toISOString().slice(0, 10)}.json`; | |
| a.click(); | |
| URL.revokeObjectURL(url); | |
| } catch {} | |
| } | |
| } | |
| resetChatUI(); | |
| }); | |
| // ββ Auto-resize textarea ββββββββββββββββββββββββββββββββββββ | |
| chatInput.addEventListener('input', () => { | |
| chatInput.style.height = 'auto'; | |
| chatInput.style.height = Math.min(chatInput.scrollHeight, 140) + 'px'; | |
| }); | |
| // ββ Model UI visibility βββββββββββββββββββββββββββββββββββ | |
| // Capability gates read from LocalMind.runtime.capabilities() so | |
| // that any future runtime adapter (WebLLM/wllama) can advertise | |
| // its own caps without the UI needing to know which backend is | |
| // active. The runtime returns a snapshot β re-read on every call. | |
| function updateModelUI() { | |
| const caps = LocalMind.runtime.capabilities(); | |
| const isMultimodal = !!(caps && (caps.image || caps.audio || caps.video)); | |
| const isAgent = !!(caps && caps.toolCalling); | |
| inputAffordances.classList.toggle('visible', isMultimodal && modelReady); | |
| thinkingRow.classList.toggle('hidden', !isMultimodal); | |
| plannerRow.classList.toggle('hidden', !isAgent); | |
| searchSettingsSection.style.display = isAgent ? '' : 'none'; | |
| samSettingsSection.style.display = isAgent ? '' : 'none'; | |
| updateSearchUI(); | |
| } | |
| // ββ Attachment management βββββββββββββββββββββββββββββββββββ | |
| function addAttachment(att) { | |
| attachments.push(att); | |
| renderAttachmentBar(); | |
| } | |
| function removeAttachment(index) { | |
| const att = attachments[index]; | |
| if (att.thumb) URL.revokeObjectURL(att.thumb); | |
| attachments.splice(index, 1); | |
| renderAttachmentBar(); | |
| } | |
| function clearAttachments() { | |
| for (const att of attachments) { | |
| if (att.thumb) URL.revokeObjectURL(att.thumb); | |
| } | |
| attachments = []; | |
| renderAttachmentBar(); | |
| } | |
| function renderAttachmentBar() { | |
| attachmentBar.innerHTML = ''; | |
| if (attachments.length === 0) { | |
| attachmentBar.classList.remove('visible'); | |
| return; | |
| } | |
| attachmentBar.classList.add('visible'); | |
| attachments.forEach((att, i) => { | |
| const chip = document.createElement('div'); | |
| chip.className = 'attachment-chip'; | |
| if (att.type === 'image') { | |
| const img = document.createElement('img'); | |
| img.src = att.thumb; | |
| img.alt = att.name; | |
| chip.appendChild(img); | |
| } else if (att.type === 'video') { | |
| chip.classList.add('video-chip'); | |
| const vid = document.createElement('video'); | |
| vid.src = att.thumb; | |
| vid.muted = true; | |
| vid.preload = 'metadata'; | |
| // Show first frame as poster | |
| vid.addEventListener('loadeddata', () => { vid.currentTime = 0.1; }); | |
| chip.appendChild(vid); | |
| const badge = document.createElement('span'); | |
| badge.className = 'video-badge'; | |
| badge.textContent = '\u25B6 Video'; | |
| chip.appendChild(badge); | |
| } else if (att.type === 'audio') { | |
| chip.classList.add('audio-chip'); | |
| chip.style.flexDirection = 'column'; | |
| const label = document.createElement('span'); | |
| label.className = 'audio-label'; | |
| label.textContent = att.name; | |
| chip.appendChild(label); | |
| const audio = document.createElement('audio'); | |
| audio.src = att.thumb || URL.createObjectURL(att.blob); | |
| audio.controls = true; | |
| audio.preload = 'metadata'; | |
| chip.appendChild(audio); | |
| } | |
| const removeBtn = document.createElement('button'); | |
| removeBtn.className = 'remove-btn'; | |
| removeBtn.textContent = '\u00D7'; | |
| removeBtn.addEventListener('click', () => removeAttachment(i)); | |
| chip.appendChild(removeBtn); | |
| attachmentBar.appendChild(chip); | |
| }); | |
| } | |
| // ββ File input handler ββββββββββββββββββββββββββββββββββββββ | |
| attachBtn.addEventListener('click', () => fileInput.click()); | |
| fileInput.addEventListener('change', () => { | |
| handleFiles(fileInput.files); | |
| fileInput.value = ''; | |
| }); | |
| function handleFiles(files) { | |
| for (const file of files) { | |
| if (file.type.startsWith('image/')) { | |
| addAttachment({ | |
| type: 'image', | |
| blob: file, | |
| thumb: URL.createObjectURL(file), | |
| name: file.name, | |
| }); | |
| } else if (file.type.startsWith('audio/')) { | |
| addAttachment({ | |
| type: 'audio', | |
| blob: file, | |
| thumb: URL.createObjectURL(file), | |
| name: file.name, | |
| }); | |
| } else if (file.type === 'video/mp4' || file.name.endsWith('.mp4')) { | |
| addAttachment({ | |
| type: 'video', | |
| blob: file, | |
| thumb: URL.createObjectURL(file), | |
| name: file.name, | |
| }); | |
| } else if (/\.(txt|md|json|csv)$/i.test(file.name) || file.type.startsWith('text/')) { | |
| // Text document β embed into RAG memory | |
| ingestDocument(file.name, file.text()); | |
| } else if (/\.pdf$/i.test(file.name) || file.type === 'application/pdf') { | |
| // PDF β extract text β embed | |
| (async () => { | |
| try { | |
| showToast(`Extracting text from ${file.name}...`); | |
| const { text, pageCount } = await extractPDFText(file); | |
| if (!text.trim()) { showToast('PDF appears to be empty or image-only'); return; } | |
| const count = await embedAndStore(text, 'document', file.name); | |
| const summary = extractiveSummary(text); | |
| await embedAndStore(summary, 'document_summary', file.name); | |
| showToast(`Stored "${file.name}" (${pageCount} pages, ${count} chunks) β ${summary.slice(0, 80)}...`); | |
| } catch (e) { | |
| console.error('PDF error:', e); | |
| showToast('Failed to process PDF: ' + e.message); | |
| } | |
| })(); | |
| } else if (/\.docx$/i.test(file.name) || file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') { | |
| // DOCX β extract text β embed | |
| (async () => { | |
| try { | |
| showToast(`Extracting text from ${file.name}...`); | |
| const text = await extractDOCXText(file); | |
| if (!text.trim()) { showToast('Document appears to be empty'); return; } | |
| const count = await embedAndStore(text, 'document', file.name); | |
| const summary = extractiveSummary(text); | |
| await embedAndStore(summary, 'document_summary', file.name); | |
| showToast(`Stored "${file.name}" (${count} chunks) β ${summary.slice(0, 80)}...`); | |
| } catch (e) { | |
| console.error('DOCX error:', e); | |
| showToast('Failed to process DOCX: ' + e.message); | |
| } | |
| })(); | |
| } | |
| } | |
| } | |
| // ββ Folder ingestion (FS API) ββββββββββββββββββββββββββββββ | |
| const folderBtn = document.getElementById('folderBtn'); | |
| folderBtn.addEventListener('click', async () => { | |
| if (dirHandle) { | |
| dirHandle = null; | |
| folderBtn.classList.remove('folder-open'); | |
| folderBtn.title = 'Open folder β ingest all .md/.txt/.pdf files into memory'; | |
| showToast('Folder closed'); | |
| return; | |
| } | |
| if (!window.showDirectoryPicker) { | |
| showToast('Folder picker not supported in this browser'); | |
| return; | |
| } | |
| try { | |
| dirHandle = await window.showDirectoryPicker({ mode: 'readwrite' }); | |
| folderBtn.classList.add('folder-open'); | |
| folderBtn.title = `Folder: ${dirHandle.name} (click to close)`; | |
| await ingestFolder(dirHandle); | |
| } catch (e) { | |
| if (e.name !== 'AbortError') showToast('Could not open folder: ' + e.message); | |
| dirHandle = null; | |
| folderBtn.classList.remove('folder-open'); | |
| } | |
| }); | |
| async function ingestFolder(handle) { | |
| const SUPPORTED = /\.(md|txt|pdf|docx)$/i; | |
| const fpKey = 'lm_folder_fp'; | |
| const fingerprints = JSON.parse(localStorage.getItem(fpKey) || '{}'); | |
| let ingested = 0, skipped = 0, failed = 0; | |
| async function walk(dir, prefix) { | |
| for await (const [name, entry] of dir.entries()) { | |
| if (name.startsWith('.')) continue; | |
| if (entry.kind === 'directory') { | |
| await walk(entry, prefix ? `${prefix}/${name}` : name); | |
| } else if (SUPPORTED.test(name)) { | |
| const fullPath = prefix ? `${prefix}/${name}` : name; | |
| const file = await entry.getFile(); | |
| const fp = `${file.lastModified}-${file.size}`; | |
| if (fingerprints[fullPath] === fp) { skipped++; continue; } | |
| try { | |
| if (/\.pdf$/i.test(name)) { | |
| const { text, pageCount } = await extractPDFText(file); | |
| if (text.trim()) { | |
| const count = await embedAndStore(text, 'document', fullPath); | |
| const summary = extractiveSummary(text); | |
| if (summary) await embedAndStore(summary, 'document_summary', fullPath); | |
| } | |
| } else if (/\.docx$/i.test(name)) { | |
| const text = await extractDOCXText(file); | |
| if (text.trim()) { | |
| const count = await embedAndStore(text, 'document', fullPath); | |
| const summary = extractiveSummary(text); | |
| if (summary) await embedAndStore(summary, 'document_summary', fullPath); | |
| } | |
| } else { | |
| const text = await file.text(); | |
| if (text.trim()) { | |
| const count = await embedAndStore(text, 'document', fullPath); | |
| const summary = extractiveSummary(text); | |
| if (summary) await embedAndStore(summary, 'document_summary', fullPath); | |
| } | |
| } | |
| fingerprints[fullPath] = fp; | |
| ingested++; | |
| } catch (e) { | |
| console.error(`Failed to ingest ${fullPath}:`, e); | |
| failed++; | |
| } | |
| } | |
| } | |
| } | |
| showToast(`Scanning "${handle.name}"β¦`); | |
| await walk(handle, ''); | |
| localStorage.setItem(fpKey, JSON.stringify(fingerprints)); | |
| const parts = [`"${handle.name}": ${ingested} file(s) ingested`]; | |
| if (skipped) parts.push(`${skipped} unchanged`); | |
| if (failed) parts.push(`${failed} failed`); | |
| showToast(parts.join(', ')); | |
| } | |
| // ββ Document extractors (lazy-loaded) βββββββββββββββββββββ | |
| let pdfjsLoaded = false; | |
| async function ensurePDFJS() { | |
| if (pdfjsLoaded) return; | |
| return new Promise((resolve, reject) => { | |
| const script = document.createElement('script'); | |
| script.src = 'https://cdn.jsdelivr.net/npm/pdfjs-dist@4.4.168/build/pdf.min.mjs'; | |
| script.type = 'module'; | |
| // SRI on the main module. Chrome 89+ / Firefox 113+ enforce this for | |
| // module scripts the same as classic scripts. | |
| script.integrity = 'sha384-fzqD3KLclV6zyTSuToNuSG370CipiqN572vfT2zti7ORKnQPaJrvsz8deY+/Tgwe'; | |
| script.crossOrigin = 'anonymous'; | |
| script.onload = async () => { | |
| try { | |
| // The PDF.js worker is loaded by pdf.js via `new Worker(src)`, which | |
| // does not accept an SRI attribute. Fetch it with Fetch-API SRI | |
| // instead, turn it into a blob URL, and hand that to workerSrc. | |
| // Fetch will throw if the hash doesn't match. | |
| const workerUrl = 'https://cdn.jsdelivr.net/npm/pdfjs-dist@4.4.168/build/pdf.worker.min.mjs'; | |
| const res = await fetch(workerUrl, { | |
| integrity: 'sha384-RTiLhGDoA0SEd0K0E9qrg2HwSEEnAo91MwUh79PteR4v01eEByhHOdEkFKxeaeqw', | |
| }); | |
| if (!res.ok) throw new Error('HTTP ' + res.status); | |
| const blob = await res.blob(); | |
| window.pdfjsLib.GlobalWorkerOptions.workerSrc = URL.createObjectURL(blob); | |
| pdfjsLoaded = true; | |
| resolve(); | |
| } catch (e) { | |
| reject(new Error('Failed to load or verify PDF.js worker: ' + e.message)); | |
| } | |
| }; | |
| script.onerror = () => reject(new Error('Failed to load or verify PDF.js')); | |
| document.head.appendChild(script); | |
| }); | |
| } | |
| let mammothLoaded = false; | |
| async function ensureMammoth() { | |
| if (mammothLoaded) return; | |
| return new Promise((resolve, reject) => { | |
| const script = document.createElement('script'); | |
| script.src = 'https://cdn.jsdelivr.net/npm/mammoth@1.8.0/mammoth.browser.min.js'; | |
| script.integrity = 'sha384-/cXAMbzovUIKbBERjPmR3SnPTh8siWr5lsvFYj1Uq4XP0yaJUZJmsh0YXyGv5P0y'; | |
| script.crossOrigin = 'anonymous'; | |
| script.onload = () => { mammothLoaded = true; resolve(); }; | |
| script.onerror = () => reject(new Error('Failed to load or verify mammoth.js')); | |
| document.head.appendChild(script); | |
| }); | |
| } | |
| async function extractPDFText(blob) { | |
| await ensurePDFJS(); | |
| const arrayBuffer = await blob.arrayBuffer(); | |
| const pdf = await window.pdfjsLib.getDocument({ data: arrayBuffer }).promise; | |
| const pages = []; | |
| for (let i = 1; i <= pdf.numPages; i++) { | |
| const page = await pdf.getPage(i); | |
| const content = await page.getTextContent(); | |
| pages.push(content.items.map(item => item.str).join(' ')); | |
| } | |
| return { text: pages.join('\n\n'), pageCount: pdf.numPages }; | |
| } | |
| async function extractDOCXText(blob) { | |
| await ensureMammoth(); | |
| const arrayBuffer = await blob.arrayBuffer(); | |
| const result = await mammoth.extractRawText({ arrayBuffer }); | |
| return result.value; | |
| } | |
| function extractiveSummary(text, maxSentences = 3) { | |
| const sentences = text.match(/[^.!?\n]+[.!?]+/g) || []; | |
| if (sentences.length <= maxSentences) return text.slice(0, 300); | |
| // Score by word count (longer = more informative) and position (earlier = more important) | |
| const scored = sentences.map((s, i) => ({ | |
| text: s.trim(), | |
| score: s.split(/\s+/).length * (1 - i / sentences.length * 0.3), | |
| })); | |
| scored.sort((a, b) => b.score - a.score); | |
| return scored.slice(0, maxSentences).map(s => s.text).join(' '); | |
| } | |
| async function ingestDocument(fileName, textPromise) { | |
| try { | |
| const text = await textPromise; | |
| if (!text || !text.trim()) { showToast(`${fileName} appears to be empty`); return; } | |
| const count = await embedAndStore(text, 'document', fileName); | |
| const summary = extractiveSummary(text); | |
| if (summary) await embedAndStore(summary, 'document_summary', fileName); | |
| showToast(`Stored "${fileName}" (${count} chunks) β ${summary.slice(0, 80)}...`); | |
| } catch (e) { | |
| console.error('Document ingest error:', e); | |
| showToast('Failed to store document: ' + e.message); | |
| } | |
| } | |
| function showToast(msg) { | |
| const toast = document.createElement('div'); | |
| toast.textContent = msg; | |
| toast.style.cssText = 'position:fixed;bottom:80px;left:50%;transform:translateX(-50%);background:var(--gray-800);color:white;padding:10px 20px;border-radius:8px;font-size:0.82rem;z-index:1000;opacity:0;transition:opacity 0.3s'; | |
| document.body.appendChild(toast); | |
| requestAnimationFrame(() => toast.style.opacity = '1'); | |
| setTimeout(() => { toast.style.opacity = '0'; setTimeout(() => toast.remove(), 300); }, 3000); | |
| } | |
| // ββ Clipboard paste (images) ββββββββββββββββββββββββββββββββ | |
| chatInput.addEventListener('paste', (e) => { | |
| const m = MODELS[activeModelKey]; | |
| if (!m || !m.multimodal) return; | |
| const items = e.clipboardData?.items; | |
| if (!items) return; | |
| for (const item of items) { | |
| if (item.type.startsWith('image/')) { | |
| e.preventDefault(); | |
| const file = item.getAsFile(); | |
| if (file) { | |
| addAttachment({ | |
| type: 'image', | |
| blob: file, | |
| thumb: URL.createObjectURL(file), | |
| name: 'pasted-image.png', | |
| }); | |
| } | |
| } | |
| } | |
| }); | |
| // ββ Drag and drop βββββββββββββββββββββββββββββββββββββββββββ | |
| let dragCounter = 0; | |
| const card = document.querySelector('.card'); | |
| card.addEventListener('dragenter', (e) => { | |
| e.preventDefault(); | |
| const m = MODELS[activeModelKey]; | |
| if (!m || !m.multimodal) return; | |
| dragCounter++; | |
| dragOverlay.classList.add('visible'); | |
| }); | |
| card.addEventListener('dragleave', (e) => { | |
| e.preventDefault(); | |
| dragCounter--; | |
| if (dragCounter <= 0) { | |
| dragCounter = 0; | |
| dragOverlay.classList.remove('visible'); | |
| } | |
| }); | |
| card.addEventListener('dragover', (e) => e.preventDefault()); | |
| card.addEventListener('drop', (e) => { | |
| e.preventDefault(); | |
| dragCounter = 0; | |
| dragOverlay.classList.remove('visible'); | |
| const m = MODELS[activeModelKey]; | |
| if (!m || !m.multimodal) return; | |
| if (e.dataTransfer?.files?.length) { | |
| handleFiles(e.dataTransfer.files); | |
| } | |
| }); | |
| // ββ Camera capture ββββββββββββββββββββββββββββββββββββββββββ | |
| cameraBtn.addEventListener('click', async () => { | |
| try { | |
| cameraStream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'environment' } }); | |
| cameraPreview.srcObject = cameraStream; | |
| cameraOverlay.classList.add('open'); | |
| } catch (err) { | |
| console.error('Camera error:', err); | |
| alert('Could not access camera: ' + err.message); | |
| } | |
| }); | |
| camCaptureBtn.addEventListener('click', () => { | |
| const video = cameraPreview; | |
| const canvas = document.createElement('canvas'); | |
| canvas.width = video.videoWidth; | |
| canvas.height = video.videoHeight; | |
| canvas.getContext('2d').drawImage(video, 0, 0); | |
| canvas.toBlob((blob) => { | |
| if (blob) { | |
| addAttachment({ | |
| type: 'image', | |
| blob, | |
| thumb: URL.createObjectURL(blob), | |
| name: 'camera-photo.jpg', | |
| }); | |
| } | |
| closeCamera(); | |
| }, 'image/jpeg', 0.9); | |
| }); | |
| camCancelBtn.addEventListener('click', closeCamera); | |
| function closeCamera() { | |
| cameraOverlay.classList.remove('open'); | |
| if (cameraStream) { | |
| cameraStream.getTracks().forEach(t => t.stop()); | |
| cameraStream = null; | |
| } | |
| cameraPreview.srcObject = null; | |
| } | |
| // ββ Voice to text (Whisper WebGPU, on-device) ββββββββββββββ | |
| // Separate from the mic-for-audio-attachment flow above: this records | |
| // audio, decodes it to 16 kHz mono PCM, and runs local Whisper to | |
| // transcribe text back into the input field. Works on any model. | |
| const voiceBtn = document.getElementById('voiceBtn'); | |
| let voiceRecorder = null; | |
| let voiceStream = null; | |
| let voiceWorker = null; | |
| let voiceReady = false; | |
| let voiceReadyPromise = null; | |
| function ensureVoiceWorker() { | |
| if (voiceReadyPromise) return voiceReadyPromise; | |
| voiceWorker = createTranscriptionWorker(); | |
| voiceReadyPromise = new Promise((resolve, reject) => { | |
| const onMsg = (e) => { | |
| if (e.data.type === 'progress') { | |
| const p = e.data.data; | |
| if (p.status === 'downloading' && p.total) { | |
| const mb = (p.loaded / 1024 / 1024).toFixed(0); | |
| const total = (p.total / 1024 / 1024).toFixed(0); | |
| showToast('Loading Whisper ' + mb + ' / ' + total + ' MB'); | |
| } | |
| } else if (e.data.type === 'ready') { | |
| voiceReady = true; | |
| voiceWorker.removeEventListener('message', onMsg); | |
| resolve(); | |
| } else if (e.data.type === 'error') { | |
| voiceWorker.removeEventListener('message', onMsg); | |
| reject(new Error(e.data.message)); | |
| } | |
| }; | |
| voiceWorker.addEventListener('message', onMsg); | |
| voiceWorker.postMessage({ type: 'load' }); | |
| }); | |
| return voiceReadyPromise; | |
| } | |
| async function decodeToWhisperPCM(blob) { | |
| const AC = window.AudioContext || window.webkitAudioContext; | |
| if (!AC) throw new Error('AudioContext unavailable'); | |
| const audioCtx = new AC(); | |
| try { | |
| const arrayBuf = await blob.arrayBuffer(); | |
| const audioBuf = await audioCtx.decodeAudioData(arrayBuf); | |
| const targetRate = 16000; | |
| if (audioBuf.sampleRate === targetRate && audioBuf.numberOfChannels === 1) { | |
| return new Float32Array(audioBuf.getChannelData(0)); | |
| } | |
| // Resample + downmix via OfflineAudioContext | |
| const length = Math.ceil(audioBuf.duration * targetRate); | |
| const offline = new OfflineAudioContext(1, length, targetRate); | |
| const src = offline.createBufferSource(); | |
| src.buffer = audioBuf; | |
| src.connect(offline.destination); | |
| src.start(); | |
| const rendered = await offline.startRendering(); | |
| return new Float32Array(rendered.getChannelData(0)); | |
| } finally { | |
| try { audioCtx.close(); } catch {} | |
| } | |
| } | |
| async function transcribeAndInsert(blob) { | |
| try { | |
| voiceBtn.classList.remove('recording'); | |
| voiceBtn.innerHTML = '⏳'; // hourglass | |
| showToast(voiceReady ? 'Transcribing\u2026' : 'Loading Whisper \u2014 first use may take a minute'); | |
| await ensureVoiceWorker(); | |
| const pcm = await decodeToWhisperPCM(blob); | |
| const text = await new Promise((resolve, reject) => { | |
| const id = Math.random().toString(36).slice(2); | |
| const onMsg = (e) => { | |
| if (e.data.id && e.data.id !== id) return; | |
| if (e.data.type === 'transcription') { | |
| voiceWorker.removeEventListener('message', onMsg); | |
| resolve(e.data.text || ''); | |
| } else if (e.data.type === 'error') { | |
| voiceWorker.removeEventListener('message', onMsg); | |
| reject(new Error(e.data.message)); | |
| } | |
| }; | |
| voiceWorker.addEventListener('message', onMsg); | |
| const lang = localStorage.getItem('lm_voice_language') || 'en'; | |
| voiceWorker.postMessage({ type: 'transcribe', pcm, lang, id }, [pcm.buffer]); | |
| }); | |
| if (text) { | |
| const existing = chatInput.value; | |
| chatInput.value = existing ? existing + (existing.endsWith(' ') ? '' : ' ') + text : text; | |
| chatInput.dispatchEvent(new Event('input')); | |
| chatInput.focus(); | |
| showToast('Transcribed'); | |
| } else { | |
| showToast('No speech detected'); | |
| } | |
| } catch (e) { | |
| console.error('Whisper error:', e); | |
| showToast('Transcription failed: ' + (e.message || e)); | |
| } finally { | |
| voiceBtn.innerHTML = '🗣'; // speaking head | |
| } | |
| } | |
| voiceBtn.addEventListener('click', async () => { | |
| if (voiceRecorder && voiceRecorder.state === 'recording') { | |
| voiceRecorder.stop(); | |
| return; | |
| } | |
| try { | |
| voiceStream = await navigator.mediaDevices.getUserMedia({ audio: true }); | |
| const chunks = []; | |
| voiceRecorder = new MediaRecorder(voiceStream); | |
| voiceRecorder.addEventListener('dataavailable', (e) => { if (e.data.size > 0) chunks.push(e.data); }); | |
| voiceRecorder.addEventListener('stop', () => { | |
| voiceStream.getTracks().forEach(t => t.stop()); | |
| voiceStream = null; | |
| const blob = new Blob(chunks, { type: voiceRecorder.mimeType || 'audio/webm' }); | |
| voiceRecorder = null; | |
| if (blob.size === 0) { showToast('No audio captured'); voiceBtn.classList.remove('recording'); return; } | |
| transcribeAndInsert(blob); | |
| }); | |
| voiceRecorder.start(); | |
| voiceBtn.classList.add('recording'); | |
| } catch (err) { | |
| showToast('Mic error: ' + err.message); | |
| } | |
| }); | |
| // ββ Microphone recording (audio attachment for multimodal) βββ | |
| micBtn.addEventListener('click', () => { | |
| if (mediaRecorder && mediaRecorder.state === 'recording') { | |
| mediaRecorder.stop(); | |
| return; | |
| } | |
| startMicRecording(); | |
| }); | |
| async function startMicRecording() { | |
| try { | |
| const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); | |
| const chunks = []; | |
| mediaRecorder = new MediaRecorder(stream); | |
| mediaRecorder.addEventListener('dataavailable', (e) => { | |
| if (e.data.size > 0) chunks.push(e.data); | |
| }); | |
| mediaRecorder.addEventListener('stop', () => { | |
| stream.getTracks().forEach(t => t.stop()); | |
| micBtn.classList.remove('recording'); | |
| micBtn.textContent = '\u{1F3A4}'; | |
| if (chunks.length > 0) { | |
| const blob = new Blob(chunks, { type: mediaRecorder.mimeType || 'audio/webm' }); | |
| addAttachment({ | |
| type: 'audio', | |
| blob, | |
| thumb: URL.createObjectURL(blob), | |
| name: 'recording.webm', | |
| }); | |
| } | |
| mediaRecorder = null; | |
| }); | |
| mediaRecorder.start(); | |
| micBtn.classList.add('recording'); | |
| micBtn.textContent = '\u{23F9}'; | |
| } catch (err) { | |
| console.error('Mic error:', err); | |
| alert('Could not access microphone: ' + err.message); | |
| } | |
| } | |
| // ββ Markdown renderer (lightweight) βββββββββββββββββββββββββ | |
| // ββ Math & diagram rendering ββββββββββββββββββββββββββββββββ | |
| // KaTeX loads via a <script defer> in <head>. If a render happens | |
| // before it finishes, math stays literal; once it loads we re-render | |
| // any already-painted assistant bubbles. Mermaid is heavier (~2 MB), | |
| // so it's lazy-loaded on the first message that contains a ```mermaid | |
| // block. | |
| let __mermaidPromise = null; | |
| async function ensureMermaid() { | |
| if (!__mermaidPromise) { | |
| __mermaidPromise = import('https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs') | |
| .then((m) => { | |
| m.default.initialize({ startOnLoad: false, theme: 'default', securityLevel: 'strict' }); | |
| return m.default; | |
| }); | |
| } | |
| return __mermaidPromise; | |
| } | |
| // Re-render all assistant bubbles that already contain literal math | |
| // markers ($β¦$ or $$β¦$$) once KaTeX finishes loading. Called once from | |
| // the KaTeX onload hook installed below. | |
| function rerenderAssistantBubblesWithMath() { | |
| document.querySelectorAll('.msg.assistant').forEach((el) => { | |
| const idxStr = el.dataset.msgIndex; | |
| if (idxStr == null) return; | |
| const msg = messages[parseInt(idxStr, 10)]; | |
| if (!msg || msg.role !== 'assistant' || typeof msg.content !== 'string') return; | |
| if (!/\$[^$\n]/.test(msg.content)) return; | |
| const bubble = el.querySelector('.msg-bubble'); | |
| if (bubble) { bubble.innerHTML = renderMarkdown(msg.content); postProcessBubble(bubble); } | |
| }); | |
| } | |
| // Transform `<pre data-lang="mermaid">` blocks into rendered Mermaid | |
| // diagrams, and attach sandboxed preview iframes beneath `<pre>` blocks | |
| // tagged html / svg / artifact. Called after each bubble innerHTML update; | |
| // guarded with data attributes so repeated calls are cheap. | |
| async function postProcessBubble(bubble) { | |
| if (!bubble) return; | |
| // Artifact previews β cheap, pure DOM. | |
| const ARTIFACT_LANGS = new Set(['html', 'svg', 'artifact']); | |
| bubble.querySelectorAll('pre[data-lang]:not([data-af-processed])').forEach((pre) => { | |
| const lang = pre.dataset.lang; | |
| if (!ARTIFACT_LANGS.has(lang)) return; | |
| pre.dataset.afProcessed = '1'; | |
| const code = pre.querySelector('code'); | |
| const src = code ? code.textContent : pre.textContent; | |
| const doc = lang === 'svg' | |
| ? '<!DOCTYPE html><html><body style="margin:0;padding:12px">' + src + '</body></html>' | |
| : src; | |
| const wrap = document.createElement('div'); | |
| wrap.className = 'artifact-preview'; | |
| const header = document.createElement('div'); | |
| header.className = 'artifact-header'; | |
| const badge = document.createElement('span'); | |
| badge.innerHTML = 'Preview \u2014 <strong>' + escapeHtml(lang) + '</strong> <span style="color:var(--gray-400)">(sandboxed)</span>'; | |
| header.appendChild(badge); | |
| const frame = document.createElement('iframe'); | |
| frame.className = 'artifact-frame'; | |
| // allow-scripts only β no allow-same-origin so the frame runs in an | |
| // opaque origin and can't touch localStorage, IDB, cookies, or parent. | |
| frame.setAttribute('sandbox', 'allow-scripts'); | |
| frame.srcdoc = doc; | |
| wrap.appendChild(header); | |
| wrap.appendChild(frame); | |
| pre.after(wrap); | |
| }); | |
| // Mermaid β async, lazy-loaded. | |
| const pendingMermaid = bubble.querySelectorAll('pre[data-lang="mermaid"]:not([data-mm-processed])'); | |
| if (pendingMermaid.length === 0) return; | |
| const nodes = []; | |
| pendingMermaid.forEach((pre) => { | |
| pre.dataset.mmProcessed = '1'; | |
| const code = pre.querySelector('code'); | |
| const src = code ? code.textContent : pre.textContent; | |
| const div = document.createElement('div'); | |
| div.className = 'mermaid'; | |
| div.textContent = src; | |
| pre.replaceWith(div); | |
| nodes.push(div); | |
| }); | |
| try { | |
| const m = await ensureMermaid(); | |
| await m.run({ nodes }); | |
| } catch (e) { | |
| nodes.forEach((n) => { | |
| n.innerHTML = '<span class="mermaid-error">Mermaid: ' + escapeHtml(e.message || String(e)) + '</span>'; | |
| }); | |
| } | |
| } | |
| // Once KaTeX finishes loading, re-paint any already-rendered bubbles so | |
| // math shows up. No-op if katex never loads (offline, CDN blocked). | |
| if (typeof katex === 'undefined') { | |
| const poll = setInterval(() => { | |
| if (typeof katex !== 'undefined') { clearInterval(poll); rerenderAssistantBubblesWithMath(); } | |
| }, 200); | |
| // Stop polling after 30 s so we don't burn CPU forever when offline | |
| setTimeout(() => clearInterval(poll), 30000); | |
| } | |
| function renderMarkdown(text) { | |
| let thinkContent = ''; | |
| let mainContent = text; | |
| // Handle <|think|>...<|/think|> format | |
| const thinkMatch = text.match(/<\|think\|>([\s\S]*?)(<\|\/think\|>|$)/); | |
| if (thinkMatch) { | |
| thinkContent = thinkMatch[1].trim(); | |
| mainContent = text.replace(/<\|think\|>[\s\S]*?(<\|\/think\|>|$)/, '').trim(); | |
| } | |
| // Also handle Gemma 4 <|channel>thought ... <channel|> format | |
| if (!thinkContent) { | |
| const channelMatch = text.match(/<\|channel>thought\n?([\s\S]*?)(<channel\|>|$)/); | |
| if (channelMatch) { | |
| thinkContent = channelMatch[1].trim(); | |
| mainContent = text.replace(/<\|channel>thought\n?[\s\S]*?(<channel\|>|$)/, '').trim(); | |
| } | |
| } | |
| let html = ''; | |
| if (thinkContent) { | |
| const thinkId = 'think-' + Math.random().toString(36).slice(2, 8); | |
| // During streaming: mainContent is empty β thinking is in progress β show open | |
| // After streaming: mainContent has text β thinking is done β show collapsed | |
| const isStillThinking = !mainContent; | |
| const openClass = isStillThinking ? ' open' : ''; | |
| const arrow = isStillThinking ? '▼' : '▶'; | |
| const label = isStillThinking ? 'Thinking...' : 'Thought process'; | |
| html += `<div class="thinking-block"> | |
| <div class="thinking-toggle" onclick="document.getElementById('${thinkId}').classList.toggle('open'); this.querySelector('span').textContent = document.getElementById('${thinkId}').classList.contains('open') ? '▼' : '▶'"> | |
| <span>${arrow}</span> ${label} | |
| </div> | |
| <div class="thinking-content${openClass}" id="${thinkId}">${escapeHtml(thinkContent)}</div> | |
| </div>`; | |
| } | |
| if (!mainContent) return html || '<span style="color:var(--gray-400)">...</span>'; | |
| const cbNonce = Math.random().toString(36).slice(2, 12); | |
| mainContent = mainContent.replace(/```(\w*)\n?([\s\S]*?)```/g, (_, lang, code) => { | |
| // SAFETY: sanitise `lang` to [a-zA-Z0-9] so it can't break out of | |
| // the onclick handler's single-quoted string argument. | |
| const ext = (lang || 'txt').replace(/[^a-zA-Z0-9]/g, '') || 'txt'; | |
| const codeId = 'code-' + Math.random().toString(36).slice(2, 8); | |
| // data-lang carries the original language for post-processors | |
| // (mermaid, artifact preview). Separate from `ext` which feeds | |
| // the download filename. | |
| const langAttr = ext === 'txt' && lang ? '' : ' data-lang="' + ext + '"'; | |
| return `<pre class="cb-${cbNonce}"${langAttr}><code id="${codeId}">${escapeHtml(code.trim())}</code><button class="code-download-btn" onclick="downloadCodeBlock('${codeId}','${ext}')" title="Download">↓</button></pre>`; | |
| }); | |
| // Split on our nonce-stamped code-block wrapper. The nonce ensures the | |
| // split matches only code blocks WE emitted in this call, not any | |
| // <pre>-looking HTML the model may have produced verbatim via prompt | |
| // injection. The [^>]* allows for optional attributes (e.g. data-lang). | |
| const splitRe = new RegExp('(<pre class="cb-' + cbNonce + '"[^>]*>[\\s\\S]*?<\\/pre>)', 'g'); | |
| const parts = mainContent.split(splitRe); | |
| const preTag = '<pre class="cb-' + cbNonce + '"'; | |
| html += parts.map(part => { | |
| if (part.startsWith(preTag)) return part; | |
| // Extract math expressions BEFORE escapeHtml so LaTeX survives. The | |
| // `Β§Β§KMβ¦Β§Β§` marker is vanishingly unlikely in model output and | |
| // contains no HTML-meaningful characters, so it survives escape. | |
| // Display math ($$β¦$$) is checked first to avoid `$$` being eaten | |
| // by the inline rule. Inline math must contain a LaTeX-ish token | |
| // (`\`, `^`, `_`, `{`, `}`, or a Greek name) so plain prose like | |
| // "cost $5 and $10" doesn't get treated as math. | |
| const maths = []; | |
| part = part.replace(/\$\$([\s\S]+?)\$\$/g, (_, expr) => { | |
| maths.push({ expr, display: true }); | |
| return '\u00A7\u00A7KM' + (maths.length - 1) + '\u00A7\u00A7'; | |
| }); | |
| part = part.replace(/\$([^\n$]+?)\$/g, (full, expr) => { | |
| if (!/[\\^_{}]|\\(alpha|beta|gamma|delta|theta|lambda|mu|pi|sigma|sum|int|frac|sqrt)/i.test(expr)) { | |
| return full; | |
| } | |
| maths.push({ expr, display: false }); | |
| return '\u00A7\u00A7KM' + (maths.length - 1) + '\u00A7\u00A7'; | |
| }); | |
| // SAFETY: escape untrusted text before running markdown regexes. | |
| // Model output can contain raw HTML via prompt injection from fetched | |
| // pages, search results, or ingested documents. Inline markdown | |
| // regexes match literal * ` - digits \n which all survive escape. | |
| part = escapeHtml(part); | |
| part = part.replace(/`([^`]+)`/g, '<code>$1</code>'); | |
| part = part.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>'); | |
| part = part.replace(/\*(.+?)\*/g, '<em>$1</em>'); | |
| part = part.replace(/(?:^|\n)((?:[-*] .+\n?)+)/g, (_, list) => { | |
| const items = list.trim().split('\n').map(l => `<li>${l.replace(/^[-*] /, '')}</li>`).join(''); | |
| return `<ul>${items}</ul>`; | |
| }); | |
| part = part.replace(/(?:^|\n)((?:\d+\. .+\n?)+)/g, (_, list) => { | |
| const items = list.trim().split('\n').map(l => `<li>${l.replace(/^\d+\. /, '')}</li>`).join(''); | |
| return `<ol>${items}</ol>`; | |
| }); | |
| part = part.replace(/\n\n+/g, '</p><p>'); | |
| part = part.replace(/\n/g, '<br>'); | |
| if (part.trim() && !part.startsWith('<')) part = `<p>${part}</p>`; | |
| // Substitute math placeholders with KaTeX HTML (or literal $β¦ if | |
| // KaTeX isn't loaded yet; a poll re-renders once it's ready). | |
| if (maths.length) { | |
| part = part.replace(/\u00A7\u00A7KM(\d+)\u00A7\u00A7/g, (_, i) => { | |
| const m = maths[parseInt(i, 10)]; | |
| if (!m) return ''; | |
| if (typeof katex === 'undefined') { | |
| const d = m.display ? '$$' : '$'; | |
| return d + escapeHtml(m.expr) + d; | |
| } | |
| try { | |
| return katex.renderToString(m.expr, { displayMode: m.display, throwOnError: false, output: 'html' }); | |
| } catch (e) { | |
| return '<span class="katex-error">' + escapeHtml((m.display ? '$$' : '$') + m.expr) + '</span>'; | |
| } | |
| }); | |
| } | |
| return part; | |
| }).join(''); | |
| return html; | |
| } | |
| // Global: download a code block by ID | |
| window.downloadCodeBlock = function(codeId, ext) { | |
| const el = document.getElementById(codeId); | |
| if (!el) return; | |
| const text = el.textContent; | |
| const blob = new Blob([text], { type: 'text/plain' }); | |
| const url = URL.createObjectURL(blob); | |
| const a = document.createElement('a'); | |
| a.href = url; a.download = `code.${ext}`; | |
| a.click(); | |
| URL.revokeObjectURL(url); | |
| }; | |
| function escapeHtml(s) { | |
| return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"'); | |
| } | |
| // ββ Chat rendering ββββββββββββββββββββββββββββββββββββββββββ | |
| function renderRestoredMessages() { | |
| if (messages.length === 0) return; | |
| welcomeMsg.style.display = 'none'; | |
| for (let i = 0; i < messages.length; i++) { | |
| const msg = messages[i]; | |
| const role = msg.role === 'user' ? 'user' : 'assistant'; | |
| const el = document.createElement('div'); | |
| el.className = `msg ${role}`; | |
| el.dataset.msgIndex = String(i); | |
| const bubble = document.createElement('div'); | |
| bubble.className = 'msg-bubble'; | |
| if (role === 'user') { | |
| // For restored messages, content is always text (blobs not persisted) | |
| const textContent = typeof msg.content === 'string' ? msg.content : | |
| msg.content.filter(c => c.type === 'text').map(c => c.text).join(' '); | |
| bubble.textContent = textContent; | |
| } else { | |
| bubble.innerHTML = renderMarkdown(msg.content); | |
| postProcessBubble(bubble); | |
| } | |
| el.appendChild(bubble); | |
| chatArea.appendChild(el); | |
| } | |
| chatArea.scrollTop = chatArea.scrollHeight; | |
| } | |
| function addUserMessage(text, hasAttachments) { | |
| welcomeMsg.style.display = 'none'; | |
| const el = document.createElement('div'); | |
| el.className = 'msg user'; | |
| // Index stored so right-click/long-press can find the corresponding | |
| // entry in the messages array for "Branch from here". | |
| el.dataset.msgIndex = String(messages.length - 1); | |
| const bubble = document.createElement('div'); | |
| bubble.className = 'msg-bubble'; | |
| // Show attachment thumbnails in the message | |
| if (hasAttachments && attachments.length > 0) { | |
| const attDiv = document.createElement('div'); | |
| attDiv.className = 'msg-attachments'; | |
| for (const att of attachments) { | |
| if (att.type === 'image' && att.thumb) { | |
| const img = document.createElement('img'); | |
| img.src = att.thumb; | |
| img.alt = att.name; | |
| attDiv.appendChild(img); | |
| } else if (att.type === 'video' && att.thumb) { | |
| const tag = document.createElement('span'); | |
| tag.className = 'audio-tag'; | |
| tag.textContent = '\u{1F3AC} ' + att.name; | |
| attDiv.appendChild(tag); | |
| } else if (att.type === 'audio') { | |
| const tag = document.createElement('span'); | |
| tag.className = 'audio-tag'; | |
| tag.textContent = '\u{1F3B5} ' + att.name; | |
| attDiv.appendChild(tag); | |
| } | |
| } | |
| bubble.appendChild(attDiv); | |
| } | |
| const textNode = document.createElement('span'); | |
| textNode.textContent = text; | |
| bubble.appendChild(textNode); | |
| el.appendChild(bubble); | |
| chatArea.appendChild(el); | |
| chatArea.scrollTop = chatArea.scrollHeight; | |
| } | |
| function addAssistantPlaceholder() { | |
| const el = document.createElement('div'); | |
| el.className = 'msg assistant'; | |
| const bubble = document.createElement('div'); | |
| bubble.className = 'msg-bubble'; | |
| bubble.innerHTML = '<span class="typing-dot"></span><span class="typing-dot"></span><span class="typing-dot"></span>'; | |
| el.appendChild(bubble); | |
| chatArea.appendChild(el); | |
| chatArea.scrollTop = chatArea.scrollHeight; | |
| return { el, bubble }; | |
| } | |
| function updateAssistantBubble(bubble, text) { | |
| bubble.innerHTML = renderMarkdown(text); | |
| // Mermaid blocks render async once the bubble is in the DOM. Safe to | |
| // call every token β postProcessBubble guards with data-mm-processed. | |
| postProcessBubble(bubble); | |
| chatArea.scrollTop = chatArea.scrollHeight; | |
| } | |
| // Strip the model's raw <tool_call>...</tool_call> blocks from text so | |
| // the tags don't leak into the visible bubble or saved message history. | |
| // (They're still present in loopMessages when feeding the model its own | |
| // turn back, so the model continues to see its own output format.) | |
| function stripToolCallTags(text) { | |
| if (!text) return text; | |
| return text.replace(/<tool_call>[\s\S]*?<\/tool_call>/g, '').trim(); | |
| } | |
| function saveChat() { | |
| try { | |
| // Strip blobs from messages for sessionStorage (only persist text) | |
| const serializable = messages.map(m => { | |
| if (m.role === 'assistant') return m; | |
| if (typeof m.content === 'string') return m; | |
| // Multimodal user message: extract text only | |
| const textParts = m.content.filter(c => c.type === 'text'); | |
| const text = textParts.map(c => c.text).join(' '); | |
| return { role: m.role, content: text }; | |
| }); | |
| sessionStorage.setItem('localmind_chat', JSON.stringify(serializable)); | |
| } catch {} | |
| } | |
| // =========================================================== | |
| // === RUNTIME ADAPTER: Transformers.js ====================== | |
| // =========================================================== | |
| // This section is the ONLY place in the file that may reference | |
| // Transformers.js. Everything else β agent loop, chat UI, JS API, | |
| // RAG ingestion, document upload β must go through | |
| // `LocalMind.runtime.*`. The rule exists so a second backend | |
| // (WebLLM, wllama, β¦) can be slotted in by adding another adapter | |
| // section without touching anything above. If you need to add new | |
| // model-side functionality, the API surface below is the contract; | |
| // bend the adapter to fit it, not the other way around. | |
| // | |
| // Public surface: | |
| // runtime.loadModel(modelId, options) -> Promise<{ modelId, capabilities }> | |
| // runtime.unload() | |
| // runtime.chat(request) -> async iterator of events | |
| // events: { type: 'token', text } | |
| // | { type: 'tool_call', id, name, arguments } | |
| // | { type: 'done', reason, usage } | |
| // | { type: 'error', error } | |
| // runtime.embed(string) -> Promise<Float32Array> | |
| // runtime.embed(string[]) -> Promise<Float32Array[]> | |
| // runtime.embeddingsReady -> boolean (gate; embed is lazy-loaded) | |
| // runtime.capabilities() -> { text, image, audio, video, toolCalling, maxContextTokens } | |
| // | |
| // Tool calling is faked via prompt injection + parse-after-completion | |
| // (Transformers.js has no native tool API). The scaffolding and the | |
| // parser both live in this file as `_adapter*` helpers β the agent | |
| // loop above sees only the {type:'tool_call'} events. | |
| // | |
| // Multimodal preprocessing (audio decode, video frame extraction) | |
| // also lives here. The chat UI passes content blocks containing raw | |
| // Blobs; the adapter translates them to whatever the worker expects. | |
| // ----------------------------------------------------------- | |
| const LocalMind = (typeof window !== 'undefined' && window.LocalMind) || {}; | |
| if (typeof window !== 'undefined') window.LocalMind = LocalMind; | |
| // ββ Tool-calling helpers (adapter-internal) βββββββββββββββββ | |
| // Transformers.js has no native tool-calling API. The strategy is | |
| // prompt-injection: we describe the available tools and the | |
| // expected <tool_call> XML format in the system prompt, generate, | |
| // then scan the model's output for tool-call blocks. Both the | |
| // scaffolding and the parser live here so the agent loop above | |
| // never has to know how a particular runtime fakes tool calling. | |
| // A WebLLM/wllama adapter would replace these with whatever | |
| // native shape that runtime exposes. | |
| function _adapterRepairJSON(str) { | |
| try { return JSON.parse(str); } catch {} | |
| const fixed = String(str) | |
| .replace(/,\s*([}\]])/g, '$1') // trailing commas | |
| .replace(/'/g, '"') // single β double quotes | |
| .replace(/(\w+)\s*:/g, '"$1":'); // unquoted keys | |
| try { return JSON.parse(fixed); } catch {} | |
| return null; | |
| } | |
| function _adapterBuildToolPrompt(tools, userPrompt) { | |
| // tools: [{ name, description, parameters }] | |
| const toolDefs = (tools || []).map(t => ({ | |
| type: 'function', | |
| function: { name: t.name, description: t.description, parameters: t.parameters }, | |
| })); | |
| let prompt = `You are a helpful AI assistant with access to tools. Use them when they would help answer the user's question. If tools aren't needed, respond directly. | |
| <tools> | |
| ${JSON.stringify(toolDefs, null, 2)} | |
| </tools> | |
| To call a function, output EXACTLY this format (not inside thinking): | |
| <tool_call> | |
| {"name": "function_name", "arguments": {"arg1": "value1"}} | |
| </tool_call> | |
| CRITICAL: The <tool_call> block must appear in your main response, NEVER inside your thinking/reasoning. Think first, then output the <tool_call> block as your response. | |
| Rules: | |
| - Call one tool at a time. Wait for the result before calling another. | |
| - When citing sources, mention where the information came from. | |
| - If you don't need tools, just answer directly. | |
| - Never fabricate tool results. | |
| - Always use the exact <tool_call> XML format shown above. Do not invent other formats like "tool.call:" or function-call syntax. | |
| - You can translate text between 140+ languages directly in your response β no tool needed for translation.`; | |
| if (userPrompt) prompt += '\n\n' + userPrompt; | |
| return prompt; | |
| } | |
| function _adapterParseToolCalls(text, knownToolNames) { | |
| const toolCalls = []; | |
| let cleanText = text; | |
| // 1. Match <tool_call>β¦</tool_call> blocks | |
| const tagRegex = /<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/g; | |
| let match; | |
| while ((match = tagRegex.exec(text)) !== null) { | |
| const parsed = _adapterRepairJSON(match[1].trim()); | |
| const tcName = parsed && (parsed.name || parsed.function); | |
| if (tcName) { | |
| toolCalls.push({ name: tcName, arguments: parsed.arguments || {} }); | |
| } | |
| cleanText = cleanText.replace(match[0], ''); | |
| } | |
| // 2. Bare JSON without tags | |
| if (toolCalls.length === 0) { | |
| const bareRegex = /\{\s*"(?:name|function)"\s*:\s*"(\w+)"\s*,\s*"arguments"\s*:\s*(\{[\s\S]*?\})\s*\}/g; | |
| while ((match = bareRegex.exec(text)) !== null) { | |
| const args = _adapterRepairJSON(match[2]); | |
| if (args) { | |
| toolCalls.push({ name: match[1], arguments: args }); | |
| cleanText = cleanText.replace(match[0], ''); | |
| } | |
| } | |
| } | |
| // 3. Inline tool.call:name{args} buried in thinking | |
| if (toolCalls.length === 0 && knownToolNames && knownToolNames.length) { | |
| const mainContent = text | |
| .replace(/<\|think\|>[\s\S]*?(<\|\/think\|>|$)/, '') | |
| .replace(/<\|channel>thought\n?[\s\S]*?(<channel\|>|$)/, '') | |
| .trim(); | |
| if (!mainContent) { | |
| const namesAlt = knownToolNames.map(n => n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|'); | |
| const intentRegex = new RegExp(`tool[._]?call\\s*[:.]\\s*(${namesAlt})\\s*[{(]([^})]*)`, 'i'); | |
| const intentMatch = text.match(intentRegex); | |
| if (intentMatch) { | |
| const args = {}; | |
| intentMatch[2].split(/[,;]/).forEach(pair => { | |
| const [k, ...rest] = pair.split(':'); | |
| if (k && rest.length) args[k.trim()] = rest.join(':').trim().replace(/["']/g, ''); | |
| }); | |
| toolCalls.push({ name: intentMatch[1], arguments: args }); | |
| } | |
| } | |
| } | |
| return { toolCalls, cleanText: cleanText.trim() }; | |
| } | |
| function _adapterFormatToolResultMessage(name, result) { | |
| // The string the model sees as the tool result. Wrapped as a | |
| // user-role message in apply_chat_template β Gemma's convention. | |
| return `<tool_response> | |
| ${JSON.stringify({ name, result })} | |
| </tool_response>`; | |
| } | |
| // ββ Multimodal helpers (adapter-internal) βββββββββββββββββββ | |
| // The Transformers.js multimodal processor expects: | |
| // - images: ArrayBuffer of an image file (jpg/png), reconstructed | |
| // in the worker via load_image() on a blob URL | |
| // - audio: Float32Array of mono PCM at 16 kHz | |
| // - video: decomposed into N frames (images) + 1 audio track | |
| // The translation lives here so the agent loop never has to know | |
| // about decode pipelines or sample rates. | |
| async function _adapterDecodeAudioTo16k(blob) { | |
| const arrayBuffer = await blob.arrayBuffer(); | |
| const audioCtx = new AudioContext({ sampleRate: 16000 }); | |
| const decoded = await audioCtx.decodeAudioData(arrayBuffer); | |
| const mono = decoded.getChannelData(0); | |
| const pcm = new Float32Array(mono); | |
| audioCtx.close(); | |
| return pcm; | |
| } | |
| async function _adapterDecomposeVideo(blob) { | |
| const url = URL.createObjectURL(blob); | |
| const video = document.createElement('video'); | |
| video.muted = true; | |
| video.preload = 'auto'; | |
| video.src = url; | |
| await new Promise((resolve, reject) => { | |
| video.addEventListener('loadedmetadata', resolve); | |
| video.addEventListener('error', reject); | |
| }); | |
| const results = { images: [], audio: null }; | |
| const duration = video.duration; | |
| const numFrames = Math.min(4, Math.max(1, Math.floor(duration / 2))); | |
| const canvas = document.createElement('canvas'); | |
| const ctx = canvas.getContext('2d'); | |
| for (let i = 0; i < numFrames; i++) { | |
| const t = (duration / (numFrames + 1)) * (i + 1); | |
| video.currentTime = t; | |
| await new Promise(r => video.addEventListener('seeked', r, { once: true })); | |
| canvas.width = video.videoWidth; | |
| canvas.height = video.videoHeight; | |
| ctx.drawImage(video, 0, 0); | |
| const frameBlob = await new Promise(r => canvas.toBlob(r, 'image/jpeg', 0.85)); | |
| if (frameBlob) { | |
| results.images.push(await frameBlob.arrayBuffer()); | |
| } | |
| } | |
| try { | |
| const arrayBuffer = await blob.arrayBuffer(); | |
| const audioCtx = new AudioContext({ sampleRate: 16000 }); | |
| const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); | |
| results.audio = new Float32Array(audioBuffer.getChannelData(0)); | |
| audioCtx.close(); | |
| } catch {} | |
| URL.revokeObjectURL(url); | |
| return results; | |
| } | |
| // Walk a messages array. For any content array containing media | |
| // blocks (image/audio/video) with `data` set, process the data and | |
| // build a parallel attData list. The returned chatMessages have | |
| // the same structure but with media blocks stripped of their data | |
| // field β that's what the chat template needs to emit positional | |
| // tokens. Stale placeholder media blocks (no data) are preserved | |
| // in chatMessages but contribute nothing to attData; this matches | |
| // the agent-loop behaviour where iteration > 0 carries position | |
| // tokens forward but skips re-processing. | |
| async function _adapterProcessContentBlocks(messages) { | |
| const attData = []; | |
| let hasMedia = false; | |
| const out = []; | |
| for (const m of messages) { | |
| if (Array.isArray(m.content)) { | |
| const cleanBlocks = []; | |
| for (const block of m.content) { | |
| if (!block || typeof block !== 'object') continue; | |
| if (block.type === 'text') { | |
| cleanBlocks.push({ type: 'text', text: block.text || '' }); | |
| continue; | |
| } | |
| if (block.type !== 'image' && block.type !== 'audio' && block.type !== 'video') { | |
| continue; | |
| } | |
| hasMedia = true; | |
| if (!block.data) { | |
| // Stale placeholder from a prior agent-loop iteration β | |
| // keep the chat-template position, no attData entry. | |
| if (block.type === 'video') { | |
| // Videos were already expanded on first pass, so they | |
| // shouldn't appear as stale; treat defensively as one | |
| // image position. | |
| cleanBlocks.push({ type: 'image' }); | |
| } else { | |
| cleanBlocks.push({ type: block.type }); | |
| } | |
| continue; | |
| } | |
| const blob = block.data instanceof Blob | |
| ? block.data | |
| : new Blob([block.data], block.mimeType ? { type: block.mimeType } : undefined); | |
| if (block.type === 'image') { | |
| const buf = await blob.arrayBuffer(); | |
| attData.push({ type: 'image', data: buf, mimeType: blob.type || 'image/jpeg' }); | |
| cleanBlocks.push({ type: 'image' }); | |
| } else if (block.type === 'audio') { | |
| const pcm = await _adapterDecodeAudioTo16k(blob); | |
| attData.push({ type: 'audio', pcmData: pcm }); | |
| cleanBlocks.push({ type: 'audio' }); | |
| } else if (block.type === 'video') { | |
| const decomp = await _adapterDecomposeVideo(blob); | |
| for (const imgBuf of decomp.images) { | |
| attData.push({ type: 'image', data: imgBuf, mimeType: 'image/jpeg' }); | |
| cleanBlocks.push({ type: 'image' }); | |
| } | |
| if (decomp.audio) { | |
| attData.push({ type: 'audio', pcmData: decomp.audio }); | |
| cleanBlocks.push({ type: 'audio' }); | |
| } | |
| } | |
| } | |
| out.push({ role: m.role, content: cleanBlocks }); | |
| } else { | |
| out.push(m); | |
| } | |
| } | |
| return { chatMessages: out, attData, hasMedia }; | |
| } | |
| LocalMind.runtime = { | |
| // --- internal state ----------------------------------------- | |
| _worker: null, | |
| _activeModelId: null, | |
| _activeCaps: { | |
| text: true, image: false, audio: false, video: false, | |
| toolCalling: false, maxContextTokens: 0, | |
| }, | |
| // --- loadModel ---------------------------------------------- | |
| // Synchronous up to the first await: terminates any previous | |
| // worker, creates a fresh one, posts the load message, and stores | |
| // the worker on `_worker` BEFORE returning the promise. This lets | |
| // legacy callers grab the worker via `_getWorker()` synchronously | |
| // after invoking loadModel and attach their own message handlers | |
| // before any worker message is dispatched (postMessage and | |
| // addEventListener inside the executor both run in the current | |
| // microtask, so handlers attached by the caller in the same tick | |
| // are guaranteed to receive every event). | |
| loadModel(modelId, options) { | |
| options = options || {}; | |
| const dtype = options.dtype || 'q4f16'; | |
| const modelType = options.modelType || 'causal'; | |
| const onProgress = options.onProgress; | |
| const caps = options.capabilities || { | |
| text: true, image: false, audio: false, video: false, | |
| toolCalling: false, maxContextTokens: 0, | |
| }; | |
| // Terminate previous worker if any | |
| if (this._worker) { | |
| try { this._worker.terminate(); } catch {} | |
| this._worker = null; | |
| } | |
| const w = createWorker(); | |
| this._worker = w; | |
| this._activeModelId = modelId; | |
| this._activeCaps = caps; | |
| const promise = new Promise((resolve, reject) => { | |
| let settled = false; | |
| const cleanup = () => { | |
| settled = true; | |
| w.removeEventListener('message', onMsg); | |
| }; | |
| const onMsg = (e) => { | |
| const d = e.data; | |
| if (settled) return; | |
| if (d.type === 'progress' && typeof onProgress === 'function') { | |
| try { onProgress(d.data); } catch {} | |
| } else if (d.type === 'ready') { | |
| cleanup(); | |
| resolve({ modelId, capabilities: caps }); | |
| } else if (d.type === 'error') { | |
| cleanup(); | |
| reject(new Error(d.message || 'Model load failed')); | |
| } | |
| }; | |
| w.addEventListener('message', onMsg); | |
| w.addEventListener('error', (ev) => { | |
| if (settled) return; | |
| cleanup(); | |
| reject(new Error(ev.message || 'Worker startup failed')); | |
| }); | |
| const resumable = localStorage.getItem('lm_resumable_downloads') !== '0'; | |
| w.postMessage({ type: 'load', modelId, dtype, modelType, resumable }); | |
| }); | |
| return promise; | |
| }, | |
| // --- unload ------------------------------------------------- | |
| // Frees the chat worker. Embedding worker is independent and | |
| // unaffected. Idempotent. | |
| unload() { | |
| if (this._worker) { | |
| try { this._worker.terminate(); } catch {} | |
| this._worker = null; | |
| } | |
| this._activeModelId = null; | |
| this._activeCaps = { | |
| text: true, image: false, audio: false, video: false, | |
| toolCalling: false, maxContextTokens: 0, | |
| }; | |
| }, | |
| // --- chat --------------------------------------------------- | |
| // Returns an async iterator that yields: | |
| // { type: 'token', text } | |
| // { type: 'tool_call', id, name, arguments } | |
| // { type: 'done', reason: 'stop'|'length'|'tool_calls'|'abort', | |
| // usage: { promptTokens, completionTokens } } | |
| // { type: 'error', error } | |
| // | |
| // Tool calling: parse-after-completion. We accumulate the text | |
| // stream, then run _adapterParseToolCalls on the full text once | |
| // generation finishes. If any calls are detected we emit them | |
| // as tool_call events BEFORE the done event, and set the done | |
| // reason to 'tool_calls'. Tokens still stream live to consumers | |
| // β the chat-UI bubble keeps painting tool-call XML in real | |
| // time, exactly as it does today. | |
| // | |
| // Multimodal: messages may carry content arrays with image/audio/ | |
| // video blocks (each with `data` set to a Blob or ArrayBuffer). | |
| // The adapter walks them via _adapterProcessContentBlocks before | |
| // posting to the worker. Stale placeholder blocks (no data field) | |
| // are kept in the chat-template message but contribute nothing | |
| // to attData β that's how follow-up agent iterations carry the | |
| // image position tokens forward without re-decoding. | |
| chat(request) { | |
| request = request || {}; | |
| const messages = Array.isArray(request.messages) ? request.messages : []; | |
| const enableThinking = !!request.enableThinking; | |
| const signal = request.signal; | |
| const tools = Array.isArray(request.tools) ? request.tools : null; | |
| const toolsEnabled = tools && tools.length > 0; | |
| const knownToolNames = toolsEnabled ? tools.map(t => t.name) : []; | |
| // Detect whether ANY message has a media block. If yes, the | |
| // multimodal processing path runs (asynchronously) before we | |
| // post to the worker; otherwise we take the fast text-only | |
| // path. The multimodal pre-processing (audio decode, video | |
| // frame extraction) requires await, so we wrap the rest of | |
| // setup in an async IIFE β the iterator is still returned | |
| // synchronously and will start emitting events as soon as the | |
| // worker begins streaming. | |
| let hasMediaInput = false; | |
| for (const m of messages) { | |
| if (Array.isArray(m.content)) { | |
| for (const b of m.content) { | |
| if (b && (b.type === 'image' || b.type === 'audio' || b.type === 'video')) { | |
| hasMediaInput = true; | |
| break; | |
| } | |
| } | |
| } | |
| if (hasMediaInput) break; | |
| } | |
| // Build genConfig: start from the active model's defaults so | |
| // model-specific tuning (top_k, repetition_penalty, etc) is | |
| // preserved, then layer request-level overrides. | |
| const activeModel = (typeof MODELS !== 'undefined' && typeof activeModelKey === 'string') ? MODELS[activeModelKey] : null; | |
| const baseGen = activeModel && activeModel.genConfig ? activeModel.genConfig : {}; | |
| const genConfig = Object.assign({}, baseGen); | |
| if (typeof request.maxTokens === 'number' && request.maxTokens > 0) { | |
| genConfig.max_new_tokens = Math.floor(request.maxTokens); | |
| } | |
| if (typeof request.temperature === 'number') { | |
| genConfig.temperature = request.temperature; | |
| } | |
| const iter = makeAsyncIterator(); | |
| let completionTokens = 0; | |
| // Approximate prompt tokens from the raw input β exact count | |
| // would require tokenising on the worker, which we don't | |
| // surface to the main thread. | |
| const promptTokens = messages.reduce((s, x) => { | |
| let txt = ''; | |
| if (typeof x.content === 'string') txt = x.content; | |
| else if (Array.isArray(x.content)) txt = x.content.filter(c => c && c.type === 'text').map(c => c.text || '').join(' '); | |
| return s + Math.max(1, Math.round(txt.length / 4)); | |
| }, 0); | |
| // Wire up cancellation: an aborted signal posts {type:'stop'} | |
| // to the worker. The legacy 'complete' handler still fires | |
| // afterwards (with whatever was generated so far), which lets | |
| // runInference resolve and the iterator finish cleanly. | |
| let aborted = false; | |
| const onAbort = () => { | |
| aborted = true; | |
| try { | |
| const w = LocalMind.runtime._worker; | |
| if (w) w.postMessage({ type: 'stop' }); | |
| } catch {} | |
| }; | |
| if (signal) { | |
| if (signal.aborted) onAbort(); | |
| else signal.addEventListener('abort', onAbort, { once: true }); | |
| } | |
| // Async IIFE: do (optional) multimodal preprocessing, then | |
| // build the final chatMessages, inject tools if needed, fire | |
| // runInference. The iterator was created above and is returned | |
| // synchronously below β events stream into it as the worker | |
| // produces tokens. | |
| (async () => { | |
| let chatMessages; | |
| let attDataForWorker = null; | |
| try { | |
| if (hasMediaInput) { | |
| // Multimodal path: keep content arrays as positional | |
| // placeholder blocks, extract binary data into attData. | |
| const processed = await _adapterProcessContentBlocks(messages); | |
| chatMessages = processed.chatMessages.map(m => { | |
| if (m.role === 'tool') { | |
| // Tool-role messages don't carry media; collapse via | |
| // the same translation as the text path. | |
| const content = typeof m.content === 'string' ? m.content : ''; | |
| const toolPayload = (() => { | |
| try { return JSON.parse(content); } catch { return content; } | |
| })(); | |
| return { | |
| role: 'user', | |
| content: _adapterFormatToolResultMessage(m.name || 'tool', toolPayload), | |
| }; | |
| } | |
| return m; | |
| }); | |
| attDataForWorker = processed.attData; | |
| } else { | |
| // Text-only path: collapse content arrays to strings, | |
| // translate tool-role messages. | |
| chatMessages = messages.map(m => { | |
| let content = m.content; | |
| if (Array.isArray(content)) { | |
| content = content | |
| .filter(c => c && c.type === 'text') | |
| .map(c => c.text || '') | |
| .join(' '); | |
| } else if (typeof content !== 'string') { | |
| content = String(content == null ? '' : content); | |
| } | |
| if (m.role === 'tool') { | |
| const toolPayload = (() => { | |
| try { return JSON.parse(content); } catch { return content; } | |
| })(); | |
| return { | |
| role: 'user', | |
| content: _adapterFormatToolResultMessage(m.name || 'tool', toolPayload), | |
| }; | |
| } | |
| return { role: m.role, content }; | |
| }); | |
| } | |
| // Tool prompt injection. Replaces (or inserts) the system | |
| // message at index 0 with the tool-scaffolded version. The | |
| // original system content (if any) becomes the suffix. | |
| if (toolsEnabled) { | |
| let userSysContent = ''; | |
| if (chatMessages.length && chatMessages[0].role === 'system') { | |
| const c = chatMessages[0].content; | |
| userSysContent = typeof c === 'string' ? c : ''; | |
| chatMessages = chatMessages.slice(1); | |
| } | |
| const toolSys = _adapterBuildToolPrompt(tools, userSysContent); | |
| chatMessages = [{ role: 'system', content: toolSys }].concat(chatMessages); | |
| } | |
| } catch (e) { | |
| const err = e instanceof Error ? e : new Error(String(e)); | |
| iter.push({ type: 'error', error: err }); | |
| iter.finish(err); | |
| return; | |
| } | |
| const inferencePromise = runInference({ | |
| chatMessages, | |
| attData: attDataForWorker, | |
| enableThinking, | |
| genConfig, | |
| setup: () => { | |
| // Reset the legacy accumulator so `currentAssistantText` | |
| // (which the 'complete' handler resolves runInference | |
| // with) starts this call from empty. Without this, | |
| // direct programmatic callers of runtime.chat would see | |
| // the text of the previous call prepended to theirs and | |
| // the tool-call parser could double-match. The chat UI | |
| // and JS API paths also reset this themselves β doing | |
| // it again here is harmless and makes the adapter | |
| // self-contained. | |
| currentAssistantText = ''; | |
| // Install the per-token sink. We deliberately do NOT | |
| // touch currentAssistantEl here β the legacy chat-UI | |
| // bubble keeps updating for callers that placed a bubble | |
| // before invoking us. Callers that don't want a bubble | |
| // (the JS API, sub-agents) null currentAssistantEl | |
| // themselves before the call. | |
| currentInferenceContext = { | |
| onToken: (token) => { | |
| completionTokens++; | |
| iter.push({ type: 'token', text: token }); | |
| }, | |
| }; | |
| }, | |
| }); | |
| inferencePromise.then( | |
| (fullText) => { | |
| if (signal) signal.removeEventListener && signal.removeEventListener('abort', onAbort); | |
| // Tool-call detection (parse-after-completion). Only emit | |
| // tool_call events when the caller actually asked for | |
| // tools β otherwise the same model output is just plain | |
| // text and we don't want phantom calls from regex hits. | |
| let reason = aborted ? 'abort' : 'stop'; | |
| if (toolsEnabled && !aborted && typeof fullText === 'string' && fullText.length) { | |
| const { toolCalls } = _adapterParseToolCalls(fullText, knownToolNames); | |
| if (toolCalls.length > 0) { | |
| reason = 'tool_calls'; | |
| for (const tc of toolCalls) { | |
| iter.push({ | |
| type: 'tool_call', | |
| id: 'tc-' + Math.random().toString(36).slice(2, 10), | |
| name: tc.name, | |
| arguments: tc.arguments || {}, | |
| }); | |
| } | |
| } | |
| } | |
| iter.push({ | |
| type: 'done', | |
| reason, | |
| usage: { promptTokens, completionTokens }, | |
| }); | |
| iter.finish(); | |
| }, | |
| (err) => { | |
| if (signal) signal.removeEventListener && signal.removeEventListener('abort', onAbort); | |
| const e = err instanceof Error ? err : new Error(String(err)); | |
| iter.push({ type: 'error', error: e }); | |
| iter.finish(e); | |
| } | |
| ); | |
| })(); | |
| return iter; | |
| }, | |
| // --- embed -------------------------------------------------- | |
| // Lazy: the embedding worker is created on first call, kept | |
| // alive for the lifetime of the page, and serialised internally | |
| // so concurrent callers can't race the underlying postMessage | |
| // request/response pairing. The chat worker is independent β | |
| // unload() does NOT touch this one. | |
| // | |
| // Spec contract: | |
| // embed(string) -> Promise<Float32Array> | |
| // embed(string[]) -> Promise<Float32Array[]> | |
| _embWorker: null, | |
| _embReady: false, | |
| _embQueue: Promise.resolve(), | |
| get embeddingsReady() { return this._embReady; }, | |
| _ensureEmbWorker() { | |
| if (this._embReady) return Promise.resolve(); | |
| const self = this; | |
| if (this._embWorker) { | |
| return new Promise((resolve, reject) => { | |
| const handler = (e) => { | |
| if (e.data.type === 'ready') { | |
| self._embReady = true; | |
| self._embWorker.removeEventListener('message', handler); | |
| resolve(); | |
| } else if (e.data.type === 'error') { | |
| self._embWorker.removeEventListener('message', handler); | |
| reject(new Error(e.data.message)); | |
| } | |
| }; | |
| self._embWorker.addEventListener('message', handler); | |
| }); | |
| } | |
| this._embWorker = createEmbeddingWorker(); | |
| return new Promise((resolve, reject) => { | |
| const handler = (e) => { | |
| if (e.data.type === 'ready') { | |
| self._embReady = true; | |
| self._embWorker.removeEventListener('message', handler); | |
| resolve(); | |
| } else if (e.data.type === 'error') { | |
| self._embWorker.removeEventListener('message', handler); | |
| reject(new Error(e.data.message)); | |
| } | |
| }; | |
| self._embWorker.addEventListener('message', handler); | |
| self._embWorker.postMessage({ type: 'load' }); | |
| }); | |
| }, | |
| embed(texts /*, options */) { | |
| const self = this; | |
| const wasString = typeof texts === 'string'; | |
| const arr = wasString ? [texts] : texts; | |
| if (!Array.isArray(arr) || arr.length === 0) { | |
| return Promise.resolve(wasString ? new Float32Array(0) : []); | |
| } | |
| // Serialise: each call chains onto the previous so worker | |
| // request IDs cannot collide across concurrent callers. | |
| this._embQueue = this._embQueue.then(() => self._ensureEmbWorker().then(() => { | |
| return new Promise((resolve, reject) => { | |
| const id = Math.random().toString(36).slice(2); | |
| const handler = (e) => { | |
| if (e.data.id === id) { | |
| self._embWorker.removeEventListener('message', handler); | |
| if (e.data.type === 'embeddings') { | |
| // Worker returns Array<Array<number>>; cast each to | |
| // Float32Array per the runtime contract. cosineSimilarity | |
| // and IndexedDB both handle typed arrays transparently. | |
| resolve(e.data.vectors.map(v => new Float32Array(v))); | |
| } else { | |
| reject(new Error(e.data.message || 'Embedding failed')); | |
| } | |
| } | |
| }; | |
| self._embWorker.addEventListener('message', handler); | |
| self._embWorker.postMessage({ type: 'embed', texts: arr, id }); | |
| }); | |
| })); | |
| return this._embQueue.then(vectors => wasString ? vectors[0] : vectors); | |
| }, | |
| // --- SAM segmentation ---------------------------------------- | |
| // Lazy: the SAM worker is created on first segmentImage() call. | |
| // Independent of the chat and embedding workers. Auto-switches | |
| // if the user picks a different SAM model in settings. | |
| _samWorker: null, | |
| _samReady: false, | |
| _samQueue: Promise.resolve(), | |
| _samModelId: null, | |
| _ensureSAMWorker(modelId) { | |
| const self = this; | |
| const samSection = document.getElementById('samProgressSection'); | |
| const samFill = document.getElementById('samProgressFill'); | |
| const samText = document.getElementById('samProgressText'); | |
| function showBar(text, pct) { | |
| if (samSection) samSection.classList.add('visible'); | |
| if (samText) samText.textContent = text; | |
| if (samFill && pct != null) samFill.style.width = pct + '%'; | |
| } | |
| function hideBar() { | |
| if (samSection) setTimeout(() => samSection.classList.remove('visible'), 1200); | |
| } | |
| // Model changed β tear down old worker | |
| if (this._samWorker && this._samModelId !== modelId) { | |
| try { this._samWorker.terminate(); } catch {} | |
| this._samWorker = null; | |
| this._samReady = false; | |
| this._samModelId = null; | |
| } | |
| if (this._samReady && this._samModelId === modelId) return Promise.resolve(); | |
| function handleProgress(e, resolve, reject) { | |
| if (e.data.type === 'progress' && e.data.data) { | |
| const p = e.data.data; | |
| // v4 emits 'progress_total' as the aggregate across files; the | |
| // older 'downloading' branch is kept as a belt-and-braces fallback. | |
| if ((p.status === 'progress_total' || p.status === 'downloading') && p.total) { | |
| const pct = Math.min(100, Math.round((p.loaded / p.total) * 100)); | |
| const loadedMB = (p.loaded / 1e6).toFixed(1); | |
| const totalMB = (p.total / 1e6).toFixed(1); | |
| showBar('Downloading SAM \u2014 ' + loadedMB + ' / ' + totalMB + ' MB (' + pct + '%)', pct); | |
| } else if (p.status === 'loading') { | |
| showBar('Loading SAM model\u2026', 100); | |
| } else if (p.status === 'initiate') { | |
| showBar('Fetching SAM ' + (p.file || '') + '\u2026', null); | |
| } | |
| } else if (e.data.type === 'ready') { | |
| self._samReady = true; | |
| self._samWorker.removeEventListener('message', handleProgress._bound); | |
| showBar('SAM model ready', 100); | |
| hideBar(); | |
| resolve(); | |
| } else if (e.data.type === 'error') { | |
| self._samWorker.removeEventListener('message', handleProgress._bound); | |
| self._samReady = false; | |
| showBar('SAM load failed: ' + (e.data.message || 'unknown error'), 0); | |
| reject(new Error(e.data.message)); | |
| } | |
| } | |
| if (this._samWorker) { | |
| showBar('Loading SAM model\u2026', null); | |
| return new Promise((resolve, reject) => { | |
| const bound = (e) => handleProgress(e, resolve, reject); | |
| handleProgress._bound = bound; | |
| self._samWorker.addEventListener('message', bound); | |
| }); | |
| } | |
| showBar('Preparing to download SAM model\u2026', 0); | |
| this._samWorker = createSAMWorker(); | |
| this._samModelId = modelId; | |
| return new Promise((resolve, reject) => { | |
| const bound = (e) => handleProgress(e, resolve, reject); | |
| handleProgress._bound = bound; | |
| self._samWorker.addEventListener('message', bound); | |
| self._samWorker.postMessage({ type: 'load', modelId }); | |
| }); | |
| }, | |
| segmentImage(imageBlob, points, labels) { | |
| const self = this; | |
| const modelId = localStorage.getItem('lm_sam_model') || 'Xenova/slimsam-77-uniform'; | |
| const samSection = document.getElementById('samProgressSection'); | |
| const samText = document.getElementById('samProgressText'); | |
| this._samQueue = this._samQueue.then(() => self._ensureSAMWorker(modelId).then(async () => { | |
| if (samSection) samSection.classList.add('visible'); | |
| if (samText) samText.textContent = 'Segmenting image\u2026'; | |
| const arrayBuf = await imageBlob.arrayBuffer(); | |
| return new Promise((resolve, reject) => { | |
| const id = Math.random().toString(36).slice(2); | |
| const handler = (e) => { | |
| if (e.data.id === id) { | |
| self._samWorker.removeEventListener('message', handler); | |
| if (e.data.type === 'masks') { | |
| if (samText) samText.textContent = 'Segmentation complete'; | |
| if (samSection) setTimeout(() => samSection.classList.remove('visible'), 1200); | |
| resolve({ masks: e.data.masks, scores: e.data.scores }); | |
| } else { | |
| if (samSection) samSection.classList.remove('visible'); | |
| reject(new Error(e.data.message || 'Segmentation failed')); | |
| } | |
| } | |
| }; | |
| self._samWorker.addEventListener('message', handler); | |
| self._samWorker.postMessage( | |
| { type: 'segment', imageBuffer: arrayBuf, mimeType: imageBlob.type, points, labels, id }, | |
| [arrayBuf] | |
| ); | |
| }); | |
| })); | |
| return this._samQueue; | |
| }, | |
| // --- capabilities ------------------------------------------- | |
| capabilities() { | |
| return Object.assign({}, this._activeCaps); | |
| }, | |
| // --- transitional helpers (removed in later steps) ---------- | |
| // These exist only so the legacy code that still pokes the | |
| // worker directly (attachWorkerHandlers, generateOnce, the stop | |
| // button) can keep working until Step 5 routes everything | |
| // through chat(). They are NOT part of the public surface. | |
| _getWorker() { return this._worker; }, | |
| }; | |
| // ββ Embedding worker factory (MiniLM via @huggingface/transformers) ββ | |
| // Called once, lazily, by runtime.embed() on the first embed request. | |
| // Lives inside the adapter because its blob source code imports from | |
| // the Transformers.js CDN. | |
| function createEmbeddingWorker() { | |
| const code = ` | |
| import { | |
| env, | |
| pipeline, | |
| } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4/+esm'; | |
| env.allowLocalModels = true; | |
| env.localModelPath = '/models/'; | |
| env.allowRemoteModels = true; | |
| let embedder = null; | |
| self.addEventListener('message', async (e) => { | |
| const { type, texts, id } = e.data; | |
| if (type === 'load') { | |
| try { | |
| embedder = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', { | |
| device: 'wasm', | |
| progress_callback: (p) => self.postMessage({ type: 'progress', data: p }), | |
| }); | |
| self.postMessage({ type: 'ready' }); | |
| } catch (err) { | |
| self.postMessage({ type: 'error', message: err.message || String(err) }); | |
| } | |
| } else if (type === 'embed') { | |
| try { | |
| const results = []; | |
| for (const text of texts) { | |
| const output = await embedder(text, { pooling: 'mean', normalize: true }); | |
| results.push(Array.from(output.data)); | |
| } | |
| self.postMessage({ type: 'embeddings', vectors: results, id }); | |
| } catch (err) { | |
| self.postMessage({ type: 'error', message: err.message || String(err), id }); | |
| } | |
| } | |
| }); | |
| `; | |
| const blob = new Blob([code], { type: 'application/javascript' }); | |
| const url = URL.createObjectURL(blob); | |
| const w = new Worker(url, { type: 'module' }); | |
| w.addEventListener('error', (e) => { | |
| console.error('Embedding worker error:', e.message); | |
| }); | |
| return w; | |
| } | |
| // ββ SAM segmentation worker factory ββββββββββββββββββββββββ | |
| // Lazy-loaded on first segment_image tool call. Runs on WASM | |
| // (WebGPU is occupied by Gemma). Handles both SAM 1 (SamModel) | |
| // and SAM 3 (Sam3TrackerModel) via a modelId check. | |
| function createSAMWorker() { | |
| const code = ` | |
| import { | |
| env, | |
| SamModel, | |
| Sam3TrackerModel, | |
| AutoProcessor, | |
| RawImage, | |
| } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4/+esm'; | |
| env.allowLocalModels = true; | |
| env.localModelPath = '/models/'; | |
| env.allowRemoteModels = true; | |
| let model = null; | |
| let processor = null; | |
| let loadedModelId = null; | |
| self.addEventListener('message', async (e) => { | |
| const { type } = e.data; | |
| if (type === 'load') { | |
| const { modelId } = e.data; | |
| try { | |
| const isSam3 = modelId.toLowerCase().includes('sam3'); | |
| const ModelClass = isSam3 ? Sam3TrackerModel : SamModel; | |
| const opts = { | |
| device: 'wasm', | |
| progress_callback: (p) => self.postMessage({ type: 'progress', data: p }), | |
| }; | |
| processor = await AutoProcessor.from_pretrained(modelId); | |
| model = await ModelClass.from_pretrained(modelId, opts); | |
| loadedModelId = modelId; | |
| self.postMessage({ type: 'ready' }); | |
| } catch (err) { | |
| self.postMessage({ type: 'error', message: 'SAM load failed: ' + (err.message || String(err)) }); | |
| } | |
| } | |
| else if (type === 'segment') { | |
| const { imageBuffer, mimeType, points, labels, id } = e.data; | |
| if (!model || !processor) { | |
| self.postMessage({ type: 'error', message: 'SAM model not loaded', id }); | |
| return; | |
| } | |
| try { | |
| const blob = new Blob([imageBuffer], { type: mimeType || 'image/jpeg' }); | |
| const raw_image = await RawImage.fromBlob(blob); | |
| // Point format differs by model: | |
| // SAM 1 (SamModel): input_points [[[x,y]]], input_labels [[1]] | |
| // SAM 3 (Sam3TrackerModel): input_points [[[[x,y]]]], input_labels [[[1]]] | |
| const isSam3 = loadedModelId && loadedModelId.toLowerCase().includes('sam3'); | |
| const ptsList = points.map(p => [p[0], p[1]]); | |
| const lblList = labels.map(l => l); | |
| const input_points = isSam3 ? [[ptsList]] : [ptsList]; | |
| const input_labels = isSam3 ? [[lblList]] : [lblList]; | |
| const inputs = await processor(raw_image, { input_points, input_labels }); | |
| const outputs = await model(inputs); | |
| const masks = await processor.post_process_masks( | |
| outputs.pred_masks, | |
| inputs.original_sizes, | |
| inputs.reshaped_input_sizes, | |
| ); | |
| // masks from post_process_masks: masks[0] is a Tensor. | |
| // Shape varies: could be [num_masks, H, W] or [1, num_masks, H, W]. | |
| // Normalize to always work with [num_masks, H, W]. | |
| let maskTensor = masks[0]; | |
| const dims = maskTensor.dims; | |
| // If 4D [1, num_masks, H, W], squeeze the batch dim | |
| let numMasks, maskH, maskW, dataOffset; | |
| if (dims.length === 4) { | |
| numMasks = dims[1]; | |
| maskH = dims[2]; | |
| maskW = dims[3]; | |
| dataOffset = dims[0] > 1 ? 0 : 0; // always first batch | |
| } else { | |
| numMasks = dims[0]; | |
| maskH = dims[1]; | |
| maskW = dims[2]; | |
| } | |
| const iou_scores = outputs.iou_scores?.data | |
| ? Array.from(outputs.iou_scores.data) | |
| : []; | |
| const result = []; | |
| const transferables = []; | |
| const stridePerMask = maskH * maskW; | |
| // For 4D tensor, skip the batch dimension in the flat data array | |
| const batchStart = dims.length === 4 ? 0 : 0; // batch 0 | |
| for (let i = 0; i < numMasks; i++) { | |
| const maskData = new Uint8Array(stridePerMask); | |
| const offset = batchStart + i * stridePerMask; | |
| for (let j = 0; j < stridePerMask; j++) { | |
| // Data can be booleans (true/false) or logits (floats). | |
| // Booleans: true β 255. Logits: > 0 β foreground. | |
| const val = maskTensor.data[offset + j]; | |
| maskData[j] = (val === true || val > 0) ? 255 : 0; | |
| } | |
| result.push({ width: maskW, height: maskH, data: maskData }); | |
| transferables.push(maskData.buffer); | |
| } | |
| self.postMessage( | |
| { type: 'masks', masks: result, scores: iou_scores, id }, | |
| transferables | |
| ); | |
| } catch (err) { | |
| self.postMessage({ type: 'error', message: 'Segmentation failed: ' + (err.message || String(err)), id }); | |
| } | |
| } | |
| }); | |
| `; | |
| const blob = new Blob([code], { type: 'application/javascript' }); | |
| const url = URL.createObjectURL(blob); | |
| const w = new Worker(url, { type: 'module' }); | |
| w.addEventListener('error', (e) => { | |
| console.error('SAM worker error:', e.message); | |
| }); | |
| return w; | |
| } | |
| // ββ Pyodide worker factory (run_python agent tool) ββββββ | |
| // Lazy-loaded on first tool call. Loads Pyodide + auto-loads imports | |
| // (numpy, pandas, matplotlib via loadPackagesFromImports). stdout / | |
| // stderr buffered and returned; no plots yet. | |
| function createPyodideWorker() { | |
| const code = ` | |
| import { loadPyodide } from 'https://cdn.jsdelivr.net/pyodide/v0.26.4/full/pyodide.mjs'; | |
| let pyodide = null; | |
| let pyodideReadyPromise = null; | |
| function ensurePyodide() { | |
| if (!pyodideReadyPromise) { | |
| self.postMessage({ type: 'loading' }); | |
| pyodideReadyPromise = loadPyodide({ indexURL: 'https://cdn.jsdelivr.net/pyodide/v0.26.4/full/' }).then((p) => { | |
| pyodide = p; | |
| self.postMessage({ type: 'loaded' }); | |
| return p; | |
| }); | |
| } | |
| return pyodideReadyPromise; | |
| } | |
| self.addEventListener('message', async (e) => { | |
| const { type, code, id } = e.data; | |
| if (type === 'run') { | |
| try { | |
| await ensurePyodide(); | |
| let out = ''; | |
| let err = ''; | |
| pyodide.setStdout({ batched: (s) => { out += s + '\\n'; } }); | |
| pyodide.setStderr({ batched: (s) => { err += s + '\\n'; } }); | |
| try { await pyodide.loadPackagesFromImports(code); } catch (loadErr) { | |
| err += 'Package load warning: ' + (loadErr.message || loadErr) + '\\n'; | |
| } | |
| let result = undefined; | |
| try { | |
| result = await pyodide.runPythonAsync(code); | |
| } catch (pyErr) { | |
| err += (pyErr.message || String(pyErr)) + '\\n'; | |
| } | |
| const resultStr = (result !== undefined && result !== null) ? String(result) : ''; | |
| self.postMessage({ type: 'result', stdout: out, stderr: err, result: resultStr, id }); | |
| } catch (outerErr) { | |
| self.postMessage({ type: 'result', error: outerErr.message || String(outerErr), id }); | |
| } | |
| } | |
| }); | |
| `; | |
| const blob = new Blob([code], { type: 'application/javascript' }); | |
| const url = URL.createObjectURL(blob); | |
| const w = new Worker(url, { type: 'module' }); | |
| w.addEventListener('error', (e) => { | |
| console.error('Pyodide worker error:', e.message); | |
| }); | |
| return w; | |
| } | |
| // ββ Transcription worker factory (Whisper via WebGPU) βββ | |
| // Lazy-loaded on first voice-to-text click. Expects Float32Array PCM | |
| // at 16 kHz mono (decoded+resampled on the main thread via OfflineAudioContext). | |
| // Follows the HF realtime-whisper-webgpu pattern: bypasses the pipeline() | |
| // wrapper (which drops language/task kwargs silently on v4) and calls | |
| // model.generate() directly with language/task/max_new_tokens. | |
| function createTranscriptionWorker() { | |
| const code = ` | |
| import { | |
| env, | |
| AutoProcessor, | |
| AutoTokenizer, | |
| WhisperForConditionalGeneration, | |
| full, | |
| } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4/+esm'; | |
| env.allowLocalModels = true; | |
| env.localModelPath = '/models/'; | |
| env.allowRemoteModels = true; | |
| const MODEL_ID = 'onnx-community/whisper-base'; | |
| const MAX_NEW_TOKENS = 64; | |
| let processor = null; | |
| let tokenizer = null; | |
| let model = null; | |
| async function load() { | |
| const onProgress = (p) => self.postMessage({ type: 'progress', data: p }); | |
| processor = await AutoProcessor.from_pretrained(MODEL_ID, { progress_callback: onProgress }); | |
| tokenizer = await AutoTokenizer.from_pretrained(MODEL_ID, { progress_callback: onProgress }); | |
| model = await WhisperForConditionalGeneration.from_pretrained(MODEL_ID, { | |
| // fp32 encoder + q4 decoder is the known-good combo (HF realtime demo). | |
| // fp16 encoder produced subtle feature drift that biased the decoder to | |
| // filler tokens ("and", "the") on short clips. | |
| dtype: { encoder_model: 'fp32', decoder_model_merged: 'q4' }, | |
| device: 'webgpu', | |
| progress_callback: onProgress, | |
| }); | |
| // Warmup: compile shaders against a real mel-shape tensor so the first | |
| // real transcription doesn't take the compile hit. | |
| await model.generate({ | |
| input_features: full([1, 80, 3000], 0.0), | |
| max_new_tokens: 1, | |
| }); | |
| self.postMessage({ type: 'ready' }); | |
| } | |
| async function transcribe(pcm, lang) { | |
| if (!model || !processor || !tokenizer) throw new Error('Whisper not loaded'); | |
| // processor builds log-mel features from the Float32Array PCM. | |
| const inputs = await processor(pcm); | |
| // Pass language/task/max_new_tokens directly to generate() β the pipeline | |
| // wrapper used to silently drop these kwargs in v4. | |
| const outputs = await model.generate({ | |
| ...inputs, | |
| max_new_tokens: MAX_NEW_TOKENS, | |
| language: lang || 'en', | |
| task: 'transcribe', | |
| }); | |
| const text = tokenizer.batch_decode(outputs, { skip_special_tokens: true })[0] || ''; | |
| return text.trim(); | |
| } | |
| self.addEventListener('message', async (e) => { | |
| const { type, pcm, lang, id } = e.data; | |
| try { | |
| if (type === 'load') { | |
| await load(); | |
| } else if (type === 'transcribe') { | |
| const text = await transcribe(pcm, lang); | |
| self.postMessage({ type: 'transcription', text, id }); | |
| } | |
| } catch (err) { | |
| self.postMessage({ type: 'error', message: err.message || String(err), id }); | |
| } | |
| }); | |
| `; | |
| const blob = new Blob([code], { type: 'application/javascript' }); | |
| const url = URL.createObjectURL(blob); | |
| const w = new Worker(url, { type: 'module' }); | |
| w.addEventListener('error', (e) => { | |
| console.error('Transcription worker error:', e.message); | |
| }); | |
| return w; | |
| } | |
| // ββ Generation worker factory βββββββββββββββββββββββββββββββ | |
| // Pattern from huggingface/transformers.js-examples (Llama 3.2 WebGPU). | |
| // Uses AutoModelForCausalLM + AutoTokenizer (not pipeline) for causal models. | |
| // Uses Gemma4ForConditionalGeneration + AutoProcessor for multimodal models. | |
| // GQA variant required for Gemma 3 1B (issue #1469: regular variant crashes). | |
| function createWorker() { | |
| const code = ` | |
| // ββ Resumable download cache ββββββββββββββββββββββββββββββββ | |
| // Monkey-patches self.fetch so HF CDN downloads checkpoint to IndexedDB | |
| // every 5 MB. If the tab closes mid-download, the next load resumes from | |
| // the last saved chunk via HTTP Range instead of restarting from byte 0. | |
| // Any failure in this path falls back to the original fetch, so the model | |
| // still loads β it just won't resume if re-interrupted. Toggle via the | |
| // 'resumable' flag on the load message; default on. | |
| const __RDB_NAME = 'localmind-downloads'; | |
| const __RDB_CHUNK = 5 * 1024 * 1024; | |
| let __resumableEnabled = true; | |
| function __openRDB() { | |
| return new Promise((resolve, reject) => { | |
| const r = indexedDB.open(__RDB_NAME, 1); | |
| r.onupgradeneeded = () => { | |
| const db = r.result; | |
| if (!db.objectStoreNames.contains('meta')) db.createObjectStore('meta', { keyPath: 'url' }); | |
| if (!db.objectStoreNames.contains('chunks')) db.createObjectStore('chunks', { keyPath: 'key' }); | |
| }; | |
| r.onsuccess = () => resolve(r.result); | |
| r.onerror = () => reject(r.error); | |
| }); | |
| } | |
| function __rdbGet(db, store, key) { | |
| return new Promise((resolve, reject) => { | |
| const req = db.transaction(store, 'readonly').objectStore(store).get(key); | |
| req.onsuccess = () => resolve(req.result || null); | |
| req.onerror = () => reject(req.error); | |
| }); | |
| } | |
| function __rdbPut(db, store, value) { | |
| return new Promise((resolve, reject) => { | |
| const tx = db.transaction(store, 'readwrite'); | |
| tx.objectStore(store).put(value); | |
| tx.oncomplete = () => resolve(); | |
| tx.onerror = () => reject(tx.error); | |
| }); | |
| } | |
| function __rdbClearFile(db, url) { | |
| return new Promise((resolve, reject) => { | |
| const tx = db.transaction(['chunks', 'meta'], 'readwrite'); | |
| const req = tx.objectStore('chunks').openCursor(); | |
| req.onsuccess = (e) => { | |
| const c = e.target.result; | |
| if (!c) return; | |
| if (c.value.key.indexOf(url + '|') === 0) c.delete(); | |
| c.continue(); | |
| }; | |
| tx.objectStore('meta').delete(url); | |
| tx.oncomplete = () => resolve(); | |
| tx.onerror = () => reject(tx.error); | |
| }); | |
| } | |
| // On worker start: drop IDB chunks for files already fully cached in Cache | |
| // Storage. Recovers disk after transformers.js has cache.put()'d the whole | |
| // response independently of our chunk store. | |
| async function __rdbCleanup() { | |
| let db; | |
| try { db = await __openRDB(); } catch { return; } | |
| try { | |
| const urls = await new Promise((resolve, reject) => { | |
| const r = db.transaction('meta', 'readonly').objectStore('meta').getAllKeys(); | |
| r.onsuccess = () => resolve(r.result || []); | |
| r.onerror = () => reject(r.error); | |
| }); | |
| for (const url of urls) { | |
| try { | |
| const hit = await caches.match(url); | |
| if (hit) await __rdbClearFile(db, url); | |
| } catch {} | |
| } | |
| } finally { | |
| try { db.close(); } catch {} | |
| } | |
| } | |
| function __shouldIntercept(url) { | |
| return typeof url === 'string' && url.indexOf('https://huggingface.co/') === 0; | |
| } | |
| const __origFetch = self.fetch.bind(self); | |
| self.fetch = async function(input, init) { | |
| const urlStr = typeof input === 'string' ? input : (input && input.url); | |
| const method = (init && init.method) || (input && input.method) || 'GET'; | |
| if (!__resumableEnabled || method !== 'GET' || !__shouldIntercept(urlStr)) { | |
| return __origFetch(input, init); | |
| } | |
| let db; | |
| try { db = await __openRDB(); } | |
| catch { return __origFetch(input, init); } | |
| const closeDb = () => { try { db.close(); } catch {} }; | |
| try { | |
| const headResp = await __origFetch(urlStr, { method: 'HEAD' }); | |
| if (!headResp.ok) { closeDb(); return __origFetch(input, init); } | |
| const totalSize = parseInt(headResp.headers.get('content-length'), 10); | |
| const etag = headResp.headers.get('etag') || headResp.headers.get('last-modified') || ''; | |
| const contentType = headResp.headers.get('content-type') || 'application/octet-stream'; | |
| if (!totalSize || totalSize < __RDB_CHUNK) { | |
| closeDb(); | |
| return __origFetch(input, init); | |
| } | |
| let meta = await __rdbGet(db, 'meta', urlStr); | |
| if (meta && meta.etag !== etag) { | |
| await __rdbClearFile(db, urlStr); | |
| meta = null; | |
| } | |
| const chunkCount = Math.ceil(totalSize / __RDB_CHUNK); | |
| const existing = new Array(chunkCount).fill(null); | |
| if (meta) { | |
| for (let i = 0; i < chunkCount; i++) { | |
| const rec = await __rdbGet(db, 'chunks', urlStr + '|' + i); | |
| if (rec) existing[i] = rec.data; | |
| } | |
| } | |
| const firstMissing = existing.findIndex(c => !c); | |
| if (firstMissing === -1) { | |
| closeDb(); | |
| return new Response(new Blob(existing, { type: contentType }), { | |
| status: 200, | |
| headers: new Headers({ | |
| 'content-length': String(totalSize), | |
| 'content-type': contentType, | |
| }), | |
| }); | |
| } | |
| if (!meta) { | |
| await __rdbPut(db, 'meta', { url: urlStr, etag, totalSize, chunkSize: __RDB_CHUNK, chunkCount }); | |
| } | |
| const startByte = firstMissing * __RDB_CHUNK; | |
| const rangeResp = await __origFetch(urlStr, { headers: { Range: 'bytes=' + startByte + '-' } }); | |
| if (!rangeResp.ok) { closeDb(); return __origFetch(input, init); } | |
| const stream = new ReadableStream({ | |
| async start(controller) { | |
| try { | |
| for (let i = 0; i < firstMissing; i++) { | |
| const buf = await existing[i].arrayBuffer(); | |
| controller.enqueue(new Uint8Array(buf)); | |
| } | |
| const reader = rangeResp.body.getReader(); | |
| let accum = new Uint8Array(__RDB_CHUNK); | |
| let accumLen = 0; | |
| let chunkIdx = firstMissing; | |
| while (true) { | |
| const { done, value } = await reader.read(); | |
| if (done) break; | |
| let pos = 0; | |
| while (pos < value.length) { | |
| const take = Math.min(__RDB_CHUNK - accumLen, value.length - pos); | |
| accum.set(value.subarray(pos, pos + take), accumLen); | |
| accumLen += take; | |
| pos += take; | |
| if (accumLen === __RDB_CHUNK) { | |
| const slice = accum.slice(0, accumLen); | |
| await __rdbPut(db, 'chunks', { key: urlStr + '|' + chunkIdx, data: new Blob([slice]) }); | |
| controller.enqueue(slice); | |
| chunkIdx++; | |
| accumLen = 0; | |
| accum = new Uint8Array(__RDB_CHUNK); | |
| } | |
| } | |
| } | |
| if (accumLen > 0) { | |
| const slice = accum.slice(0, accumLen); | |
| await __rdbPut(db, 'chunks', { key: urlStr + '|' + chunkIdx, data: new Blob([slice]) }); | |
| controller.enqueue(slice); | |
| } | |
| controller.close(); | |
| } catch (e) { | |
| controller.error(e); | |
| } finally { | |
| closeDb(); | |
| } | |
| }, | |
| cancel() { closeDb(); }, | |
| }); | |
| return new Response(stream, { | |
| status: 200, | |
| headers: new Headers({ | |
| 'content-length': String(totalSize), | |
| 'content-type': contentType, | |
| }), | |
| }); | |
| } catch { | |
| closeDb(); | |
| return __origFetch(input, init); | |
| } | |
| }; | |
| __rdbCleanup(); | |
| // Dynamic import *after* the fetch patch is installed. Using top-level await | |
| // would race the 'load' postMessage from the main thread β it can arrive | |
| // before the module-body handler is registered and get dropped. Kick off | |
| // the import eagerly and have the message handler await it. | |
| let env, AutoTokenizer, AutoModelForCausalLM, AutoProcessor, Gemma4ForConditionalGeneration, load_image, TextStreamer, InterruptableStoppingCriteria; | |
| let stopping_criteria; | |
| const __modReady = import('https://cdn.jsdelivr.net/npm/@huggingface/transformers@4/+esm').then((m) => { | |
| env = m.env; | |
| AutoTokenizer = m.AutoTokenizer; | |
| AutoModelForCausalLM = m.AutoModelForCausalLM; | |
| AutoProcessor = m.AutoProcessor; | |
| Gemma4ForConditionalGeneration = m.Gemma4ForConditionalGeneration; | |
| load_image = m.load_image; | |
| TextStreamer = m.TextStreamer; | |
| InterruptableStoppingCriteria = m.InterruptableStoppingCriteria; | |
| env.allowLocalModels = true; | |
| env.localModelPath = '/models/'; | |
| env.allowRemoteModels = true; | |
| // Transformers.js captures globalThis.fetch at module init via .bind(), so | |
| // overriding self.fetch alone doesn't reach it. Replace env.fetch so its | |
| // internal getFile() routes through our resumable path. | |
| env.fetch = self.fetch; | |
| stopping_criteria = new InterruptableStoppingCriteria(); | |
| }); | |
| let processor = null; | |
| let tokenizer = null; | |
| let model = null; | |
| let loadedType = null; | |
| function progressCallback(p) { | |
| self.postMessage({ type: 'progress', data: p }); | |
| } | |
| async function loadCausal(modelId, dtype) { | |
| processor = null; tokenizer = null; model = null; | |
| tokenizer = await AutoTokenizer.from_pretrained(modelId, { | |
| progress_callback: progressCallback, | |
| }); | |
| model = await AutoModelForCausalLM.from_pretrained(modelId, { | |
| dtype: dtype, | |
| device: 'webgpu', | |
| progress_callback: progressCallback, | |
| }); | |
| // Warmup: compile WebGPU shaders with a single token | |
| self.postMessage({ type: 'warmup' }); | |
| const warmupInputs = tokenizer('a'); | |
| await model.generate({ ...warmupInputs, max_new_tokens: 1 }); | |
| loadedType = 'causal'; | |
| self.postMessage({ type: 'ready' }); | |
| } | |
| async function loadMultimodal(modelId, dtype) { | |
| processor = null; tokenizer = null; model = null; | |
| processor = await AutoProcessor.from_pretrained(modelId, { | |
| progress_callback: progressCallback, | |
| }); | |
| tokenizer = processor.tokenizer; | |
| model = await Gemma4ForConditionalGeneration.from_pretrained(modelId, { | |
| dtype: dtype, | |
| device: 'webgpu', | |
| progress_callback: progressCallback, | |
| }); | |
| // Warmup | |
| self.postMessage({ type: 'warmup' }); | |
| const warmupInputs = tokenizer('a'); | |
| await model.generate({ ...warmupInputs, max_new_tokens: 1 }); | |
| loadedType = 'multimodal'; | |
| self.postMessage({ type: 'ready' }); | |
| } | |
| async function generateCausal(chatMessages, id, genConfig) { | |
| stopping_criteria.reset(); | |
| try { | |
| const inputs = tokenizer.apply_chat_template(chatMessages, { | |
| add_generation_prompt: true, | |
| return_dict: true, | |
| }); | |
| const streamer = new TextStreamer(tokenizer, { | |
| skip_prompt: true, | |
| skip_special_tokens: true, | |
| callback_function: (text) => { | |
| self.postMessage({ type: 'token', token: text, id }); | |
| }, | |
| }); | |
| const gc = genConfig || {}; | |
| await model.generate({ | |
| ...inputs, | |
| max_new_tokens: gc.max_new_tokens || 2048, | |
| do_sample: true, | |
| temperature: gc.temperature ?? 0.7, | |
| top_k: gc.top_k ?? 50, | |
| top_p: gc.top_p ?? 0.95, | |
| repetition_penalty: gc.repetition_penalty ?? 1.0, | |
| streamer, | |
| stopping_criteria, | |
| }); | |
| self.postMessage({ type: 'complete', id }); | |
| } catch (err) { | |
| self.postMessage({ type: 'error', message: err.message || String(err) }); | |
| self.postMessage({ type: 'complete', id }); | |
| } | |
| } | |
| async function generateMultimodal(chatMessages, id, attachmentData, enableThinking, genConfig) { | |
| stopping_criteria.reset(); | |
| try { | |
| // Build multimodal content blocks for the last user message | |
| // chatMessages is already in the right format with content arrays | |
| // Apply chat template via processor (returns a prompt string, not tokens) | |
| const prompt = processor.apply_chat_template(chatMessages, { | |
| add_generation_prompt: true, | |
| enable_thinking: enableThinking || false, | |
| }); | |
| // Reconstruct images and audio from transferred data | |
| const images = []; | |
| const audios = []; | |
| if (attachmentData) { | |
| for (const att of attachmentData) { | |
| if (att.type === 'image') { | |
| // Create blob URL and use load_image() which properly constructs | |
| // a RawImage with .rgb() and other methods the processor needs | |
| const blob = new Blob([att.data], { type: att.mimeType || 'image/jpeg' }); | |
| const blobUrl = URL.createObjectURL(blob); | |
| const img = await load_image(blobUrl); | |
| URL.revokeObjectURL(blobUrl); | |
| images.push(img); | |
| } else if (att.type === 'audio') { | |
| // Reconstruct Float32Array from transferred data | |
| // (postMessage may transfer as ArrayBuffer, losing typed array identity) | |
| const pcm = att.pcmData instanceof Float32Array | |
| ? att.pcmData | |
| : new Float32Array(att.pcmData); | |
| audios.push(pcm); | |
| } | |
| } | |
| } | |
| // Process multimodal inputs: processor(prompt, image|null, audio|null, options) | |
| // Must pass null for missing modalities β positional args, not named. | |
| // The processor dispatches arg[1] to vision and arg[2] to audio extractor. | |
| const imageArg = images.length > 0 ? (images.length === 1 ? images[0] : images) : null; | |
| const audioArg = audios.length > 0 ? (audios.length === 1 ? audios[0] : audios) : null; | |
| const inputs = await processor(prompt, imageArg, audioArg, { add_special_tokens: false }); | |
| const streamer = new TextStreamer(processor.tokenizer, { | |
| skip_prompt: true, | |
| skip_special_tokens: true, | |
| callback_function: (text) => { | |
| self.postMessage({ type: 'token', token: text, id }); | |
| }, | |
| }); | |
| const gc = genConfig || {}; | |
| await model.generate({ | |
| ...inputs, | |
| max_new_tokens: gc.max_new_tokens || 2048, | |
| do_sample: true, | |
| temperature: gc.temperature ?? 1.0, | |
| top_k: gc.top_k ?? 64, | |
| top_p: gc.top_p ?? 0.95, | |
| repetition_penalty: gc.repetition_penalty ?? 1.0, | |
| streamer, | |
| stopping_criteria, | |
| }); | |
| self.postMessage({ type: 'complete', id }); | |
| } catch (err) { | |
| self.postMessage({ type: 'error', message: err.message || String(err) }); | |
| self.postMessage({ type: 'complete', id }); | |
| } | |
| } | |
| self.addEventListener('message', async (e) => { | |
| await __modReady; // Transformers.js may still be loading on first message | |
| const { type, messages, id, modelId, dtype, modelType, attachments: attData, enableThinking, generationConfig, resumable } = e.data; | |
| if (type === 'load') { | |
| if (resumable === false) __resumableEnabled = false; | |
| try { | |
| if (modelType === 'multimodal') { | |
| await loadMultimodal(modelId, dtype); | |
| } else { | |
| await loadCausal(modelId, dtype); | |
| } | |
| } catch (err) { | |
| self.postMessage({ type: 'error', message: 'Failed to load model: ' + (err.message || String(err)) }); | |
| } | |
| } else if (type === 'generate') { | |
| if (loadedType === 'multimodal') { | |
| await generateMultimodal(messages, id, attData, enableThinking, generationConfig); | |
| } else { | |
| await generateCausal(messages, id, generationConfig); | |
| } | |
| } else if (type === 'stop') { | |
| stopping_criteria.interrupt(); | |
| } | |
| }); | |
| `; | |
| const blob = new Blob([code], { type: 'application/javascript' }); | |
| const url = URL.createObjectURL(blob); | |
| return new Worker(url, { type: 'module' }); | |
| } | |
| // =========================================================== | |
| // === END RUNTIME ADAPTER =================================== | |
| // =========================================================== | |
| // ββ Worker message handling βββββββββββββββββββββββββββββββββ | |
| function attachWorkerHandlers(w) { | |
| // Catch silent worker failures (e.g. bad import, missing export) | |
| w.addEventListener('error', (e) => { | |
| console.error('Worker error event:', e.message, e); | |
| statusBadge.className = 'status-badge error'; | |
| statusSpinner.style.display = 'none'; | |
| statusText.textContent = 'Worker error'; | |
| progressSection.classList.remove('visible'); | |
| progressText.textContent = e.message || 'Worker failed to start'; | |
| modelSelect.disabled = false; | |
| }); | |
| w.addEventListener('message', (e) => { | |
| const data = e.data; | |
| if (data.type === 'progress') { | |
| const p = data.data; | |
| progressSection.classList.add('visible'); | |
| const fileName = p.file ? p.file.split('/').pop() : ''; | |
| if (fileName) currentDownloadFile = fileName; | |
| // Transformers.js v4 emits: 'initiate' (fetch starting), 'progress' | |
| // (per-file bytes), 'progress_total' (aggregate bytes across all | |
| // files β no `file` field), and 'loading' (file parsing into | |
| // memory after download). We drive the bar off progress_total so | |
| // it moves monotonically across the whole model instead of | |
| // resetting per file. | |
| if (p.status === 'initiate') { | |
| progressText.textContent = `Fetching ${fileName}...`; | |
| } else if (p.status === 'progress_total' && p.loaded && p.total) { | |
| const pct = Math.min(100, Math.round((p.loaded / p.total) * 100)); | |
| progressFill.style.width = pct + '%'; | |
| const loadedMB = (p.loaded / 1024 / 1024).toFixed(0); | |
| const totalMB = (p.total / 1024 / 1024).toFixed(0); | |
| const label = currentDownloadFile ? ` ${currentDownloadFile}` : ''; | |
| progressText.textContent = `Downloading${label} β ${loadedMB} / ${totalMB} MB (${pct}%)`; | |
| statusText.textContent = 'Downloading...'; | |
| } else if (p.status === 'loading') { | |
| progressText.textContent = `Loading ${fileName}...`; | |
| statusText.textContent = 'Loading into memory...'; | |
| } | |
| } | |
| else if (data.type === 'warmup') { | |
| progressText.textContent = 'Compiling shaders and warming up...'; | |
| statusText.textContent = 'Warming up...'; | |
| } | |
| else if (data.type === 'ready') { | |
| modelReady = true; | |
| progressSection.classList.remove('visible'); | |
| statusBadge.className = 'status-badge ready'; | |
| statusSpinner.style.display = 'none'; | |
| const m = MODELS[activeModelKey]; | |
| statusText.textContent = m.label; | |
| chatInput.disabled = false; | |
| const readyCaps = LocalMind.runtime.capabilities(); | |
| const supportsMedia = !!(readyCaps && (readyCaps.image || readyCaps.audio || readyCaps.video)); | |
| chatInput.placeholder = supportsMedia ? 'Type a message, or attach images/audio...' : 'Type a message...'; | |
| sendBtn.disabled = false; searchSendBtn.disabled = false; | |
| clearBtn.disabled = false; | |
| newChatBtn.disabled = false; | |
| modelSelect.disabled = false; | |
| updateModelUI(); | |
| updateBatchCount(); | |
| chatInput.focus(); | |
| // Resolve any pending window.localmind.load() promise | |
| if (loadResolvers) { | |
| const r = loadResolvers; loadResolvers = null; | |
| r.resolve(); | |
| } | |
| } | |
| else if (data.type === 'token') { | |
| // Always accumulate so non-streaming callers (chat UI, non-streaming | |
| // API) still receive the full text via the 'complete' handler. | |
| currentAssistantText += data.token; | |
| if (currentAssistantEl) { | |
| updateAssistantBubble(currentAssistantEl.bubble, currentAssistantText); | |
| } | |
| // Streaming consumer (window.localmind.chat.completions.create with | |
| // stream: true) β route this token to its async iterator. | |
| if (currentInferenceContext && typeof currentInferenceContext.onToken === 'function') { | |
| try { currentInferenceContext.onToken(data.token); } catch (e) { console.error('streaming onToken threw:', e); } | |
| } | |
| } | |
| else if (data.type === 'complete') { | |
| // If agentic loop is awaiting, resolve promise (loop manages state) | |
| if (generateResolve) { | |
| const resolve = generateResolve; | |
| generateResolve = null; | |
| resolve(currentAssistantText); | |
| } else { | |
| // Fallback: no agentic loop (shouldn't happen, but safe) | |
| generating = false; | |
| if (currentAssistantText) { | |
| messages.push({ role: 'assistant', content: currentAssistantText }); | |
| saveChat(); | |
| } | |
| currentAssistantEl = null; | |
| currentAssistantText = ''; | |
| sendBtn.innerHTML = '▶'; | |
| sendBtn.classList.remove('stop'); | |
| sendBtn.disabled = false; searchSendBtn.disabled = false; | |
| chatInput.disabled = false; | |
| modelSelect.disabled = false; | |
| chatInput.focus(); | |
| } | |
| } | |
| else if (data.type === 'error') { | |
| if (!modelReady) { | |
| statusBadge.className = 'status-badge error'; | |
| statusSpinner.style.display = 'none'; | |
| statusText.textContent = 'Error'; | |
| progressSection.classList.remove('visible'); | |
| modelSelect.disabled = false; | |
| } | |
| console.error('Worker error:', data.message); | |
| if (generating && currentAssistantEl) { | |
| currentAssistantText += `\n\n*Error: ${data.message}*`; | |
| updateAssistantBubble(currentAssistantEl.bubble, currentAssistantText); | |
| } | |
| // Reject any pending window.localmind.load() promise | |
| if (loadResolvers) { | |
| const r = loadResolvers; loadResolvers = null; | |
| r.reject(new Error(data.message || 'Model load failed')); | |
| } | |
| } | |
| }); | |
| } | |
| // ββ Load model ββββββββββββββββββββββββββββββββββββββββββββββ | |
| // Thin shell over LocalMind.runtime.loadModel that drives the | |
| // existing UI state machine. The runtime owns worker lifecycle; | |
| // we still attach the legacy message handlers to the freshly | |
| // created worker so streaming tokens, progress, and 'ready' | |
| // events update the UI exactly as before. Step 5 collapses the | |
| // legacy handlers in favour of the chat() async iterator. | |
| function loadModel(key) { | |
| const m = MODELS[key]; | |
| if (!m) return; | |
| // Any in-flight API load promise is now stale β reject it before the | |
| // new load starts so the caller is notified and the new caller (if any) | |
| // can attach fresh resolvers right after this returns. | |
| if (loadResolvers) { | |
| const r = loadResolvers; loadResolvers = null; | |
| try { r.reject(new Error('Load superseded by another load')); } catch {} | |
| } | |
| activeModelKey = key; | |
| modelReady = false; | |
| // Clear attachments when switching away from multimodal | |
| if (!m.multimodal) clearAttachments(); | |
| // Reset aggregate-progress state for this load | |
| currentDownloadFile = ''; | |
| // Reset UI | |
| statusBadge.className = 'status-badge loading'; | |
| statusSpinner.style.display = ''; | |
| statusText.textContent = 'Loading...'; | |
| progressFill.style.width = '0%'; | |
| progressText.textContent = `Preparing to download ${m.label} (${m.size})...`; | |
| progressSection.classList.add('visible'); | |
| chatInput.disabled = true; | |
| chatInput.placeholder = `Loading ${m.label}...`; | |
| sendBtn.disabled = true; searchSendBtn.disabled = true; | |
| // modelSelect stays enabled so users can hot-swap to a different model | |
| // mid-download (runtime.loadModel terminates the existing worker first). | |
| // Partial chunks already written to IDB stay available for resume. | |
| updateModelUI(); | |
| // Delegate worker creation + load message to the runtime adapter. | |
| // We deliberately do NOT await the returned promise here β the | |
| // legacy attachWorkerHandlers() below already drives the UI from | |
| // worker events (progress / ready / error). The .catch is purely | |
| // defensive against unhandled rejections; UI error state is | |
| // already handled by the legacy 'error' message handler. | |
| LocalMind.runtime.loadModel(m.id, { | |
| dtype: m.dtype, | |
| modelType: m.type, | |
| capabilities: { | |
| text: true, | |
| image: !!m.multimodal, | |
| audio: !!m.multimodal, | |
| video: !!m.multimodal, | |
| toolCalling: !!m.agentCapable, | |
| maxContextTokens: m.contextSize || 0, | |
| }, | |
| }).catch((e) => { | |
| console.warn('runtime.loadModel rejected:', e && e.message); | |
| }); | |
| // Pull the freshly created worker from the runtime and attach | |
| // the legacy UI handlers. Synchronous after loadModel() returns, | |
| // so handlers are wired before any worker message dispatches. | |
| worker = LocalMind.runtime._getWorker(); | |
| attachWorkerHandlers(worker); | |
| } | |
| // ββ Model selector ββββββββββββββββββββββββββββββββββββββββββ | |
| modelSelect.addEventListener('change', () => { | |
| if (generating) return; | |
| const key = modelSelect.value; | |
| if (key === activeModelKey && modelReady) return; | |
| loadModel(key); | |
| }); | |
| // ββ Cancel button on the download progress panel ββββββββββββ | |
| // Terminates the worker (which aborts in-flight fetches) and returns | |
| // the app to an idle state. Partial chunks already saved to IDB stay | |
| // there, so reselecting the same model picks up roughly where we left | |
| // off via the resumable-fetch path. | |
| const progressCancelBtn = document.getElementById('progressCancelBtn'); | |
| progressCancelBtn.addEventListener('click', () => { | |
| try { LocalMind.runtime.unload(); } catch {} | |
| worker = null; | |
| modelReady = false; | |
| activeModelKey = null; | |
| progressSection.classList.remove('visible'); | |
| // Base `.status-badge` class with no state modifier β neutral grey | |
| // instead of red (this is user-initiated, not an error). | |
| statusBadge.className = 'status-badge'; | |
| statusSpinner.style.display = 'none'; | |
| statusText.textContent = 'Cancelled β pick a model'; | |
| chatInput.disabled = true; | |
| chatInput.placeholder = 'Select a model to load'; | |
| sendBtn.disabled = true; | |
| searchSendBtn.disabled = true; | |
| showToast('Download cancelled. Partial chunks kept for resume.'); | |
| }); | |
| // Attachment preprocessing (audio decode, video frame extraction, | |
| // image blob reads) has moved inside the runtime adapter β see | |
| // _adapterDecodeAudioTo16k / _adapterDecomposeVideo / | |
| // _adapterProcessContentBlocks. The chat UI now passes raw Blobs | |
| // in content-block form and the adapter handles the translation. | |
| // ββ Send message ββββββββββββββββββββββββββββββββββββββββββββ | |
| function generateOnce(chatMessages, attData, enableThinking, genConfig) { | |
| return new Promise((resolve) => { | |
| generateResolve = resolve; | |
| const id = ++msgIdCounter; | |
| worker.postMessage({ | |
| type: 'generate', | |
| messages: chatMessages, | |
| id, | |
| attachments: attData || null, | |
| enableThinking, | |
| generationConfig: genConfig, | |
| }); | |
| }); | |
| } | |
| // FIFO wrapper around generateOnce. All inference callers β the chat UI, | |
| // the agentic loop, and (soon) the window.localmind API β route through | |
| // here so two callers can't race and clobber each other's generateResolve | |
| // promise. Chat UI is already serialised by the `generating` flag; the | |
| // queue adds correctness so that does not have to be the load-bearing | |
| // guarantee. Bounded length keeps a misbehaving caller from piling up | |
| // jobs. Items past MAX_INFER_QUEUE are rejected synchronously. | |
| const inferQueue = []; | |
| const MAX_INFER_QUEUE = 4; | |
| let inferDraining = false; | |
| // opts: { chatMessages, attData, enableThinking, genConfig, setup? } | |
| // The optional `setup` callback runs INSIDE the queue critical section, | |
| // right before the worker job posts. This is how the API path safely | |
| // resets currentAssistantEl/Text without racing the chat path's own | |
| // setup that lives in sendMessage(). | |
| function runInference(opts) { | |
| return new Promise((resolve, reject) => { | |
| if (inferQueue.length >= MAX_INFER_QUEUE) { | |
| reject(new Error('inference queue full (max ' + MAX_INFER_QUEUE + ')')); | |
| return; | |
| } | |
| inferQueue.push({ ...opts, resolve, reject }); | |
| drainInferQueue(); | |
| }); | |
| } | |
| async function drainInferQueue() { | |
| if (inferDraining) return; | |
| inferDraining = true; | |
| try { | |
| while (inferQueue.length) { | |
| const job = inferQueue.shift(); | |
| // Reset per-job context so an earlier streaming job's token sink | |
| // can't leak into this one. The job's setup() may install a new | |
| // currentInferenceContext on top. | |
| currentInferenceContext = null; | |
| try { | |
| if (typeof job.setup === 'function') job.setup(); | |
| const result = await generateOnce(job.chatMessages, job.attData, job.enableThinking, job.genConfig); | |
| job.resolve(result); | |
| } catch (e) { | |
| job.reject(e); | |
| } | |
| } | |
| currentInferenceContext = null; | |
| } finally { | |
| inferDraining = false; | |
| } | |
| } | |
| function renderToolCallBlock(bubble, toolName, args, result) { | |
| const blockId = 'tc-' + Math.random().toString(36).slice(2, 8); | |
| const argsStr = Object.entries(args).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join(', '); | |
| const resultStr = typeof result === 'string' ? result : JSON.stringify(result, null, 2); | |
| const block = document.createElement('div'); | |
| block.className = 'tool-call-block'; | |
| block.innerHTML = ` | |
| <div class="tool-call-toggle" onclick="document.getElementById('${blockId}').classList.toggle('open'); this.querySelector('span').textContent = document.getElementById('${blockId}').classList.contains('open') ? '▼' : '▶'"> | |
| <span>▶</span> <strong>${escapeHtml(toolName)}</strong>(${escapeHtml(argsStr)}) | |
| </div> | |
| <div class="tool-call-result" id="${blockId}"><pre>${escapeHtml(resultStr)}</pre></div> | |
| `; | |
| bubble.appendChild(block); | |
| chatArea.scrollTop = chatArea.scrollHeight; | |
| } | |
| function renderSegmentationOverlay(bubble, imageBlob, masks, scores) { | |
| if (!masks || masks.length === 0) return; | |
| // Distinct colors for up to 6 masks | |
| const COLORS = [ | |
| [234, 88, 12], // orange | |
| [59, 130, 246], // blue | |
| [16, 185, 129], // emerald | |
| [168, 85, 247], // purple | |
| [239, 68, 68], // red | |
| [245, 158, 11], // amber | |
| ]; | |
| const container = document.createElement('div'); | |
| container.className = 'segmentation-result'; | |
| const img = document.createElement('img'); | |
| const imgUrl = URL.createObjectURL(imageBlob); | |
| img.src = imgUrl; | |
| img.alt = 'Segmented image'; | |
| container.appendChild(img); | |
| img.onload = () => { | |
| const displayW = img.naturalWidth; | |
| const displayH = img.naturalHeight; | |
| const canvas = document.createElement('canvas'); | |
| canvas.width = displayW; | |
| canvas.height = displayH; | |
| container.appendChild(canvas); | |
| const ctx = canvas.getContext('2d'); | |
| // Each mask = one object (best-of-3 already selected in execute). | |
| // Draw fill + bounding box per object in distinct colors. | |
| const imgData = ctx.createImageData(displayW, displayH); | |
| const boxes = []; // collect {minX, minY, maxX, maxY, colorIdx, score} per mask | |
| for (let mi = 0; mi < masks.length; mi++) { | |
| const mask = masks[mi]; | |
| const [r, g, b] = COLORS[mi % COLORS.length]; | |
| let bMinX = displayW, bMinY = displayH, bMaxX = 0, bMaxY = 0; | |
| for (let y = 0; y < displayH; y++) { | |
| for (let x = 0; x < displayW; x++) { | |
| const mx = Math.floor((x / displayW) * mask.width); | |
| const my = Math.floor((y / displayH) * mask.height); | |
| if (mask.data[my * mask.width + mx] > 128) { | |
| const idx = (y * displayW + x) * 4; | |
| imgData.data[idx] = r; | |
| imgData.data[idx + 1] = g; | |
| imgData.data[idx + 2] = b; | |
| imgData.data[idx + 3] = 80; | |
| if (x < bMinX) bMinX = x; | |
| if (x > bMaxX) bMaxX = x; | |
| if (y < bMinY) bMinY = y; | |
| if (y > bMaxY) bMaxY = y; | |
| } | |
| } | |
| } | |
| if (bMaxX > bMinX && bMaxY > bMinY) { | |
| boxes.push({ minX: bMinX, minY: bMinY, maxX: bMaxX, maxY: bMaxY, ci: mi, score: scores[mi] || 0 }); | |
| } | |
| } | |
| ctx.putImageData(imgData, 0, 0); | |
| // Draw bounding boxes and labels on top | |
| const lw = Math.max(3, Math.round(displayW / 200)); | |
| const fontSize = Math.max(14, Math.round(displayW / 35)); | |
| ctx.font = `bold ${fontSize}px sans-serif`; | |
| for (const box of boxes) { | |
| const [r, g, b] = COLORS[box.ci % COLORS.length]; | |
| ctx.strokeStyle = `rgb(${r},${g},${b})`; | |
| ctx.lineWidth = lw; | |
| ctx.strokeRect(box.minX, box.minY, box.maxX - box.minX, box.maxY - box.minY); | |
| const label = `#${box.ci + 1} ${Math.round(box.score * 100)}%`; | |
| const tm = ctx.measureText(label); | |
| const labelH = fontSize + 6; | |
| const labelY = box.minY > labelH + 4 ? box.minY - labelH - 2 : box.minY + 2; | |
| ctx.fillStyle = `rgb(${r},${g},${b})`; | |
| ctx.fillRect(box.minX, labelY, tm.width + 12, labelH); | |
| ctx.fillStyle = '#fff'; | |
| ctx.fillText(label, box.minX + 6, labelY + fontSize - 2); | |
| } | |
| // Download button | |
| const dlBtn = document.createElement('button'); | |
| dlBtn.className = 'segmentation-download'; | |
| dlBtn.textContent = '\u2913 Download segmented image'; | |
| dlBtn.addEventListener('click', () => { | |
| const exportCanvas = document.createElement('canvas'); | |
| exportCanvas.width = displayW; | |
| exportCanvas.height = displayH; | |
| const ectx = exportCanvas.getContext('2d'); | |
| ectx.drawImage(img, 0, 0); | |
| ectx.drawImage(canvas, 0, 0); | |
| exportCanvas.toBlob((blob) => { | |
| const url = URL.createObjectURL(blob); | |
| const a = document.createElement('a'); | |
| a.href = url; | |
| a.download = 'segmented-' + new Date().toISOString().slice(0, 19).replace(/:/g, '') + '.png'; | |
| a.click(); | |
| URL.revokeObjectURL(url); | |
| }, 'image/png'); | |
| }); | |
| container.appendChild(dlBtn); | |
| chatArea.scrollTop = chatArea.scrollHeight; | |
| }; | |
| bubble.appendChild(container); | |
| } | |
| // ββ Multi-step planner ββββββββββββββββββββββββββββββββββββββ | |
| // parsePlan is tolerant: picks up any line that starts with "N." or "N)" | |
| // or "- N." and ignores everything else (prefaces, trailing commentary). | |
| function parsePlan(text) { | |
| const steps = []; | |
| const lines = String(text || '').split(/\r?\n/); | |
| for (const line of lines) { | |
| const m = line.match(/^[\s\-*]*(\d+)[.):\]]\s+(.+?)\s*$/); | |
| if (m && m[2]) steps.push(m[2].trim()); | |
| } | |
| return steps; | |
| } | |
| // Append a collapsible block to the msg container (parent of bubble, so | |
| // bubble.innerHTML updates from streaming tokens don't wipe it). | |
| function renderPlannerBlock(container, title, body) { | |
| const id = 'pb-' + Math.random().toString(36).slice(2, 8); | |
| const block = document.createElement('div'); | |
| block.className = 'planner-block'; | |
| const safeBody = escapeHtml(body || '(no output)'); | |
| block.innerHTML = | |
| '<div class="planner-toggle" onclick="document.getElementById(\'' + id + | |
| '\').classList.toggle(\'open\'); this.querySelector(\'span\').textContent = ' + | |
| 'document.getElementById(\'' + id + '\').classList.contains(\'open\') ? \'\\u25bc\' : \'\\u25b6\'">' + | |
| '<span>\u25b6</span> <strong>' + escapeHtml(title) + '</strong></div>' + | |
| '<div class="planner-body" id="' + id + '">' + safeBody + '</div>'; | |
| container.appendChild(block); | |
| chatArea.scrollTop = chatArea.scrollHeight; | |
| } | |
| // Drive one runtime.chat() call, collect text + any tool_calls. Pure β | |
| // does no tool execution. Used by the planner for plan/synthesis passes | |
| // and as the inner loop of runStepWithTools. | |
| async function runPlannerChatTurn(messages, thinking, tools, genConfig) { | |
| let fullText = ''; | |
| const toolCalls = []; | |
| let error = null; | |
| try { | |
| for await (const ev of LocalMind.runtime.chat({ | |
| messages, | |
| tools: tools || undefined, | |
| enableThinking: thinking, | |
| maxTokens: genConfig && genConfig.max_new_tokens, | |
| temperature: genConfig && genConfig.temperature, | |
| })) { | |
| if (ev.type === 'token') fullText += ev.text; | |
| else if (ev.type === 'tool_call') toolCalls.push(ev); | |
| else if (ev.type === 'done') break; | |
| else if (ev.type === 'error') { error = ev.error; break; } | |
| } | |
| } catch (e) { error = e; } | |
| return { fullText, toolCalls, error }; | |
| } | |
| // Run a planner step: up to 2 tool iterations, returns the text of the | |
| // final (non-tool) assistant response along with collected sources and | |
| // a flag for whether any tool was actually called. | |
| async function runPlannerStepWithTools(stepMessages, enableThinking, tools, genConfig, lastImageBlob, userQuery) { | |
| const STEP_MAX_ITER = 2; | |
| let loopMessages = [...stepMessages]; | |
| let finalText = ''; | |
| let usedTools = false; | |
| const sources = []; | |
| for (let iter = 0; iter < STEP_MAX_ITER; iter++) { | |
| if (!generating) return { fullText: finalText, usedTools, sources }; | |
| const turn = await runPlannerChatTurn(loopMessages, iter === 0 ? enableThinking : false, tools, genConfig); | |
| if (turn.error) return { fullText: finalText || stripToolCallTags(turn.fullText) || '', usedTools, sources }; | |
| finalText = stripToolCallTags(turn.fullText); | |
| if (turn.toolCalls.length === 0) return { fullText: finalText, usedTools, sources }; | |
| // Execute ALL tool calls emitted this turn, not just the first. | |
| const stepResults = []; | |
| for (const tc of turn.toolCalls) { | |
| const tool = TOOL_REGISTRY[tc.name]; | |
| let toolResult; | |
| if (!tool) { | |
| toolResult = { error: 'Unknown tool: ' + tc.name }; | |
| } else { | |
| try { | |
| toolResult = await tool.execute(tc.arguments, { userQuery, imageBlob: lastImageBlob }); | |
| } catch (e) { | |
| toolResult = { error: e.message }; | |
| } | |
| } | |
| usedTools = true; | |
| if (tc.name === 'web_search' && toolResult && toolResult.results) { | |
| sources.push(...toolResult.results); | |
| } | |
| stepResults.push({ tc, toolResult }); | |
| } | |
| loopMessages.push({ role: 'assistant', content: turn.fullText }); | |
| for (const { tc, toolResult } of stepResults) { | |
| loopMessages.push({ role: 'tool', name: tc.name, content: JSON.stringify(toolResult) }); | |
| } | |
| if (iter === STEP_MAX_ITER - 1) { | |
| loopMessages.push({ role: 'user', content: '[System: Tool limit reached β give final answer for this step now.]' }); | |
| const finalTurn = await runPlannerChatTurn(loopMessages, false, null, genConfig); | |
| finalText = stripToolCallTags(finalTurn.fullText) || finalText; | |
| } | |
| } | |
| return { fullText: finalText, usedTools, sources }; | |
| } | |
| // Plan β execute per step β synthesize. Returns { handled, fullText, | |
| // usedTools, sources }. handled=false means the caller should fall back | |
| // to the single-pass agent loop (bad plan, plan phase errored, etc.). | |
| async function runPlannerAgent(opts) { | |
| const { text, chatMessages, tools, genConfig, enableThinking, msgEl, bubble, lastImageBlob, searchSources } = opts; | |
| // Suppress tokenβbubble streaming during planner phases so the | |
| // planner-block DOM we render to msgEl isn't wiped by each token | |
| // arrival. Restore savedEl for the synthesis phase to stream | |
| // normally into the bubble. | |
| const savedEl = currentAssistantEl; | |
| currentAssistantEl = null; | |
| bubble.innerHTML = '<div class="planner-status">Planning\u2026</div>'; | |
| const plannerSystem = | |
| 'You are a task planner. Given the user task, break it into 2 to 5 sequential steps.\n' + | |
| 'Format: a numbered list, one step per line, each step written as a short imperative ' + | |
| '("Find X", "Compare Y and Z", "Summarize results").\n' + | |
| 'Do NOT explain or add commentary. Do NOT perform the task. Only list the steps.'; | |
| const planTurn = await runPlannerChatTurn( | |
| [{ role: 'system', content: plannerSystem }, { role: 'user', content: text }], | |
| false, null, genConfig, | |
| ); | |
| if (!generating) { currentAssistantEl = savedEl; return { handled: true, fullText: null }; } | |
| if (planTurn.error) { currentAssistantEl = savedEl; return { handled: false }; } | |
| const steps = parsePlan(planTurn.fullText); | |
| if (steps.length < 2) { | |
| // Bad / too-short plan β fall back to single-pass | |
| currentAssistantEl = savedEl; | |
| return { handled: false }; | |
| } | |
| renderPlannerBlock(msgEl, 'Plan (' + steps.length + ' steps)', | |
| steps.map((s, i) => (i + 1) + '. ' + s).join('\n')); | |
| const stepOutputs = []; | |
| const allSources = [...(searchSources || [])]; | |
| let anyToolsUsed = false; | |
| for (let i = 0; i < steps.length; i++) { | |
| if (!generating) { currentAssistantEl = savedEl; return { handled: true, fullText: null }; } | |
| bubble.innerHTML = '<div class="planner-status">Step ' + (i + 1) + ' / ' + steps.length + ': ' + escapeHtml(steps[i]) + '</div>'; | |
| const stepMessages = [...chatMessages]; | |
| const priorBlock = stepOutputs.length | |
| ? '[Prior step outputs]\n' + stepOutputs.map((o, j) => 'Step ' + (j + 1) + ' result: ' + o).join('\n\n') + '\n\n' | |
| : ''; | |
| stepMessages.push({ role: 'user', content: priorBlock + '[Current step]\n' + steps[i] }); | |
| const result = await runPlannerStepWithTools(stepMessages, enableThinking, tools, genConfig, lastImageBlob, text); | |
| anyToolsUsed = anyToolsUsed || result.usedTools; | |
| if (result.sources && result.sources.length) allSources.push(...result.sources); | |
| stepOutputs.push(result.fullText || ''); | |
| renderPlannerBlock(msgEl, 'Step ' + (i + 1) + ': ' + steps[i], result.fullText || '(no output)'); | |
| } | |
| if (!generating) { currentAssistantEl = savedEl; return { handled: true, fullText: null }; } | |
| // Synthesis β restore bubble so tokens stream into it normally. | |
| currentAssistantEl = savedEl; | |
| currentAssistantText = ''; | |
| bubble.innerHTML = ''; | |
| const synthSystem = | |
| 'You just completed a multi-step task. Using the step outputs provided, write a direct final ' + | |
| 'answer for the user. Do not repeat the steps β deliver the answer. Cite sources by number if present.'; | |
| const synthMessages = [ | |
| { role: 'system', content: synthSystem }, | |
| { role: 'user', content: text }, | |
| { role: 'assistant', content: 'Step outputs:\n\n' + stepOutputs.map((o, i) => 'Step ' + (i + 1) + ' (' + steps[i] + '): ' + (o || '(none)')).join('\n\n') }, | |
| { role: 'user', content: 'Now give me the final answer.' }, | |
| ]; | |
| const synthTurn = await runPlannerChatTurn(synthMessages, false, null, genConfig); | |
| return { | |
| handled: true, | |
| fullText: synthTurn.fullText || '', | |
| usedTools: anyToolsUsed, | |
| sources: allSources, | |
| }; | |
| } | |
| async function sendMessage() { | |
| const text = chatInput.value.trim(); | |
| if ((!text && attachments.length === 0) || !modelReady || generating) return; | |
| const m = MODELS[activeModelKey]; | |
| const caps = LocalMind.runtime.capabilities(); | |
| const supportsMedia = !!(caps && (caps.image || caps.audio || caps.video)); | |
| const isMultimodal = supportsMedia && attachments.length > 0; | |
| const hasAttachments = attachments.length > 0; | |
| const isAgent = !!(caps && caps.toolCalling); | |
| // Snapshot the attachment Blobs into a content-block array with | |
| // real data β this is what runtime.chat() will translate into | |
| // worker-ready inputs. The stored chat-history message stays as | |
| // text-only ('(attachments only)') because Blobs can't be | |
| // serialised to sessionStorage anyway, and the chat history | |
| // display only renders attachments via the input bar at send | |
| // time, not from the messages array. | |
| let runtimeUserContent = null; | |
| if (isMultimodal) { | |
| const blocks = []; | |
| for (const att of attachments) { | |
| if (att.type === 'audio' && att.pcmData) { | |
| // Pre-decoded mic recording β wrap as a Blob so the adapter | |
| // re-decoder gets consistent input. The decoder will redo | |
| // the work; we could pass pcmData directly via a custom | |
| // block shape, but Blob keeps the contract simple. | |
| blocks.push({ type: 'audio', data: att.blob }); | |
| } else if (att.type === 'image' || att.type === 'audio' || att.type === 'video') { | |
| blocks.push({ type: att.type, data: att.blob }); | |
| } | |
| } | |
| if (text) blocks.push({ type: 'text', text }); | |
| runtimeUserContent = blocks; | |
| } | |
| // Build user message content for storage / chat history. Stored | |
| // shape is text-only β the data-bearing version above is used | |
| // only for the runtime.chat call. | |
| const userContent = text || (hasAttachments ? '(attachments only)' : ''); | |
| messages.push({ role: 'user', content: userContent }); | |
| addUserMessage(text || '(attachments)', hasAttachments); | |
| saveChat(); | |
| chatInput.value = ''; | |
| chatInput.style.height = 'auto'; | |
| // Build system prompt. The tool-calling scaffolding is added | |
| // by runtime.chat() when we pass it a `tools` array β we just | |
| // build the user-facing portion (custom prompt + optional RAG | |
| // + optional web search results) here. | |
| const userSysPrompt = systemPrompt.value.trim(); | |
| let sysPrompt = userSysPrompt; | |
| // Auto-inject RAG context if embedding model is ready and agent-capable | |
| if (isAgent && LocalMind.runtime.embeddingsReady) { | |
| try { | |
| const userText = typeof userContent === 'string' ? userContent : text; | |
| if (userText) { | |
| const ragResults = await searchMemory(userText, 3); | |
| if (ragResults.length > 0) { | |
| const ragBlock = ragResults.map(r => `[${r.category}] ${r.text}`).join('\n\n'); | |
| sysPrompt += `\n\n[Retrieved from memory β use if relevant]\n${ragBlock}`; | |
| } | |
| } | |
| } catch (e) { | |
| console.warn('RAG retrieval failed:', e); | |
| } | |
| } | |
| // Web-enriched mode: pre-search and inject results into context | |
| const isWebEnriched = webEnrichedMode && isAgent && isSearchConfigured(); | |
| let searchSources = []; | |
| webEnrichedMode = false; // reset flag | |
| if (isWebEnriched) { | |
| try { | |
| const userText = typeof userContent === 'string' ? userContent : text; | |
| const results = await executeWebSearch(userText); | |
| if (results.length > 0) { | |
| searchSources = results; | |
| const searchBlock = results.map((r, i) => `[${i + 1}] ${r.title}\n${r.snippet}\nURL: ${r.url}`).join('\n\n'); | |
| sysPrompt += `\n\n[Web search results for "${userText}" β cite sources by number]\n${searchBlock}`; | |
| // Cache in RAG | |
| if (LocalMind.runtime.embeddingsReady) { | |
| const snippetText = results.map(r => `${r.title}: ${r.snippet}`).join('\n'); | |
| embedAndStore(snippetText, 'finding', 'web-search').catch(() => {}); | |
| } | |
| } | |
| } catch (e) { | |
| console.warn('Web search failed:', e); | |
| showToast(e.message || 'Web search failed'); | |
| } | |
| } | |
| // Build context-windowed chat history | |
| const chatMessages = buildContextMessages(messages, sysPrompt, activeModelKey); | |
| // For multimodal sends, replace the most-recent user message in | |
| // chatMessages with the data-bearing content array we built | |
| // above. The runtime adapter will detect the media blocks and | |
| // process them. After the first agent-loop iteration, we strip | |
| // the data field (see scrubMediaForFollowup below) so the chat | |
| // template still emits position tokens but the adapter doesn't | |
| // re-decode the same media on every pass. | |
| if (runtimeUserContent) { | |
| for (let i = chatMessages.length - 1; i >= 0; i--) { | |
| if (chatMessages[i].role === 'user') { | |
| chatMessages[i] = { role: 'user', content: runtimeUserContent }; | |
| break; | |
| } | |
| } | |
| } | |
| // UI: enter generating state | |
| currentAssistantEl = addAssistantPlaceholder(); | |
| currentAssistantText = ''; | |
| generating = true; | |
| sendBtn.innerHTML = '■'; | |
| sendBtn.classList.add('stop'); | |
| chatInput.disabled = true; | |
| modelSelect.disabled = true; | |
| const genConfig = m.genConfig; | |
| const enableThinking = thinkingToggle.checked; | |
| // Snapshot last image blob for segment_image tool before clearing | |
| const lastImageBlob = runtimeUserContent | |
| ? runtimeUserContent.filter(b => b.type === 'image').map(b => b.data).pop() | |
| : null; | |
| // Clear attachments after capturing data | |
| clearAttachments(); | |
| if (!isAgent) { | |
| // Non-agent path: single generation, no tool calling. | |
| // Routed through LocalMind.runtime.chat β the legacy bubble | |
| // updater (attachWorkerHandlers token handler) still paints | |
| // currentAssistantEl as tokens arrive; we consume the iterator | |
| // purely to collect the final text and the done signal. | |
| let fullText = ''; | |
| let chatErr = null; | |
| try { | |
| for await (const ev of LocalMind.runtime.chat({ | |
| messages: chatMessages, | |
| enableThinking, | |
| maxTokens: genConfig && genConfig.max_new_tokens, | |
| temperature: genConfig && genConfig.temperature, | |
| })) { | |
| if (ev.type === 'token') { | |
| fullText += ev.text; | |
| } else if (ev.type === 'done') { | |
| break; | |
| } else if (ev.type === 'error') { | |
| chatErr = ev.error; | |
| break; | |
| } | |
| } | |
| } catch (e) { | |
| chatErr = e; | |
| } | |
| if (chatErr) { | |
| fullText = (fullText || '') + `\n\n*Error: ${chatErr.message || chatErr}*`; | |
| } | |
| finishGeneration(fullText || null, searchSources); | |
| return; | |
| } | |
| // ββ Multi-step planner (opt-in, agent-capable only) ββββββ | |
| // Returns handled=false on a malformed/short plan β in that case we | |
| // fall through to the single-pass agent loop below. | |
| if (plannerEnabled) { | |
| const plannerTools = buildToolsForRuntime(); | |
| const msgElForPlanner = currentAssistantEl.bubble.parentElement; | |
| const result = await runPlannerAgent({ | |
| text, | |
| chatMessages, | |
| tools: plannerTools, | |
| genConfig, | |
| enableThinking, | |
| msgEl: msgElForPlanner, | |
| bubble: currentAssistantEl.bubble, | |
| lastImageBlob, | |
| searchSources, | |
| }); | |
| if (result.handled) { | |
| finishGeneration(result.fullText || null, result.sources || searchSources, result.usedTools); | |
| return; | |
| } | |
| // Fall through to single-pass agent loop on bad plan | |
| } | |
| // ββ Agentic loop ββββββββββββββββββββββββββββββββββββββββββ | |
| // The agent loop is now runtime-agnostic: it calls | |
| // LocalMind.runtime.chat() with the tool list, consumes | |
| // {token, tool_call, done} events from the iterator, executes | |
| // each tool through TOOL_REGISTRY, and feeds results back as | |
| // tool-role messages on the next iteration. | |
| const MAX_TOOL_ITERATIONS = 3; | |
| let loopMessages = [...chatMessages]; | |
| const allSources = [...searchSources]; | |
| let toolsUsedInLoop = false; | |
| let pendingSegOverlay = null; | |
| const tools = buildToolsForRuntime(); | |
| // After iter 0, strip the `data` field from media blocks in any | |
| // user message β the chat template still emits position tokens | |
| // for the placeholder shape, but the adapter won't re-decode the | |
| // same audio/video on every iteration (and the worker receives | |
| // null images/audio, matching the pre-refactor iter>0 behaviour). | |
| function scrubMediaForFollowup(msgs) { | |
| return msgs.map(msg => { | |
| if (!Array.isArray(msg.content)) return msg; | |
| return { | |
| ...msg, | |
| content: msg.content.map(b => { | |
| if (b && (b.type === 'image' || b.type === 'audio' || b.type === 'video')) { | |
| return { type: b.type }; | |
| } | |
| return b; | |
| }), | |
| }; | |
| }); | |
| } | |
| // Helper: drive one chat() call to completion, collecting text | |
| // and any tool_call events from the iterator. Returns | |
| // { fullText, toolCalls, doneReason, error }. | |
| async function runOneAgentTurn(turnMessages, turnThinking) { | |
| let fullText = ''; | |
| const toolCalls = []; | |
| let doneReason = null; | |
| let error = null; | |
| try { | |
| for await (const ev of LocalMind.runtime.chat({ | |
| messages: turnMessages, | |
| tools, | |
| enableThinking: turnThinking, | |
| maxTokens: genConfig && genConfig.max_new_tokens, | |
| temperature: genConfig && genConfig.temperature, | |
| })) { | |
| if (ev.type === 'token') { | |
| fullText += ev.text; | |
| } else if (ev.type === 'tool_call') { | |
| toolCalls.push(ev); | |
| } else if (ev.type === 'done') { | |
| doneReason = ev.reason; | |
| break; | |
| } else if (ev.type === 'error') { | |
| error = ev.error; | |
| break; | |
| } | |
| } | |
| } catch (e) { | |
| error = e; | |
| } | |
| return { fullText, toolCalls, doneReason, error }; | |
| } | |
| for (let iter = 0; iter < MAX_TOOL_ITERATIONS; iter++) { | |
| // Disable thinking on follow-up iterations to avoid tool | |
| // calls inside thinking blocks. | |
| const iterThinking = iter === 0 ? enableThinking : false; | |
| const turn = await runOneAgentTurn(loopMessages, iterThinking); | |
| // After the first iteration, scrub data from media blocks so | |
| // subsequent passes carry the chat-template position tokens | |
| // forward without re-decoding the same audio/video. | |
| if (iter === 0) loopMessages = scrubMediaForFollowup(loopMessages); | |
| function finishWithOverlay(text, srcs, tools) { | |
| // Capture the message container (parent of bubble) before finishGeneration nulls currentAssistantEl.next | |
| // Append overlay to the msg container, NOT inside the bubble β innerHTML updates during | |
| // streaming destroy bubble children, but the msg container is stable. | |
| const msgEl = currentAssistantEl ? currentAssistantEl.bubble.parentElement : null; | |
| finishGeneration(text, srcs, tools); | |
| if (pendingSegOverlay && msgEl) { | |
| // Short delay to ensure all pending DOM updates (streaming, badges) have completed | |
| setTimeout(() => { | |
| renderSegmentationOverlay(msgEl, pendingSegOverlay.imageBlob, pendingSegOverlay.masks, pendingSegOverlay.scores); | |
| chatArea.scrollTop = chatArea.scrollHeight; | |
| }, 50); | |
| } | |
| } | |
| // User clicked stop while we were generating | |
| if (!generating) { | |
| finishWithOverlay(turn.fullText || null, allSources, toolsUsedInLoop); | |
| return; | |
| } | |
| if (turn.error) { | |
| finishWithOverlay(stripToolCallTags(turn.fullText || '') + `\n\n*Error: ${turn.error.message || turn.error}*`, allSources, toolsUsedInLoop); | |
| return; | |
| } | |
| if (turn.toolCalls.length === 0) { | |
| // No tool calls β we're done | |
| finishWithOverlay(turn.fullText, allSources, toolsUsedInLoop); | |
| return; | |
| } | |
| // Clean the raw <tool_call> tags out of the live bubble display so | |
| // users see the model's prose (if any), not its raw call syntax. | |
| const cleanedTurnText = stripToolCallTags(turn.fullText); | |
| currentAssistantText = cleanedTurnText; | |
| if (currentAssistantEl) updateAssistantBubble(currentAssistantEl.bubble, cleanedTurnText); | |
| // Execute ALL tool calls the model emitted this turn (Qwen3-family | |
| // models including Ternary Bonsai often batch multiple calls into | |
| // one response). Blocks are appended to a tool-trace container | |
| // inserted BEFORE the bubble β collapsed by default, expanded via | |
| // a toggle added in finishGeneration. Container is a sibling of | |
| // the bubble so streaming updates to bubble.innerHTML don't wipe it. | |
| toolsUsedInLoop = true; | |
| const msgContainer = currentAssistantEl ? currentAssistantEl.bubble.parentElement : null; | |
| let traceContainer = msgContainer && msgContainer.querySelector('.tool-trace-container'); | |
| if (msgContainer && !traceContainer) { | |
| traceContainer = document.createElement('div'); | |
| traceContainer.className = 'tool-trace-container collapsed'; | |
| msgContainer.insertBefore(traceContainer, currentAssistantEl.bubble); | |
| } | |
| const toolResults = []; | |
| for (let tcIdx = 0; tcIdx < turn.toolCalls.length; tcIdx++) { | |
| const tc = turn.toolCalls[tcIdx]; | |
| const tool = TOOL_REGISTRY[tc.name]; | |
| let toolResult; | |
| if (!tool) { | |
| toolResult = { error: `Unknown tool: ${tc.name}` }; | |
| } else { | |
| try { | |
| toolResult = await tool.execute(tc.arguments, { userQuery: text, imageBlob: lastImageBlob }); | |
| } catch (e) { | |
| toolResult = { error: e.message }; | |
| } | |
| } | |
| if (tc.name === 'web_search' && toolResult && toolResult.results) { | |
| allSources.push(...toolResult.results); | |
| } | |
| if (traceContainer) { | |
| const stepLabel = document.createElement('div'); | |
| stepLabel.style.cssText = 'font-size:0.68rem;color:var(--gray-400);margin:6px 0 2px;font-weight:500'; | |
| const stepSuffix = turn.toolCalls.length > 1 ? ` \u00B7 Call ${tcIdx + 1}/${turn.toolCalls.length}` : ''; | |
| stepLabel.textContent = `Step ${iter + 1}/${MAX_TOOL_ITERATIONS}${stepSuffix}`; | |
| traceContainer.appendChild(stepLabel); | |
| renderToolCallBlock(traceContainer, tc.name, tc.arguments, toolResult); | |
| } | |
| if (tc.name === 'segment_image' && toolResult._rawMasks) { | |
| pendingSegOverlay = { imageBlob: toolResult._imageBlob, masks: toolResult._rawMasks, scores: toolResult._scores }; | |
| delete toolResult._rawMasks; | |
| delete toolResult._imageBlob; | |
| delete toolResult._scores; | |
| } | |
| toolResults.push({ tc, toolResult }); | |
| } | |
| // Append the assistant turn once (with raw tags so the model sees | |
| // its own format), then all tool-role results in order. | |
| loopMessages.push({ role: 'assistant', content: turn.fullText }); | |
| for (const { tc, toolResult } of toolResults) { | |
| loopMessages.push({ role: 'tool', name: tc.name, content: JSON.stringify(toolResult) }); | |
| } | |
| // Reset streaming state for next generation pass | |
| currentAssistantText = ''; | |
| // If this is the last iteration, force a final response | |
| if (iter === MAX_TOOL_ITERATIONS - 1) { | |
| loopMessages.push({ role: 'user', content: '[System: Tool call limit reached. Please provide your final answer now.]' }); | |
| const finalTurn = await runOneAgentTurn(loopMessages, enableThinking); | |
| finishWithOverlay(stripToolCallTags(finalTurn.fullText), allSources, toolsUsedInLoop); | |
| return; | |
| } | |
| } | |
| } | |
| function finishGeneration(response, sources, usedTools) { | |
| generating = false; | |
| if (response) { | |
| messages.push({ role: 'assistant', content: response }); | |
| saveChat(); | |
| } | |
| // Force-collapse any open thinking blocks (they expand during streaming) | |
| if (currentAssistantEl) { | |
| const bubble = currentAssistantEl.bubble; | |
| bubble.querySelectorAll('.thinking-content.open').forEach(el => { | |
| el.classList.remove('open'); | |
| const toggle = el.previousElementSibling; | |
| if (toggle) { | |
| const arrow = toggle.querySelector('span'); | |
| if (arrow) arrow.innerHTML = '▶'; | |
| toggle.childNodes.forEach(n => { if (n.nodeType === 3 && n.textContent.includes('Thinking')) n.textContent = ' Thought process'; }); | |
| } | |
| }); | |
| } | |
| // Transparency: add source badge and links to the assistant bubble | |
| if (currentAssistantEl) { | |
| const bubble = currentAssistantEl.bubble; | |
| const badge = document.createElement('div'); | |
| if (sources && sources.length > 0) { | |
| badge.innerHTML = `<span class="msg-source-badge web-enriched">Web-enriched · ${sources.length} sources</span>`; | |
| const linksDiv = document.createElement('div'); | |
| linksDiv.className = 'source-links'; | |
| linksDiv.innerHTML = sources.map(s => `<a href="${escapeHtml(s.url)}" target="_blank" rel="noopener">${escapeHtml(s.title || s.url)}</a>`).join(''); | |
| bubble.appendChild(linksDiv); | |
| } else if (usedTools) { | |
| badge.innerHTML = `<span class="msg-source-badge" style="background:rgba(102,126,234,0.1);color:var(--indigo-600)">Agent</span>`; | |
| } else { | |
| badge.innerHTML = `<span class="msg-source-badge on-device">On-device</span>`; | |
| } | |
| bubble.insertBefore(badge, bubble.firstChild); | |
| // Save as Markdown button | |
| if (response && response.length > 20) { | |
| const saveBtn = document.createElement('button'); | |
| saveBtn.className = 'save-md-btn'; | |
| saveBtn.textContent = dirHandle ? `Save to ${dirHandle.name}` : 'Save as MD'; | |
| saveBtn.addEventListener('click', async () => { | |
| const filename = `response-${new Date().toISOString().slice(0, 16).replace(/:/g, '')}.md`; | |
| if (dirHandle) { | |
| try { | |
| const fh = await dirHandle.getFileHandle(filename, { create: true }); | |
| const writable = await fh.createWritable(); | |
| await writable.write(response); | |
| await writable.close(); | |
| showToast(`Saved to ${dirHandle.name}/${filename}`); | |
| } catch (e) { | |
| showToast('Write failed: ' + e.message); | |
| } | |
| } else { | |
| const blob = new Blob([response], { type: 'text/markdown' }); | |
| const url = URL.createObjectURL(blob); | |
| const a = document.createElement('a'); | |
| a.href = url; a.download = filename; | |
| a.click(); | |
| URL.revokeObjectURL(url); | |
| } | |
| }); | |
| bubble.appendChild(saveBtn); | |
| } | |
| // Tool-trace toggle: if this msg has a collapsed trace container | |
| // above the bubble, expose a discreet button that expands it. | |
| const msgEl = bubble.parentElement; | |
| const traceContainer = msgEl && msgEl.querySelector('.tool-trace-container'); | |
| const callCount = traceContainer ? traceContainer.querySelectorAll('.tool-call-block').length : 0; | |
| if (callCount > 0) { | |
| const traceBtn = document.createElement('button'); | |
| traceBtn.className = 'save-md-btn'; | |
| traceBtn.style.marginLeft = '6px'; | |
| const label = () => (traceContainer.classList.contains('collapsed') ? '\u25B6' : '\u25BC') + | |
| ' ' + callCount + ' tool call' + (callCount > 1 ? 's' : ''); | |
| traceBtn.innerHTML = label(); | |
| traceBtn.addEventListener('click', () => { | |
| traceContainer.classList.toggle('collapsed'); | |
| traceBtn.innerHTML = label(); | |
| }); | |
| bubble.appendChild(traceBtn); | |
| } | |
| } | |
| currentAssistantEl = null; | |
| currentAssistantText = ''; | |
| sendBtn.innerHTML = '▶'; | |
| sendBtn.classList.remove('stop'); | |
| sendBtn.disabled = false; searchSendBtn.disabled = false; | |
| chatInput.disabled = false; | |
| modelSelect.disabled = false; | |
| chatInput.focus(); | |
| } | |
| sendBtn.addEventListener('click', () => { | |
| if (generating) { | |
| // Stop generation β interrupt worker, let complete handler resolve the promise. | |
| // Set generating=false so the agentic loop knows to exit after awaiting. | |
| worker.postMessage({ type: 'stop' }); | |
| generating = false; | |
| } else { | |
| sendMessage(); | |
| } | |
| }); | |
| chatInput.addEventListener('keydown', (e) => { | |
| if (e.key === 'Enter' && !e.shiftKey) { | |
| e.preventDefault(); | |
| if (!generating) sendMessage(); | |
| } | |
| }); | |
| // ββ Clear chat (delete, no archive) ββββββββββββββββββββββββ | |
| clearBtn.addEventListener('click', () => { | |
| if (generating) return; | |
| if (messages.length > 0) showToast('Chat cleared (not saved to History)'); | |
| resetChatUI(); | |
| }); | |
| // ββ Batch prompts ββββββββββββββββββββββββββββββββββββββββββββ | |
| const batchBtn = document.getElementById('batchBtn'); | |
| const batchPanel = document.getElementById('batchPanel'); | |
| const batchTextarea = document.getElementById('batchTextarea'); | |
| const batchCount = document.getElementById('batchCount'); | |
| const batchProgress = document.getElementById('batchProgress'); | |
| const batchRunBtn = document.getElementById('batchRunBtn'); | |
| const batchStopBtn = document.getElementById('batchStopBtn'); | |
| const batchChainToggle = document.getElementById('batchChainToggle'); | |
| let batchRunning = false; | |
| let batchShouldStop = false; | |
| batchBtn.addEventListener('click', () => { | |
| batchPanel.classList.toggle('open'); | |
| settingsPanel.classList.remove('open'); | |
| memoryPanel.classList.remove('open'); | |
| closeHistorySidebar(); | |
| }); | |
| function parseBatchPrompts() { | |
| return batchTextarea.value.split('\n').map(l => l.trim()).filter(Boolean); | |
| } | |
| function updateBatchCount() { | |
| const n = parseBatchPrompts().length; | |
| batchCount.textContent = `${n} prompt${n === 1 ? '' : 's'}`; | |
| batchRunBtn.disabled = n === 0 || !modelReady || batchRunning; | |
| } | |
| batchTextarea.addEventListener('input', updateBatchCount); | |
| batchStopBtn.addEventListener('click', () => { | |
| batchShouldStop = true; | |
| batchStopBtn.disabled = true; | |
| batchProgress.textContent = 'Stoppingβ¦'; | |
| }); | |
| batchRunBtn.addEventListener('click', async () => { | |
| if (!modelReady) { showToast('Model not ready yet'); return; } | |
| if (generating) { showToast('Already generating β wait for current response'); return; } | |
| const prompts = parseBatchPrompts(); | |
| if (!prompts.length) return; | |
| batchRunning = true; | |
| batchShouldStop = false; | |
| batchRunBtn.disabled = true; | |
| batchStopBtn.disabled = false; | |
| batchTextarea.disabled = true; | |
| let lastResponse = ''; | |
| let ranCount = 0; | |
| for (let i = 0; i < prompts.length; i++) { | |
| if (batchShouldStop) break; | |
| ranCount = i + 1; | |
| let prompt = prompts[i]; | |
| // {{previous}} substitution (always β user put it there explicitly) | |
| if (i > 0 && lastResponse) { | |
| prompt = prompt.replace(/\{\{previous\}\}/g, lastResponse); | |
| } | |
| // Auto-inject: if chain toggle is on and no explicit {{previous}}, append previous as context | |
| if (batchChainToggle.checked && i > 0 && lastResponse && !prompts[i].includes('{{previous}}')) { | |
| prompt = `${prompt}\n\n[Previous response for context:\n${lastResponse}\n]`; | |
| } | |
| batchProgress.textContent = `${i + 1} / ${prompts.length}`; | |
| chatInput.value = prompt; | |
| await sendMessage(); | |
| // Grab the last assistant turn | |
| const last = messages[messages.length - 1]; | |
| if (last?.role === 'assistant') { | |
| lastResponse = typeof last.content === 'string' ? last.content : ''; | |
| } | |
| } | |
| const wasStopped = batchShouldStop; | |
| batchRunning = false; | |
| batchShouldStop = false; | |
| batchRunBtn.disabled = false; | |
| batchStopBtn.disabled = true; | |
| batchTextarea.disabled = false; | |
| const ran = ranCount; | |
| batchProgress.textContent = wasStopped ? `Stopped at ${ran}/${prompts.length}.` : `Done (${prompts.length} prompts)`; | |
| showToast(wasStopped ? 'Batch stopped' : 'Batch complete'); | |
| }); | |
| // ββ Encrypted share links βββββββββββββββββββββββββββββββββββ | |
| // URL format: | |
| // #lm:<base64> plain | |
| // #lme:<b64salt>.<b64iv>.<b64ciphertext> AES-256-GCM + PBKDF2 | |
| function b64ToArr(b64) { | |
| return Uint8Array.from(atob(b64), c => c.charCodeAt(0)); | |
| } | |
| function arrToB64(arr) { | |
| return btoa(String.fromCharCode(...new Uint8Array(arr))); | |
| } | |
| async function deriveKey(passphrase, salt) { | |
| const enc = new TextEncoder(); | |
| const keyMat = await crypto.subtle.importKey( | |
| 'raw', enc.encode(passphrase), { name: 'PBKDF2' }, false, ['deriveKey'] | |
| ); | |
| return crypto.subtle.deriveKey( | |
| { name: 'PBKDF2', salt, hash: 'SHA-256', iterations: 200000 }, | |
| keyMat, { name: 'AES-GCM', length: 256 }, false, ['encrypt', 'decrypt'] | |
| ); | |
| } | |
| async function encryptPayload(json, passphrase) { | |
| const salt = crypto.getRandomValues(new Uint8Array(16)); | |
| const iv = crypto.getRandomValues(new Uint8Array(12)); | |
| const key = await deriveKey(passphrase, salt); | |
| const data = new TextEncoder().encode(json); | |
| const cipher = await crypto.subtle.encrypt({ name: 'AES-GCM', iv }, key, data); | |
| return `${arrToB64(salt)}.${arrToB64(iv)}.${arrToB64(cipher)}`; | |
| } | |
| async function decryptPayload(encoded, passphrase) { | |
| const parts = encoded.split('.'); | |
| if (parts.length !== 3) throw new Error('Invalid format'); | |
| const [saltB64, ivB64, cipherB64] = parts; | |
| const salt = b64ToArr(saltB64), iv = b64ToArr(ivB64), cipher = b64ToArr(cipherB64); | |
| const key = await deriveKey(passphrase, salt); | |
| const plain = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, key, cipher); | |
| return new TextDecoder().decode(plain); | |
| } | |
| function buildConversationPayload() { | |
| // Strip blobs; keep text content only | |
| const clean = messages.map(m => { | |
| if (typeof m.content === 'string') return { role: m.role, content: m.content }; | |
| if (Array.isArray(m.content)) { | |
| const text = m.content.filter(p => p.type === 'text').map(p => p.text).join('\n'); | |
| return { role: m.role, content: text }; | |
| } | |
| return { role: m.role, content: String(m.content || '') }; | |
| }).filter(m => m.content); | |
| return { v: 1, model: activeModelKey, created: new Date().toISOString(), messages: clean }; | |
| } | |
| // Share modal | |
| const shareBackdrop = document.getElementById('shareBackdrop'); | |
| const shareMetaText = document.getElementById('shareMetaText'); | |
| const sharePassphrase = document.getElementById('sharePassphrase'); | |
| const shareUsePassphrase = document.getElementById('shareUsePassphrase'); | |
| const shareUrlInput = document.getElementById('shareUrlInput'); | |
| const shareGenerateBtn = document.getElementById('shareGenerateBtn'); | |
| const shareCopyBtn = document.getElementById('shareCopyBtn'); | |
| const shareCloseBtn = document.getElementById('shareCloseBtn'); | |
| const shareBtn = document.getElementById('shareBtn'); | |
| shareBtn.addEventListener('click', () => { | |
| if (!messages.length) { showToast('Nothing to share yet'); return; } | |
| const userTurns = messages.filter(m => m.role === 'user').length; | |
| shareMetaText.textContent = `${messages.length} message(s) Β· ${userTurns} turn(s)`; | |
| shareUrlInput.value = ''; | |
| shareCopyBtn.disabled = true; | |
| shareUsePassphrase.checked = false; | |
| sharePassphrase.style.display = 'none'; | |
| sharePassphrase.value = ''; | |
| shareBackdrop.classList.add('open'); | |
| }); | |
| shareUsePassphrase.addEventListener('change', () => { | |
| sharePassphrase.style.display = shareUsePassphrase.checked ? '' : 'none'; | |
| if (shareUsePassphrase.checked) sharePassphrase.focus(); | |
| shareUrlInput.value = ''; | |
| shareCopyBtn.disabled = true; | |
| }); | |
| shareGenerateBtn.addEventListener('click', async () => { | |
| const usePass = shareUsePassphrase.checked; | |
| const pass = sharePassphrase.value.trim(); | |
| if (usePass && !pass) { showToast('Enter a passphrase first'); sharePassphrase.focus(); return; } | |
| shareGenerateBtn.disabled = true; | |
| shareGenerateBtn.textContent = 'Generatingβ¦'; | |
| try { | |
| const payload = buildConversationPayload(); | |
| const json = JSON.stringify(payload); | |
| let fragment; | |
| if (usePass) { | |
| const enc = await encryptPayload(json, pass); | |
| fragment = '#lme:' + enc; | |
| } else { | |
| fragment = '#lm:' + btoa(unescape(encodeURIComponent(json))); | |
| } | |
| const url = location.origin + location.pathname + fragment; | |
| shareUrlInput.value = url; | |
| shareCopyBtn.disabled = false; | |
| } catch (e) { | |
| showToast('Failed to generate link: ' + e.message); | |
| } finally { | |
| shareGenerateBtn.disabled = false; | |
| shareGenerateBtn.textContent = 'Generate'; | |
| } | |
| }); | |
| shareCopyBtn.addEventListener('click', async () => { | |
| try { | |
| await navigator.clipboard.writeText(shareUrlInput.value); | |
| shareCopyBtn.textContent = 'Copied!'; | |
| setTimeout(() => { shareCopyBtn.textContent = 'Copy Link'; }, 1800); | |
| } catch { | |
| shareUrlInput.select(); | |
| document.execCommand('copy'); | |
| } | |
| }); | |
| shareCloseBtn.addEventListener('click', () => shareBackdrop.classList.remove('open')); | |
| shareBackdrop.addEventListener('click', e => { if (e.target === shareBackdrop) shareBackdrop.classList.remove('open'); }); | |
| // Import banner (on load, detect share link in URL hash) | |
| const importBanner = document.getElementById('importBanner'); | |
| const importBannerMsg = document.getElementById('importBannerMsg'); | |
| const importPassphrase = document.getElementById('importPassphrase'); | |
| const importConfirmBtn = document.getElementById('importConfirmBtn'); | |
| const importDismissBtn = document.getElementById('importDismissBtn'); | |
| let _pendingSharePayload = null; // decoded plain payload | |
| let _pendingShareEncoded = null; // raw encoded string for encrypted links | |
| async function checkShareLink() { | |
| const hash = location.hash; | |
| if (!hash) return; | |
| if (hash.startsWith('#lm:')) { | |
| try { | |
| const json = decodeURIComponent(escape(atob(hash.slice(4)))); | |
| _pendingSharePayload = JSON.parse(json); | |
| importBannerMsg.textContent = `Shared conversation (${_pendingSharePayload.messages?.length ?? '?'} messages) β load it?`; | |
| importBanner.classList.add('open'); | |
| } catch (e) { console.warn('Bad share link', e); } | |
| } else if (hash.startsWith('#lme:')) { | |
| _pendingShareEncoded = hash.slice(5); | |
| importBannerMsg.textContent = 'Encrypted shared conversation β enter passphrase to load:'; | |
| importPassphrase.style.display = ''; | |
| importBanner.classList.add('open'); | |
| } | |
| } | |
| importConfirmBtn.addEventListener('click', async () => { | |
| try { | |
| let payload = _pendingSharePayload; | |
| if (!payload) { | |
| const pass = importPassphrase.value; | |
| if (!pass) { showToast('Enter the passphrase'); importPassphrase.focus(); return; } | |
| const json = await decryptPayload(_pendingShareEncoded, pass); | |
| payload = JSON.parse(json); | |
| } | |
| if (!payload?.messages?.length) { showToast('No messages found in link'); return; } | |
| // Load into current session | |
| resetChatUI(); | |
| for (const msg of payload.messages) { | |
| if (msg.role === 'user') { | |
| addUserMessage(msg.content, false); | |
| } else if (msg.role === 'assistant') { | |
| const { bubble } = addAssistantPlaceholder(); | |
| updateAssistantBubble(bubble, msg.content); | |
| } | |
| messages.push(msg); | |
| } | |
| saveChat(); | |
| importBanner.classList.remove('open'); | |
| history.replaceState(null, '', location.pathname); | |
| showToast(`Loaded ${payload.messages.length} messages`); | |
| } catch (e) { | |
| showToast('Could not load: ' + e.message); | |
| } | |
| }); | |
| importDismissBtn.addEventListener('click', () => { | |
| importBanner.classList.remove('open'); | |
| history.replaceState(null, '', location.pathname); | |
| }); | |
| checkShareLink(); | |
| // ββ Init ββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| renderRestoredMessages(); | |
| loadModel(modelSelect.value); | |
| </script> | |
| </body> | |
| </html> | |