step 2

scripts/migrate-to-postgres.js

@@ -1,573 +1,6 @@
-import fs from 'fs/promises';
-import path from 'path';
-import { createClient } from '@supabase/supabase-js';
-import { loadEncryptedJson } from '../server/cryptoUtils.js';
-import {
-  POSTGRES_STORAGE_DIR,
-  POSTGRES_STORAGE_MANIFEST,
-  DATA_ROOT,
-} from '../server/dataPaths.js';
-import {
-  createStandalonePostgresPool,
-  encryptJsonPayload,
-  makeLookupToken,
-  makeOwnerLookup,
-} from '../server/postgres.js';
-import { POSTGRES_SCHEMA_SQL } from '../server/postgresSchema.js';
-import { SUPABASE_URL } from '../server/config.js';
+import { runPostgresMigrationCli } from '../server/postgresMigration.js';
 
-const TEMP_TTL_MS = 24 * 60 * 60 * 1000;
-const TEMP_INACTIVITY = 12 * 60 * 60 * 1000;
-const FEEDBACK_AAD = 'feedback_tickets_v1';
-const VERSION_FILE = path.join(DATA_ROOT, 'version.json');
-const WEB_SEARCH_USAGE_FILE = path.join(DATA_ROOT, 'web-search-usage.json');
-const MEDIA_INDEX_FILE = path.join(DATA_ROOT, 'media', 'index.json');
-const MEDIA_BLOBS_DIR = path.join(DATA_ROOT, 'media', 'blobs');
-const TEMP_STORE_FILE = path.join(DATA_ROOT, 'temp_sessions.json');
-const MEMORIES_FILE = path.join(DATA_ROOT, 'memories', 'index.json');
-const DELETED_CHATS_FILE = path.join(DATA_ROOT, 'deleted_chats', 'index.json');
-const SYSTEM_PROMPTS_FILE = path.join(DATA_ROOT, 'system-prompts', 'index.json');
-const FEEDBACK_FILE = path.join(DATA_ROOT, 'feedback_tickets.json');
-const GUEST_REQUEST_FILE = path.join(DATA_ROOT, 'guest_request_counts.json');
-
-function nowIso() {
-  return new Date().toISOString();
-}
-
-function sessionAad(scopeType, sessionId) {
-  return `chat-session:${scopeType}:${sessionId}`;
-}
-
-function guestStateLookup(tempId) {
-  return makeLookupToken('guest-state', tempId);
-}
-
-function guestStateAad(tempId) {
-  return `guest-state:${tempId}`;
-}
-
-function guestExpiryRecord(tempData) {
-  const createdExpires = (tempData.created || Date.now()) + TEMP_TTL_MS;
-  const inactiveExpires = (tempData.lastActive || Date.now()) + TEMP_INACTIVITY;
-  return new Date(Math.min(createdExpires, inactiveExpires)).toISOString();
-}
-
-function shareTokenLookup(token) {
-  return makeLookupToken('session-share-token', token);
-}
-
-function shareAad(recordId) {
-  return `session-share:${recordId}`;
-}
-
-function promptLookup(userId) {
-  return makeLookupToken('system-prompt', userId);
-}
-
-function promptAad(userId) {
-  return `system-prompt:${userId}`;
-}
-
-function mediaEntryAad(id) {
-  return `media-entry:${id}`;
-}
-
-function feedbackAad(id) {
-  return `feedback:${id}`;
-}
-
-function usernameLookup(username) {
-  return makeLookupToken('username', username);
-}
-
-function versionLookup(publicUrl) {
-  return makeLookupToken('app-version', publicUrl);
-}
-
-function versionAad(publicUrl) {
-  return `app-version:${publicUrl}`;
-}
-
-function requestLookup(ip) {
-  return makeLookupToken('guest-request', ip);
-}
-
-function webUsageLookup(key) {
-  return makeLookupToken('web-search-usage', key);
-}
-
-function webUsageAad(key, day) {
-  return `web-search-usage:${key}:${day}`;
-}
-
-async function fileExists(filePath) {
-  try {
-    await fs.access(filePath);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-async function readJsonIfExists(filePath) {
-  try {
-    return JSON.parse(await fs.readFile(filePath, 'utf8'));
-  } catch {
-    return null;
-  }
-}
-
-async function fetchAllSupabaseRows(client, tableName) {
-  const pageSize = 1000;
-  const rows = [];
-  let from = 0;
-  while (true) {
-    const { data, error } = await client
-      .from(tableName)
-      .select('*')
-      .range(from, from + pageSize - 1);
-    if (error) throw error;
-    rows.push(...(data || []));
-    if (!data || data.length < pageSize) break;
-    from += pageSize;
-  }
-  return rows;
-}
-
-async function truncateTargetTables(pool) {
-  await pool.query(`
-    TRUNCATE TABLE
-      media_blobs,
-      media_entries,
-      deleted_chats,
-      memories,
-      system_prompts,
-      feedback_tickets,
-      guest_request_counters,
-      web_search_usage,
-      user_settings,
-      user_profiles,
-      device_sessions,
-      session_shares,
-      chat_sessions,
-      guest_state,
-      app_versions
-    RESTART IDENTITY
-    CASCADE
-  `);
-}
-
-async function migrateVersions(pool, report) {
-  const data = await readJsonIfExists(VERSION_FILE);
-  const entries = Array.isArray(data) ? data : [];
-  let count = 0;
-  for (const entry of entries) {
-    for (const [publicUrl, sha] of Object.entries(entry || {})) {
-      await pool.query(
-        `INSERT INTO app_versions (public_url_lookup, updated_at, payload)
-         VALUES ($1, $2, $3::jsonb)`,
-        [
-          versionLookup(publicUrl),
-          nowIso(),
-          JSON.stringify(encryptJsonPayload({ publicUrl, sha }, versionAad(publicUrl))),
-        ]
-      );
-      count += 1;
-    }
-  }
-  report.migrated.versionEntries = count;
-}
-
-async function migrateTempSessions(pool, report) {
-  const data = await loadEncryptedJson(TEMP_STORE_FILE);
-  const records = data || {};
-  let ownerCount = 0;
-  let sessionCount = 0;
-
-  for (const [tempId, tempData] of Object.entries(records)) {
-    ownerCount += 1;
-    const owner = { type: 'guest', id: tempId };
-    await pool.query(
-      `INSERT INTO guest_state (owner_lookup, expires_at, updated_at, payload)
-       VALUES ($1, $2, $3, $4::jsonb)`,
-      [
-        guestStateLookup(tempId),
-        guestExpiryRecord(tempData),
-        nowIso(),
-        JSON.stringify(encryptJsonPayload({
-          tempId,
-          msgCount: tempData.msgCount || 0,
-          created: tempData.created || Date.now(),
-          lastActive: tempData.lastActive || Date.now(),
-        }, guestStateAad(tempId))),
-      ]
-    );
-
-    for (const session of Object.values(tempData.sessions || {})) {
-      await pool.query(
-        `INSERT INTO chat_sessions (id, scope_type, owner_lookup, created_at, updated_at, expires_at, payload)
-         VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
-        [
-          session.id,
-          'guest',
-          makeOwnerLookup(owner),
-          new Date(session.created || Date.now()).toISOString(),
-          nowIso(),
-          guestExpiryRecord(tempData),
-          JSON.stringify(encryptJsonPayload(session, sessionAad('guest', session.id))),
-        ]
-      );
-      sessionCount += 1;
-    }
-  }
-
-  report.migrated.tempOwners = ownerCount;
-  report.migrated.tempSessions = sessionCount;
-}
-
-async function migrateMemories(pool, report) {
-  const data = await loadEncryptedJson(MEMORIES_FILE);
-  const memories = Object.values(data?.memories || {});
-  for (const memory of memories) {
-    await pool.query(
-      `INSERT INTO memories (id, owner_lookup, created_at, updated_at, payload)
-       VALUES ($1, $2, $3, $4, $5::jsonb)`,
-      [
-        memory.id,
-        makeOwnerLookup({ type: memory.ownerType, id: memory.ownerId }),
-        memory.createdAt,
-        memory.updatedAt,
-        JSON.stringify(encryptJsonPayload(memory, `memory:${memory.id}`)),
-      ]
-    );
-  }
-  report.migrated.memories = memories.length;
-}
-
-async function migrateDeletedChats(pool, report) {
-  const data = await loadEncryptedJson(DELETED_CHATS_FILE);
-  const deletedChats = Object.values(data?.deletedChats || {});
-  for (const record of deletedChats) {
-    await pool.query(
-      `INSERT INTO deleted_chats (id, owner_lookup, purge_at, deleted_at, payload)
-       VALUES ($1, $2, $3, $4, $5::jsonb)`,
-      [
-        record.id,
-        makeOwnerLookup({ type: record.ownerType, id: record.ownerId }),
-        record.purgeAt || null,
-        record.deletedAt,
-        JSON.stringify(encryptJsonPayload(record, `deleted-chat:${record.id}`)),
-      ]
-    );
-  }
-  report.migrated.deletedChats = deletedChats.length;
-}
-
-async function migrateSystemPrompts(pool, report) {
-  const data = await loadEncryptedJson(SYSTEM_PROMPTS_FILE, 'system-prompts');
-  const prompts = Object.entries(data?.prompts || {});
-  for (const [userId, prompt] of prompts) {
-    const record = {
-      userId,
-      markdown: prompt.markdown,
-      updatedAt: prompt.updatedAt || nowIso(),
-    };
-    await pool.query(
-      `INSERT INTO system_prompts (owner_lookup, updated_at, payload)
-       VALUES ($1, $2, $3::jsonb)`,
-      [
-        promptLookup(userId),
-        record.updatedAt,
-        JSON.stringify(encryptJsonPayload(record, promptAad(userId))),
-      ]
-    );
-  }
-  report.migrated.systemPrompts = prompts.length;
-}
-
-async function migrateFeedback(pool, report) {
-  const data = await loadEncryptedJson(FEEDBACK_FILE, FEEDBACK_AAD);
-  const tickets = Array.isArray(data?.tickets) ? data.tickets : [];
-  for (const ticket of tickets) {
-    await pool.query(
-      `INSERT INTO feedback_tickets (id, status, submitted_at, payload)
-       VALUES ($1, $2, $3, $4::jsonb)`,
-      [
-        ticket.id,
-        ticket.status || 'open',
-        ticket.submittedAt,
-        JSON.stringify(encryptJsonPayload(ticket, feedbackAad(ticket.id))),
-      ]
-    );
-  }
-  report.migrated.feedbackTickets = tickets.length;
-}
-
-async function migrateGuestRequestCounters(pool, report) {
-  const data = await loadEncryptedJson(GUEST_REQUEST_FILE);
-  const entries = Object.entries(data || {});
-  for (const [ip, entry] of entries) {
-    await pool.query(
-      `INSERT INTO guest_request_counters (key_lookup, expires_at, updated_at, payload)
-       VALUES ($1, $2, $3, $4::jsonb)`,
-      [
-        requestLookup(ip),
-        new Date(entry.resetAt || Date.now()).toISOString(),
-        nowIso(),
-        JSON.stringify(encryptJsonPayload({
-          ip,
-          count: entry.count || 0,
-          resetAt: entry.resetAt || Date.now(),
-        }, 'guest-request-row')),
-      ]
-    );
-  }
-  report.migrated.guestRequestCounters = entries.length;
-}
-
-async function migrateWebSearchUsage(pool, report) {
-  const data = await readJsonIfExists(WEB_SEARCH_USAGE_FILE);
-  const days = data?.days && typeof data.days === 'object' ? data.days : {};
-  let count = 0;
-  for (const [dayKey, keys] of Object.entries(days)) {
-    for (const [key, used] of Object.entries(keys || {})) {
-      await pool.query(
-        `INSERT INTO web_search_usage (key_lookup, day_key, updated_at, payload)
-         VALUES ($1, $2, $3, $4::jsonb)`,
-        [
-          webUsageLookup(key),
-          dayKey,
-          nowIso(),
-          JSON.stringify(encryptJsonPayload({ used }, webUsageAad(key, dayKey))),
-        ]
-      );
-      count += 1;
-    }
-  }
-  report.migrated.webSearchUsageRows = count;
-}
-
-async function migrateMedia(pool, report) {
-  const data = await loadEncryptedJson(MEDIA_INDEX_FILE);
-  const entries = Object.values(data?.entries || {});
-  let blobCount = 0;
-  for (const entry of entries) {
-    await pool.query(
-      `INSERT INTO media_entries (
-         id, owner_lookup, parent_id, entry_type, updated_at, created_at,
-         trashed_at, purge_at, expires_at, size_bytes, payload
-       )
-       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb)`,
-      [
-        entry.id,
-        makeOwnerLookup({ type: entry.ownerType, id: entry.ownerId }),
-        entry.parentId || null,
-        entry.type,
-        entry.updatedAt || entry.createdAt,
-        entry.createdAt,
-        entry.trashedAt || null,
-        entry.purgeAt || null,
-        entry.expiresAt || null,
-        entry.size || 0,
-        JSON.stringify(encryptJsonPayload(entry, mediaEntryAad(entry.id))),
-      ]
-    );
-
-    if (entry.type === 'file') {
-      const blobPath = path.join(MEDIA_BLOBS_DIR, `${entry.id}.bin`);
-      if (await fileExists(blobPath)) {
-        const blob = await fs.readFile(blobPath);
-        await pool.query(
-          `INSERT INTO media_blobs (entry_id, updated_at, payload)
-           VALUES ($1, $2, $3)`,
-          [entry.id, entry.updatedAt || entry.createdAt, blob]
-        );
-        blobCount += 1;
-      }
-    }
-  }
-  report.migrated.mediaEntries = entries.length;
-  report.migrated.mediaBlobs = blobCount;
-}
-
-async function migrateSupabaseData(pool, report) {
-  const serviceRoleKey = process.env.SUPABASE_SERVICE_ROLE_KEY || '';
-  if (!serviceRoleKey) {
-    if (process.env.ALLOW_PARTIAL_SQL_MIGRATION === '1') {
-      report.pending.push(
-        'Supabase-backed user tables were not exported because SUPABASE_SERVICE_ROLE_KEY is not set.'
-      );
-      return;
-    }
-    throw new Error(
-      'SUPABASE_SERVICE_ROLE_KEY is required to migrate existing Supabase-backed sessions, shares, settings, and profiles before enabling PostgreSQL-only mode. Set ALLOW_PARTIAL_SQL_MIGRATION=1 only if you intentionally want to skip those records.'
-    );
-  }
-
-  const supabase = createClient(SUPABASE_URL, serviceRoleKey, {
-    auth: { persistSession: false },
-  });
-
-  const [webSessions, sharedSessions, userSettings, profiles] = await Promise.all([
-    fetchAllSupabaseRows(supabase, 'web_sessions'),
-    fetchAllSupabaseRows(supabase, 'shared_sessions'),
-    fetchAllSupabaseRows(supabase, 'user_settings'),
-    fetchAllSupabaseRows(supabase, 'profiles'),
-  ]);
-
-  for (const row of webSessions) {
-    const session = {
-      id: row.id,
-      name: row.name,
-      created: new Date(row.created_at).getTime(),
-      history: row.history || [],
-      model: row.model || null,
-    };
-    await pool.query(
-      `INSERT INTO chat_sessions (id, scope_type, owner_lookup, created_at, updated_at, expires_at, payload)
-       VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
-      [
-        row.id,
-        'user',
-        makeOwnerLookup({ type: 'user', id: row.user_id }),
-        row.created_at,
-        row.updated_at || nowIso(),
-        null,
-        JSON.stringify(encryptJsonPayload(session, sessionAad('user', row.id))),
-      ]
-    );
-  }
-
-  for (const row of sharedSessions) {
-    const lookup = shareTokenLookup(row.token);
-    const id = row.id || `share_${lookup.slice(0, 24)}`;
-    await pool.query(
-      `INSERT INTO session_shares (id, token_lookup, owner_lookup, created_at, payload)
-       VALUES ($1, $2, $3, $4, $5::jsonb)`,
-      [
-        id,
-        lookup,
-        makeOwnerLookup({ type: 'user', id: row.owner_id }),
-        row.created_at || nowIso(),
-        JSON.stringify(encryptJsonPayload({
-          id,
-          ownerId: row.owner_id,
-          sessionSnapshot: row.session_snapshot,
-          createdAt: row.created_at || nowIso(),
-        }, shareAad(id))),
-      ]
-    );
-  }
-
-  for (const row of userSettings) {
-    await pool.query(
-      `INSERT INTO user_settings (owner_lookup, updated_at, payload)
-       VALUES ($1, $2, $3::jsonb)`,
-      [
-        makeOwnerLookup({ type: 'user', id: row.user_id }),
-        row.updated_at || nowIso(),
-        JSON.stringify(encryptJsonPayload({
-          userId: row.user_id,
-          settings: row.settings || {},
-          updatedAt: row.updated_at || nowIso(),
-        }, `user-settings:${row.user_id}`)),
-      ]
-    );
-  }
-
-  for (const row of profiles) {
-    if (!row.id || !row.username) continue;
-    await pool.query(
-      `INSERT INTO user_profiles (owner_lookup, username_lookup, updated_at, payload)
-       VALUES ($1, $2, $3, $4::jsonb)`,
-      [
-        makeOwnerLookup({ type: 'user', id: row.id }),
-        usernameLookup(row.username),
-        row.updated_at || nowIso(),
-        JSON.stringify(encryptJsonPayload({
-          userId: row.id,
-          username: row.username,
-          updatedAt: row.updated_at || nowIso(),
-        }, `user-profile:${row.id}`)),
-      ]
-    );
-  }
-
-  report.migrated.supabaseWebSessions = webSessions.length;
-  report.migrated.supabaseSharedSessions = sharedSessions.length;
-  report.migrated.supabaseUserSettings = userSettings.length;
-  report.migrated.supabaseProfiles = profiles.length;
-}
-
-async function writeStorageFolder(report) {
-  const tempDir = `${POSTGRES_STORAGE_DIR}.tmp`;
-  await fs.rm(tempDir, { recursive: true, force: true });
-  await fs.mkdir(tempDir, { recursive: true });
-  await fs.writeFile(
-    path.join(tempDir, path.basename(POSTGRES_STORAGE_MANIFEST)),
-    JSON.stringify({
-      createdAt: report.completedAt,
-      storageMode: 'postgres',
-      status: report.status,
-      pending: report.pending,
-    }, null, 2),
-    'utf8'
-  );
-  await fs.writeFile(path.join(tempDir, 'schema.sql'), POSTGRES_SCHEMA_SQL, 'utf8');
-  await fs.writeFile(path.join(tempDir, 'migration-report.json'), JSON.stringify(report, null, 2), 'utf8');
-  await fs.writeFile(
-    path.join(tempDir, 'README.txt'),
-    [
-      'This folder marks PostgreSQL storage as active for the backend.',
-      'The server checks for this folder on startup and uses PostgreSQL-only mode when it exists.',
-      'schema.sql contains the schema used for the migrated encrypted SQL backend.',
-    ].join('\n'),
-    'utf8'
-  );
-  await fs.rename(tempDir, POSTGRES_STORAGE_DIR);
-}
-
-async function main() {
-  if (await fileExists(POSTGRES_STORAGE_DIR)) {
-    throw new Error(`SQL storage folder already exists at ${POSTGRES_STORAGE_DIR}. Remove it before running the migration again.`);
-  }
-
-  await fs.mkdir(DATA_ROOT, { recursive: true });
-
-  const pool = await createStandalonePostgresPool();
-  const report = {
-    startedAt: nowIso(),
-    targetFolder: POSTGRES_STORAGE_DIR,
-    migrated: {},
-    pending: [],
-  };
-
-  try {
-    await truncateTargetTables(pool);
-    await migrateVersions(pool, report);
-    await migrateTempSessions(pool, report);
-    await migrateMemories(pool, report);
-    await migrateDeletedChats(pool, report);
-    await migrateSystemPrompts(pool, report);
-    await migrateFeedback(pool, report);
-    await migrateGuestRequestCounters(pool, report);
-    await migrateWebSearchUsage(pool, report);
-    await migrateMedia(pool, report);
-    await migrateSupabaseData(pool, report);
-
-    report.completedAt = nowIso();
-    report.status = report.pending.length ? 'completed_with_pending_items' : 'completed';
-
-    await writeStorageFolder(report);
-
-    console.log('PostgreSQL migration complete.');
-    console.log(JSON.stringify(report, null, 2));
-  } finally {
-    await pool.end();
-  }
-}
-
-main().catch((err) => {
+runPostgresMigrationCli().catch((err) => {
   console.error('PostgreSQL migration failed:', err);
   process.exitCode = 1;
 });

server/dataPaths.js

@@ -6,7 +6,25 @@ export const POSTGRES_STORAGE_DIR_NAME = 'postgresql-store';
 export const POSTGRES_STORAGE_DIR = path.join(DATA_ROOT, POSTGRES_STORAGE_DIR_NAME);
 export const POSTGRES_STORAGE_MANIFEST = path.join(POSTGRES_STORAGE_DIR, 'manifest.json');
 
-export const STORAGE_MODE = fs.existsSync(POSTGRES_STORAGE_DIR) ? 'postgres' : 'legacy';
+function detectStorageMode() {
+  return fs.existsSync(POSTGRES_STORAGE_DIR) ? 'postgres' : 'legacy';
+}
+
+export let STORAGE_MODE = detectStorageMode();
+
+export function refreshStorageMode() {
+  STORAGE_MODE = detectStorageMode();
+  return STORAGE_MODE;
+}
+
+export function setStorageMode(mode) {
+  STORAGE_MODE = mode === 'postgres' ? 'postgres' : 'legacy';
+  return STORAGE_MODE;
+}
+
+export function getStorageMode() {
+  return STORAGE_MODE;
+}
 
 export function isPostgresStorageMode() {
   return STORAGE_MODE === 'postgres';

server/index.js

@@ -9,22 +9,17 @@ import fetch from 'node-fetch';
 import rateLimit from 'express-rate-limit';
 import fs from 'fs';
 import { registerFeedbackRoutes } from './handleFeedback.js';
-import { abortActiveStream, handleWsMessage } from './wsHandler.js';
-import { sessionStore, initStoreConfig } from './sessionStore.js';
 import { SUPABASE_URL, SUPABASE_ANON_KEY } from './config.js';
 import { safeSend } from './helpers.js';
-import { verifySupabaseToken } from './auth.js';
-import { mediaStore } from './mediaStore.js';
 import { initializePostgresStorage } from './postgres.js';
-import { POSTGRES_STORAGE_DIR, STORAGE_MODE } from './dataPaths.js';
+import { POSTGRES_STORAGE_DIR, refreshStorageMode } from './dataPaths.js';
+import { migrateLegacyDataToPostgres } from './postgresMigration.js';
 import { loadStoredSHA, saveStoredSHA } from './versionStore.js';
 
 export { SUPABASE_URL, SUPABASE_ANON_KEY };
 export { LIGHTNING_BASE, PUBLIC_URL } from './config.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-initStoreConfig(SUPABASE_URL, SUPABASE_ANON_KEY);
 export const supabaseAnon = createClient(SUPABASE_URL, SUPABASE_ANON_KEY);
 
 const PORT = process.env.PORT || 7860;
@@ -63,10 +58,36 @@ const verifyLimiter = rateLimit({
 
 const PUBLIC_URL = process.env.PUBLIC_URL || 'default';
 
-if (STORAGE_MODE === 'postgres') {
-  await initializePostgresStorage();
+async function ensureStorageBackendReady() {
+  let mode = refreshStorageMode();
+  const autoBootstrap = process.env.POSTGRES_AUTO_BOOTSTRAP !== '0';
+
+  if (mode !== 'postgres' && autoBootstrap) {
+    console.log(`[storage] PostgreSQL marker folder missing at ${POSTGRES_STORAGE_DIR}; starting automatic bootstrap.`);
+    const report = await migrateLegacyDataToPostgres({ skipIfFolderExists: true });
+    mode = refreshStorageMode();
+    if (report?.skipped) {
+      console.log(`[storage] PostgreSQL marker folder became available during bootstrap check.`);
+    } else {
+      console.log(`[storage] automatic PostgreSQL bootstrap ${report.status} at ${POSTGRES_STORAGE_DIR}`);
+    }
+  }
+
+  if (mode === 'postgres') {
+    await initializePostgresStorage();
+  }
+
+  console.log(`[storage] mode=${mode} target=${mode === 'postgres' ? POSTGRES_STORAGE_DIR : '/data'}`);
+  return mode;
 }
-console.log(`[storage] mode=${STORAGE_MODE} target=${STORAGE_MODE === 'postgres' ? POSTGRES_STORAGE_DIR : '/data'}`);
+
+await ensureStorageBackendReady();
+
+const { sessionStore, initStoreConfig } = await import('./sessionStore.js');
+const { abortActiveStream, handleWsMessage } = await import('./wsHandler.js');
+const { verifySupabaseToken } = await import('./auth.js');
+const { mediaStore } = await import('./mediaStore.js');
+initStoreConfig(SUPABASE_URL, SUPABASE_ANON_KEY);
 
 function getRequestIp(req) {
   return (req.headers['x-forwarded-for'] || '').split(',')[0].trim()

server/postgres.js

@@ -30,7 +30,7 @@ export function buildPoolConfig() {
   const { PGHOST, PGPORT, PGUSER, PGPASSWORD, PGDATABASE } = process.env;
   if (!PGHOST || !PGUSER || !PGDATABASE) {
     throw new Error(
-      `PostgreSQL storage is enabled by ${POSTGRES_STORAGE_DIR}, but no database connection is configured. Set DATABASE_URL or PGHOST/PGUSER/PGDATABASE.`
+      `PostgreSQL bootstrap/storage for ${POSTGRES_STORAGE_DIR} requires a database connection. Set DATABASE_URL or PGHOST/PGUSER/PGDATABASE.`
     );
   }
 

server/postgresMigration.js

@@ -0,0 +1,585 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { createClient } from '@supabase/supabase-js';
+import { loadEncryptedJson } from './cryptoUtils.js';
+import {
+  POSTGRES_STORAGE_DIR,
+  POSTGRES_STORAGE_MANIFEST,
+  DATA_ROOT,
+  refreshStorageMode,
+} from './dataPaths.js';
+import {
+  createStandalonePostgresPool,
+  encryptJsonPayload,
+  makeLookupToken,
+  makeOwnerLookup,
+} from './postgres.js';
+import { POSTGRES_SCHEMA_SQL } from './postgresSchema.js';
+import { SUPABASE_URL } from './config.js';
+
+const TEMP_TTL_MS = 24 * 60 * 60 * 1000;
+const TEMP_INACTIVITY = 12 * 60 * 60 * 1000;
+const FEEDBACK_AAD = 'feedback_tickets_v1';
+const VERSION_FILE = path.join(DATA_ROOT, 'version.json');
+const WEB_SEARCH_USAGE_FILE = path.join(DATA_ROOT, 'web-search-usage.json');
+const MEDIA_INDEX_FILE = path.join(DATA_ROOT, 'media', 'index.json');
+const MEDIA_BLOBS_DIR = path.join(DATA_ROOT, 'media', 'blobs');
+const TEMP_STORE_FILE = path.join(DATA_ROOT, 'temp_sessions.json');
+const MEMORIES_FILE = path.join(DATA_ROOT, 'memories', 'index.json');
+const DELETED_CHATS_FILE = path.join(DATA_ROOT, 'deleted_chats', 'index.json');
+const SYSTEM_PROMPTS_FILE = path.join(DATA_ROOT, 'system-prompts', 'index.json');
+const FEEDBACK_FILE = path.join(DATA_ROOT, 'feedback_tickets.json');
+const GUEST_REQUEST_FILE = path.join(DATA_ROOT, 'guest_request_counts.json');
+
+function nowIso() {
+  return new Date().toISOString();
+}
+
+function sessionAad(scopeType, sessionId) {
+  return `chat-session:${scopeType}:${sessionId}`;
+}
+
+function guestStateLookup(tempId) {
+  return makeLookupToken('guest-state', tempId);
+}
+
+function guestStateAad(tempId) {
+  return `guest-state:${tempId}`;
+}
+
+function guestExpiryRecord(tempData) {
+  const createdExpires = (tempData.created || Date.now()) + TEMP_TTL_MS;
+  const inactiveExpires = (tempData.lastActive || Date.now()) + TEMP_INACTIVITY;
+  return new Date(Math.min(createdExpires, inactiveExpires)).toISOString();
+}
+
+function shareTokenLookup(token) {
+  return makeLookupToken('session-share-token', token);
+}
+
+function shareAad(recordId) {
+  return `session-share:${recordId}`;
+}
+
+function promptLookup(userId) {
+  return makeLookupToken('system-prompt', userId);
+}
+
+function promptAad(userId) {
+  return `system-prompt:${userId}`;
+}
+
+function mediaEntryAad(id) {
+  return `media-entry:${id}`;
+}
+
+function feedbackAad(id) {
+  return `feedback:${id}`;
+}
+
+function usernameLookup(username) {
+  return makeLookupToken('username', username);
+}
+
+function versionLookup(publicUrl) {
+  return makeLookupToken('app-version', publicUrl);
+}
+
+function versionAad(publicUrl) {
+  return `app-version:${publicUrl}`;
+}
+
+function requestLookup(ip) {
+  return makeLookupToken('guest-request', ip);
+}
+
+function webUsageLookup(key) {
+  return makeLookupToken('web-search-usage', key);
+}
+
+function webUsageAad(key, day) {
+  return `web-search-usage:${key}:${day}`;
+}
+
+async function fileExists(filePath) {
+  try {
+    await fs.access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function readJsonIfExists(filePath) {
+  try {
+    return JSON.parse(await fs.readFile(filePath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+async function fetchAllSupabaseRows(client, tableName) {
+  const pageSize = 1000;
+  const rows = [];
+  let from = 0;
+  while (true) {
+    const { data, error } = await client
+      .from(tableName)
+      .select('*')
+      .range(from, from + pageSize - 1);
+    if (error) throw error;
+    rows.push(...(data || []));
+    if (!data || data.length < pageSize) break;
+    from += pageSize;
+  }
+  return rows;
+}
+
+async function truncateTargetTables(pool) {
+  await pool.query(`
+    TRUNCATE TABLE
+      media_blobs,
+      media_entries,
+      deleted_chats,
+      memories,
+      system_prompts,
+      feedback_tickets,
+      guest_request_counters,
+      web_search_usage,
+      user_settings,
+      user_profiles,
+      device_sessions,
+      session_shares,
+      chat_sessions,
+      guest_state,
+      app_versions
+    RESTART IDENTITY
+    CASCADE
+  `);
+}
+
+async function migrateVersions(pool, report) {
+  const data = await readJsonIfExists(VERSION_FILE);
+  const entries = Array.isArray(data) ? data : [];
+  let count = 0;
+  for (const entry of entries) {
+    for (const [publicUrl, sha] of Object.entries(entry || {})) {
+      await pool.query(
+        `INSERT INTO app_versions (public_url_lookup, updated_at, payload)
+         VALUES ($1, $2, $3::jsonb)`,
+        [
+          versionLookup(publicUrl),
+          nowIso(),
+          JSON.stringify(encryptJsonPayload({ publicUrl, sha }, versionAad(publicUrl))),
+        ]
+      );
+      count += 1;
+    }
+  }
+  report.migrated.versionEntries = count;
+}
+
+async function migrateTempSessions(pool, report) {
+  const data = await loadEncryptedJson(TEMP_STORE_FILE);
+  const records = data || {};
+  let ownerCount = 0;
+  let sessionCount = 0;
+
+  for (const [tempId, tempData] of Object.entries(records)) {
+    ownerCount += 1;
+    const owner = { type: 'guest', id: tempId };
+    await pool.query(
+      `INSERT INTO guest_state (owner_lookup, expires_at, updated_at, payload)
+       VALUES ($1, $2, $3, $4::jsonb)`,
+      [
+        guestStateLookup(tempId),
+        guestExpiryRecord(tempData),
+        nowIso(),
+        JSON.stringify(encryptJsonPayload({
+          tempId,
+          msgCount: tempData.msgCount || 0,
+          created: tempData.created || Date.now(),
+          lastActive: tempData.lastActive || Date.now(),
+        }, guestStateAad(tempId))),
+      ]
+    );
+
+    for (const session of Object.values(tempData.sessions || {})) {
+      await pool.query(
+        `INSERT INTO chat_sessions (id, scope_type, owner_lookup, created_at, updated_at, expires_at, payload)
+         VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
+        [
+          session.id,
+          'guest',
+          makeOwnerLookup(owner),
+          new Date(session.created || Date.now()).toISOString(),
+          nowIso(),
+          guestExpiryRecord(tempData),
+          JSON.stringify(encryptJsonPayload(session, sessionAad('guest', session.id))),
+        ]
+      );
+      sessionCount += 1;
+    }
+  }
+
+  report.migrated.tempOwners = ownerCount;
+  report.migrated.tempSessions = sessionCount;
+}
+
+async function migrateMemories(pool, report) {
+  const data = await loadEncryptedJson(MEMORIES_FILE);
+  const memories = Object.values(data?.memories || {});
+  for (const memory of memories) {
+    await pool.query(
+      `INSERT INTO memories (id, owner_lookup, created_at, updated_at, payload)
+       VALUES ($1, $2, $3, $4, $5::jsonb)`,
+      [
+        memory.id,
+        makeOwnerLookup({ type: memory.ownerType, id: memory.ownerId }),
+        memory.createdAt,
+        memory.updatedAt,
+        JSON.stringify(encryptJsonPayload(memory, `memory:${memory.id}`)),
+      ]
+    );
+  }
+  report.migrated.memories = memories.length;
+}
+
+async function migrateDeletedChats(pool, report) {
+  const data = await loadEncryptedJson(DELETED_CHATS_FILE);
+  const deletedChats = Object.values(data?.deletedChats || {});
+  for (const record of deletedChats) {
+    await pool.query(
+      `INSERT INTO deleted_chats (id, owner_lookup, purge_at, deleted_at, payload)
+       VALUES ($1, $2, $3, $4, $5::jsonb)`,
+      [
+        record.id,
+        makeOwnerLookup({ type: record.ownerType, id: record.ownerId }),
+        record.purgeAt || null,
+        record.deletedAt,
+        JSON.stringify(encryptJsonPayload(record, `deleted-chat:${record.id}`)),
+      ]
+    );
+  }
+  report.migrated.deletedChats = deletedChats.length;
+}
+
+async function migrateSystemPrompts(pool, report) {
+  const data = await loadEncryptedJson(SYSTEM_PROMPTS_FILE, 'system-prompts');
+  const prompts = Object.entries(data?.prompts || {});
+  for (const [userId, prompt] of prompts) {
+    const record = {
+      userId,
+      markdown: prompt.markdown,
+      updatedAt: prompt.updatedAt || nowIso(),
+    };
+    await pool.query(
+      `INSERT INTO system_prompts (owner_lookup, updated_at, payload)
+       VALUES ($1, $2, $3::jsonb)`,
+      [
+        promptLookup(userId),
+        record.updatedAt,
+        JSON.stringify(encryptJsonPayload(record, promptAad(userId))),
+      ]
+    );
+  }
+  report.migrated.systemPrompts = prompts.length;
+}
+
+async function migrateFeedback(pool, report) {
+  const data = await loadEncryptedJson(FEEDBACK_FILE, FEEDBACK_AAD);
+  const tickets = Array.isArray(data?.tickets) ? data.tickets : [];
+  for (const ticket of tickets) {
+    await pool.query(
+      `INSERT INTO feedback_tickets (id, status, submitted_at, payload)
+       VALUES ($1, $2, $3, $4::jsonb)`,
+      [
+        ticket.id,
+        ticket.status || 'open',
+        ticket.submittedAt,
+        JSON.stringify(encryptJsonPayload(ticket, feedbackAad(ticket.id))),
+      ]
+    );
+  }
+  report.migrated.feedbackTickets = tickets.length;
+}
+
+async function migrateGuestRequestCounters(pool, report) {
+  const data = await loadEncryptedJson(GUEST_REQUEST_FILE);
+  const entries = Object.entries(data || {});
+  for (const [ip, entry] of entries) {
+    await pool.query(
+      `INSERT INTO guest_request_counters (key_lookup, expires_at, updated_at, payload)
+       VALUES ($1, $2, $3, $4::jsonb)`,
+      [
+        requestLookup(ip),
+        new Date(entry.resetAt || Date.now()).toISOString(),
+        nowIso(),
+        JSON.stringify(encryptJsonPayload({
+          ip,
+          count: entry.count || 0,
+          resetAt: entry.resetAt || Date.now(),
+        }, 'guest-request-row')),
+      ]
+    );
+  }
+  report.migrated.guestRequestCounters = entries.length;
+}
+
+async function migrateWebSearchUsage(pool, report) {
+  const data = await readJsonIfExists(WEB_SEARCH_USAGE_FILE);
+  const days = data?.days && typeof data.days === 'object' ? data.days : {};
+  let count = 0;
+  for (const [dayKey, keys] of Object.entries(days)) {
+    for (const [key, used] of Object.entries(keys || {})) {
+      await pool.query(
+        `INSERT INTO web_search_usage (key_lookup, day_key, updated_at, payload)
+         VALUES ($1, $2, $3, $4::jsonb)`,
+        [
+          webUsageLookup(key),
+          dayKey,
+          nowIso(),
+          JSON.stringify(encryptJsonPayload({ used }, webUsageAad(key, dayKey))),
+        ]
+      );
+      count += 1;
+    }
+  }
+  report.migrated.webSearchUsageRows = count;
+}
+
+async function migrateMedia(pool, report) {
+  const data = await loadEncryptedJson(MEDIA_INDEX_FILE);
+  const entries = Object.values(data?.entries || {});
+  let blobCount = 0;
+  for (const entry of entries) {
+    await pool.query(
+      `INSERT INTO media_entries (
+         id, owner_lookup, parent_id, entry_type, updated_at, created_at,
+         trashed_at, purge_at, expires_at, size_bytes, payload
+       )
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb)`,
+      [
+        entry.id,
+        makeOwnerLookup({ type: entry.ownerType, id: entry.ownerId }),
+        entry.parentId || null,
+        entry.type,
+        entry.updatedAt || entry.createdAt,
+        entry.createdAt,
+        entry.trashedAt || null,
+        entry.purgeAt || null,
+        entry.expiresAt || null,
+        entry.size || 0,
+        JSON.stringify(encryptJsonPayload(entry, mediaEntryAad(entry.id))),
+      ]
+    );
+
+    if (entry.type === 'file') {
+      const blobPath = path.join(MEDIA_BLOBS_DIR, `${entry.id}.bin`);
+      if (await fileExists(blobPath)) {
+        const blob = await fs.readFile(blobPath);
+        await pool.query(
+          `INSERT INTO media_blobs (entry_id, updated_at, payload)
+           VALUES ($1, $2, $3)`,
+          [entry.id, entry.updatedAt || entry.createdAt, blob]
+        );
+        blobCount += 1;
+      }
+    }
+  }
+  report.migrated.mediaEntries = entries.length;
+  report.migrated.mediaBlobs = blobCount;
+}
+
+async function migrateSupabaseData(pool, report) {
+  const serviceRoleKey = process.env.SUPABASE_SERVICE_ROLE_KEY || '';
+  if (!serviceRoleKey) {
+    if (process.env.ALLOW_PARTIAL_SQL_MIGRATION === '1') {
+      report.pending.push(
+        'Supabase-backed user tables were not exported because SUPABASE_SERVICE_ROLE_KEY is not set.'
+      );
+      return;
+    }
+    throw new Error(
+      'SUPABASE_SERVICE_ROLE_KEY is required to migrate existing Supabase-backed sessions, shares, settings, and profiles before enabling PostgreSQL-only mode. Set ALLOW_PARTIAL_SQL_MIGRATION=1 only if you intentionally want to skip those records.'
+    );
+  }
+
+  const supabase = createClient(SUPABASE_URL, serviceRoleKey, {
+    auth: { persistSession: false },
+  });
+
+  const [webSessions, sharedSessions, userSettings, profiles] = await Promise.all([
+    fetchAllSupabaseRows(supabase, 'web_sessions'),
+    fetchAllSupabaseRows(supabase, 'shared_sessions'),
+    fetchAllSupabaseRows(supabase, 'user_settings'),
+    fetchAllSupabaseRows(supabase, 'profiles'),
+  ]);
+
+  for (const row of webSessions) {
+    const session = {
+      id: row.id,
+      name: row.name,
+      created: new Date(row.created_at).getTime(),
+      history: row.history || [],
+      model: row.model || null,
+    };
+    await pool.query(
+      `INSERT INTO chat_sessions (id, scope_type, owner_lookup, created_at, updated_at, expires_at, payload)
+       VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
+      [
+        row.id,
+        'user',
+        makeOwnerLookup({ type: 'user', id: row.user_id }),
+        row.created_at,
+        row.updated_at || nowIso(),
+        null,
+        JSON.stringify(encryptJsonPayload(session, sessionAad('user', row.id))),
+      ]
+    );
+  }
+
+  for (const row of sharedSessions) {
+    const lookup = shareTokenLookup(row.token);
+    const id = row.id || `share_${lookup.slice(0, 24)}`;
+    await pool.query(
+      `INSERT INTO session_shares (id, token_lookup, owner_lookup, created_at, payload)
+       VALUES ($1, $2, $3, $4, $5::jsonb)`,
+      [
+        id,
+        lookup,
+        makeOwnerLookup({ type: 'user', id: row.owner_id }),
+        row.created_at || nowIso(),
+        JSON.stringify(encryptJsonPayload({
+          id,
+          ownerId: row.owner_id,
+          sessionSnapshot: row.session_snapshot,
+          createdAt: row.created_at || nowIso(),
+        }, shareAad(id))),
+      ]
+    );
+  }
+
+  for (const row of userSettings) {
+    await pool.query(
+      `INSERT INTO user_settings (owner_lookup, updated_at, payload)
+       VALUES ($1, $2, $3::jsonb)`,
+      [
+        makeOwnerLookup({ type: 'user', id: row.user_id }),
+        row.updated_at || nowIso(),
+        JSON.stringify(encryptJsonPayload({
+          userId: row.user_id,
+          settings: row.settings || {},
+          updatedAt: row.updated_at || nowIso(),
+        }, `user-settings:${row.user_id}`)),
+      ]
+    );
+  }
+
+  for (const row of profiles) {
+    if (!row.id || !row.username) continue;
+    await pool.query(
+      `INSERT INTO user_profiles (owner_lookup, username_lookup, updated_at, payload)
+       VALUES ($1, $2, $3, $4::jsonb)`,
+      [
+        makeOwnerLookup({ type: 'user', id: row.id }),
+        usernameLookup(row.username),
+        row.updated_at || nowIso(),
+        JSON.stringify(encryptJsonPayload({
+          userId: row.id,
+          username: row.username,
+          updatedAt: row.updated_at || nowIso(),
+        }, `user-profile:${row.id}`)),
+      ]
+    );
+  }
+
+  report.migrated.supabaseWebSessions = webSessions.length;
+  report.migrated.supabaseSharedSessions = sharedSessions.length;
+  report.migrated.supabaseUserSettings = userSettings.length;
+  report.migrated.supabaseProfiles = profiles.length;
+}
+
+async function writeStorageFolder(report) {
+  const tempDir = `${POSTGRES_STORAGE_DIR}.tmp`;
+  await fs.rm(tempDir, { recursive: true, force: true });
+  await fs.mkdir(tempDir, { recursive: true });
+  await fs.writeFile(
+    path.join(tempDir, path.basename(POSTGRES_STORAGE_MANIFEST)),
+    JSON.stringify({
+      createdAt: report.completedAt,
+      storageMode: 'postgres',
+      status: report.status,
+      pending: report.pending,
+    }, null, 2),
+    'utf8'
+  );
+  await fs.writeFile(path.join(tempDir, 'schema.sql'), POSTGRES_SCHEMA_SQL, 'utf8');
+  await fs.writeFile(path.join(tempDir, 'migration-report.json'), JSON.stringify(report, null, 2), 'utf8');
+  await fs.writeFile(
+    path.join(tempDir, 'README.txt'),
+    [
+      'This folder marks PostgreSQL storage as active for the backend.',
+      'The server checks for this folder on startup and uses PostgreSQL-only mode when it exists.',
+      'schema.sql contains the schema used for the migrated encrypted SQL backend.',
+    ].join('\n'),
+    'utf8'
+  );
+  await fs.rename(tempDir, POSTGRES_STORAGE_DIR);
+}
+
+export async function migrateLegacyDataToPostgres({ skipIfFolderExists = true } = {}) {
+  if (skipIfFolderExists && await fileExists(POSTGRES_STORAGE_DIR)) {
+    refreshStorageMode();
+    return {
+      status: 'already_active',
+      targetFolder: POSTGRES_STORAGE_DIR,
+      skipped: true,
+      migrated: {},
+      pending: [],
+    };
+  }
+
+  if (!skipIfFolderExists && await fileExists(POSTGRES_STORAGE_DIR)) {
+    throw new Error(`SQL storage folder already exists at ${POSTGRES_STORAGE_DIR}. Remove it before running the migration again.`);
+  }
+
+  await fs.mkdir(DATA_ROOT, { recursive: true });
+
+  const pool = await createStandalonePostgresPool();
+  const report = {
+    startedAt: nowIso(),
+    targetFolder: POSTGRES_STORAGE_DIR,
+    migrated: {},
+    pending: [],
+  };
+
+  try {
+    await truncateTargetTables(pool);
+    await migrateVersions(pool, report);
+    await migrateTempSessions(pool, report);
+    await migrateMemories(pool, report);
+    await migrateDeletedChats(pool, report);
+    await migrateSystemPrompts(pool, report);
+    await migrateFeedback(pool, report);
+    await migrateGuestRequestCounters(pool, report);
+    await migrateWebSearchUsage(pool, report);
+    await migrateMedia(pool, report);
+    await migrateSupabaseData(pool, report);
+
+    report.completedAt = nowIso();
+    report.status = report.pending.length ? 'completed_with_pending_items' : 'completed';
+
+    await writeStorageFolder(report);
+    refreshStorageMode();
+    return report;
+  } finally {
+    await pool.end();
+  }
+}
+
+export async function runPostgresMigrationCli() {
+  const report = await migrateLegacyDataToPostgres({ skipIfFolderExists: false });
+  console.log('PostgreSQL migration complete.');
+  console.log(JSON.stringify(report, null, 2));
+}