piclets / src /lib /db /trainerScanning.ts
Fraser's picture
more logs
b195c80
import { db } from './index';
import type { TrainerScanProgress } from './schema';
// Initialize trainer scan progress records from paths
export async function initializeTrainerScanProgress(imagePaths: string[]): Promise<void> {
let processedCount = 0;
let skippedCount = 0;
for (let i = 0; i < imagePaths.length; i++) {
const imagePath = imagePaths[i];
try {
if (typeof imagePath !== 'string') {
console.error(`❌ Path at index ${i} is not a string:`, imagePath, typeof imagePath);
skippedCount++;
continue;
}
// Extract trainer name and image index from path
// Format: "trainer_images/001_Willow_Snap/image_001.jpg"
const pathParts = imagePath.split('/');
if (pathParts.length < 3) {
console.warn(`⚠️ Skipping invalid path format: ${imagePath}`);
skippedCount++;
continue;
}
const trainerName = pathParts[1]?.trim();
const imageFile = pathParts[2]?.trim();
if (!trainerName || !imageFile) {
console.warn(`⚠️ Skipping path with missing parts: ${imagePath}`);
skippedCount++;
continue;
}
const imageMatch = imageFile.match(/image_(\d+)\.jpg/);
const imageIndex = imageMatch ? parseInt(imageMatch[1]) : 1;
// Build remote URL without trainer_images folder
// From: trainer_images/001_Willow_Snap/image_001.jpg
// To: https://huggingface.co/datasets/Fraser/piclets/resolve/main/001_Willow_Snap/image_001.jpg
const remoteUrl = `https://huggingface.co/datasets/Fraser/piclets/resolve/main/${trainerName}/${imageFile}`;
// Check if this path already exists to avoid duplicates
const existing = await db.trainerScanProgress.get(imagePath);
if (!existing) {
const progressRecord: Omit<TrainerScanProgress, 'id'> = {
imagePath,
trainerName,
imageIndex,
status: 'pending',
remoteUrl
};
await db.trainerScanProgress.add(progressRecord);
processedCount++;
} else {
processedCount++;
}
// Log progress every 500 items instead of every 100
if (i % 500 === 0 && i > 0) {
console.log(`Progress: ${i}/${imagePaths.length} (${Math.round((i/imagePaths.length)*100)}%)`);
}
} catch (error) {
console.error(`❌ ERROR processing path ${i}: "${imagePath}"`, error);
if (error instanceof Error && error.message.includes('replace')) {
console.error('❌ REPLACE ERROR FOUND at path:', imagePath);
console.error('❌ Path parts:', imagePath.split('/'));
console.error('❌ Full error:', error);
}
skippedCount++;
}
}
console.log(`βœ… Initialization complete: ${processedCount} processed, ${skippedCount} skipped`);
}
// Get next pending image to process
export async function getNextPendingImage(): Promise<TrainerScanProgress | null> {
try {
const pendingRecord = await db.trainerScanProgress.where('status').equals('pending').first();
return pendingRecord || null;
} catch (error) {
console.error('❌ Failed to get next pending image:', error);
return null;
}
}
// Update scan progress status
export async function updateScanProgress(
imagePath: string,
updates: Partial<Omit<TrainerScanProgress, 'id' | 'imagePath'>>
): Promise<void> {
try {
await db.trainerScanProgress.update(imagePath, updates);
} catch (error) {
console.error(`❌ Failed to update scan progress for ${imagePath}:`, error);
throw error;
}
}
// Mark image processing as started
export async function markImageProcessingStarted(imagePath: string): Promise<void> {
await updateScanProgress(imagePath, {
status: 'processing',
startedAt: new Date()
});
}
// Mark image processing as completed successfully
export async function markImageProcessingCompleted(
imagePath: string,
picletInstanceId: number
): Promise<void> {
await updateScanProgress(imagePath, {
status: 'completed',
picletInstanceId,
completedAt: new Date()
});
}
// Mark image processing as failed
export async function markImageProcessingFailed(
imagePath: string,
errorMessage: string
): Promise<void> {
await updateScanProgress(imagePath, {
status: 'failed',
errorMessage,
completedAt: new Date()
});
}
// Get scanning statistics
export async function getScanningStats(): Promise<{
total: number;
pending: number;
processing: number;
completed: number;
failed: number;
}> {
try {
const [total, pending, processing, completed, failed] = await Promise.all([
db.trainerScanProgress.count(),
db.trainerScanProgress.where('status').equals('pending').count(),
db.trainerScanProgress.where('status').equals('processing').count(),
db.trainerScanProgress.where('status').equals('completed').count(),
db.trainerScanProgress.where('status').equals('failed').count(),
]);
return {
total,
pending,
processing,
completed,
failed
};
} catch (error) {
console.error('❌ Failed to get scanning stats:', error);
return {
total: 0,
pending: 0,
processing: 0,
completed: 0,
failed: 0
};
}
}
// Get all completed scans for a specific trainer
export async function getCompletedScansForTrainer(trainerName: string): Promise<TrainerScanProgress[]> {
try {
return await db.trainerScanProgress
.where('trainerName').equals(trainerName)
.and(record => record.status === 'completed')
.toArray();
} catch (error) {
console.error(`❌ Failed to get completed scans for trainer ${trainerName}:`, error);
return [];
}
}
// Reset all failed scans back to pending (for retry)
export async function resetFailedScans(): Promise<number> {
try {
const failedRecords = await db.trainerScanProgress.where('status').equals('failed').toArray();
for (const record of failedRecords) {
await db.trainerScanProgress.update(record.imagePath, {
status: 'pending',
errorMessage: undefined,
startedAt: undefined,
completedAt: undefined
});
}
return failedRecords.length;
} catch (error) {
console.error('❌ Failed to reset failed scans:', error);
return 0;
}
}
// Get current processing status (for resuming interrupted sessions)
export async function getCurrentProcessingImage(): Promise<TrainerScanProgress | null> {
try {
const processingRecord = await db.trainerScanProgress.where('status').equals('processing').first();
return processingRecord || null;
} catch (error) {
console.error('❌ Failed to get current processing image:', error);
return null;
}
}