Repository: appsmithorg/appsmith
File: app/client/packages/rts/src/ctl/verify-migration.mjs (JavaScript)

Code before:
      
      if (collectionName.startsWith('mongock')) {
        continue;
      }

      console.log(`\nVerifying collection: ${collectionName}`);
      const pgTableName = collectionName.toLowerCase();
      
      // Get all MongoDB documents and filter out archived ones
      const allMongoDocs = await mongoDb
        .collection(collectionName)
        .find({})
        .toArray();
        
      const mongoDocs = allMongoDocs.filter(doc => !isArchivedObject(doc));

      const missingInPostgres = [];
      
      // Verify each document exists in PostgreSQL
      for (const mongoDoc of mongoDocs) {
        transformFields(mongoDoc);

        const pgRecord = await pgClient.query(
          `SELECT id FROM ${pgTableName} WHERE id = $1 AND "deletedAt" IS NULL`,
          [mongoDoc.id]
        );

        if (pgRecord.rows.length === 0) {
          missingInPostgres.push(mongoDoc.id);
          hasDiscrepancy = true;
        }
      }

      // Get PostgreSQL documents not in MongoDB
      const pgDocs = await pgClient.query(
        `SELECT id FROM ${pgTableName} WHERE "deletedAt" IS NULL`
      );
      
      const pgIds = new Set(pgDocs.rows.map(row => row.id));
      const mongoIds = new Set(mongoDocs.map(doc => doc.id));
      
      const missingInMongo = [...pgIds].filter(id => !mongoIds.has(id));
      
      if (missingInMongo.length > 0 || missingInPostgres.length > 0) {
        verificationResults.push({
          collectionName,
          mongoCount: mongoDocs.length,
          pgCount: pgIds.size,
          missingInPostgres,
          missingInMongo
        });

Review comment (performance):
Let's make this a batch process. Can we introduce the validation on specific fields as well specifically which are introduced as a part of migration after pg went live e.g. `workflows.schedules`?

Code after:
      const collectionName = collection.name;
      
      if (collectionName.startsWith('mongock')) {
        continue;
      }

      console.log(`\nVerifying collection: ${collectionName}`);
      const pgTableName = collectionName.toLowerCase();
      
      // Get total count for progress tracking
      const totalCount = await mongoDb
        .collection(collectionName)
        .countDocuments({ deleted: { $ne: true }, deletedAt: null });
      
      let processedCount = 0;
      const missingInPostgres = [];
      const fieldDiscrepancies = [];

      // Process in batches
      while (processedCount < totalCount) {
        const mongoDocs = await mongoDb
          .collection(collectionName)
          .find({ deleted: { $ne: true }, deletedAt: null })
          .skip(processedCount)
          .limit(BATCH_SIZE)
          .toArray();

        for (const mongoDoc of mongoDocs) {
          transformFields(mongoDoc);
          
          // Get full PostgreSQL record
          const pgRecord = await pgClient.query(
            `SELECT * FROM ${pgTableName} WHERE id = $1 AND "deletedAt" IS NULL`,
            [mongoDoc.id]
          );

          if (pgRecord.rows.length === 0) {
            missingInPostgres.push(mongoDoc.id);
            hasDiscrepancy = true;
            continue;
          }

          // Compare all fields
          const pgDoc = pgRecord.rows[0];
          const differences = compareDocuments(mongoDoc, pgDoc);
          
          if (differences.length > 0) {
            fieldDiscrepancies.push({
              id: mongoDoc.id,
              differences
            });