const fs = require('fs');
const path = require('path');
const { PredictionServiceClient } = require('@google-cloud/aiplatform').v1;
const { helpers } = require('@google-cloud/aiplatform');
const Bottleneck = require('bottleneck');


async function loadPainPoints() {
  console.log("Loading pain points...");
  const data = fs.readFileSync('pain_points_results.json', 'utf8');
  const results = JSON.parse(data);

  const painPoints = [];
  for (const demographic in results) {
    for (const environment in results[demographic]) {
      const points = results[demographic][environment].pain_points;
      points.forEach(point => {
        painPoints.push({
          demographic,
          environment,
          point
        });
      });
    }
  }
  console.log(`Total pain points loaded: ${painPoints.length}`);
  return painPoints;
}


async function getEmbeddings(painPoints) {
  console.log("Generating embeddings...");
  const clientOptions = { apiEndpoint: 'us-central1-aiplatform.googleapis.com' };
  const client = new PredictionServiceClient(clientOptions);

  const project = 'language-school-integration';
  const location = 'us-central1';
  const model = 'textembedding-gecko'; // Updated to match the actual model being used

  const endpoint = `projects/${project}/locations/${location}/publishers/google/models/${model}`;

  let batchSize = 100; // Start with a larger batch size
  const embeddings = [];
  let successfulRequestsInLastMinute = 0;
  let lastMinuteTimestamp = Date.now();

  // Set up the rate limiter
  const limiter = new Bottleneck({
    reservoir: 5, // Start with the known quota
    reservoirRefreshAmount: 5,
    reservoirRefreshInterval: 60 * 1000, // 1 minute in milliseconds
    minTime: 100, // Minimum time between requests
    maxConcurrent: 1, // Process one request at a time
  });

  for (let i = 0; i < painPoints.length; i += batchSize) {
    const batchPoints = painPoints.slice(i, i + batchSize);
    const instances = batchPoints.map(p => ({ content: p.point }));

    const request = {
      endpoint: endpoint,
      instances: instances.map(e => helpers.toValue(e)),
    };

    try {
      await limiter.schedule(async () => {
        const [response] = await client.predict(request);
        const predictions = response.predictions;

        predictions.forEach(prediction => {
          const embeddingsProto = prediction.structValue.fields.embeddings;
          const valuesProto = embeddingsProto.structValue.fields.values;
          const embeddingValues = valuesProto.listValue.values.map(v => v.numberValue);
          embeddings.push(embeddingValues);
        });

        console.log(`Processed ${Math.min(i + batchSize, painPoints.length)} / ${painPoints.length} pain points`);
        
        // Update successful requests count
        successfulRequestsInLastMinute++;
        
        // Adjust batch size and limiter if we're under-utilizing the quota
        const currentTime = Date.now();
        if (currentTime - lastMinuteTimestamp >= 60000) {
          if (successfulRequestsInLastMinute > limiter.reservoir) {
            limiter.reservoir = successfulRequestsInLastMinute;
            batchSize = Math.min(batchSize + 10, 200); // Increase batch size, max 200
            console.log(`Increasing batch size to ${batchSize} and reservoir to ${limiter.reservoir}`);
          }
          successfulRequestsInLastMinute = 0;
          lastMinuteTimestamp = currentTime;
        }
      });
    } catch (error) {
      console.error(`Error processing batch starting at index ${i}: ${error.message}`);
      if (error.message.includes('RESOURCE_EXHAUSTED')) {
        // Reduce batch size and limiter reservoir on resource exhaustion
        batchSize = Math.max(Math.floor(batchSize * 0.8), 10); // Reduce batch size, min 10
        limiter.reservoir = Math.max(Math.floor(limiter.reservoir * 0.8), 1); // Reduce reservoir, min 1
        console.log(`Reducing batch size to ${batchSize} and reservoir to ${limiter.reservoir}`);
        i -= batchSize; // Retry this batch
        await new Promise(resolve => setTimeout(resolve, 60000)); // Wait for 1 minute
      } else {
        // For other errors, skip this batch
        console.error(`Skipping batch due to error: ${error.message}`);
      }
    }

    // Add a small delay between batches
    await new Promise(resolve => setTimeout(resolve, 1000));
  }

  console.log(`Total embeddings generated: ${embeddings.length}`);
  return embeddings;
}

function clusterEmbeddings(embeddings, numClusters = 100) {
  console.log("Clustering embeddings...");
  const kmeansResult = KMeans(embeddings, numClusters);
  return kmeansResult.clusters;
}

function reduceDimensions(embeddings) {
  console.log("Reducing dimensions for visualization...");
  const pca = new PCA(embeddings);
  const reducedData = pca.predict(embeddings, { nComponents: 2 }).to2DArray();
  return reducedData;
}

function visualizeClusters(reducedData, clusters) {
  console.log("Visualizing clusters...");

  const data = [];
  const uniqueClusters = Array.from(new Set(clusters));

  uniqueClusters.forEach(clusterId => {
    const indices = clusters.reduce((arr, clustId, idx) => {
      if (clustId === clusterId) {
        arr.push(idx);
      }
      return arr;
    }, []);

    const x = indices.map(i => reducedData[i][0]);
    const y = indices.map(i => reducedData[i][1]);

    data.push({
      x: x,
      y: y,
      mode: 'markers',
      type: 'scatter',
      name: `Cluster ${clusterId}`
    });
  });

  const layout = {
    title: 'Pain Points Clusters',
    xaxis: {
      title: 'Principal Component 1'
    },
    yaxis: {
      title: 'Principal Component 2'
    }
  };

  const plotHtml = `
    <!DOCTYPE html>
    <html>
      <head>
        <meta charset="utf-8">
        <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
      </head>
      <body>
        <div id="plot"></div>
        <script>
          var data = ${JSON.stringify(data)};
          var layout = ${JSON.stringify(layout)};
          Plotly.newPlot('plot', data, layout);
        </script>
      </body>
    </html>
  `;

  fs.writeFileSync('pain_points_clusters.html', plotHtml);
  console.log("Visualization saved to 'pain_points_clusters.html'");
}

async function main() {
  try {
    const painPoints = await loadPainPoints();

    const embeddings = await getEmbeddings(painPoints);

    const clusters = clusterEmbeddings(embeddings);

    const reducedData = reduceDimensions(embeddings);

    visualizeClusters(reducedData, clusters);

    console.log("Saving clustered pain points...");
    const clusteredPainPoints = painPoints.map((point, index) => ({
      ...point,
      cluster: clusters[index]
    }));

    fs.writeFileSync('clustered_pain_points.json', JSON.stringify(clusteredPainPoints, null, 2));

    console.log("Process completed successfully.");
  } catch (error) {
    console.error(`An error occurred: ${error.message}`);
  }
}

main();

The code above is an unfortunate disappointment, after running for 5 hrs to finish, i got this error:
Total embeddings generated: 151008
Clustering embeddings...
An error occurred: KMeans is not defined

And the whole progress was lost.

So there are two things that i need done, no more no less: 1. Implement an embedding saving system so that it can continue from where it stopped it if fail 2.defined KMeans

Dont change anything else