const RATE_LIMIT = 999; // 999 requests per minute
const MAX_PAYLOAD_SIZE = 9000; // Bytes
const INITIAL_BATCH_SIZE = 100;
const CHECKPOINT_INTERVAL = 50; // Save checkpoint every 50 processed items

function createRateLimiter(requestsPerMinute) {
    const interval = 60000 / requestsPerMinute;
    let lastRequest = Date.now();
    return async () => {
        const now = Date.now();
        const timeToWait = interval - (now - lastRequest);
        if (timeToWait > 0) {
            await new Promise(resolve => setTimeout(resolve, timeToWait));
        }
        lastRequest = Date.now();
    };
}

const rateLimiter = createRateLimiter(RATE_LIMIT);

function estimateBatchSize(points, currentBatchSize) {
    const totalSize = points.reduce((sum, point) => sum + Buffer.byteLength(point, 'utf8'), 0);
    const averageSize = totalSize / points.length;
    return Math.floor(Math.min(currentBatchSize, MAX_PAYLOAD_SIZE / averageSize));
}

async function embedPainPointsBatch(batch) {
    await rateLimiter();
    try {
        const result = await model.embedContent(batch);
        return result.embedding.values;
    } catch (error) {
        if (error.message.includes('Request payload size exceeds the limit') && batch.length > 1) {
            const halfBatchSize = Math.ceil(batch.length / 2);
            const firstHalf = await embedPainPointsBatch(batch.slice(0, halfBatchSize));
            const secondHalf = await embedPainPointsBatch(batch.slice(halfBatchSize));
            return [...firstHalf, ...secondHalf];
        } else {
            console.error(`Error embedding batch: ${error.message}`);
            return batch.map(() => null);
        }
    }
}

function loadCheckpoint() {
    if (fs.existsSync('checkpoint.json')) {
        console.log("Loading from checkpoint...");
        const checkpoint = JSON.parse(fs.readFileSync('checkpoint.json', 'utf8'));
        return {
            painPoints: checkpoint.painPoints,
            embeddings: checkpoint.embeddings,
            processedCount: checkpoint.processedCount,
            lastProcessedDemographic: checkpoint.lastProcessedDemographic,
            lastProcessedEnvironment: checkpoint.lastProcessedEnvironment
        };
    }
    return { painPoints: [], embeddings: [], processedCount: 0, lastProcessedDemographic: '', lastProcessedEnvironment: '' };
}

function saveCheckpoint(painPoints, embeddings, processedCount, lastProcessedDemographic, lastProcessedEnvironment) {
    console.log("Saving checkpoint...");
    const checkpoint = {
        painPoints,
        embeddings,
        processedCount,
        lastProcessedDemographic,
        lastProcessedEnvironment
    };
    fs.writeFileSync('checkpoint.json', JSON.stringify(checkpoint));
}

async function processResults() {
    const resultsFile = fs.readFileSync('pain_points_results.json', 'utf8');
    const results = JSON.parse(resultsFile);

    let { painPoints, embeddings, processedCount, lastProcessedDemographic, lastProcessedEnvironment } = loadCheckpoint();

    let batchSize = INITIAL_BATCH_SIZE;

    for (const demographic in results) {
        if (demographic < lastProcessedDemographic) continue;
        
        for (const environment in results[demographic]) {
            if (demographic === lastProcessedDemographic && environment < lastProcessedEnvironment) continue;

            const points = results[demographic][environment].pain_points;
            for (let i = 0; i < points.length; i += batchSize) {
                const batch = points.slice(i, i + batchSize);
                const newPainPoints = batch.map(point => ({ demographic, environment, point }));
                painPoints.push(...newPainPoints);
                
                const batchEmbeddings = await embedPainPointsBatch(batch);
                embeddings.push(...batchEmbeddings);

                processedCount += batch.length;
                console.log(`Processed ${processedCount} pain points. Current batch size: ${batchSize}`);

                if (processedCount % CHECKPOINT_INTERVAL === 0) {
                    saveCheckpoint(painPoints, embeddings, processedCount, demographic, environment);
                }

                batchSize = estimateBatchSize(batch, batchSize);
            }
            lastProcessedEnvironment = environment;
        }
        lastProcessedDemographic = demographic;
        lastProcessedEnvironment = '';
    }

    console.log(`Total processed: ${processedCount} pain points.`);
    fs.writeFileSync('pain_points_embeddings.json', JSON.stringify({ painPoints, embeddings }));

    return { painPoints, embeddings };
}

function clusterPainPoints(embeddings, numClusters = 100) {
    console.log("Clustering pain points...");
    const result = kmeans(embeddings, numClusters);
    return result.clusters;
}

function visualizeClusters(embeddings, clusters) {
    console.log("Preparing data for visualization...");
    const pca = new PCA(embeddings);
    const reducedData = pca.predict(embeddings, { nComponents: 2 });

    const data = clusters.map((cluster, index) => ({
        x: reducedData[index][0],
        y: reducedData[index][1],
        cluster: cluster
    }));

    console.log("Creating scatter plot...");
    const traces = Array.from({ length: Math.max(...clusters) + 1 }, (_, i) => ({
        x: data.filter(d => d.cluster === i).map(d => d.x),
        y: data.filter(d => d.cluster === i).map(d => d.y),
        mode: 'markers',
        type: 'scatter',
        name: `Cluster ${i}`
    }));

    plot(traces, { title: 'Pain Points Clusters' });
}

async function main() {
    console.log("Processing pain points and generating embeddings...");
    const startTime = Date.now();
    const { painPoints, embeddings } = await processResults();
    const embeddingTime = Date.now();
    console.log(`Embedding completed in ${(embeddingTime - startTime) / 1000} seconds`);

    const clusters = clusterPainPoints(embeddings);
    const clusteringTime = Date.now();
    console.log(`Clustering completed in ${(clusteringTime - embeddingTime) / 1000} seconds`);

    visualizeClusters(embeddings, clusters);

    console.log("Saving clustered results...");
    const clusteredResults = painPoints.filter((_, index) => embeddings[index] !== null).map((point, index) => ({
        ...point,
        cluster: clusters[index]
    }));
    fs.writeFileSync('clustered_pain_points.json', JSON.stringify(clusteredResults, null, 2));

    const endTime = Date.now();
    console.log(`Total process completed in ${(endTime - startTime) / 1000} seconds`);
}

main().catch(console.error);
------------------------

documentation of embeddings:

Generative AI on Vertex AI 
Documentation 
API reference
Was this helpful?

Send feedbackText embeddings API 

bookmark_border
The Text embeddings API converts textual data into numerical vectors. These vector representations are designed to capture the semantic meaning and context of the words they represent.

Supported Models:

English models	Multilingual models
textembedding-gecko@001	textembedding-gecko-multilingual@001
textembedding-gecko@003	text-multilingual-embedding-002
text-embedding-004	
text-embedding-preview-0815	
Syntax
curl
Python


PROJECT_ID = PROJECT_ID
REGION = us-central1
MODEL_ID = MODEL_ID

curl -X POST \
  -H "Authorization: Bearer $(gcloud auth print-access-token)" \
  -H "Content-Type: application/json" \
  https://${REGION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${REGION}/publishers/google/models/${MODEL_ID}:predict -d \
  '{
    "instances": [
      ...
    ],
    "parameters": {
      ...
    }
  }'
Parameter list
Parameters
texts

list of union[string, TextEmbeddingInput]

Each instance represents a single piece of text to be embedded.

TextEmbeddingInput

string

The text that you want to generate embeddings for.

auto_truncate

Optional: bool

When set to true, input text will be truncated. When set to false, an error is returned if the input text is longer than the maximum length supported by the model. Defaults to true.

output_dimensionality

Optional: int

Used to specify output embedding size. If set, output embeddings will be truncated to the size specified.

Request body


{
  "instances": [
    { 
      "task_type": "RETRIEVAL_DOCUMENT",
      "title": "document title",
      "content": "I would like embeddings for this text!"
    },
  ]
}
Parameters
content

string

The text that you want to generate embeddings for.

task_type

Optional: string

Used to convey intended downstream application to help the model produce better embeddings. If left blank, the default used is RETRIEVAL_QUERY.

RETRIEVAL_QUERY
RETRIEVAL_DOCUMENT
SEMANTIC_SIMILARITY
CLASSIFICATION
CLUSTERING
QUESTION_ANSWERING
FACT_VERIFICATION
CODE_RETRIEVAL_QUERY
The task_type parameter is not supported for the textembedding-gecko@001 model.

For more information about task types, see Choose an embeddings task type.

title

Optional: string

Used to help the model produce better embeddings. Only valid with task_type=RETRIEVAL_DOCUMENT.

taskType
The following table describes the task_type parameter values and their use cases:

task_type	Description
RETRIEVAL_QUERY	Specifies the given text is a query in a search or retrieval setting.
RETRIEVAL_DOCUMENT	Specifies the given text is a document in a search or retrieval setting.
SEMANTIC_SIMILARITY	Specifies the given text is used for Semantic Textual Similarity (STS).
CLASSIFICATION	Specifies that the embedding is used for classification.
CLUSTERING	Specifies that the embedding is used for clustering.
QUESTION_ANSWERING	Specifies that the query embedding is used for answering questions. Use RETRIEVAL_DOCUMENT for the document side.
FACT_VERIFICATION	Specifies that the query embedding is used for fact verification.
CODE_RETRIEVAL_QUERY	Specifies that the query embedding is used for code retrieval for Java and Python.
Retrieval Tasks:

Query: Use task_type=RETRIEVAL_QUERY to indicate that the input text is a search query. Corpus: Use task_type=RETRIEVAL_DOCUMENT to indicate that the input text is part of the document collection being searched.

Similarity Tasks:

Semantic similarity: Use task_type= SEMANTIC_SIMILARITY for both input texts to assess their overall meaning similarity.

Response body


{
  "predictions": [
    {
      "embeddings": {
        "statistics": {
          "truncated": boolean,
          "token_count": integer
        },
        "values": [ number ]
      }
    }
  ]
}
Response element	Description
embeddings	The result generated from input text.
statistics	The statistics computed from the input text.
truncated	Indicates if the input text was longer than max allowed tokens and truncated.
tokenCount	Number of tokens of the input text.
values	The values field contains the embedding vectors corresponding to the words in the input text.
Sample response


{
  "predictions": [
    {
      "embeddings": {
        "values": [
          0.0058424929156899452,
          0.011848051100969315,
          0.032247550785541534,
          -0.031829461455345154,
          -0.055369812995195389,
          ...
        ],
        "statistics": {
          "token_count": 4,
          "truncated": false
        }
      }
    }
  ]
}
Examples
Embed a text string
Basic use case
The following example shows how to obtain the embedding of a text string.

REST
Python
Go
Java
Node.js
After you set up your environment, you can use REST to test a text prompt. The following sample sends a request to the publisher model endpoint.

Before using any of the request data, make the following replacements:

PROJECT_ID: Your project ID.
TEXT: The text that you want to generate embeddings for. Limit: five texts of up to 2,048 tokens per text for all models except textembedding-gecko@001. The max input token length for textembedding-gecko@001 is 3072.
AUTO_TRUNCATE: If set to false, text that exceeds the token limit causes the request to fail. The default value is true.
HTTP method and URL:



POST https://us-central1-aiplatform.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/publishers/google/models/text-embedding-004:predict
Request JSON body:



{
  "instances": [
    { "content": "TEXT"}
  ],
  "parameters": { 
    "autoTruncate": AUTO_TRUNCATE 
  }
}
To send your request, choose one of these options:

curl
PowerShell
Note: The following command assumes that you have logged in to the gcloud CLI with your user account by running gcloud init or gcloud auth login , or by using Cloud Shell, which automatically logs you into the gcloud CLI . You can check the currently active account by running gcloud auth list.
Save the request body in a file named request.json, and execute the following command:



curl -X POST \
     -H "Authorization: Bearer $(gcloud auth print-access-token)" \
     -H "Content-Type: application/json; charset=utf-8" \
     -d @request.json \
     "https://us-central1-aiplatform.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/publishers/google/models/text-embedding-004:predict"
You should receive a JSON response similar to the following. Note that values has been truncated to save space.

Response
Note the following in the URL for this sample:
Use the generateContent method to request that the response is returned after it's fully generated. To reduce the perception of latency to a human audience, stream the response as it's being generated by using the streamGenerateContent method.
The multimodal model ID is located at the end of the URL before the method (for example, gemini-1.5-flash or gemini-1.0-pro-vision). This sample may support other models as well.
Advanced Use Case
The following example demonstrates some advanced features

Use task_type and title to improve embedding quality.
Use parameters to control the behavior of the API.
Note: Feature support varies by model version. For example, task_type and title fields are not supported by textembedding-gecko@001. For best results, choose the latest available version.
REST
Python
Go
Java
Node.js
Before using any of the request data, make the following replacements:

PROJECT_ID: Your project ID.
TEXT: The text that you want to generate embeddings for. Limit: five texts of up to 3,072 tokens per text.
TASK_TYPE: Used to convey the intended downstream application to help the model produce better embeddings.
TITLE: Used to help the model produce better embeddings.
AUTO_TRUNCATE: If set to false, text that exceeds the token limit causes the request to fail. The default value is true.
OUTPUT_DIMENSIONALITY: Used to specify output embedding size. If set, output embeddings will be truncated to the size specified.
HTTP method and URL:



POST https://us-central1-aiplatform.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/publishers/google/models/textembedding-gecko@003:predict
Request JSON body:



{
  "instances": [
    { "content": "TEXT",
      "task_type": "TASK_TYPE",
      "title": "TITLE"
    },
  ],
  "parameters": {
    "autoTruncate": AUTO_TRUNCATE,
    "outputDimensionality": OUTPUT_DIMENSIONALITY
  }
}
To send your request, choose one of these options:

curl
PowerShell
Note: The following command assumes that you have logged in to the gcloud CLI with your user account by running gcloud init or gcloud auth login , or by using Cloud Shell, which automatically logs you into the gcloud CLI . You can check the currently active account by running gcloud auth list.
Save the request body in a file named request.json, and execute the following command:



curl -X POST \
     -H "Authorization: Bearer $(gcloud auth print-access-token)" \
     -H "Content-Type: application/json; charset=utf-8" \
     -d @request.json \
     "https://us-central1-aiplatform.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/publishers/google/models/textembedding-gecko@003:predict"
You should receive a JSON response similar to the following. Note that values has been truncated to save space.

Response

------------------------------


Given the script above and the documentation for the embeddings used, write a script that is going to go over each pain point of all demographs, save it and embed it. The goal is to group the most similar pain points to have a general sense of the most common pain points in general. Also make sure that the results are vizualized in an intuitive way