const RATE_LIMIT = 999; // 999 requests per minute
const MAX_PAYLOAD_SIZE = 9000; // Bytes
const INITIAL_BATCH_SIZE = 100;
const CHECKPOINT_INTERVAL = 50; // Save checkpoint every 50 processed items

function createRateLimiter(requestsPerMinute) {
const interval = 60000 / requestsPerMinute;
let lastRequest = Date.now();
return async () => {
const now = Date.now();
const timeToWait = interval - (now - lastRequest);
if (timeToWait > 0) {
await new Promise(resolve => setTimeout(resolve, timeToWait));
}
lastRequest = Date.now();
};
}

const rateLimiter = createRateLimiter(RATE_LIMIT);

function estimateBatchSize(points, currentBatchSize) {
const totalSize = points.reduce((sum, point) => sum + Buffer.byteLength(point, 'utf8'), 0);
const averageSize = totalSize / points.length;
return Math.floor(Math.min(currentBatchSize, MAX_PAYLOAD_SIZE / averageSize));
}

async function embedPainPointsBatch(batch) {
await rateLimiter();
try {
const result = await model.embedContent(batch);
return result.embedding.values;
} catch (error) {
if (error.message.includes('Request payload size exceeds the limit') && batch.length > 1) {
const halfBatchSize = Math.ceil(batch.length / 2);
const firstHalf = await embedPainPointsBatch(batch.slice(0, halfBatchSize));
const secondHalf = await embedPainPointsBatch(batch.slice(halfBatchSize));
return [...firstHalf, ...secondHalf];
} else {
console.error(Error embedding batch: ${error.message});
return batch.map(() => null);
}
}
}

function loadCheckpoint() {
if (fs.existsSync('checkpoint.json')) {
console.log("Loading from checkpoint...");
const checkpoint = JSON.parse(fs.readFileSync('checkpoint.json', 'utf8'));
return {
painPoints: checkpoint.painPoints,
embeddings: checkpoint.embeddings,
processedCount: checkpoint.processedCount,
lastProcessedDemographic: checkpoint.lastProcessedDemographic,
lastProcessedEnvironment: checkpoint.lastProcessedEnvironment
};
}
return { painPoints: [], embeddings: [], processedCount: 0, lastProcessedDemographic: '', lastProcessedEnvironment: '' };
}

function saveCheckpoint(painPoints, embeddings, processedCount, lastProcessedDemographic, lastProcessedEnvironment) {
console.log("Saving checkpoint...");
const checkpoint = {
painPoints,
embeddings,
processedCount,
lastProcessedDemographic,
lastProcessedEnvironment
};
fs.writeFileSync('checkpoint.json', JSON.stringify(checkpoint));
}

async function processResults() {
const resultsFile = fs.readFileSync('pain_points_results.json', 'utf8');
const results = JSON.parse(resultsFile);

let { painPoints, embeddings, processedCount, lastProcessedDemographic, lastProcessedEnvironment } = loadCheckpoint();

let batchSize = INITIAL_BATCH_SIZE;

for (const demographic in results) {
if (demographic < lastProcessedDemographic) continue;

for (const environment in results[demographic]) {
if (demographic === lastProcessedDemographic && environment < lastProcessedEnvironment) continue;

const points = results[demographic][environment].pain_points;
for (let i = 0; i < points.length; i += batchSize) {
const batch = points.slice(i, i + batchSize);
const newPainPoints = batch.map(point => ({ demographic, environment, point }));
painPoints.push(...newPainPoints);

const batchEmbeddings = await embedPainPointsBatch(batch);
embeddings.push(...batchEmbeddings);

processedCount += batch.length;
console.log(Processed ${processedCount} pain points. Current batch size: ${batchSize});

if (processedCount % CHECKPOINT_INTERVAL === 0) {
saveCheckpoint(painPoints, embeddings, processedCount, demographic, environment);
}

batchSize = estimateBatchSize(batch, batchSize);
}
lastProcessedEnvironment = environment;
}
lastProcessedDemographic = demographic;
lastProcessedEnvironment = '';
}

console.log(Total processed: ${processedCount} pain points.);
fs.writeFileSync('pain_points_embeddings.json', JSON.stringify({ painPoints, embeddings }));

return { painPoints, embeddings };
}

function clusterPainPoints(embeddings, numClusters = 100) {
console.log("Clustering pain points...");
const result = kmeans(embeddings, numClusters);
return result.clusters;
}

function visualizeClusters(embeddings, clusters) {
console.log("Preparing data for visualization...");
const pca = new PCA(embeddings);
const reducedData = pca.predict(embeddings, { nComponents: 2 });

const data = clusters.map((cluster, index) => ({
x: reducedData[index][0],
y: reducedData[index][1],
cluster: cluster
}));

console.log("Creating scatter plot...");
const traces = Array.from({ length: Math.max(...clusters) + 1 }, (, i) => ({
x: data.filter(d => d.cluster === i).map(d => d.x),
y: data.filter(d => d.cluster === i).map(d => d.y),
mode: 'markers',
type: 'scatter',
name: Cluster ${i}
}));

plot(traces, { title: 'Pain Points Clusters' });
}

async function main() {
console.log("Processing pain points and generating embeddings...");
const startTime = Date.now();
const { painPoints, embeddings } = await processResults();
const embeddingTime = Date.now();
console.log(Embedding completed in ${(embeddingTime - startTime) / 1000} seconds);

const clusters = clusterPainPoints(embeddings);
const clusteringTime = Date.now();
console.log(Clustering completed in ${(clusteringTime - embeddingTime) / 1000} seconds);

visualizeClusters(embeddings, clusters);

console.log("Saving clustered results...");
const clusteredResults = painPoints.filter((, index) => embeddings[index] !== null).map((point, index) => ({
...point,
cluster: clusters[index]
}));
fs.writeFileSync('clustered_pain_points.json', JSON.stringify(clusteredResults, null, 2));

const endTime = Date.now();
console.log(Total process completed in ${(endTime - startTime) / 1000} seconds);
}

main().catch(console.error);
------------------------

documentation of embeddings:

Generative AI on Vertex AI 
Documentation 
API reference
Was this helpful?

Send feedbackText embeddings API 

bookmark_border
The Text embeddings API converts textual data into numerical vectors. These vector representations are designed to capture the semantic meaning and context of the words they represent.

Supported Models:

English models	Multilingual models
textembedding-gecko@001	textembedding-gecko-multilingual@001
textembedding-gecko@003	text-multilingual-embedding-002
text-embedding-004	
text-embedding-preview-0815	
Syntax
curl
Python


PROJECT_ID = PROJECT_ID
REGION = us-central1
MODEL_ID = MODEL_ID

curl -X POST \
  -H "Authorization: Bearer $(gcloud auth print-access-token)" \
  -H "Content-Type: application/json" \
  https://${REGION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${REGION}/publishers/google/models/${MODEL_ID}:predict -d \
  '{
    "instances": [
      ...
    ],
    "parameters": {
      ...
    }
  }'
Parameter list
Parameters
texts

list of union[string, TextEmbeddingInput]

Each instance represents a single piece of text to be embedded.

TextEmbeddingInput

string

The text that you want to generate embeddings for.

auto_truncate

Optional: bool

When set to true, input text will be truncated. When set to false, an error is returned if the input text is longer than the maximum length supported by the model. Defaults to true.

output_dimensionality

Optional: int

Used to specify output embedding size. If set, output embeddings will be truncated to the size specified.

Request body


{
  "instances": [
    { 
      "task_type": "RETRIEVAL_DOCUMENT",
      "title": "document title",
      "content": "I would like embeddings for this text!"
    },
  ]
}
Parameters
content

string

The text that you want to generate embeddings for.

task_type

Optional: string

Used to convey intended downstream application to help the model produce better embeddings. If left blank, the default used is RETRIEVAL_QUERY.

RETRIEVAL_QUERY
RETRIEVAL_DOCUMENT
SEMANTIC_SIMILARITY
CLASSIFICATION
CLUSTERING
QUESTION_ANSWERING
FACT_VERIFICATION
CODE_RETRIEVAL_QUERY
The task_type parameter is not supported for the textembedding-gecko@001 model.

For more information about task types, see Choose an embeddings task type.

title

Optional: string

Used to help the model produce better embeddings. Only valid with task_type=RETRIEVAL_DOCUMENT.

taskType
The following table describes the task_type parameter values and their use cases:

task_type	Description
RETRIEVAL_QUERY	Specifies the given text is a query in a search or retrieval setting.
RETRIEVAL_DOCUMENT	Specifies the given text is a document in a search or retrieval setting.
SEMANTIC_SIMILARITY	Specifies the given text is used for Semantic Textual Similarity (STS).
CLASSIFICATION	Specifies that the embedding is used for classification.
CLUSTERING	Specifies that the embedding is used for clustering.
QUESTION_ANSWERING	Specifies that the query embedding is used for answering questions. Use RETRIEVAL_DOCUMENT for the document side.
FACT_VERIFICATION	Specifies that the query embedding is used for fact verification.
CODE_RETRIEVAL_QUERY	Specifies that the query embedding is used for code retrieval for Java and Python.
Retrieval Tasks:

Query: Use task_type=RETRIEVAL_QUERY to indicate that the input text is a search query. Corpus: Use task_type=RETRIEVAL_DOCUMENT to indicate that the input text is part of the document collection being searched.

Similarity Tasks:

Semantic similarity: Use task_type= SEMANTIC_SIMILARITY for both input texts to assess their overall meaning similarity.

Response body


{
  "predictions": [
    {
      "embeddings": {
        "statistics": {
          "truncated": boolean,
          "token_count": integer
        },
        "values": [ number ]
      }
    }
  ]
}
Response element	Description
embeddings	The result generated from input text.
statistics	The statistics computed from the input text.
truncated	Indicates if the input text was longer than max allowed tokens and truncated.
tokenCount	Number of tokens of the input text.
values	The values field contains the embedding vectors corresponding to the words in the input text.
Sample response


{
  "predictions": [
    {
      "embeddings": {
        "values": [
          0.0058424929156899452,
          0.011848051100969315,
          0.032247550785541534,
          -0.031829461455345154,
          -0.055369812995195389,
          ...
        ],
        "statistics": {
          "token_count": 4,
          "truncated": false
        }
      }
    }
  ]
}
Examples
Embed a text string
Basic use case
The following example shows how to obtain the embedding of a text string.

REST
Python
Go
Java
Node.js
Before trying this sample, follow the Node.js setup instructions in the Vertex AI quickstart using client libraries. For more information, see the Vertex AI Node.js API reference documentation.

To authenticate to Vertex AI, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.



async function main(
  project,
  model = 'text-embedding-004',
  texts = 'banana bread?;banana muffins?',
  task = 'QUESTION_ANSWERING',
  dimensionality = 0,
  apiEndpoint = 'us-central1-aiplatform.googleapis.com'
) {
  const aiplatform = require('@google-cloud/aiplatform');
  const {PredictionServiceClient} = aiplatform.v1;
  const {helpers} = aiplatform; // helps construct protobuf.Value objects.
  const clientOptions = {apiEndpoint: apiEndpoint};
  const location = 'us-central1';
  const endpoint = `projects/${project}/locations/${location}/publishers/google/models/${model}`;

  async function callPredict() {
    const instances = texts
      .split(';')
      .map(e => helpers.toValue({content: e, task_type: task}));
    const parameters = helpers.toValue(
      dimensionality > 0 ? {outputDimensionality: parseInt(dimensionality)} : {}
    );
    const request = {endpoint, instances, parameters};
    const client = new PredictionServiceClient(clientOptions);
    const [response] = await client.predict(request);
    const predictions = response.predictions;
    const embeddings = predictions.map(p => {
      const embeddingsProto = p.structValue.fields.embeddings;
      const valuesProto = embeddingsProto.structValue.fields.values;
      return valuesProto.listValue.values.map(v => v.numberValue);
    });
    console.log('Got embeddings: \n' + JSON.stringify(embeddings));
  }

  callPredict();
}
Advanced Use Case
The following example demonstrates some advanced features

Use task_type and title to improve embedding quality.
Use parameters to control the behavior of the API.
Note: Feature support varies by model version. For example, task_type and title fields are not supported by textembedding-gecko@001. For best results, choose the latest available version.
REST
Python
Go
Java
Node.js
Before trying this sample, follow the Node.js setup instructions in the Vertex AI quickstart using client libraries. For more information, see the Vertex AI Node.js API reference documentation.

To authenticate to Vertex AI, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.



async function main(
  project,
  model = 'text-embedding-004',
  texts = 'banana bread?;banana muffins?',
  task = 'QUESTION_ANSWERING',
  dimensionality = 0,
  apiEndpoint = 'us-central1-aiplatform.googleapis.com'
) {
  const aiplatform = require('@google-cloud/aiplatform');
  const {PredictionServiceClient} = aiplatform.v1;
  const {helpers} = aiplatform; // helps construct protobuf.Value objects.
  const clientOptions = {apiEndpoint: apiEndpoint};
  const location = 'us-central1';
  const endpoint = `projects/${project}/locations/${location}/publishers/google/models/${model}`;

  async function callPredict() {
    const instances = texts
      .split(';')
      .map(e => helpers.toValue({content: e, task_type: task}));
    const parameters = helpers.toValue(
      dimensionality > 0 ? {outputDimensionality: parseInt(dimensionality)} : {}
    );
    const request = {endpoint, instances, parameters};
    const client = new PredictionServiceClient(clientOptions);
    const [response] = await client.predict(request);
    const predictions = response.predictions;
    const embeddings = predictions.map(p => {
      const embeddingsProto = p.structValue.fields.embeddings;
      const valuesProto = embeddingsProto.structValue.fields.values;
      return valuesProto.listValue.values.map(v => v.numberValue);
    });
    console.log('Got embeddings: \n' + JSON.stringify(embeddings));
  }

  callPredict();
}


------------------------------


Given the script above and the documentation for the embeddings used, write a node.js script that is going to go over each pain point of all demographs, save it and embed it. The goal is to group the most similar pain points to have a general sense of the most common pain points in general. Also make sure that the results are vizualized in an intuitive way