vania-janet/multiturn-rag-retrieval-data huggingface dataset:
For more information, see the API documentation's [Authentication section](https://cloud.ibm.com/apidocs/cloudant?code=goauthentication-with-external-configuration) for examples.





 Uploading partitioned index by device ID and timestamp 

Upload the index by device ID and timestamp to the database by using this command:



* Curl
* Java
* Node
* Python
* Go



curl -X POST "$SERVICE_URL/readings/_index" -H 'Content-Type: application/json' --data '{
"index": {
"fields": [
{"deviceID": "asc"},
{"ts": "asc"}
]
},
"name": "deviceID-readings",
"type": "json",
"partitioned": true
}'

import com.ibm.cloud.cloudant.v1.Cloudant;
import com.ibm.cloud.cloudant.v1.model.IndexDefinition;
import com.ibm.cloud.cloudant.v1.model.IndexField;
import com.ibm.cloud.cloudant.v1.model.IndexResult;
import com.ibm.cloud.cloudant.v1.model.PostIndexOptions;

import java.util.Arrays;

Cloudant service = Cloudant.newInstance();

IndexField indexField1 = new IndexField.Builder()
.add("deviceID", "asc")
.build();

IndexField indexField2 = new IndexField.Builder()
.add("ts", "asc")
.build();

IndexDefinition index = new IndexDefinition.Builder()
.fields(Arrays.asList(indexField1, indexField2))
.build();

PostIndexOptions indexOptions = new PostIndexOptions.Builder()
.db("readings")
.index(index)
documentResult, response, err := service.PutDocument(putDocumentOptions)
if err != nil {
panic(err)
}

b, _ := json.MarshalIndent(documentResult, "", " ")
fmt.Println(string(b))

const { CloudantV1 } = require('@ibm-cloud/cloudant');

const service = CloudantV1.newInstance({});

const eventDoc = {
type: 'event',
userid: 'abc123',
eventType: 'addedToBasket',
productId: '1000042',
date: '2019-01-28T10:44:22.000Z'
};

service.putDocument({
db: 'events',
docId: '0007241142412418284',
document: eventDoc
}).then(response => {
console.log(response.result);
});

The previous Go example requires the following import block:

import (
"encoding/json"
"fmt"
"github.com/IBM/cloudant-go-sdk/cloudantv1"
)

All Go examples require the service object to be initialized. For more information, see the API documentation's [Authentication section](https://cloud.ibm.com/apidocs/cloudant?code=goauthentication-with-external-configuration) for examples.

The response is a JSON document that contains the ID of the created document, the revision string, and "ok": true.

If you didn't provide an _id field, IBM Cloudant generates one automatically as a [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).

A failure to create the document results in a response that contains a description of the error.

See an example response after successfully creating a document:

{
youyou@cloudshell:$ ibmcloud resource service-keys --instance-name $YOURINITIALS-cloudant-service --output json
[
{
"guid": "01234560-902d-4078-9a7f-20446a639aeb",
"id": "crn:v1:bluemix:public:cloudantnosqldb:us-south:a/0123456789507a53135fe6793c37cc74:SECRET",
"url": "/v2/resource_keys/01234560-902d-4078-9a7f-20446a639aeb",
"created_at": "2020-05-06T23:03:43.484872077Z",
"updated_at": "2020-05-06T23:03:43.484872077Z",
"deleted_at": null,
"name": "cloudant-binding",
"account_id": "0123456789507a53135fe6793c37cc74",
"resource_group_id": "01234567836d49029966ab5be7fe50b5",
"source_crn": "crn:v1:bluemix:public:cloudantnosqldb:us-south:a/0123456789507a53135fe6793c37cc74:SECRET",
"state": "active",
"credentials": {
"apikey": "SECRET",
"host": "SECRET",
"iam_apikey_description": "Auto-generated for key SECRET",
"iam_apikey_name": "cloudant-binding",
"iam_role_crn": "SECRET",
"iam_serviceid_crn": "SECRET",
"password": "SECRET",
b, _ := json.MarshalIndent(findResult, "", " ")
fmt.Println(string(b))

The previous Go example requires the following import block:

import (
"encoding/json"
"fmt"
"github.com/IBM/cloudant-go-sdk/cloudantv1"
)

All Go examples require the service object to be initialized. For more information, see the API documentation's [Authentication section](https://cloud.ibm.com/apidocs/cloudant?code=goauthentication-with-external-configuration) for examples.
b, _ := json.MarshalIndent(documentResult, "", " ")
fmt.Println(string(b))

const { CloudantV1 } = require('@ibm-cloud/cloudant');
const service = CloudantV1.newInstance({});

service.deleteDocument({
db: 'events',
docId: '0007241142412418284',
rev: '2-9a0d1cd9f40472509e9aac6461837367'
}).then(response => {
console.log(response.result);
});

The previous Go example requires the following import block:

import (
"encoding/json"
"fmt"
"github.com/IBM/cloudant-go-sdk/cloudantv1"
)

All Go examples require the service object to be initialized. For more information, see the API documentation's [Authentication section](https://cloud.ibm.com/apidocs/cloudant?code=goauthentication-with-external-configuration) for examples.

Now, conflicts affecting the document are resolved. You can verify the status by running GET to the document again with the conflicts parameter set to true.
from ibmcloudant.cloudant_v1 import CloudantV1

service = CloudantV1.new_instance()

response = service.put_database(db='products', partitioned=True).get_result()

print(response)

putDatabaseOptions := service.NewPutDatabaseOptions(
"products",
)
putDatabaseOptions.SetPartitioned(true)

ok, response, err := service.PutDatabase(putDatabaseOptions)
if err != nil {
panic(err)
}

b, _ := json.MarshalIndent(ok, "", " ")
fmt.Println(string(b))

const { CloudantV1 } = require('@ibm-cloud/cloudant');

const service = CloudantV1.newInstance({});

service.putDatabase({
db: 'products',
partitioned: true
}).then(response => {
console.log(response.result);
});

The previous Go example requires the following import block:

import (
"encoding/json"
"fmt"
"github.com/IBM/cloudant-go-sdk/cloudantv1"
)

All Go examples require the service object to be initialized. For more information, see the API documentation's [Authentication section](https://cloud.ibm.com/apidocs/cloudant?code=goauthentication-with-external-configuration) for examples.





 Example - Creating a nonpartitioned database 

To create a nonpartitioned database, see the following example:



* Curl
* Java
* Python
* Go
* Node



curl -H "Authorization: Bearer $API_BEARER_TOKEN" -X PUT "$SERVICE_URL/products"

import com.ibm.cloud.cloudant.v1.Cloudant;
The [POST /{db}/_explain](https://cloud.ibm.com/apidocs/cloudant?code=javapostexplain) API endpoint when passed a JSON object that is usually sent to the [POST /{db}/_find](https://cloud.ibm.com/apidocs/cloudantpostfind) endpoint, explains how such a query is handled and which indexes, if any, might be used.

If the index object in the response indicates that "all_docs" is being used, a full database scan is required to service the query. We recommend that you use the _explain mechanism to check each IBM Cloudant query to ensure it is using an index before you deploy to production.

For example, a type=json index on firstname, surname and date is suitable for finding documents for:



* A known firstname, lastname, and date.
* A known firstname, lastname, and a range of date values (that use $lt, $lte, $gt, $gte operators).
* A known firstname and lastname sorted by date.



It can also be used to assist queries on firstname, surname, date, and other attributes. In other words, it might be able to answer only part of the query but it can help reduce the number of documents that are scanned to find the answer.





 How can I ensure that my query is efficent? 

Ideally, an IBM Cloudant Query execution would need to scan only one document for each document returned. If a query has to scan a million documents for each one returned, it is clearly not optimal, and is in need of a secondary index to help.

When you execute a query, passing execution_stats: true as an extra parameter forces IBM Cloudant to enumerate the number of documents it scanned in performing the query, for example:

{
"selector": {
"firstname": "Charles",
"surname": "Dickens"
},
"sort": ["date"],
"limit": 10,
"execution_stats": true
}

The returned data now includes an extra JSON object:

{
...
if err != nil {
panic(err)
}

b, _ := json.MarshalIndent(ok, "", " ")
fmt.Println(string(b))

const { CloudantV1 } = require('@ibm-cloud/cloudant');

const service = CloudantV1.newInstance({});

service.putDatabase({
db: 'products'
}).then(response => {
console.log(response.result);
});

The previous Go example requires the following import block:

import (
"encoding/json"
"fmt"
"github.com/IBM/cloudant-go-sdk/cloudantv1"
)

All Go examples require the service object to be initialized. For more information, see the API documentation's [Authentication section](https://cloud.ibm.com/apidocs/cloudant?code=goauthentication-with-external-configuration) for examples.





 Query arguments 



Table 1. Query arguments

 Argument Description Optional Type Default Supported values 

 partitioned Determines whether the database is partitioned. Yes Boolean false true, false 







 Database naming 

The database name must start with a lowercase letter, and contain only the following characters:



* Lowercase characters (a-z)
* Digits (0-9)
* Any of the following special characters: _, $, (, ), +, -, and /



If your database is successfully created, you get a 201 or 202 response. An error response uses the HTTP status code to indicate what went wrong.



Table 2. HTTP status codes

 Code Description 

 201 Database created successfully. 
 202 The database was successfully created on some nodes, but the number of nodes is less than the write quorum. 
 400 Invalid database name. 
 412 Database exists. 



See the following example response that is received after a database is created successfully:

HTTP/1.1 201 Created

{
"ok": true
}





 Database topology
db: 'orders',
}).then(response => {
console.log(response.result);
});

from ibmcloudant.cloudant_v1 import CloudantV1

service = CloudantV1.new_instance()

response = service.post_all_docs(
db='orders',
).get_result()

print(response)

postAllDocsOptions := service.NewPostAllDocsOptions(
"orders",
)

allDocsResult, response, err := service.PostAllDocs(postAllDocsOptions)
if err != nil {
panic(err)
}

b, _ := json.MarshalIndent(allDocsResult, "", " ")
fmt.Println(string(b))

The previous Go example requires the following import block:

import (
"encoding/json"
"fmt"
"github.com/IBM/cloudant-go-sdk/cloudantv1"
)

All Go examples require the service object to be initialized. For more information, see the API documentation's [Authentication section](https://cloud.ibm.com/apidocs/cloudant?code=goauthentication-with-external-configuration) for examples.

{
"total_rows": 11,
"offset": 0,
"rows": [ ! ! ! ! !!!
{
"id": "4eee973603bf77f30b1f880ed83df76a",
"key": "4eee973603bf77f30b1f880ed83df76a",
"value": {
"rev": "1-3b5e6b73e57745787ad5627fe8f268c1"
}
},
{
[Activate the alias](https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-working-with-curlactivate-the-alias).



If you're using a Windows™ computer, you can specify your username and password from the command line.



 Encode username and password 

First, you base64-encode your IBM Cloudant username and password. This encoding gives us a base64 character sequence as output.

$ACCOUNT is the username field in your service credentials. For more information, see [Locating your service credentials](https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-locating-your-service-credentials).

The command to base64-encode some data is similar to the following example:

python3 -c 'import base64; print(base64.urlsafe_b64encode("$ACCOUNT:$PASSWORD".encode("utf-8")).decode("utf-8"))'

The output is called <OUTPUT-OF-BASE64>.

For example, if you use the following command:

python3 -c 'import base64; print(base64.urlsafe_b64encode("$ACCOUNT:$PASSWORD".encode("utf-8")).decode("utf-8"))'

You then get the following output:

NTFkZGM5YTAtZmE2MC00M2Q1LTgyNmJeKGNmYjBhNTVkMzFiLWJsdWVtaXguY2xvdWRhbnQuY29tOjY4ODIyZGQ5YTU5YzNhZjA1NDY5YzRhMGRjODUzZjVhYjQzMmQxMDI0NTFiNTQ0ZTUxZjA5MjkwODU2NDcxNWM=

Remember your password is still stored in plain text on your computer. Base64-encoding isn't encryption. If you use base64-encode on the same character sequence, you always get the same corresponding character output sequence.





 Create an alias 

Now, you create an alias for curl that includes these credentials, so you don’t have to enter them every time you write a curl command.

Add the following line to your /.bashrc or /.bash_profile:

alias acurl="curl -s --proto '=https' -g -H 'Authorization: Basic <OUTPUT-OF-BASE64>'"

This alias adds an authorization header instead of including the authorization credentials in the URL you enter on the command line.
import com.ibm.cloud.cloudant.v1.model.PutDatabaseOptions;

Cloudant service = Cloudant.newInstance();

PutDatabaseOptions databaseOptions = new PutDatabaseOptions.Builder()
.db("_replicator")
.build();

Ok response =
service.putDatabase(databaseOptions).execute()
.getResult();

System.out.println(response);

const { CloudantV1 } = require('@ibm-cloud/cloudant');

const service = CloudantV1.newInstance({});

service.putDatabase({
db: '_replicator'
}).then(response => {
console.log(response.result);
});

from ibmcloudant.cloudant_v1 import CloudantV1

service = CloudantV1.new_instance()

response = service.put_database(db='_replicator').get_result()

print(response)

putDatabaseOptions := service.NewPutDatabaseOptions(
"_replicator",
)

ok, response, err := service.PutDatabase(putDatabaseOptions)
if err != nil {
panic(err)
}

b, _ := json.MarshalIndent(ok, "", " ")
fmt.Println(string(b))

The previous Go example requires the following import block:

import (
"encoding/json"
"fmt"
"github.com/IBM/cloudant-go-sdk/cloudantv1"
)

All Go examples require the service object to be initialized. For more information, see the API documentation's [Authentication section](https://cloud.ibm.com/apidocs/cloudant?code=goauthentication-with-external-configuration) for examples.







 Many simultaneous replications
If you need to access objects within documents, you can use standard dot notation for example, address.zipcode to access a postal code string inside an address object.

We can also add the following parameters:

Fields
: Specifies the document attributes that we want returned (the default is the entire document).

Sort
: Defines how the data is to be sorted. Sort is an array, allowing the sort to be calculated on multiple attributes.

Limit
: The number of documents to return.

If you are from a relational database background, this query is the equivalent SQL query to that last IBM Cloudant query example.

The WHERE clause is the equivalent of SELECTOR in IBM Cloudant Query. ORDER and LIMIT are exactly equivalent, and the IBM Cloudant Query FIELDS list is equivalent to the comma-separated list of attributes after the SELECT keyword.

The JSON syntax might take a bit of getting used to, but MongoDB users might find it familiar.

IBM Cloudant queries can be executed in the IBM Cloudant Dashboard. Select the database that you are working with, for example, books then choose the Query tab.

Enter your IBM Cloudant Query JSON in the box that is provided, and click Run Query when you're ready. The result set appears on the page.

The Explain button is used to provide an explanation on how the database interprets the supplied query. This explanation becomes more important when we get to Indexing in the next part.

Queries can be triggered from curl too. The Query JSON, in this case, is stored in a file and we POST to the _find endpoint by using the -d@ command-line syntax.

The Node.js code is similar. The Query is a standard JavaScript object, which is passed to the db.find function which it POSTs to the _find endpoint on your behalf.

Now, time for a practical exercise. Devise your own IBM Cloudant Query that finds the titles of books that are written in the 20th Century. The IBM Cloudant Query documentation is at the on-screen URL if you need it.

Pause the presentation here if you don't want to know the answer...

See one solution:
Now you know that IBM Cloudant doesn’t enforce any normalization and that it has no transactions of the type you’re used to from, say, [Postgres](https://www.postgresql.org/). The temptation can be to cram as much as possible into each document, which would also save on HTTP usage.

This practice is often a bad idea.

If your model groups information that doesn’t change together, you’re more likely to suffer from update conflicts.

Consider a situation where you have users, each with a set of orders associated with them. One way might be to represent the orders as an array in the user document:

{ // DON'T DO THIS
"customer_id": 65522389,
"orders": [ {
"order_id": 887865,
"items": {
"item_id": 9982,
"item_name": "Iron sprocket",
"cost": 53.0
}, {
"item_id": 2932,
"item_name": "Rubber wedge",
"cost": 3.0
}
]
}
]
}
Show more

To add an order, I need to fetch the complete document, unmarshal the JSON, add the item, marshal the new JSON, and send it back as an update. If I’m the only one doing so, it might work for a while. If the document is being updated concurrently, or being replicated, we might likely see update conflicts.

Instead, keep orders separate as their own document type, referencing the customer ID. Now the model is immutable. To add an order, I create a new order document in the database, which cannot generate conflicts.

To be able to retrieve all orders for a specific customer, we can employ a view, which we cover later.

Avoid constructs that rely on updates to parts of existing documents, where possible. Bad data models are often hard to change after you’re in production.
For the surname fields, we call the [POST /{db}/_index](https://cloud.ibm.com/apidocs/cloudantpostindex) endpoint to pass the JSON index definition as the HTTP POST body:

{
"index": {
"fields": ["firstname", "surname"]
},
"ddoc": "jsonindexes",
"name": "byName",
"type": "json"
}

For more information, see [Creating an Index](https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-querycreating-an-index).





 Can I sort my search results with IBM Cloudant Query? 

Yes! The _find JSON syntax allows for a sort parameter to be provided listing the attribute or attributes to sort by. In this case, we are sorting by date:

{
"selector": {
"firstname": "Charles",
"surname": "Dickens"
},
"sort": ["date"],
"limit": 10
}

A suitable index must be present that contains the selector fields and the sort fields. Otherwise, IBM Cloudant refuses to execute the query. A suitable index definition for the previous query is shown next:

{
"index": {
"fields": [
"firstname",
"surname",
"date"
]
},
"ddoc": "jsonindexes",
"name": "byNameAndDate",
"type": "json"
}





 Can I sort in reverse order? 

Yes! IBM Cloudant Query supports sorting the result set in ascending or descending order, but not a combination of the two. For example, a query that sorts some fields in ascending order, and a query where descending is not allowed.

This query returns documents that match firstname and surname and sorts by surname/firstname/date descending:

{
"selector": {
* IBM Cloud Object Storage
* IBM Cloud Data Engine (Spark)
* IBM Cloudant
* Secure Gateway



IBM Cloud Kubernetes Service provides scalable compute resources and the associated DevOps dashboards to create, scale, and tear down apps and services as needed. Using industry-standard containers, apps can initially be re-hosted on IBM Cloud Kubernetes Service quickly without major architectural changes.

This solution provides the immediate benefit of scalability. By using Kubernetes's rich set of deployment and runtime objects, the mortgage company monitors and manages the upgrades to apps reliably. They're also able to replicate and scale the apps that use defined rules and the automated Kubernetes orchestrator.

Secure Gateway is used to create a secure pipeline to on-premises databases and documents for apps that are re-hosted to run in IBM Cloud Kubernetes Service.

IBM Cloud Object Storage is for all raw document and data storage as they go forward. For Monte Carlo simulations, a workflow pipeline is put in place where simulation data is in structured files that are stored in IBM Cloud Object Storage. A trigger to start the simulation scales compute services in IBM Cloud Kubernetes Service to split the data of the files into N event buckets for simulation processing. IBM Cloud Kubernetes Service automatically scales to N associated service executions and writes intermediate results to IBM Cloud Object Storage. Those results are processed by another set of the IBM Cloud Kubernetes Service compute services to produce the final results.

IBM Cloudant is a modern NoSQL database that is useful for many data-driven use cases: from key-value to complex document-oriented data storage and query. To manage the growing set of regulatory and management report rules, the mortgage company uses IBM Cloudant to store documents that are associated with raw regulatory data the come into the firm. Compute processes on IBM Cloud Kubernetes Service are triggered to compile, process, and publish the data in various reporting formats. Intermediate results common across reports are stored as IBM Cloudant documents so template-driven processes can be used to produce the necessary reports.





 Results 



* Complex financial simulations are completed in 25% of the time than was previously possible with the existing on-premises systems.
Several of the fields have default values:



Table 2. Default values for monitoring API request fields

 Field Default value 

 DURATION 5 minutes 
 END No default value 
 START The current time 





 Results format 

By default, the monitoring results are returned in [JSON format](https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-monitoring-an-ibm-cloudant-clusterwith-format=json-default). If you prefer, you can choose to receive the results in [raw format](https://cloud.ibm.com/docs/Cloudant?topic=Cloudant-monitoring-an-ibm-cloudant-clusterwith-format=raw).

The results include a text string that identifies the metric that is stored on the server that provides the API capability, for example:

sumSeries(net.cloudant.mycustomer001.db.df.srv.used)

The results include cluster-level data.

IBM Cloudant stores the queried data at the following resolutions: 10 seconds for the past 24 hours; 1 minute for the past 7 days; and 1 hour for the past 2 years. As a result, and to ensure that IBM Cloudant always stores the higher resolution interval length, deltas on the boundary of these resolutions are trimmed by one interval's length.





 With format=json (default) 

Unless you specify otherwise, the metric data that is returned is in JSON format. Each value that is returned consists of [datapoint, timestamp] values.

See an example monitoring request for disk use data returned in JSON format:

curl "https://$ACCOUNT.cloudant.com/_api/v2/monitoring/disk_use?cluster=myclustername&format=json"

See an example result after you request disk use data in JSON format:

[
{
"target": "sumSeries(net.cloudant.mycustomer001.db.df.srv.used)",
"datapoints":
523562172416.0, 1391019360],