# migrations

dir: embeddings_queue
filename: 00001-embeddings.sqlite.sql
hash: d3755dfd232be8e8301f4d7fcfb3a486
sql:
CREATE TABLE embeddings_queue (
    seq_id INTEGER PRIMARY KEY,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    operation INTEGER NOT NULL,
    topic TEXT NOT NULL,
    id TEXT NOT NULL,
    vector BLOB,
    encoding TEXT,
    metadata TEXT
);


dir: sysdb
filename: 00001-collections.sqlite.sql
hash: 38352d725ad1c16074fac420b22b4633
sql:
CREATE TABLE collections (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    topic TEXT NOT NULL,
    UNIQUE (name)
);

CREATE TABLE collection_metadata (
    collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (collection_id, key)
);


dir: sysdb
filename: 00002-segments.sqlite.sql
hash: 2913cb6a503055a95f625448037e8912
sql:
CREATE TABLE segments (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    topic TEXT,
    collection TEXT REFERENCES collection(id)
);

CREATE TABLE segment_metadata (
    segment_id TEXT  REFERENCES segments(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (segment_id, key)
);


dir: sysdb
filename: 00004-tenants-databases.sqlite.sql
hash: 048867ce8fcdefe4023c7110e4433591
sql:
CREATE TABLE IF NOT EXISTS tenants (
    id TEXT PRIMARY KEY,
    UNIQUE (id)
);

CREATE TABLE IF NOT EXISTS databases (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per tenant
    tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
    UNIQUE (tenant_id, name) -- Ensure that a tenant has only one database with a given name
);

CREATE TABLE IF NOT EXISTS collections_tmp (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per database
    topic TEXT NOT NULL,
    dimension INTEGER,
    database_id TEXT NOT NULL REFERENCES databases(id) ON DELETE CASCADE,
    UNIQUE (name, database_id)
);

-- Create default tenant and database
INSERT OR REPLACE INTO tenants (id) VALUES ('default_tenant'); -- The default tenant id is 'default_tenant' others are UUIDs
INSERT OR REPLACE INTO databases (id, name, tenant_id) VALUES ('00000000-0000-0000-0000-000000000000', 'default_database', 'default_tenant');

INSERT OR REPLACE INTO collections_tmp (id, name, topic, dimension, database_id)
    SELECT id, name, topic, dimension, '00000000-0000-0000-0000-000000000000' FROM collections;
DROP TABLE collections;
ALTER TABLE collections_tmp RENAME TO collections;


dir: metadb
filename: 00001-embedding-metadata.sqlite.sql
hash: 2b4cf52c4bb2676e21d6860a4409f856
sql:
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    segment_id TEXT NOT NULL,
    embedding_id TEXT NOT NULL,
    seq_id BLOB NOT NULL,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    UNIQUE (segment_id, embedding_id)
);

CREATE TABLE embedding_metadata (
    id INTEGER REFERENCES embeddings(id),
    key TEXT NOT NULL,
    string_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (id, key)
);

CREATE TABLE max_seq_id (
    segment_id TEXT PRIMARY KEY,
    seq_id BLOB NOT NULL
);

CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value);


dir: metadb
filename: 00002-embedding-metadata.sqlite.sql
hash: 12a570f7121b3a8ce750a2a7c36da20f
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER


dir: metadb
filename: 00003-full-text-tokenize.sqlite.sql
hash: f97ad6334aeaa8f419f01110b648b97a
sql:
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram');
INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata;
DROP TABLE embedding_fulltext;


# embeddings_queue

created_at: 2023-12-14 10:40:09
topic: persistent://default/default/0b83760d-14ff-44e5-9217-bd1e79bd985b
id: 26c92d85-9a6d-11ee-98af-74d83eba396a
encoding: FLOAT32
metadata:
{"source": "C:/Users/itzad/Downloads/Collections (List & Tuple).pdf", "page": 0, "chroma:document": "Collections\n:\nThey\nare\nused\nto\nstore\nmultiples\nin\na\nsingle\nvariable.\n-\nList\n-\nTuple\n-\nSet\n-\nDictionary\nList\n:\n-\nIt\nis\ndenoted\nby\nsquare\nbrackets\n[\n]\nEx\n:\nfruits\n=\n[\u201cApple\u201d,\u201dOrange\u201d,\u201dBanana\u201d]\n-\nIt\nallows\nduplicate\nvalues\n-\nIt\nis\nmutable/changeable\n(add,remove\nor\nreplace).\n-\nIt\nmaintains\nthe\norder\nof\nthe\ndata\n-\nEach\nelement/value\nwill\nbe\nassigned\nan\nindex\nposition\nno.\nstarting\nfrom\n0.\n-\nIt\nallows\nindexing\nIndexing\n:\nIt\nis\nto\naccess/view\nan\nelement\nusing\nindex\nposition.\nIt\ngenerally\nstarts\nfrom\n0.\nEx\n:\ncollection_name[index_position_no]\nReverse\nIndexing\n:\nIt\nis\na\nconcept\nto\naccess\nlast\nelements.\nIt\nis\nused\nusing\n\u201c-\u201d.\nEx\n:\ncollection_name[-\nindex_position_no]\nMethods\n:\n-\nThey\nare\nthe\nsame\nlike\nfunctions\n(perform\nsome\nfunctionality\nand\ngive\nan\noutput)\n-\nMethods\nare\nspecific\nto\na\nparticular\nobject\nand\ncannot\nbe\nused\noutside\nof\nthat\nobject\n-\nFunctions\nare\ngeneral\nand\ncan\nbe\napplied\non\nany\nobject."}


created_at: 2023-12-14 10:40:09
topic: persistent://default/default/0b83760d-14ff-44e5-9217-bd1e79bd985b
id: 26c92d86-9a6d-11ee-810e-74d83eba396a
encoding: FLOAT32
metadata:
{"source": "C:/Users/itzad/Downloads/Collections (List & Tuple).pdf", "page": 1, "chroma:document": "How\nto\naccess\na\nmethod\nof\nany\nobject\n:\nObject_name.method_name(inputs)\nOperations\non\nlist\n:\nIt\ncan\nbe\ndone\nusing\ndifferent\nmethods\nof\nlist.\n-\nTo\nadd\nan\nnew\nelement\ninside\na\nlist\nAppend()\n-\nTo\nadd\na\nnew\nelement\nat\na\nspecific\nposition\ninsert()\n-\nRemove\nan\nelement\nusing\nindex\nposition\npop()\n-\nRemove\nan\nelement\nthrough\nits\nvalue\nremove()\n-\nTo\nsee\nno.of\noccurrence\nof\na\nvalue\ncount()\n-\nTo\nsee\nindex\nposition\nof\na\nvalue\nindex()\n-\nTo\nremove\nall\nelements\nof\na\nlist\nclear()\nTo\ndelete\nany\nobject\nfrom\nthe\nmemory\nin\npython,\nthere\nis\na\ndel\nkeyword\n>>\ndel\nobject_name"}


created_at: 2023-12-14 10:40:09
topic: persistent://default/default/0b83760d-14ff-44e5-9217-bd1e79bd985b
id: 26c92d87-9a6d-11ee-8999-74d83eba396a
encoding: FLOAT32
metadata:
{"source": "C:/Users/itzad/Downloads/Collections (List & Tuple).pdf", "page": 2, "chroma:document": "Tuple\n:\n-\nIt\nis\ndenoted\nby\nparenthesis\n(\n)\n-\nIt\nis\na\ndefault\ncollection.\nT\n=\n(1,2,3,4,5)\n-\nIt\nis\nimmutable\n(cannot\nadd,\nremove\nor\nreplace\nan\nelement)\n-\nIt\nallows\nindexing.\nTuple\nhas\nonly\ntwo\nmethods\n:\n-\ncount()\n:\nIt\nis\nused\nto\nsee\nno.of\noccurrence\nof\nan\nelement\n-\nindex()\n:\nIt\nis\nused\nto\nsee\nindex\nno.\nof\nan\nelement"}


# embedding_metadata

key: chroma:document
string_value:
Collections
:
They
are
used
to
store
multiples
in
a
single
variable.
-
List
-
Tuple
-
Set
-
Dictionary
List
:
-
It
is
denoted
by
square
brackets
[
]
Ex
:
fruits
=
[“Apple”,”Orange”,”Banana”]
-
It
allows
duplicate
values
-
It
is
mutable/changeable
(add,remove
or
replace).
-
It
maintains
the
order
of
the
data
-
Each
element/value
will
be
assigned
an
index
position
no.
starting
from
0.
-
It
allows
indexing
Indexing
:
It
is
to
access/view
an
element
using
index
position.
It
generally
starts
from
0.
Ex
:
collection_name[index_position_no]
Reverse
Indexing
:
It
is
a
concept
to
access
last
elements.
It
is
used
using
“-”.
Ex
:
collection_name[-
index_position_no]
Methods
:
-
They
are
the
same
like
functions
(perform
some
functionality
and
give
an
output)
-
Methods
are
specific
to
a
particular
object
and
cannot
be
used
outside
of
that
object
-
Functions
are
general
and
can
be
applied
on
any
object.


key: chroma:document
string_value:
How
to
access
a
method
of
any
object
:
Object_name.method_name(inputs)
Operations
on
list
:
It
can
be
done
using
different
methods
of
list.
-
To
add
an
new
element
inside
a
list
Append()
-
To
add
a
new
element
at
a
specific
position
insert()
-
Remove
an
element
using
index
position
pop()
-
Remove
an
element
through
its
value
remove()
-
To
see
no.of
occurrence
of
a
value
count()
-
To
see
index
position
of
a
value
index()
-
To
remove
all
elements
of
a
list
clear()
To
delete
any
object
from
the
memory
in
python,
there
is
a
del
keyword
>>
del
object_name


key: chroma:document
string_value:
Tuple
:
-
It
is
denoted
by
parenthesis
(
)
-
It
is
a
default
collection.
T
=
(1,2,3,4,5)
-
It
is
immutable
(cannot
add,
remove
or
replace
an
element)
-
It
allows
indexing.
Tuple
has
only
two
methods
:
-
count()
:
It
is
used
to
see
no.of
occurrence
of
an
element
-
index()
:
It
is
used
to
see
index
no.
of
an
element


