Repository: weaviate/weaviate
Pull request #7316: Add tokeniser metrics
Author: login: donomii
Created: 2025-02-20T14:41:39Z | Merged: 2025-03-19T13:45:44Z

### What's being changed:

More metrics

### Review checklist

- [ ] Documentation has been updated, if necessary. Link to changed documentation:
- [ ] Chaos pipeline run or not necessary. Link to pipeline:
- [ ] All new code is covered by tests where it is reasonable.
- [ ] Performance tests have been run or not necessary.

<!-- Uncomment the following section if this PR requires changes in related projects (e.g., documentation, client libraries).

GitHub actions will automatically create an issue in the corresponding repository for each checked box below. (See `.github/workflows/create-cross-functional-issues.yml`)

### Cross-functional impact

- [ ] This change requires public documentation (weaviate-io) to be updated. Check the box to automatically create a corresponding issue.
- Does it require a change in the client libraries? If yes, please check the boxes for the affected client libraries.
    - [ ] Python (weaviate-python-client)
    - [ ] JavaScript/TypeScript (typescript-client)
    - [ ] Go (weaviate-go-client)
    - [ ] Java (java-client)

-->



Diff:
diff --git a/adapters/repos/db/helpers/tokenizer.go b/adapters/repos/db/helpers/tokenizer.go
index 94bc5c26dc4..1876d92ed7c 100644
--- a/adapters/repos/db/helpers/tokenizer.go
+++ b/adapters/repos/db/helpers/tokenizer.go
@@ -17,6 +17,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 	"unicode"
 
 	entcfg "github.com/weaviate/weaviate/entities/config"
@@ -26,6 +27,7 @@ import (
 	"github.com/ikawaha/kagome-dict/ipa"
 	kagomeTokenizer "github.com/ikawaha/kagome/v2/tokenizer"
 	"github.com/weaviate/weaviate/entities/models"
+	"github.com/weaviate/weaviate/usecases/monitoring"
 )
 
 var (
@@ -88,19 +90,16 @@ func init() {
 }
 
 func init_gse() {
-	if entcfg.Enabled(os.Getenv("USE_GSE")) || entcfg.Enabled(os.Getenv("ENABLE_TOKENIZER_GSE")) {
-		UseGse = true
-	}
-	if UseGse {
-		gseLock.Lock()
-		defer gseLock.Unlock()
-		if gseTokenizer == nil {
-			seg, err := gse.New("ja")
-			if err != nil {
-				return //[]string{}
-			}
-			gseTokenizer = &seg
+	gseLock.Lock()
+	defer gseLock.Unlock()
+	if gseTokenizer == nil {
+		startTime := time.Now()
+		seg, err := gse.New("ja")
+		if err != nil {
+			return
 		}
+		gseTokenizer = &seg
+		monitoring.GetMetrics().TokenizerInitializeDuration.WithLabelValues("gse").Observe(time.Since(startTime).Seconds())
 	}
 }
 
@@ -108,11 +107,13 @@ func init_gse_ch() {
 	gseLock.Lock()
 	defer gseLock.Unlock()
 	if gseTokenizerCh == nil {
+		startTime := time.Now()
 		seg, err := gse.New("zh")
 		if err != nil {
 			return
 		}
 		gseTokenizerCh = &seg
+		monitoring.GetMetrics().TokenizerInitializeDuration.WithLabelValues("gse").Observe(time.Since(startTime).Seconds())
 	}
 }
 
@@ -195,32 +196,51 @@ func removeEmptyStrings(terms []string) []string {
 // tokenizeField trims white spaces
 // (former DataTypeString/Field)
 func tokenizeField(in string) []string {
-	return []string{strings.TrimFunc(in, unicode.IsSpace)}
+	startTime := time.Now()
+	ret := []string{strings.TrimFunc(in, unicode.IsSpace)}
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("field").Observe(float64(time.Since(startTime).Seconds()))
+	monitoring.GetMetrics().TokenCount.WithLabelValues("field").Add(float64(len(ret)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("field").Observe(float64(len(ret)))
+	return ret
 }
 
 // tokenizeWhitespace splits on white spaces, does not alter casing
 // (former DataTypeString/Word)
 func tokenizeWhitespace(in string) []string {
-	return strings.FieldsFunc(in, unicode.IsSpace)
+	startTime := time.Now()
+	ret := strings.FieldsFunc(in, unicode.IsSpace)
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("whitespace").Observe(float64(time.Since(startTime).Seconds()))
+	monitoring.GetMetrics().TokenCount.WithLabelValues("whitespace").Add(float64(len(ret)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("whitespace").Observe(float64(len(ret)))
+	return ret
 }
 
 // tokenizeLowercase splits on white spaces and lowercases the words
 func tokenizeLowercase(in string) []string {
+	startTime := time.Now()
 	terms := tokenizeWhitespace(in)
-	return lowercase(terms)
+	ret := lowercase(terms)
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("lowercase").Observe(float64(time.Since(startTime).Seconds()))
+	return ret
 }
 
 // tokenizeWord splits on any non-alphanumerical and lowercases the words
 // (former DataTypeText/Word)
 func tokenizeWord(in string) []string {
+	startTime := time.Now()
 	terms := strings.FieldsFunc(in, func(r rune) bool {
 		return !unicode.IsLetter(r) && !unicode.IsNumber(r)
 	})
-	return lowercase(terms)
+	ret := lowercase(terms)
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("word").Observe(float64(time.Since(startTime).Seconds()))
+	monitoring.GetMetrics().TokenCount.WithLabelValues("word").Add(float64(len(ret)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("word").Observe(float64(len(ret)))
+	return ret
 }
 
 // tokenizetrigram splits on any non-alphanumerical and lowercases the words, joins them together, then groups them into trigrams
 func tokenizetrigram(in string) []string {
+	startTime := time.Now()
 	// Strip whitespace and punctuation from the input string
 	inputString := strings.ToLower(strings.Join(strings.FieldsFunc(in, func(r rune) bool {
 		return !unicode.IsLetter(r) && !unicode.IsNumber(r)
@@ -235,6 +255,9 @@ func tokenizetrigram(in string) []string {
 	for _, trirune := range trirunes {
 		trigrams = append(trigrams, string(trirune))
 	}
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("trigram").Observe(float64(time.Since(startTime).Seconds()))
+	monitoring.GetMetrics().TokenCount.WithLabelValues("trigram").Add(float64(len(trigrams)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("trigram").Observe(float64(len(trigrams)))
 	return trigrams
 }
 
@@ -243,13 +266,17 @@ func tokenizeGSE(in string) []string {
 	if !UseGse {
 		return []string{}
 	}
+	startTime := time.Now()
 	gseLock.Lock()
 	defer gseLock.Unlock()
 	terms := gseTokenizer.CutAll(in)
 
-	terms = removeEmptyStrings(terms)
+	ret := removeEmptyStrings(terms)
 
-	return terms
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("gse").Observe(float64(time.Since(startTime).Seconds()))
+	monitoring.GetMetrics().TokenCount.WithLabelValues("gse").Add(float64(len(ret)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("gse").Observe(float64(len(ret)))
+	return ret
 }
 
 // tokenizeGSE uses the gse tokenizer to tokenise Chinese
@@ -259,10 +286,14 @@ func tokenizeGseCh(in string) []string {
 	}
 	gseLock.Lock()
 	defer gseLock.Unlock()
+	startTime := time.Now()
 	terms := gseTokenizerCh.CutAll(in)
-	terms = removeEmptyStrings(terms)
+	ret := removeEmptyStrings(terms)
 
-	return terms
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("gse").Observe(float64(time.Since(startTime).Seconds()))
+	monitoring.GetMetrics().TokenCount.WithLabelValues("gse").Add(float64(len(ret)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("gse").Observe(float64(len(ret)))
+	return ret
 }
 
 func initializeKagomeTokenizerKr() error {
@@ -274,6 +305,7 @@ func initializeKagomeTokenizerKr() error {
 		if tokenizers.Korean != nil {
 			return nil
 		}
+		startTime := time.Now()
 
 		dictInstance := koDict.Dict()
 		tokenizer, err := kagomeTokenizer.New(dictInstance)
@@ -283,6 +315,7 @@ func initializeKagomeTokenizerKr() error {
 
 		tokenizers.Korean = tokenizer
 		KagomeKrEnabled = true
+		monitoring.GetMetrics().TokenizerInitializeDuration.WithLabelValues("kagome_kr").Observe(float64(time.Since(startTime).Seconds()))
 		return nil
 	}
 
@@ -294,6 +327,7 @@ func tokenizeKagomeKr(in string) []string {
 	if tokenizer == nil || !KagomeKrEnabled {
 		return []string{}
 	}
+	startTime := time.Now()
 
 	kagomeTokens := tokenizer.Tokenize(in)
 	terms := make([]string, 0, len(kagomeTokens))
@@ -304,7 +338,11 @@ func tokenizeKagomeKr(in string) []string {
 		}
 	}
 
-	return removeEmptyStrings(terms)
+	ret := removeEmptyStrings(terms)
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("kagome_kr").Observe(float64(time.Since(startTime).Seconds()))
+	monitoring.GetMetrics().TokenCount.WithLabelValues("kagome_kr").Add(float64(len(ret)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("kagome_kr").Observe(float64(len(ret)))
+	return ret
 }
 
 func initializeKagomeTokenizerJa() error {
@@ -316,7 +354,7 @@ func initializeKagomeTokenizerJa() error {
 		if tokenizers.Japanese != nil {
 			return nil
 		}
-
+		startTime := time.Now()
 		dictInstance := ipa.Dict()
 		tokenizer, err := kagomeTokenizer.New(dictInstance)
 		if err != nil {
@@ -325,6 +363,7 @@ func initializeKagomeTokenizerJa() error {
 
 		tokenizers.Japanese = tokenizer
 		KagomeJaEnabled = true
+		monitoring.GetMetrics().TokenizerInitializeDuration.WithLabelValues("kagome_ja").Observe(float64(time.Since(startTime).Seconds()))
 		return nil
 	}
 
@@ -337,6 +376,7 @@ func tokenizeKagomeJa(in string) []string {
 		return []string{}
 	}
 
+	startTime := time.Now()
 	kagomeTokens := tokenizer.Analyze(in, kagomeTokenizer.Search)
 	terms := make([]string, 0, len(kagomeTokens))
 
@@ -346,27 +386,40 @@ func tokenizeKagomeJa(in string) []string {
 		}
 	}
 
-	return removeEmptyStrings(terms)
+	ret := removeEmptyStrings(terms)
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("kagome_ja").Observe(float64(time.Since(startTime).Seconds()))
+	monitoring.GetMetrics().TokenCount.WithLabelValues("kagome_ja").Add(float64(len(ret)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("kagome_ja").Observe(float64(len(ret)))
+	return ret
 }
 
 // tokenizeWordWithWildcards splits on any non-alphanumerical except wildcard-symbols and
 // lowercases the words
 func tokenizeWordWithWildcards(in string) []string {
+	startTime := time.Now()
 	terms := strings.FieldsFunc(in, func(r rune) bool {
 		return !unicode.IsLetter(r) && !unicode.IsNumber(r) && r != '?' && r != '*'
 	})
-	return lowercase(terms)
+	ret := lowercase(terms)
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("word_with_wildcards").Observe(float64(time.Since(startTime).Seconds()))
+	monitoring.GetMetrics().TokenCount.WithLabelValues("word_with_wildcards").Add(float64(len(ret)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("word_with_wildcards").Observe(float64(len(ret)))
+	return ret
 }
 
 // tokenizetrigramWithWildcards splits on any non-alphanumerical and lowercases the words, applies any wildcards, then joins them together, then groups them into trigrams
 // this is unlikely to be useful, but is included for completeness
 func tokenizetrigramWithWildcards(in string) []string {
+	startTime := time.Now()
 	terms := tokenizeWordWithWildcards(in)
 	inputString := strings.Join(terms, "")
 	var trigrams []string
 	for i := 0; i < len(inputString)-2; i++ {
 		trigrams = append(trigrams, inputString[i:i+3])
 	}
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("trigram_with_wildcards").Observe(float64(time.Since(startTime).Seconds()))
+	monitoring.GetMetrics().TokenCount.WithLabelValues("trigram_with_wildcards").Add(float64(len(trigrams)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("trigram_with_wildcards").Observe(float64(len(trigrams)))
 	return trigrams
 }
 
@@ -374,6 +427,8 @@ func lowercase(terms []string) []string {
 	for i := range terms {
 		terms[i] = strings.ToLower(terms[i])
 	}
+	monitoring.GetMetrics().TokenCount.WithLabelValues("lowercase").Add(float64(len(terms)))
+	monitoring.GetMetrics().TokenCountPerRequest.WithLabelValues("lowercase").Observe(float64(len(terms)))
 	return terms
 }
 
diff --git a/adapters/repos/db/helpers/tokenizer_test.go b/adapters/repos/db/helpers/tokenizer_test.go
index 02f0de85d77..ce19c613a3c 100644
--- a/adapters/repos/db/helpers/tokenizer_test.go
+++ b/adapters/repos/db/helpers/tokenizer_test.go
@@ -26,6 +26,10 @@ func TestTokeniseParallel(t *testing.T) {
 	// Kagome tokenizer for Korean
 	t.Setenv("ENABLE_TOKENIZER_KAGOME_KR", "true")
 	_ = initializeKagomeTokenizerKr()
+
+	// Kagome tokenizer for Japanese
+	t.Setenv("ENABLE_TOKENIZER_KAGOME_JA", "true")
+	_ = initializeKagomeTokenizerJa()
 	for i := 0; i < 1000; i++ {
 		go SingleTokenise(t)
 	}
@@ -93,10 +97,6 @@ func SingleTokenise(t *testing.T) {
 	tokens = Tokenize(models.PropertyTokenizationKagomeKr, "한국어를 처리하는 예시입니다")
 	assert.Equal(t, []string{"한국어", "를", "처리", "하", "는", "예시", "입니다"}, tokens)
 
-	// Kagome tokenizer for Japanese
-	t.Setenv("ENABLE_TOKENIZER_KAGOME_JA", "true")
-	_ = initializeKagomeTokenizerJa()
-
 	tokens = Tokenize(models.PropertyTokenizationKagomeJa, `春の夜の夢はうつつよりもかなしき
 	夏の夜の夢はうつつに似たり
 	秋の夜の夢はうつつを超え
diff --git a/usecases/monitoring/prometheus.go b/usecases/monitoring/prometheus.go
index 467a83be1ff..d2298eae15d 100644
--- a/usecases/monitoring/prometheus.go
+++ b/usecases/monitoring/prometheus.go
@@ -138,6 +138,12 @@ type PrometheusMetrics struct {
 	T2VRateLimitStats     *prometheus.GaugeVec
 	T2VRepeatStats        *prometheus.GaugeVec
 	T2VRequestsPerBatch   *prometheus.HistogramVec
+
+	TokenizerDuration           *prometheus.HistogramVec
+	TokenizerRequests           *prometheus.CounterVec
+	TokenizerInitializeDuration *prometheus.HistogramVec
+	TokenCount                  *prometheus.CounterVec
+	TokenCountPerRequest        *prometheus.HistogramVec
 }
 
 func NewTenantOffloadMetrics(cfg Config, reg prometheus.Registerer) *TenantOffloadMetrics {
@@ -714,6 +720,29 @@ func newPrometheusMetrics() *PrometheusMetrics {
 			Help:    "Number of requests required to process an entire (user) batch",
 			Buckets: []float64{1, 2, 5, 10, 100, 1000},
 		}, []string{"vectorizer"}),
+		TokenizerDuration: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "tokenizer_duration_seconds",
+			Help:    "Duration of a tokenizer operation",
+			Buckets: LatencyBuckets,
+		}, []string{"tokenizer"}),
+		TokenizerRequests: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "tokenizer_requests_total",
+			Help: "Number of tokenizer requests",
+		}, []string{"tokenizer"}),
+		TokenizerInitializeDuration: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "tokenizer_initialize_duration_seconds",
+			Help:    "Duration of a tokenizer initialization operation",
+			Buckets: []float64{0.05, 0.1, 0.5, 1, 2, 5, 10},
+		}, []string{"tokenizer"}),
+		TokenCount: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "token_count_total",
+			Help: "Number of tokens processed",
+		}, []string{"tokenizer"}),
+		TokenCountPerRequest: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "token_count_per_request",
+			Help:    "Number of tokens processed per request",
+			Buckets: []float64{1, 10, 50, 100, 500, 1000, 10000, 100000, 1000000},
+		}, []string{"tokenizer"}),
 	}
 }
 


Reviews:
user: login: orca-security-eu[bot]
body: ### Orca Security Scan Summary
| Status  | Check | Issues by priority |   |
| ------- | ----- | ------------------ | - |
| <img width="16" alt="Passed" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/status/passed.png" title="Passed"> Passed | Infrastructure as Code | <img width="12" alt="high" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/high.png" title="High"> 0 &emsp; <img width="12" alt="medium" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/medium.png" title="Medium"> 0 &emsp; <img width="12" alt="low" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/low.png" title="Low"> 0 &emsp; <img width="12" alt="info" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/info.png" title="Info"> 0 | <a href="https://eu.app.orcasecurity.io/shift-left/iac/scan-log/b22e2992-336b-4878-896a-2089d3d9b66f" target="_blank">View in Orca</a> |
| <img width="16" alt="Passed" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/status/passed.png" title="Passed"> Passed | SAST | <img width="12" alt="high" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/high.png" title="High"> 0 &emsp; <img width="12" alt="medium" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/medium.png" title="Medium"> 0 &emsp; <img width="12" alt="low" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/low.png" title="Low"> 0 &emsp; <img width="12" alt="info" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/info.png" title="Info"> 0 | <a href="https://eu.app.orcasecurity.io/shift-left/sast/scan-log/76014cdf-d7b0-438a-b157-08644c15aa4a" target="_blank">View in Orca</a> |
| <img width="16" alt="Passed" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/status/passed.png" title="Passed"> Passed | Secrets | <img width="12" alt="high" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/high.png" title="High"> 0 &emsp; <img width="12" alt="medium" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/medium.png" title="Medium"> 0 &emsp; <img width="12" alt="low" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/low.png" title="Low"> 0 &emsp; <img width="12" alt="info" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/info.png" title="Info"> 0 | <a href="https://eu.app.orcasecurity.io/shift-left/file_system_secret_detection/scan-log/4f44619b-91b2-449a-9969-348628952157" target="_blank">View in Orca</a> |
| <img width="16" alt="Passed" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/status/passed.png" title="Passed"> Passed | Vulnerabilities | <img width="12" alt="high" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/high.png" title="High"> 0 &emsp; <img width="12" alt="medium" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/medium.png" title="Medium"> 0 &emsp; <img width="12" alt="low" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/low.png" title="Low"> 0 &emsp; <img width="12" alt="info" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/info.png" title="Info"> 0 | <a href="https://eu.app.orcasecurity.io/shift-left/file_system_vulnerabilities/scan-log/15dfe73c-6937-4e04-96bb-f6bf53d4cfaa" target="_blank">View in Orca</a> |
state: COMMENTED
author_association: NONE
submitted_at: 2025-02-20T14:42:38Z
user: login: moogacs
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T02:51:00Z
user: login: kavirajk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T13:18:42Z
user: login: faustuzas
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T13:21:16Z
user: login: donomii
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T13:24:35Z
user: login: faustuzas
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T13:30:27Z
user: login: kavirajk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T13:31:32Z
user: login: donomii
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T13:39:04Z
user: login: faustuzas
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T13:45:43Z

Review comments:
diff_hunk: @@ -135,32 +139,51 @@ func removeEmptyStrings(terms []string) []string {
 // tokenizeField trims white spaces
 // (former DataTypeString/Field)
 func tokenizeField(in string) []string {
-	return []string{strings.TrimFunc(in, unicode.IsSpace)}
+	startTime := time.Now()
+	ret := []string{strings.TrimFunc(in, unicode.IsSpace)}
+	monitoring.GetMetrics().TokenizerDuration.WithLabelValues("field").Observe(float64(time.Since(startTime).Microseconds()))
path: adapters/repos/db/helpers/tokenizer.go
user: login: moogacs
body: **issue:** monitoring/metrics systems (like Prometheus), durations are typically expected to be in seconds as conventions, also would be, see https://prometheus.io/docs/practices/naming/#metric-and-label-naming and this because it will show large numbers in graphs if it's in microseconds 

also as a suggestion should have a suffix describing the unit in the label as best practice and convention  to avoid confusion 

same for next metrics in the same file

created_at: 2025-03-18T02:51:00Z
author_association: MEMBER
diff_hunk: @@ -714,6 +720,29 @@ func newPrometheusMetrics() *PrometheusMetrics {
 			Help:    "Number of requests required to process an entire (user) batch",
 			Buckets: []float64{1, 2, 5, 10, 100, 1000},
 		}, []string{"vectorizer"}),
+		TokenizerDuration: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "tokenizer_duration_microseconds",
path: usecases/monitoring/prometheus.go
user: login: kavirajk
body: Can we keep this in seconds? and adjust the bucket size to capture lower resolution.

Rationale is to keep all the metrics in seconds (standadize). Couldn't over state it's purpose when using promql joins to compare different metrics.

[Prometheus recommends it](https://prometheus.io/docs/practices/naming/#base-units)

[Even Otel recommends it.](https://github.com/open-telemetry/opentelemetry-specification/issues/2977#issue-1460359959)
created_at: 2025-03-19T13:12:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -88,31 +90,30 @@ func init() {
 }
 
 func init_gse() {
-	if entcfg.Enabled(os.Getenv("USE_GSE")) || entcfg.Enabled(os.Getenv("ENABLE_TOKENIZER_GSE")) {
-		UseGse = true
-	}
-	if UseGse {
-		gseLock.Lock()
-		defer gseLock.Unlock()
-		if gseTokenizer == nil {
-			seg, err := gse.New("ja")
-			if err != nil {
-				return //[]string{}
-			}
-			gseTokenizer = &seg
+	gseLock.Lock()
+	defer gseLock.Unlock()
+	if gseTokenizer == nil {
+		startTime := time.Now()
+		seg, err := gse.New("ja")
+		if err != nil {
+			return //[]string{}
path: adapters/repos/db/helpers/tokenizer.go
user: login: faustuzas
body: why this comment `//[]string{}`? 
created_at: 2025-03-19T13:17:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -138,6 +138,12 @@ type PrometheusMetrics struct {
 	T2VRateLimitStats     *prometheus.GaugeVec
 	T2VRepeatStats        *prometheus.GaugeVec
 	T2VRequestsPerBatch   *prometheus.HistogramVec
+
+	TokenizerDuration           *prometheus.HistogramVec
path: usecases/monitoring/prometheus.go
user: login: kavirajk
body: Curious can we add those metrics close to the tokenizer itself?

Similar to [grpc metrics.](https://github.com/weaviate/weaviate/blob/main/cluster/rpc/server.go#L55) (close to the actual server). Rationale being, keeping the metrics close to the actual components makes the component self contained (easy to move, mock and reason about and avoid coupling this single huge struct to all the components). 

Same reason [we marked this global Prometheus struct as deprecated and new metrics are added to the corresponding components.](https://github.com/weaviate/weaviate/blob/main/usecases/monitoring/prometheus.go#L36-L38)


created_at: 2025-03-19T13:18:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -714,6 +720,29 @@ func newPrometheusMetrics() *PrometheusMetrics {
 			Help:    "Number of requests required to process an entire (user) batch",
 			Buckets: []float64{1, 2, 5, 10, 100, 1000},
 		}, []string{"vectorizer"}),
+		TokenizerDuration: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "tokenizer_duration_microseconds",
path: usecases/monitoring/prometheus.go
user: login: faustuzas
body: why `microseconds`? The usual practice is to use `_seconds` and specify appropriate buckets for your data resolution
created_at: 2025-03-19T13:19:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -138,6 +138,12 @@ type PrometheusMetrics struct {
 	T2VRateLimitStats     *prometheus.GaugeVec
 	T2VRepeatStats        *prometheus.GaugeVec
 	T2VRequestsPerBatch   *prometheus.HistogramVec
+
+	TokenizerDuration           *prometheus.HistogramVec
path: usecases/monitoring/prometheus.go
user: login: donomii
body: Let's do it later, we're going to have to rework that entire file anyway
created_at: 2025-03-19T13:24:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -714,6 +720,29 @@ func newPrometheusMetrics() *PrometheusMetrics {
 			Help:    "Number of requests required to process an entire (user) batch",
 			Buckets: []float64{1, 2, 5, 10, 100, 1000},
 		}, []string{"vectorizer"}),
+		TokenizerDuration: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "tokenizer_duration_seconds",
+			Help:    "Duration of a tokenizer operation",
+			Buckets: []float64{1, 10, 50, 100, 1000, 5000, 10000, 100000, 500000, 1000000, 10000000},
path: usecases/monitoring/prometheus.go
user: login: faustuzas
body: Is this resolution sufficient for you? I would expect initialization to take from hundreds of milliseconds to tens of seconds? Why do we need a bucket for `10000000` seconds which is 115 days?
created_at: 2025-03-19T13:30:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -714,6 +720,29 @@ func newPrometheusMetrics() *PrometheusMetrics {
 			Help:    "Number of requests required to process an entire (user) batch",
 			Buckets: []float64{1, 2, 5, 10, 100, 1000},
 		}, []string{"vectorizer"}),
+		TokenizerDuration: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "tokenizer_duration_seconds",
+			Help:    "Duration of a tokenizer operation",
+			Buckets: []float64{1, 10, 50, 100, 1000, 5000, 10000, 100000, 500000, 1000000, 10000000},
path: usecases/monitoring/prometheus.go
user: login: kavirajk
body: Agree with @faustuzas . I created this bucket to be used for such purposes.
https://github.com/weaviate/weaviate/blob/kavirajk%2Fmake-auto-schema-disabled-runtimeconfig/usecases/monitoring/prometheus.go#L334-L340
created_at: 2025-03-19T13:31:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -714,6 +720,29 @@ func newPrometheusMetrics() *PrometheusMetrics {
 			Help:    "Number of requests required to process an entire (user) batch",
 			Buckets: []float64{1, 2, 5, 10, 100, 1000},
 		}, []string{"vectorizer"}),
+		TokenizerDuration: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "tokenizer_duration_seconds",
+			Help:    "Duration of a tokenizer operation",
+			Buckets: []float64{1, 10, 50, 100, 1000, 5000, 10000, 100000, 500000, 1000000, 10000000},
path: usecases/monitoring/prometheus.go
user: login: donomii
body: What if someone runs it on a Raspberry PI, and it takes that long to start? 😂 

It was microseconds, so fixed
created_at: 2025-03-19T13:39:04Z
author_association: CONTRIBUTOR

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-03-19T13:41:13Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=weaviate_weaviate&pullRequest=7316) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=weaviate_weaviate&pullRequest=7316&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=weaviate_weaviate&pullRequest=7316&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=weaviate_weaviate&pullRequest=7316&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=weaviate_weaviate&pullRequest=7316&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=weaviate_weaviate&pullRequest=7316&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=weaviate_weaviate&pullRequest=7316)


author_association: NONE