Repository: owncloud/ocis
Pull request #6371 (2023-05-23 13:42:17)

Improve complex search
This PR fixes a problem when searching for the `:` character. It also makes field based search case-insensitive to make it consistent with simple searches.

sonarcloud[bot]: Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=owncloud_ocis&pullRequest=6371)

[![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=owncloud_ocis&pullRequest=6371&resolved=false&types=BUG) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=owncloud_ocis&pullRequest=6371&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=owncloud_ocis&pullRequest=6371&resolved=false&types=BUG)  
[![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=owncloud_ocis&pullRequest=6371&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=owncloud_ocis&pullRequest=6371&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=owncloud_ocis&pullRequest=6371&resolved=false&types=VULNERABILITY)  
[![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=owncloud_ocis&pullRequest=6371&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=owncloud_ocis&pullRequest=6371&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=owncloud_ocis&pullRequest=6371&resolved=false&types=SECURITY_HOTSPOT)  
[![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=owncloud_ocis&pullRequest=6371&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=owncloud_ocis&pullRequest=6371&resolved=false&types=CODE_SMELL) [0 Code Smells](https://sonarcloud.io/project/issues?id=owncloud_ocis&pullRequest=6371&resolved=false&types=CODE_SMELL)

[![100.0%](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/100-16px.png '100.0%')](https://sonarcloud.io/component_measures?id=owncloud_ocis&pullRequest=6371&metric=new_coverage&view=list) [100.0% Coverage](https://sonarcloud.io/component_measures?id=owncloud_ocis&pullRequest=6371&metric=new_coverage&view=list)  
[![0.0%](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/3-16px.png '0.0%')](https://sonarcloud.io/component_measures?id=owncloud_ocis&pullRequest=6371&metric=new_duplicated_lines_density&view=list) [0.0% Duplication](https://sonarcloud.io/component_measures?id=owncloud_ocis&pullRequest=6371&metric=new_duplicated_lines_density&view=list)


ownclouders: :boom: Acceptance test [<strong>localApiTests-apiEmailNotification-ocis</strong>](https://drone.owncloud.com/owncloud/ocis/22660/56/1) failed. Further test are cancelled...
<!-- id: pr-6371 -->


Patch:
diff --git a/services/search/pkg/engine/bleve.go b/services/search/pkg/engine/bleve.go
index e330f7bb087..ec4e0d89335 100644
--- a/services/search/pkg/engine/bleve.go
+++ b/services/search/pkg/engine/bleve.go
@@ -6,6 +6,7 @@ import (
 	"math"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -143,6 +144,7 @@ func (b *Bleve) Search(_ context.Context, sir *searchService.SearchIndexRequest)
 	}
 
 	bleveReq := bleve.NewSearchRequest(q)
+	bleveReq.Highlight = bleve.NewHighlight()
 
 	switch {
 	case sir.PageSize == -1:
@@ -364,10 +366,27 @@ func formatQuery(q string) string {
 		cq = strings.ReplaceAll(cq, strings.ToLower(field)+":", field+":")
 	}
 
-	if strings.Contains(cq, ":") {
+	fieldRe := regexp.MustCompile(`\w+:[^ ]+`)
+	if fieldRe.MatchString(cq) {
+		parts := strings.Split(cq, " ")
+
+		cq = ""
+		for _, part := range parts {
+			fieldParts := strings.SplitN(part, ":", 2)
+			if len(fieldParts) > 1 {
+				value := fieldParts[1]
+				if value != "T" && value != "F" {
+					value = strings.ToLower(value) // do a lowercase query unless this is a boolean flag
+				}
+				cq += fieldParts[0] + ":" + value + " "
+			} else {
+				cq += part + " "
+			}
+		}
 		return cq // Sophisticated field based search
 	}
 
 	// this is a basic filename search
+	cq = strings.ReplaceAll(cq, ":", `\:`)
 	return "Name:*" + strings.ReplaceAll(strings.ToLower(cq), " ", `\ `) + "*"
 }
diff --git a/services/search/pkg/search/service.go b/services/search/pkg/search/service.go
index 9c2eeb0dc5b..8407ae67f12 100644
--- a/services/search/pkg/search/service.go
+++ b/services/search/pkg/search/service.go
@@ -30,6 +30,7 @@ import (
 
 const (
 	_spaceStateTrashed = "trashed"
+	_slowQueryDuration = 500 * time.Millisecond
 )
 
 // Searcher is the interface to the SearchService
@@ -255,19 +256,26 @@ func (s *Service) searchIndex(ctx context.Context, req *searchsvc.SearchRequest,
 		permissions = space.GetRootInfo().GetPermissionSet()
 	}
 
-	res, err := s.engine.Search(ctx, &searchsvc.SearchIndexRequest{
+	searchRequest := &searchsvc.SearchIndexRequest{
 		Query: req.Query,
 		Ref: &searchmsg.Reference{
 			ResourceId: searchRootID,
 			Path:       mountpointPrefix,
 		},
 		PageSize: req.PageSize,
-	})
+	}
+	start := time.Now()
+	res, err := s.engine.Search(ctx, searchRequest)
+	duration := time.Since(start)
 	if err != nil {
-		s.logger.Error().Err(err).Str("space", space.Id.OpaqueId).Msg("failed to search the index")
+		s.logger.Error().Err(err).Str("duration", fmt.Sprint(duration)).Str("space", space.Id.OpaqueId).Msg("failed to search the index")
 		return nil, err
 	}
-	s.logger.Debug().Str("space", space.Id.OpaqueId).Int("hits", len(res.Matches)).Msg("space search done")
+	if duration > _slowQueryDuration {
+		s.logger.Info().Interface("searchRequest", searchRequest).Str("duration", fmt.Sprint(duration)).Str("space", space.Id.OpaqueId).Int("hits", len(res.Matches)).Msg("slow space search")
+	} else {
+		s.logger.Debug().Interface("searchRequest", searchRequest).Str("duration", fmt.Sprint(duration)).Str("space", space.Id.OpaqueId).Int("hits", len(res.Matches)).Msg("space search done")
+	}
 
 	for _, match := range res.Matches {
 		if mountpointPrefix != "" {


Test patch:
diff --git a/services/search/pkg/engine/bleve_test.go b/services/search/pkg/engine/bleve_test.go
index 29048ba0b69..63ca73b5e97 100644
--- a/services/search/pkg/engine/bleve_test.go
+++ b/services/search/pkg/engine/bleve_test.go
@@ -215,14 +215,14 @@ var _ = Describe("Bleve", func() {
 				}
 			})
 
-			It("uses a lower-case index", func() {
+			It("does a case-insensitive search", func() {
 				parentResource.Document.Name = "foo.pdf"
 
 				err := eng.Upsert(parentResource.ID, parentResource)
 				Expect(err).ToNot(HaveOccurred())
 
 				assertDocCount(rootResource.ID, "Name:foo*", 1)
-				assertDocCount(rootResource.ID, "Name:Foo*", 0)
+				assertDocCount(rootResource.ID, "Name:Foo*", 1)
 			})
 
 			Context("and an additional file in a subdirectory", func() {