Repository: blevesearch/bleve
Pull request #1533 (2021-01-05 19:31:15)

Accommodate field names with whitespaces in query string syntax
+ Update the the yacc rules while parsing query string syntax to
  accommodate a "phrase" as a field name. This will allow for the
  user to use field names that have white spaces in them.

  For example:
    - Index definition has a field "with JSON" searchable as "with JSON"
    - With a match query - {"field": "with JSON", "match": "reserved"}
    - This is previously not supported within the query string syntax,
      but is now possible ..
            +------------------------+
            | "with JSON":"reserved" |
            +------------------------+

+ query_string.y.go is generated using goyacc based on new rules added
  to query_string.y.
+ This addresses https://issues.couchbase.com/browse/MB-41264.

abhinavdangeti: Ya fair, let me give that a try tomorrow.

Patch:
diff --git a/search/query/query_string.y b/search/query/query_string.y
index d3e5ac9d..aeec8560 100644
--- a/search/query/query_string.y
+++ b/search/query/query_string.y
@@ -28,10 +28,11 @@ tEQUAL tTILDE
 %type 
                tPHRASE
 %type 
                tNUMBER
 %type 
                posOrNegNumber
+%type 
                fieldName
 %type 
                tTILDE
 %type 
                tBOOST
 %type <q>                searchBase
-%type <pf>                searchSuffix
+%type <pf>               searchSuffix
 %type <n>                searchPrefix
 
 %%
@@ -111,7 +112,7 @@ tSTRING tTILDE {
 	$$ = q
 }
 |
-tSTRING tCOLON tSTRING tTILDE {
+fieldName tCOLON tSTRING tTILDE {
 	field := $1
 	str := $3
 	fuzziness, err := strconv.ParseFloat($4, 64)
@@ -147,7 +148,7 @@ tPHRASE {
 	$$ = q
 }
 |
-tSTRING tCOLON tSTRING {
+fieldName tCOLON tSTRING {
 	field := $1
 	str := $3
 	logDebugGrammar("FIELD - %s STRING - %s", field, str)
@@ -163,7 +164,7 @@ tSTRING tCOLON tSTRING {
 	$$ = q
 }
 |
-tSTRING tCOLON posOrNegNumber {
+fieldName tCOLON posOrNegNumber {
 	field := $1
 	str := $3
 	logDebugGrammar("FIELD - %s STRING - %s", field, str)
@@ -181,7 +182,7 @@ tSTRING tCOLON posOrNegNumber {
 	$$ = q
 }
 |
-tSTRING tCOLON tPHRASE {
+fieldName tCOLON tPHRASE {
 	field := $1
 	phrase := $3
 	logDebugGrammar("FIELD - %s PHRASE - %s", field, phrase)
@@ -190,7 +191,7 @@ tSTRING tCOLON tPHRASE {
 	$$ = q
 }
 |
-tSTRING tCOLON tGREATER posOrNegNumber {
+fieldName tCOLON tGREATER posOrNegNumber {
 	field := $1
 	min, err := strconv.ParseFloat($4, 64)
 	if err != nil {
@@ -203,7 +204,7 @@ tSTRING tCOLON tGREATER posOrNegNumber {
 	$$ = q
 }
 |
-tSTRING tCOLON tGREATER tEQUAL posOrNegNumber {
+fieldName tCOLON tGREATER tEQUAL posOrNegNumber {
 	field := $1
 	min, err := strconv.ParseFloat($5, 64)
 	if err != nil {
@@ -216,7 +217,7 @@ tSTRING tCOLON tGREATER tEQUAL posOrNegNumber {
 	$$ = q
 }
 |
-tSTRING tCOLON tLESS posOrNegNumber {
+fieldName tCOLON tLESS posOrNegNumber {
 	field := $1
 	max, err := strconv.ParseFloat($4, 64)
 	if err != nil {
@@ -229,7 +230,7 @@ tSTRING tCOLON tLESS posOrNegNumber {
 	$$ = q
 }
 |
-tSTRING tCOLON tLESS tEQUAL posOrNegNumber {
+fieldName tCOLON tLESS tEQUAL posOrNegNumber {
 	field := $1
 	max, err := strconv.ParseFloat($5, 64)
 	if err != nil {
@@ -242,7 +243,7 @@ tSTRING tCOLON tLESS tEQUAL posOrNegNumber {
 	$$ = q
 }
 |
-tSTRING tCOLON tGREATER tPHRASE {
+fieldName tCOLON tGREATER tPHRASE {
 	field := $1
 	minInclusive := false
 	phrase := $4
@@ -257,7 +258,7 @@ tSTRING tCOLON tGREATER tPHRASE {
 	$$ = q
 }
 |
-tSTRING tCOLON tGREATER tEQUAL tPHRASE {
+fieldName tCOLON tGREATER tEQUAL tPHRASE {
 	field := $1
 	minInclusive := true
 	phrase := $5
@@ -272,7 +273,7 @@ tSTRING tCOLON tGREATER tEQUAL tPHRASE {
 	$$ = q
 }
 |
-tSTRING tCOLON tLESS tPHRASE {
+fieldName tCOLON tLESS tPHRASE {
 	field := $1
 	maxInclusive := false
 	phrase := $4
@@ -287,7 +288,7 @@ tSTRING tCOLON tLESS tPHRASE {
 	$$ = q
 }
 |
-tSTRING tCOLON tLESS tEQUAL tPHRASE {
+fieldName tCOLON tLESS tEQUAL tPHRASE {
 	field := $1
 	maxInclusive := true
 	phrase := $5
@@ -326,3 +327,12 @@ tNUMBER {
 tMINUS tNUMBER {
 	$$ = "-" + $2
 };
+
+fieldName:
+tPHRASE {
+    $$ = $1
+}
+|
+tSTRING {
+    $$ = $1
+};
diff --git a/search/query/query_string.y.go b/search/query/query_string.y.go
index ac2d3222..3a2abc13 100644
--- a/search/query/query_string.y.go
+++ b/search/query/query_string.y.go
@@ -1,3 +1,6 @@
+// Code generated by goyacc -o query_string.y.go query_string.y. DO NOT EDIT.
+
+//line query_string.y:2
 package query
 
 import __yyfmt__ "fmt"
@@ -54,6 +57,7 @@ var yyToknames = [...]string{
 	"tEQUAL",
 	"tTILDE",
 }
+
 var yyStatenames = [...]string{}
 
 const yyEofCode = 1
@@ -68,70 +72,72 @@ var yyExca = [...]int{
 	-1, 3,
 	1, 3,
 	-2, 5,
+	-1, 9,
+	8, 29,
+	-2, 8,
+	-1, 12,
+	8, 28,
+	-2, 12,
 }
 
-const yyNprod = 28
 const yyPrivate = 57344
 
-var yyTokenNames []string
-var yyStates []string
-
-const yyLast = 42
+const yyLast = 43
 
 var yyAct = [...]int{
-
-	17, 16, 18, 23, 22, 30, 3, 21, 19, 20,
-	29, 26, 22, 22, 1, 21, 21, 15, 28, 25,
-	24, 27, 34, 14, 22, 13, 31, 21, 32, 33,
-	22, 9, 11, 21, 5, 6, 2, 10, 4, 12,
-	7, 8,
+	18, 17, 19, 24, 23, 15, 31, 22, 20, 21,
+	30, 27, 23, 23, 3, 22, 22, 14, 29, 26,
+	16, 25, 28, 35, 33, 23, 23, 32, 22, 22,
+	34, 9, 12, 1, 5, 6, 2, 11, 4, 13,
+	7, 8, 10,
 }
-var yyPact = [...]int{
 
-	28, -1000, -1000, 28, 27, -1000, -1000, -1000, 16, 9,
-	-1000, -1000, -1000, -1000, -1000, -3, -11, -1000, -1000, 6,
-	5, -1000, -5, -1000, -1000, 23, -1000, -1000, 17, -1000,
-	-1000, -1000, -1000, -1000, -1000,
+var yyPact = [...]int{
+	28, -1000, -1000, 28, 27, -1000, -1000, -1000, 8, -9,
+	12, -1000, -1000, -1000, -1000, -1000, -3, -11, -1000, -1000,
+	6, 5, -1000, -4, -1000, -1000, 19, -1000, -1000, 18,
+	-1000, -1000, -1000, -1000, -1000, -1000,
 }
-var yyPgo = [...]int{
 
-	0, 0, 41, 39, 38, 14, 36, 6,
+var yyPgo = [...]int{
+	0, 0, 42, 41, 39, 38, 33, 36, 14,
 }
-var yyR1 = [...]int{
 
-	0, 5, 6, 6, 7, 4, 4, 4, 2, 2,
-	2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-	2, 2, 2, 2, 3, 3, 1, 1,
+var yyR1 = [...]int{
+	0, 6, 7, 7, 8, 5, 5, 5, 3, 3,
+	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+	3, 3, 3, 3, 4, 4, 1, 1, 2, 2,
 }
-var yyR2 = [...]int{
 
+var yyR2 = [...]int{
 	0, 1, 2, 1, 3, 0, 1, 1, 1, 2,
 	4, 1, 1, 3, 3, 3, 4, 5, 4, 5,
-	4, 5, 4, 5, 0, 1, 1, 2,
+	4, 5, 4, 5, 0, 1, 1, 2, 1, 1,
 }
-var yyChk = [...]int{
 
-	-1000, -5, -6, -7, -4, 6, 7, -6, -2, 4,
-	10, 5, -3, 9, 14, 8, 4, -1, 5, 11,
-	12, 10, 7, 14, -1, 13, 5, -1, 13, 5,
-	10, -1, 5, -1, 5,
+var yyChk = [...]int{
+	-1000, -6, -7, -8, -5, 6, 7, -7, -3, 4,
+	-2, 10, 5, -4, 9, 14, 8, 4, -1, 5,
+	11, 12, 10, 7, 14, -1, 13, 5, -1, 13,
+	5, 10, -1, 5, -1, 5,
 }
-var yyDef = [...]int{
 
-	5, -2, 1, -2, 0, 6, 7, 2, 24, 8,
-	11, 12, 4, 25, 9, 0, 13, 14, 15, 0,
-	0, 26, 0, 10, 16, 0, 20, 18, 0, 22,
-	27, 17, 21, 19, 23,
+var yyDef = [...]int{
+	5, -2, 1, -2, 0, 6, 7, 2, 24, -2,
+	0, 11, -2, 4, 25, 9, 0, 13, 14, 15,
+	0, 0, 26, 0, 10, 16, 0, 20, 18, 0,
+	22, 27, 17, 21, 19, 23,
 }
-var yyTok1 = [...]int{
 
+var yyTok1 = [...]int{
 	1,
 }
-var yyTok2 = [...]int{
 
+var yyTok2 = [...]int{
 	2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
 	12, 13, 14,
 }
+
 var yyTok3 = [...]int{
 	0,
 }
@@ -475,25 +481,25 @@ yydefault:
 
 	case 1:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line query_string.y:40
+//line query_string.y:41
 		{
 			logDebugGrammar("INPUT")
 		}
 	case 2:
 		yyDollar = yyS[yypt-2 : yypt+1]
-		//line query_string.y:45
+//line query_string.y:46
 		{
 			logDebugGrammar("SEARCH PARTS")
 		}
 	case 3:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line query_string.y:49
+//line query_string.y:50
 		{
 			logDebugGrammar("SEARCH PART")
 		}
 	case 4:
 		yyDollar = yyS[yypt-3 : yypt+1]
-		//line query_string.y:54
+//line query_string.y:55
 		{
 			query := yyDollar[2].q
 			if yyDollar[3].pf != nil {
@@ -512,27 +518,27 @@ yydefault:
 		}
 	case 5:
 		yyDollar = yyS[yypt-0 : yypt+1]
-		//line query_string.y:73
+//line query_string.y:74
 		{
 			yyVAL.n = queryShould
 		}
 	case 6:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line query_string.y:77
+//line query_string.y:78
 		{
 			logDebugGrammar("PLUS")
 			yyVAL.n = queryMust
 		}
 	case 7:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line query_string.y:82
+//line query_string.y:83
 		{
 			logDebugGrammar("MINUS")
 			yyVAL.n = queryMustNot
 		}
 	case 8:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line query_string.y:88
+//line query_string.y:89
 		{
 			str := yyDollar[1].s
 			logDebugGrammar("STRING - %s", str)
@@ -548,7 +554,7 @@ yydefault:
 		}
 	case 9:
 		yyDollar = yyS[yypt-2 : yypt+1]
-		//line query_string.y:102
+//line query_string.y:103
 		{
 			str := yyDollar[1].s
 			fuzziness, err := strconv.ParseFloat(yyDollar[2].s, 64)
@@ -562,7 +568,7 @@ yydefault:
 		}
 	case 10:
 		yyDollar = yyS[yypt-4 : yypt+1]
-		//line query_string.y:114
+//line query_string.y:115
 		{
 			field := yyDollar[1].s
 			str := yyDollar[3].s
@@ -578,7 +584,7 @@ yydefault:
 		}
 	case 11:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line query_string.y:128
+//line query_string.y:129
 		{
 			str := yyDollar[1].s
 			logDebugGrammar("STRING - %s", str)
@@ -595,7 +601,7 @@ yydefault:
 		}
 	case 12:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line query_string.y:143
+//line query_string.y:144
 		{
 			phrase := yyDollar[1].s
 			logDebugGrammar("PHRASE - %s", phrase)
@@ -604,7 +610,7 @@ yydefault:
 		}
 	case 13:
 		yyDollar = yyS[yypt-3 : yypt+1]
-		//line query_string.y:150
+//line query_string.y:151
 		{
 			field := yyDollar[1].s
 			str := yyDollar[3].s
@@ -622,7 +628,7 @@ yydefault:
 		}
 	case 14:
 		yyDollar = yyS[yypt-3 : yypt+1]
-		//line query_string.y:166
+//line query_string.y:167
 		{
 			field := yyDollar[1].s
 			str := yyDollar[3].s
@@ -642,7 +648,7 @@ yydefault:
 		}
 	case 15:
 		yyDollar = yyS[yypt-3 : yypt+1]
-		//line query_string.y:184
+//line query_string.y:185
 		{
 			field := yyDollar[1].s
 			phrase := yyDollar[3].s
@@ -653,7 +659,7 @@ yydefault:
 		}
 	case 16:
 		yyDollar = yyS[yypt-4 : yypt+1]
-		//line query_string.y:193
+//line query_string.y:194
 		{
 			field := yyDollar[1].s
 			min, err := strconv.ParseFloat(yyDollar[4].s, 64)
@@ -668,7 +674,7 @@ yydefault:
 		}
 	case 17:
 		yyDollar = yyS[yypt-5 : yypt+1]
-		//line query_string.y:206
+//line query_string.y:207
 		{
 			field := yyDollar[1].s
 			min, err := strconv.ParseFloat(yyDollar[5].s, 64)
@@ -683,7 +689,7 @@ yydefault:
 		}
 	case 18:
 		yyDollar = yyS[yypt-4 : yypt+1]
-		//line query_string.y:219
+//line query_string.y:220
 		{
 			field := yyDollar[1].s
 			max, err := strconv.ParseFloat(yyDollar[4].s, 64)
@@ -698,7 +704,7 @@ yydefault:
 		}
 	case 19:
 		yyDollar = yyS[yypt-5 : yypt+1]
-		//line query_string.y:232
+//line query_string.y:233
 		{
 			field := yyDollar[1].s
 			max, err := strconv.ParseFloat(yyDollar[5].s, 64)
@@ -713,7 +719,7 @@ yydefault:
 		}
 	case 20:
 		yyDollar = yyS[yypt-4 : yypt+1]
-		//line query_string.y:245
+//line query_string.y:246
 		{
 			field := yyDollar[1].s
 			minInclusive := false
@@ -730,7 +736,7 @@ yydefault:
 		}
 	case 21:
 		yyDollar = yyS[yypt-5 : yypt+1]
-		//line query_string.y:260
+//line query_string.y:261
 		{
 			field := yyDollar[1].s
 			minInclusive := true
@@ -747,7 +753,7 @@ yydefault:
 		}
 	case 22:
 		yyDollar = yyS[yypt-4 : yypt+1]
-		//line query_string.y:275
+//line query_string.y:276
 		{
 			field := yyDollar[1].s
 			maxInclusive := false
@@ -764,7 +770,7 @@ yydefault:
 		}
 	case 23:
 		yyDollar = yyS[yypt-5 : yypt+1]
-		//line query_string.y:290
+//line query_string.y:291
 		{
 			field := yyDollar[1].s
 			maxInclusive := true
@@ -781,13 +787,13 @@ yydefault:
 		}
 	case 24:
 		yyDollar = yyS[yypt-0 : yypt+1]
-		//line query_string.y:306
+//line query_string.y:307
 		{
 			yyVAL.pf = nil
 		}
 	case 25:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line query_string.y:310
+//line query_string.y:311
 		{
 			yyVAL.pf = nil
 			boost, err := strconv.ParseFloat(yyDollar[1].s, 64)
@@ -800,16 +806,28 @@ yydefault:
 		}
 	case 26:
 		yyDollar = yyS[yypt-1 : yypt+1]
-		//line query_string.y:322
+//line query_string.y:323
 		{
 			yyVAL.s = yyDollar[1].s
 		}
 	case 27:
 		yyDollar = yyS[yypt-2 : yypt+1]
-		//line query_string.y:326
+//line query_string.y:327
 		{
 			yyVAL.s = "-" + yyDollar[2].s
 		}
+	case 28:
+		yyDollar = yyS[yypt-1 : yypt+1]
+//line query_string.y:332
+		{
+			yyVAL.s = yyDollar[1].s
+		}
+	case 29:
+		yyDollar = yyS[yypt-1 : yypt+1]
+//line query_string.y:336
+		{
+			yyVAL.s = yyDollar[1].s
+		}
 	}
 	goto yystack /* stack new state and value */
 }


Test patch:
diff --git a/search/query/query_string_parser_test.go b/search/query/query_string_parser_test.go
index f10170ed..fa0f3ae0 100644
--- a/search/query/query_string_parser_test.go
+++ b/search/query/query_string_parser_test.go
@@ -811,6 +811,36 @@ func TestQuerySyntaxParserValid(t *testing.T) {
 				},
 				nil),
 		},
+
+		// field names as phrases
+		{
+			input:   `"fie ld":test`,
+			mapping: mapping.NewIndexMapping(),
+			result: NewBooleanQueryForQueryString(
+				nil,
+				[]Query{
+					func() Query {
+						q := NewMatchQuery("test")
+						q.SetField("fie ld")
+						return q
+					}(),
+				},
+				nil),
+		},
+		{
+			input:   `"fie ld":"test"`,
+			mapping: mapping.NewIndexMapping(),
+			result: NewBooleanQueryForQueryString(
+				nil,
+				[]Query{
+					func() Query {
+						q := NewMatchPhraseQuery("test")
+						q.SetField("fie ld")
+						return q
+					}(),
+				},
+				nil),
+		},
 	}
 
 	// turn on lexer debugging