Kevin Hu
commited on
Commit
·
2bf6c61
1
Parent(s):
82f39fc
refine token similarity (#2824)
Browse files### What problem does this PR solve?
### Type of change
- [x] Performance Improvement
- agent/canvas.py +1 -1
- rag/nlp/query.py +2 -5
agent/canvas.py
CHANGED
@@ -262,7 +262,7 @@ class Canvas(ABC):
|
|
262 |
convs = []
|
263 |
for role, obj in self.history[(window_size + 1) * -1:]:
|
264 |
convs.append({"role": role, "content": (obj if role == "user" else
|
265 |
-
|
266 |
return convs
|
267 |
|
268 |
def add_user_input(self, question):
|
|
|
262 |
convs = []
|
263 |
for role, obj in self.history[(window_size + 1) * -1:]:
|
264 |
convs.append({"role": role, "content": (obj if role == "user" else
|
265 |
+
'\n'.join([str(s) for s in pd.DataFrame(obj)['content']]))})
|
266 |
return convs
|
267 |
|
268 |
def add_user_input(self, question):
|
rag/nlp/query.py
CHANGED
@@ -186,8 +186,5 @@ class EsQueryer:
|
|
186 |
s += v # * dtwt[k]
|
187 |
q = 1e-9
|
188 |
for k, v in qtwt.items():
|
189 |
-
q += v
|
190 |
-
|
191 |
-
# for k, v in dtwt.items():
|
192 |
-
# d += v * v
|
193 |
-
return s / q / max(1, math.sqrt(math.log10(max(len(qtwt.keys()), len(dtwt.keys())))))# math.sqrt(q) / math.sqrt(d)
|
|
|
186 |
s += v # * dtwt[k]
|
187 |
q = 1e-9
|
188 |
for k, v in qtwt.items():
|
189 |
+
q += v
|
190 |
+
return s / q
|
|
|
|
|
|