Kevin Hu commited on
Commit
2bf6c61
·
1 Parent(s): 82f39fc

refine token similarity (#2824)

Browse files

### What problem does this PR solve?


### Type of change

- [x] Performance Improvement

Files changed (2) hide show
  1. agent/canvas.py +1 -1
  2. rag/nlp/query.py +2 -5
agent/canvas.py CHANGED
@@ -262,7 +262,7 @@ class Canvas(ABC):
262
  convs = []
263
  for role, obj in self.history[(window_size + 1) * -1:]:
264
  convs.append({"role": role, "content": (obj if role == "user" else
265
- '\n'.join(pd.DataFrame(obj)['content']))})
266
  return convs
267
 
268
  def add_user_input(self, question):
 
262
  convs = []
263
  for role, obj in self.history[(window_size + 1) * -1:]:
264
  convs.append({"role": role, "content": (obj if role == "user" else
265
+ '\n'.join([str(s) for s in pd.DataFrame(obj)['content']]))})
266
  return convs
267
 
268
  def add_user_input(self, question):
rag/nlp/query.py CHANGED
@@ -186,8 +186,5 @@ class EsQueryer:
186
  s += v # * dtwt[k]
187
  q = 1e-9
188
  for k, v in qtwt.items():
189
- q += v # * v
190
- #d = 1e-9
191
- # for k, v in dtwt.items():
192
- # d += v * v
193
- return s / q / max(1, math.sqrt(math.log10(max(len(qtwt.keys()), len(dtwt.keys())))))# math.sqrt(q) / math.sqrt(d)
 
186
  s += v # * dtwt[k]
187
  q = 1e-9
188
  for k, v in qtwt.items():
189
+ q += v
190
+ return s / q