Update digestor.py

digestor.py

@@ -158,10 +158,10 @@ class Digestor:
     
         # Finally, chunk the piece, adjusting the chunks if too long.
         for i, j in range_list:
-            if (tokenized_len := len(tokenizer(chunk := ' '.join(fractured[i:j]).replace('\n',' ')))) <= self.token_limit:
+            if (tokenized_len := len(tokenizer(chunk := ' '.join(fractured[i:j])))) <= self.token_limit:  # d[i:j]).replace('\n',' ')))) <= self.token_limit:
                 chunk_list.append(chunk)
             else: # if chunks of <limit> words are too long, back them off.
-                chunk_list.append(' '.join(chunk.split(' ')[: self.token_limit - tokenized_len ]).replace('\n',' '))
+                chunk_list.append(' '.join(chunk.split(' ')[: self.token_limit - tokenized_len ]))  # tokenized_len ]).replace('\n',' '))
         
         return chunk_list