Fix web search

corpora/sourcer.py

@@ -36,20 +36,17 @@ def search_web(query: str) -> list:
 
     texts = []
     responses = [None] * len(links)
-    download_threads = [None] * len(links)
-    processing_threads = [None] * len(links)
-    # dowload_threads[0] = threading.Thread(target=download_page, args=(links[0], responses, 0))
-    download_page(links[0], responses, 0)
-    for i in range(1, len(links), 2):
-        # new page processing thread
-        processing_thread = threading.Thread(target=process_page, args=(texts, responses, i-1))
-        # new download thread
-        download_thread = threading.Thread(target=download_page, args=(links[i], responses, i))
+    download_page(links[0], responses, 0) # download first page
+    for i in range(1, len(links)):
+        # process previous page
+        processing_thread = threading.Thread(target=process_page, args=(texts, responses, i-1), name='processing'+str(i-1))
+        # dowload new page
+        download_thread = threading.Thread(target=download_page, args=(links[i], responses, i), name='download'+str(i))
         # start threads
         download_thread.start()
         processing_thread.start()
+        # wait for threads
         download_thread.join()
         processing_thread.join()
 
-
     return texts