Add proxy handling to NLP functions and update .gitignore to exclude JSON files

.gitignore

@@ -3,5 +3,5 @@ __pycache__
 /downloads
 *.html
 .gradio
-output.json
+*.json
 test*

nlp_processes.py

@@ -1,8 +1,12 @@
 from g4f.client import Client
 from g4f.Provider import RetryProvider, Blackbox, Airforce, AmigoChat, ChatGptEs, ChatGpt, Copilot, DDG, Liaobots, Mhystical, PollinationsAI
+from proxy import get_proxy
 import threading
 
 def generate_nlp_summary(temp_summary):
+    proxy = get_proxy()
+    while not get_proxy():
+        proxy = get_proxy()
     try:
         try:
             client = Client(
@@ -12,6 +16,7 @@ def generate_nlp_summary(temp_summary):
                 ),
             )
             completion = client.chat.completions.create(
+                proxy=proxy,
                 model="llama-3.1-405b",
                 messages=[
                     {"role": "system", "content": "You are a helpful research assistant for generating well-formatted summaries from scientific research papers."},
@@ -23,6 +28,7 @@ def generate_nlp_summary(temp_summary):
             print(str(e))
             client = Client()
             completion = client.chat.completions.create(
+                proxy=proxy,
                 provider=RetryProvider(
                     providers=[Blackbox, ChatGptEs, ChatGpt, Copilot, DDG, Liaobots, Mhystical, PollinationsAI],
                     max_retries=8,
@@ -40,6 +46,9 @@ def generate_nlp_summary(temp_summary):
         return False
 
 def generate_nlp_mindmap(temp_summary):
+    proxy = get_proxy()
+    while not get_proxy():
+        proxy = get_proxy()
     try:
         try:
             client = Client(
@@ -49,6 +58,7 @@ def generate_nlp_mindmap(temp_summary):
                 ),
             )
             completion = client.chat.completions.create(
+                proxy=proxy,
                 model="llama-3.1-405b",
                 messages=[
                     {"role": "system", "content": "You are a helpful research assistant for generating well-formatted mindmaps from scientific research papers."},
@@ -60,6 +70,7 @@ def generate_nlp_mindmap(temp_summary):
             print(str(e))
             client = Client()
             completion = client.chat.completions.create(
+                proxy=proxy,
                 provider=RetryProvider(
                     providers=[Blackbox, ChatGptEs, ChatGpt, Copilot, DDG, Liaobots, Mhystical, PollinationsAI],
                     max_retries=8,
@@ -77,6 +88,9 @@ def generate_nlp_mindmap(temp_summary):
         return False
 
 def fix_title(title):
+    proxy = get_proxy()
+    while not get_proxy():
+        proxy = get_proxy()
     try:
         try:
             client = Client(
@@ -86,6 +100,7 @@ def fix_title(title):
                 ),
             )
             completion = client.chat.completions.create(
+                proxy=proxy,
                 model="llama-3.1-405b",
                 messages=[
                     {
@@ -120,6 +135,7 @@ def fix_title(title):
             print(str(e))
             client = Client()
             completion = client.chat.completions.create(
+                proxy=proxy,
                 provider=RetryProvider(
                     providers=[Blackbox, ChatGptEs, ChatGpt, Copilot, DDG, Liaobots, Mhystical, PollinationsAI],
                     max_retries=8,
@@ -160,6 +176,9 @@ def fix_title(title):
         return False
 
 def fix_citation(citation):
+    proxy = get_proxy()
+    while not get_proxy():
+        proxy = get_proxy()
     try:
         try:
             client = Client(
@@ -169,6 +188,7 @@ def fix_citation(citation):
                 ),
             )
             completion = client.chat.completions.create(
+                proxy=proxy,
                 model="llama-3.1-405b",
                 messages=[
                     {
@@ -201,6 +221,7 @@ def fix_citation(citation):
             print(str(e))
             client = Client()
             completion = client.chat.completions.create(
+                proxy=proxy,
                 provider=RetryProvider(
                     providers=[Blackbox, ChatGptEs, ChatGpt, Copilot, DDG, Liaobots, Mhystical, PollinationsAI],
                     max_retries=8,

proxy.py

@@ -0,0 +1,28 @@
+from swiftshadow.classes import Proxy
+from requests import get, exceptions
+
+def get_proxy():
+    proxies = {}
+    max_retries = 10  # Maximum number of attempts to find a valid proxy
+    attempts = 0
+    while attempts < max_retries:
+        try:
+            proxy = Proxy(
+                autoRotate=True,
+                cachePeriod=3,
+            ).proxy()
+            proxies = {
+                'http': proxy[0],
+                'https': proxy[0]
+            }
+            resp = get('https://checkip.amazonws.com', proxies=proxies, timeout=10)
+            ip_address = resp.text.strip()
+            if ip_address == proxy[0].split(':')[0]:
+                print(f"Valid proxy found: {proxy[0]}")
+                return proxies
+            else:
+                print(f"Proxy {proxy[0]} did not match the response IP ({ip_address}). Retrying...")
+        except exceptions.RequestException as e:
+            print(f"Error testing proxy: {e}. Retrying...")
+        attempts += 1
+    raise Exception("Failed to find a valid proxy after multiple attempts.")