add model param, -inurl to exclude unhelpful websites

sheet.py

@@ -90,11 +90,12 @@ def test_get_condensed_result():
     res = get_serp(query)
     cond_res = get_condensed_result(res)
 
-def compose_analysis( client, query, search_results):
+def compose_analysis( client, query, search_results, model: str = 'gpt-3.5-turbo-0125'):
     """
     Argument
         query: str
         search_results: str
+        model: "gpt-4-0125-preview" or 'gpt-3.5-turbo-0125'
     Return
         response: str
     """
@@ -119,7 +120,7 @@ def compose_analysis( client, query, search_results):
                 ''',
             }
         ],
-        model = "gpt-4-0125-preview", 
+        model = model, 
         response_format = {"type": "json_object"},
         temperature = 0,
         # stream = True
@@ -151,12 +152,14 @@ def compose_classication(
         evidence, 
         classes: list = ['小吃店', '日式料理(含居酒屋，串燒)', '火(鍋／爐)', '東南亞料理(不含日韓)', '海鮮熱炒',  '特色餐廳(含雞、鵝、牛、羊肉)', '傳統餐廳', '燒烤', '韓式料理(含火鍋，烤肉)', '西餐廳(含美式，義式，墨式)'],
         backup_classes: list = [ '中式', '西式'],
+        model: str = 'gpt-3.5-turbo-0125'
     ) -> str:
     """
     Argument
         client: 
         evidence: str
         classes: list
+        model: 'gpt-3.5-turbo-0125', 'gpt-4-0125-preview'
     Return
         response: str
     """
@@ -187,7 +190,7 @@ def compose_classication(
                 ''',
             }
         ],
-        model = "gpt-4-0125-preview", 
+        model = model, 
         response_format = {"type": "json_object"},
         temperature = 0,
         # stream = True
@@ -278,7 +281,7 @@ def test_get_evidence_classification():
     analysis_results = classify_results( analysis_results)
     patch_analysis_results = classify_results( patch_analysis_results)
 
-def compose_query( address, name, with_index: bool = True):
+def compose_query( address, name, with_index: bool = True, exclude: str = "-inurl:twincn.com -inurl:findcompany.com.tw -inurl:iyp.com.tw -inurl:twypage.com -inurl:alltwcompany.com -inurl:zhupiter.com -inurl:twinc.com.tw"):
     """
     Argumemnt
         # d: series with d[1]: 地址, d[4]: 營業人名稱 #
@@ -292,7 +295,7 @@ def compose_query( address, name, with_index: bool = True):
     #     query = f"{d[1][:3]} {d[4]}"
     # else:
     #     query = f"{d[0][:3]} {d[3]}"
-    query = f"{address[:3]} {name}"
+    query = f"{address[:3]} {name} {exclude}" 
     return query
 
 def crawl_results( data: pd.DataFrame, google_domain: str = 'google.com.tw', gl: str = 'tw', lr: str = 'lang_zh-TW'):
@@ -563,7 +566,7 @@ def main(args):
     """
     
     ## 讀取資料名單 ##
-    data = get_leads(args.data_path)
+    data = get_leads(args.data_path).head(20)
     
     ## 進行爬蟲與分析 ##
     # crawled_results = crawl_results(data)
@@ -611,7 +614,7 @@ def main(args):
     )
 
     formatted_results = format_output( postprossed_results, input_column = 'evidence', output_column = 'formatted_evidence', format_func = format_evidence)
-    formatted_results.to_csv("data/formatted_results.csv", index=False)
+    formatted_results.to_csv( args.formatted_results, index=False)
 
 
 category2supercategory = {
@@ -658,13 +661,15 @@ if __name__=='__main__':
 
     parser = argparse.ArgumentParser()
     parser.add_argument("--data_path", type=str, default="data/餐廳類型分類.xlsx - 測試清單.csv")
-    parser.add_argument("--classified_file_path", type=str, default="data/classified_results.joblib")
-    parser.add_argument("--extracted_file_path", type=str, default="data/extracted_results.joblib")
-    parser.add_argument("--crawled_file_path", type=str, default="data/crawled_results.joblib")
-    parser.add_argument("--combined_file_path", type=str, default="data/combined_results.joblib")
-    parser.add_argument("--postprocessed_results", type=str, default="data/postprocessed_results.joblib")
+    parser.add_argument("--classified_file_path", type=str, default="data/gpt3.5/classified_results.joblib")
+    parser.add_argument("--extracted_file_path", type=str, default="data/gpt3.5/extracted_results.joblib")
+    parser.add_argument("--crawled_file_path", type=str, default="data/gpt3.5/crawled_results.joblib")
+    parser.add_argument("--combined_file_path", type=str, default="data/gpt3.5/combined_results.joblib")
+    parser.add_argument("--postprocessed_results", type=str, default="data/gpt3.5/postprocessed_results.joblib")
+    parser.add_argument("--formatted_results", type=str, default="data/gpt3.5/formatted_results.csv")
     parser.add_argument("--classes", type=list, default=['小吃店', '日式料理(含居酒屋，串燒)', '火(鍋／爐)', '東南亞料理(不含日韓)', '海鮮熱炒',  '特色餐廳(含雞、鵝、牛、羊肉)', '傳統餐廳', '燒烤', '韓式料理(含火鍋，烤肉)', '西餐廳(含美式，義式，墨式)'])
     parser.add_argument("--backup_classes", type=list, default=['中式', '西式'])
+    parser.add_argument("--strategy", type=str, default='replace', choices=['replace', 'patch'])
     parser.add_argument("--n_processes", type=int, default=4)
     args = parser.parse_args()