Update with h2oGPT hash 1b295baace42908075b47f31a84b359d8c6b1e52

client_test.py

@@ -3,7 +3,7 @@ Client test.
 
 Run server:
 
-python generate.py  --base_model=h2oai/h2ogpt-oig-oasst1-512-6.9b
+python generate.py  --base_model=h2oai/h2ogpt-oig-oasst1-512-6_9b
 
 NOTE: For private models, add --use-auth_token=True
 
@@ -39,6 +39,7 @@ Loaded as API: https://gpt.h2o.ai ✔
 import time
 import os
 import markdown  # pip install markdown
+import pytest
 from bs4 import BeautifulSoup  # pip install beautifulsoup4
 
 debug = False
@@ -79,6 +80,7 @@ def get_args(prompt, prompt_type, chat=False, stream_output=False, max_new_token
                          instruction_nochat=prompt if not chat else '',
                          iinput_nochat='',  # only for chat=False
                          langchain_mode='Disabled',
+                         document_choice=['All'],
                          )
     if chat:
         # add chatbot output on end.  Assumes serialize=False
@@ -87,6 +89,7 @@ def get_args(prompt, prompt_type, chat=False, stream_output=False, max_new_token
     return kwargs, list(kwargs.values())
 
 
+@pytest.mark.skip(reason="For manual use against some server, no server launched")
 def test_client_basic():
     return run_client_nochat(prompt='Who are you?', prompt_type='human_bot', max_new_tokens=50)
 
@@ -106,6 +109,7 @@ def run_client_nochat(prompt, prompt_type, max_new_tokens):
     return res_dict
 
 
+@pytest.mark.skip(reason="For manual use against some server, no server launched")
 def test_client_chat():
     return run_client_chat(prompt='Who are you?', prompt_type='human_bot', stream_output=False, max_new_tokens=50)
 

finetune.py

@@ -26,7 +26,7 @@ def train(
         save_code: bool = False,
         run_id: int = None,
 
-        base_model: str = 'h2oai/h2ogpt-oig-oasst1-512-6.9b',
+        base_model: str = 'h2oai/h2ogpt-oig-oasst1-512-6_9b',
         # base_model: str = 'h2oai/h2ogpt-oasst1-512-12b',
         # base_model: str = 'h2oai/h2ogpt-oasst1-512-20b',
         # base_model: str = 'EleutherAI/gpt-neox-20b',

generate.py

@@ -297,7 +297,7 @@ def main(
         if psutil.virtual_memory().available < 94 * 1024 ** 3:
             # 12B uses ~94GB
             # 6.9B uses ~47GB
-            base_model = 'h2oai/h2ogpt-oig-oasst1-512-6.9b' if not base_model else base_model
+            base_model = 'h2oai/h2ogpt-oig-oasst1-512-6_9b' if not base_model else base_model
 
     # get defaults
     model_lower = base_model.lower()
@@ -864,6 +864,7 @@ eval_func_param_names = ['instruction',
                          'instruction_nochat',
                          'iinput_nochat',
                          'langchain_mode',
+                         'document_choice',
                          ]
 
 
@@ -891,6 +892,7 @@ def evaluate(
         instruction_nochat,
         iinput_nochat,
         langchain_mode,
+        document_choice,
         # END NOTE: Examples must have same order of parameters
         src_lang=None,
         tgt_lang=None,
@@ -1010,6 +1012,7 @@ def evaluate(
                            chunk=chunk,
                            chunk_size=chunk_size,
                            langchain_mode=langchain_mode,
+                           document_choice=document_choice,
                            db_type=db_type,
                            k=k,
                            temperature=temperature,
@@ -1446,7 +1449,7 @@ y = np.random.randint(0, 1, 100)
 
     # move to correct position
     for example in examples:
-        example += [chat, '', '', 'Disabled']
+        example += [chat, '', '', 'Disabled', ['All']]
         # adjust examples if non-chat mode
         if not chat:
             example[eval_func_param_names.index('instruction_nochat')] = example[
@@ -1546,6 +1549,6 @@ if __name__ == "__main__":
     can also pass --prompt_type='human_bot' and model can somewhat handle instructions without being instruct tuned
     python generate.py --base_model=decapoda-research/llama-65b-hf --load_8bit=False --infer_devices=False --prompt_type='human_bot'
 
-    python generate.py --base_model=h2oai/h2ogpt-oig-oasst1-512-6.9b
+    python generate.py --base_model=h2oai/h2ogpt-oig-oasst1-512-6_9b
     """
     fire.Fire(main)

gpt_langchain.py

@@ -150,7 +150,7 @@ def get_llm(use_openai_model=False, model_name=None, model=None,
             assert model_name is None
             assert tokenizer is None
             model_name = 'h2oai/h2ogpt-oasst1-512-12b'
-            # model_name = 'h2oai/h2ogpt-oig-oasst1-512-6.9b'
+            # model_name = 'h2oai/h2ogpt-oig-oasst1-512-6_9b'
             # model_name = 'h2oai/h2ogpt-oasst1-512-20b'
             tokenizer = AutoTokenizer.from_pretrained(model_name)
             device, torch_dtype, context_class = get_device_dtype()
@@ -593,7 +593,7 @@ def path_to_docs(path_or_paths, verbose=False, fail_any_exception=False, n_jobs=
                  ):
     globs_image_types = []
     globs_non_image_types = []
-    if path_or_paths is None:
+    if not path_or_paths and not url and not text:
         return []
     elif url:
         globs_non_image_types = [url]
@@ -846,6 +846,7 @@ def _run_qa_db(query=None,
                top_k=40,
                top_p=0.7,
                langchain_mode=None,
+               document_choice=['All'],
                n_jobs=-1):
     """
 
@@ -917,7 +918,23 @@ def _run_qa_db(query=None,
     k_db = 1000 if db_type == 'chroma' else k  # k=100 works ok too for
 
     if db and use_context:
-        docs_with_score = db.similarity_search_with_score(query, k=k_db)[:k]
+        if isinstance(document_choice, str):
+            # support string as well
+            document_choice = [document_choice]
+        if not isinstance(db, Chroma) or len(document_choice) <= 1 and document_choice[0].lower() == 'all':
+            # treat empty list as All for now, not 'None'
+            filter_kwargs = {}
+        else:
+            if len(document_choice) >= 2:
+                or_filter = [{"source": {"$eq": x}} for x in document_choice]
+                filter_kwargs = dict(filter={"$or": or_filter})
+            else:
+                one_filter = [{"source": {"$eq": x}} for x in document_choice][0]
+                filter_kwargs = dict(filter=one_filter)
+            if len(document_choice) == 1 and document_choice[0].lower() == 'none':
+                k_db = 1
+                k = 0
+        docs_with_score = db.similarity_search_with_score(query, k=k_db, **filter_kwargs)[:k]
         # cut off so no high distance docs/sources considered
         docs = [x[0] for x in docs_with_score if x[1] < cut_distanct]
         scores = [x[1] for x in docs_with_score if x[1] < cut_distanct]
@@ -939,7 +956,7 @@ def _run_qa_db(query=None,
         reduced_query_words = reduced_query.split(' ')
         set_common = set(df['Lemma'].values.tolist())
         num_common = len([x.lower() in set_common for x in reduced_query_words])
-        frac_common = num_common / len(reduced_query)
+        frac_common = num_common / len(reduced_query) if reduced_query else 0
         # FIXME: report to user bad query that uses too many common words
         print("frac_common: %s" % frac_common, flush=True)
 

gradio_runner.py

@@ -96,7 +96,13 @@ def go_gradio(**kwargs):
         css_code = """footer {visibility: hidden}"""
     css_code += """
 body.dark{#warning {background-color: #555555};}
-"""
+#small_btn {
+    margin: 0.6em 0em 0.55em 0;
+    max-width: 20em;
+    min-width: 5em !important;
+    height: 5em;
+    font-size: 14px !important
+}"""
 
     if kwargs['gradio_avoid_processing_markdown']:
         from gradio_client import utils as client_utils
@@ -167,6 +173,7 @@ body.dark{#warning {background-color: #555555};}
         lora_options_state = gr.State([lora_options])
         my_db_state = gr.State([None, None])
         chat_state = gr.State({})
+        docs_state = gr.State(['All'])
         gr.Markdown(f"""
             {get_h2o_title(title) if kwargs['h2ocolors'] else get_simple_title(title)}
 
@@ -175,7 +182,7 @@ body.dark{#warning {background-color: #555555};}
             """)
         if is_hf:
             gr.HTML(
-                )
+            )
 
         # go button visible if
         base_wanted = kwargs['base_model'] != no_model_str and kwargs['login_mode_if_model0']
@@ -220,7 +227,7 @@ body.dark{#warning {background-color: #555555};}
                                 submit = gr.Button(value='Submit').style(full_width=False, size='sm')
                                 stop_btn = gr.Button(value="Stop").style(full_width=False, size='sm')
                         with gr.Row():
-                            clear = gr.Button("Save, New Conversation")
+                            clear = gr.Button("Save Chat / New Chat")
                             flag_btn = gr.Button("Flag")
                             if not kwargs['auto_score']:  # FIXME: For checkbox model2
                                 with gr.Column(visible=kwargs['score_model']):
@@ -251,19 +258,16 @@ body.dark{#warning {background-color: #555555};}
                     radio_chats = gr.Radio(value=None, label="Saved Chats", visible=True, interactive=True,
                                            type='value')
                     with gr.Row():
-                        remove_chat_btn = gr.Button(value="Remove Selected Chat", visible=True)
                         clear_chat_btn = gr.Button(value="Clear Chat", visible=True)
-                    chats_row = gr.Row(visible=True).style(equal_height=False)
-                    with chats_row:
-                        export_chats_btn = gr.Button(value="Export Chats")
-                        chats_file = gr.File(interactive=False, label="Download File")
-                    chats_row2 = gr.Row(visible=True).style(equal_height=False)
-                    with chats_row2:
+                        export_chats_btn = gr.Button(value="Export Chats to Download")
+                        remove_chat_btn = gr.Button(value="Remove Selected Chat", visible=True)
+                        add_to_chats_btn = gr.Button("Import Chats from Upload")
+                    with gr.Row():
+                        chats_file = gr.File(interactive=False, label="Download Exported Chats")
                         chatsup_output = gr.File(label="Upload Chat File(s)",
                                                  file_types=['.json'],
                                                  file_count='multiple',
                                                  elem_id="warning", elem_classes="feedback")
-                        add_to_chats_btn = gr.Button("Add File(s) to Chats")
                 with gr.TabItem("Data Source"):
                     langchain_readme = get_url('https://github.com/h2oai/h2ogpt/blob/main/README_LangChain.md',
                                                from_str=True)
@@ -275,8 +279,8 @@ body.dark{#warning {background-color: #555555};}
                             <p>
                             For more options see: {langchain_readme}""",
                             visible=kwargs['langchain_mode'] == 'Disabled', interactive=False)
-                    data_row = gr.Row(visible=kwargs['langchain_mode'] != 'Disabled')
-                    with data_row:
+                    data_row1 = gr.Row(visible=kwargs['langchain_mode'] != 'Disabled')
+                    with data_row1:
                         if is_hf:
                             # don't show 'wiki' since only usually useful for internal testing at moment
                             no_show_modes = ['Disabled', 'wiki']
@@ -292,77 +296,92 @@ body.dark{#warning {background-color: #555555};}
                         langchain_mode = gr.Radio(
                             [x for x in langchain_modes if x in allowed_modes and x not in no_show_modes],
                             value=kwargs['langchain_mode'],
-                            label="Data Source",
+                            label="Data Collection of Sources",
                             visible=kwargs['langchain_mode'] != 'Disabled')
+                    data_row2 = gr.Row(visible=kwargs['langchain_mode'] != 'Disabled')
+                    with data_row2:
+                        with gr.Column(scale=50):
+                            document_choice = gr.Dropdown(docs_state.value,
+                                                          label="Choose Subset of Doc(s) in Collection [click get to update]",
+                                                          value=docs_state.value[0],
+                                                          interactive=True,
+                                                          multiselect=True,
+                                                          )
+                        with gr.Row(visible=kwargs['langchain_mode'] != 'Disabled' and enable_sources_list):
+                            get_sources_btn = gr.Button(value="Get Sources",
+                                                        ).style(full_width=False, size='sm')
+                            show_sources_btn = gr.Button(value="Show Sources",
+                                                         ).style(full_width=False, size='sm')
 
-                        def upload_file(files, x):
-                            file_paths = [file.name for file in files]
-                            return files, file_paths
-
-                    upload_row = gr.Row(visible=kwargs['langchain_mode'] != 'Disabled' and allow_upload).style(
-                        equal_height=False)
                     # import control
                     if kwargs['langchain_mode'] != 'Disabled':
                         from gpt_langchain import file_types, have_arxiv
                     else:
                         have_arxiv = False
                         file_types = []
-                    with upload_row:
-                        file_types_str = '[' + ' '.join(file_types) + ']'
-                        fileup_output = gr.File(label=f'Upload {file_types_str}',
-                                                file_types=file_types,
-                                                file_count="multiple",
-                                                elem_id="warning", elem_classes="feedback")
-                        with gr.Row():
-                            upload_button = gr.UploadButton("Upload %s" % file_types_str,
-                                                            file_types=file_types,
-                                                            file_count="multiple",
-                                                            visible=False,
-                                                            )
-                            # add not visible until upload something
-                            with gr.Column():
-                                add_to_shared_db_btn = gr.Button("Add File(s) to Shared UserData DB",
-                                                                 visible=allow_upload_to_user_data)  # and False)
-                                add_to_my_db_btn = gr.Button("Add File(s) to Scratch MyData DB",
-                                                             visible=allow_upload_to_my_data)  # and False)
-                    url_row = gr.Row(
-                        visible=kwargs['langchain_mode'] != 'Disabled' and allow_upload and enable_url_upload).style(
+
+                    upload_row = gr.Row(visible=kwargs['langchain_mode'] != 'Disabled' and allow_upload).style(
                         equal_height=False)
-                    with url_row:
-                        url_label = 'URL (http/https) or ArXiv:' if have_arxiv else 'URL (http/https)'
-                        url_text = gr.Textbox(label=url_label, interactive=True)
+                    with upload_row:
                         with gr.Column():
-                            url_user_btn = gr.Button(value='Add URL content to Shared UserData DB',
-                                                     visible=allow_upload_to_user_data)
-                            url_my_btn = gr.Button(value='Add URL content to Scratch MyData DB',
-                                                   visible=allow_upload_to_my_data)
-                    text_row = gr.Row(
-                        visible=kwargs['langchain_mode'] != 'Disabled' and allow_upload and enable_text_upload).style(
+                            file_types_str = '[' + ' '.join(file_types) + ']'
+                            fileup_output = gr.File(label=f'Upload {file_types_str}',
+                                                    file_types=file_types,
+                                                    file_count="multiple",
+                                                    elem_id="warning", elem_classes="feedback")
+                            with gr.Row():
+                                add_to_shared_db_btn = gr.Button("Add File(s) to UserData",
+                                                                 visible=allow_upload_to_user_data, elem_id='small_btn')
+                                add_to_my_db_btn = gr.Button("Add File(s) to Scratch MyData",
+                                                             visible=allow_upload_to_my_data,
+                                                             elem_id='small_btn' if allow_upload_to_user_data else None,
+                                                             ).style(
+                                    size='sm' if not allow_upload_to_user_data else None)
+                        with gr.Column(
+                                visible=kwargs['langchain_mode'] != 'Disabled' and allow_upload and enable_url_upload):
+                            url_label = 'URL (http/https) or ArXiv:' if have_arxiv else 'URL (http/https)'
+                            url_text = gr.Textbox(label=url_label, interactive=True)
+                            with gr.Row():
+                                url_user_btn = gr.Button(value='Add URL content to Shared UserData',
+                                                         visible=allow_upload_to_user_data, elem_id='small_btn')
+                                url_my_btn = gr.Button(value='Add URL content to Scratch MyData',
+                                                       visible=allow_upload_to_my_data,
+                                                       elem_id='small_btn' if allow_upload_to_user_data else None,
+                                                       ).style(size='sm' if not allow_upload_to_user_data else None)
+                        with gr.Column(
+                                visible=kwargs['langchain_mode'] != 'Disabled' and allow_upload and enable_text_upload):
+                            user_text_text = gr.Textbox(label='Paste Text [Shift-Enter more lines]', interactive=True)
+                            with gr.Row():
+                                user_text_user_btn = gr.Button(value='Add Text to Shared UserData',
+                                                               visible=allow_upload_to_user_data,
+                                                               elem_id='small_btn')
+                                user_text_my_btn = gr.Button(value='Add Text to Scratch MyData',
+                                                             visible=allow_upload_to_my_data,
+                                                             elem_id='small_btn' if allow_upload_to_user_data else None,
+                                                             ).style(
+                                    size='sm' if not allow_upload_to_user_data else None)
+                        with gr.Column(visible=False):
+                            # WIP:
+                            with gr.Row(visible=False).style(equal_height=False):
+                                github_textbox = gr.Textbox(label="Github URL")
+                                with gr.Row(visible=True):
+                                    github_shared_btn = gr.Button(value="Add Github to Shared UserData",
+                                                                  visible=allow_upload_to_user_data,
+                                                                  elem_id='small_btn')
+                                    github_my_btn = gr.Button(value="Add Github to Scratch MyData",
+                                                              visible=allow_upload_to_my_data, elem_id='small_btn')
+                    sources_row3 = gr.Row(visible=kwargs['langchain_mode'] != 'Disabled' and enable_sources_list).style(
                         equal_height=False)
-                    with text_row:
-                        user_text_text = gr.Textbox(label='Paste Text', interactive=True)
-                        with gr.Column():
-                            user_text_user_btn = gr.Button(value='Add Text to Shared UserData DB',
-                                                           visible=allow_upload_to_user_data)
-                            user_text_my_btn = gr.Button(value='Add Text to Scratch MyData DB',
-                                                         visible=allow_upload_to_my_data)
-                    # WIP:
-                    with gr.Row(visible=False).style(equal_height=False):
-                        github_textbox = gr.Textbox(label="Github URL")
-                        with gr.Row(visible=True):
-                            github_shared_btn = gr.Button(value="Add Github to Shared UserData DB",
-                                                          visible=allow_upload_to_user_data)
-                            github_my_btn = gr.Button(value="Add Github to Scratch MyData DB",
-                                                      visible=allow_upload_to_my_data)
+                    with sources_row3:
+                        with gr.Column(scale=1):
+                            file_source = gr.File(interactive=False,
+                                                  label="Download File with Sources [click get to make file]")
+                        with gr.Column(scale=2):
+                            pass
                     sources_row = gr.Row(visible=kwargs['langchain_mode'] != 'Disabled' and enable_sources_list).style(
                         equal_height=False)
                     with sources_row:
                         sources_text = gr.HTML(label='Sources Added', interactive=False)
-                    sources_row2 = gr.Row(visible=kwargs['langchain_mode'] != 'Disabled' and enable_sources_list).style(
-                        equal_height=False)
-                    with sources_row2:
-                        get_sources_btn = gr.Button(value="Get Sources List for Selected DB")
-                        file_source = gr.File(interactive=False, label="Download File with list of Sources")
 
                 with gr.TabItem("Expert"):
                     with gr.Row():
@@ -545,14 +564,6 @@ body.dark{#warning {background-color: #555555};}
         def make_visible():
             return gr.update(visible=True)
 
-        # add itself to output to ensure shows working and can't click again
-        upload_button.upload(upload_file, inputs=[upload_button, fileup_output],
-                             outputs=[upload_button, fileup_output], queue=queue,
-                             api_name='upload_file' if allow_api else None) \
-            .then(make_add_visible, fileup_output, add_to_shared_db_btn, queue=queue) \
-            .then(make_add_visible, fileup_output, add_to_my_db_btn, queue=queue) \
-            .then(make_invisible, outputs=upload_button, queue=queue)
-
         # Add to UserData
         update_user_db_func = functools.partial(update_user_db, dbs=dbs, db_type=db_type, langchain_mode='UserData',
                                                 use_openai_embedding=use_openai_embedding,
@@ -623,8 +634,23 @@ body.dark{#warning {background-color: #555555};}
             .then(clear_textbox, outputs=user_text_text, queue=queue)
 
         get_sources1 = functools.partial(get_sources, dbs=dbs)
-        get_sources_btn.click(get_sources1, inputs=[my_db_state, langchain_mode], outputs=file_source, queue=queue,
-                              api_name='get_sources' if allow_api else None)
+
+        # if change collection source, must clear doc selections from it to avoid inconsistency
+        def clear_doc_choice():
+            return gr.Dropdown.update(choices=['All'], value=['All'])
+
+        langchain_mode.change(clear_doc_choice, inputs=None, outputs=document_choice)
+
+        def update_dropdown(x):
+            return gr.Dropdown.update(choices=x, value='All')
+
+        show_sources1 = functools.partial(get_source_files_given_langchain_mode, dbs=dbs)
+        get_sources_btn.click(get_sources1, inputs=[my_db_state, langchain_mode], outputs=[file_source, docs_state],
+                              queue=queue,
+                              api_name='get_sources' if allow_api else None) \
+            .then(fn=update_dropdown, inputs=docs_state, outputs=document_choice)
+        # show button, else only show when add.  Could add to above get_sources for download/dropdown, but bit much maybe
+        show_sources_btn.click(fn=show_sources1, inputs=[my_db_state, langchain_mode], outputs=sources_text)
 
         def check_admin_pass(x):
             return gr.update(visible=x == admin_pass)
@@ -818,6 +844,11 @@ body.dark{#warning {background-color: #555555};}
             my_db_state1 = args_list[-2]
             history = args_list[-1]
 
+            if model_state1[0] is None or model_state1[0] == no_model_str:
+                history = []
+                yield history, ''
+                return
+
             args_list = args_list[:-3]  # only keep rest needed for evaluate()
             langchain_mode1 = args_list[eval_func_param_names.index('langchain_mode')]
             if retry and history:
@@ -827,13 +858,19 @@ body.dark{#warning {background-color: #555555};}
                     args_list[eval_func_param_names.index('do_sample')] = True
             if not history:
                 print("No history", flush=True)
-                history = [['', None]]
+                history = []
                 yield history, ''
                 return
             # ensure output will be unique to models
             _, _, _, max_prompt_length = get_cutoffs(is_low_mem, for_context=True)
             history = copy.deepcopy(history)
             instruction1 = history[-1][0]
+            if not instruction1:
+                # reject empty query, can sometimes go nuts
+                history = []
+                yield history, ''
+                return
+
             context1 = ''
             if max_prompt_length is not None and langchain_mode1 not in ['LLM']:
                 prompt_type1 = args_list[eval_func_param_names.index('prompt_type')]
@@ -867,10 +904,6 @@ body.dark{#warning {background-color: #555555};}
                     context1 += chat_sep  # ensure if terminates abruptly, then human continues on next line
             args_list[0] = instruction1  # override original instruction with history from user
             args_list[2] = context1
-            if model_state1[0] is None or model_state1[0] == no_model_str:
-                history = [['', None]]
-                yield history, ''
-                return
             fun1 = partial(evaluate,
                            model_state1,
                            my_db_state1,
@@ -1086,10 +1119,14 @@ body.dark{#warning {background-color: #555555};}
                                api_name='export_chats' if allow_api else None)
 
         def add_chats_from_file(file, chat_state1, add_btn):
+            if not file:
+                return chat_state1, add_btn
             if isinstance(file, str):
                 files = [file]
             else:
                 files = file
+            if not files:
+                return chat_state1, add_btn
             for file1 in files:
                 try:
                     if hasattr(file1, 'name'):
@@ -1350,22 +1387,28 @@ def get_inputs_list(inputs_dict, model_lower):
 def get_sources(db1, langchain_mode, dbs=None):
     if langchain_mode in ['ChatLLM', 'LLM']:
         source_files_added = "NA"
+        source_list = []
     elif langchain_mode in ['wiki_full']:
         source_files_added = "Not showing wiki_full, takes about 20 seconds and makes 4MB file." \
                              "  Ask jon.mckinney@h2o.ai for file if required."
+        source_list = []
     elif langchain_mode == 'MyData' and len(db1) > 0 and db1[0] is not None:
         db_get = db1[0].get()
-        source_files_added = '\n'.join(sorted(set([x['source'] for x in db_get['metadatas']])))
+        source_list = sorted(set([x['source'] for x in db_get['metadatas']]))
+        source_files_added = '\n'.join(source_list)
     elif langchain_mode in dbs and dbs[langchain_mode] is not None:
         db1 = dbs[langchain_mode]
         db_get = db1.get()
-        source_files_added = '\n'.join(sorted(set([x['source'] for x in db_get['metadatas']])))
+        source_list = sorted(set([x['source'] for x in db_get['metadatas']]))
+        source_files_added = '\n'.join(source_list)
     else:
+        source_list = []
         source_files_added = "None"
     sources_file = 'sources_%s_%s' % (langchain_mode, str(uuid.uuid4()))
     with open(sources_file, "wt") as f:
         f.write(source_files_added)
-    return sources_file
+    source_list = ['All'] + source_list
+    return sources_file, source_list
 
 
 def update_user_db(file, db1, x, y, *args, dbs=None, langchain_mode='UserData', **kwargs):
@@ -1465,6 +1508,20 @@ def _update_user_db(file, db1, x, y, dbs=None, db_type=None, langchain_mode='Use
             return x, y, source_files_added
 
 
+def get_source_files_given_langchain_mode(db1, langchain_mode='UserData', dbs=None):
+    with filelock.FileLock("db_%s.lock" % langchain_mode.replace(' ', '_')):
+        if langchain_mode in ['wiki_full']:
+            # NOTE: avoid showing full wiki.  Takes about 30 seconds over about 90k entries, but not useful for now
+            db = None
+        elif langchain_mode == 'MyData' and len(db1) > 0 and db1[0] is not None:
+            db = db1[0]
+        elif langchain_mode in dbs and dbs[langchain_mode] is not None:
+            db = dbs[langchain_mode]
+        else:
+            db = None
+    return get_source_files(db, exceptions=None)
+
+
 def get_source_files(db, exceptions=None):
     if exceptions is None:
         exceptions = []

prompter.py

@@ -56,6 +56,8 @@ prompt_type_to_model_name = {
         'h2oai/h2ogpt-oasst1-512-20b',
         'h2oai/h2ogpt-oig-oasst1-256-6_9b',
         'h2oai/h2ogpt-oig-oasst1-512-6_9b',
+        'h2oai/h2ogpt-oig-oasst1-256-6.9b',  # legacy
+        'h2oai/h2ogpt-oig-oasst1-512-6.9b',  # legacy
         'h2oai/h2ogpt-research-oasst1-512-30b',  # private
     ],
     'dai_faq': [],

utils.py

@@ -148,7 +148,7 @@ def _zip_data(root_dirs=None, zip_file=None, base_dir='./'):
         host_name = os.getenv('HF_HOSTNAME', 'emptyhost')
         zip_file = "data_%s_%s.zip" % (datetime_str, host_name)
     assert root_dirs is not None
-    if not os.path.isdir(os.path.dirname(zip_file)):
+    if not os.path.isdir(os.path.dirname(zip_file)) and os.path.dirname(zip_file):
         os.makedirs(os.path.dirname(zip_file), exist_ok=True)
     with zipfile.ZipFile(zip_file, "w") as expt_zip:
         for root_dir in root_dirs: