Update app.py

app.py

@@ -39,8 +39,8 @@ app.logger.setLevel(logging.DEBUG)  # Set the desired logging level
 #repo_name = "IlyaGusev/saiga2_13b_gguf"
 #model_name = "model-q4_K.gguf"
 
-repo_name = "IlyaGusev/saiga2_70b_gguf"
-model_name = "ggml-model-q4_1.gguf"
+repo_name = "dreamgen/opus-v0-70b-gguf"
+model_name = "dreamgen-opus-v0-70b-Q4_K_M.gguf"
 
 #repo_name = "IlyaGusev/saiga2_7b_gguf"
 #model_name = "model-q4_K.gguf"
@@ -55,7 +55,7 @@ model_path = snapshot_download(repo_id=repo_name, allow_patterns=model_name) + '
 app.logger.info('Model path: ' + model_path)
 
 DATASET_REPO_URL = "https://huggingface.co/datasets/muryshev/saiga-chat"
-DATA_FILENAME = "data-saiga-cuda.xml"
+DATA_FILENAME = "opus-v0-70b.xml"
 DATA_FILE = os.path.join("dataset", DATA_FILENAME)
 
 HF_TOKEN = os.environ.get("HF_TOKEN")
@@ -190,46 +190,6 @@ def generate_unknown_response():
         app.logger.info('payload empty')
 
     return Response('What do you want?', content_type='text/plain')
-    
-@app.route('/search_request', methods=['POST'])
-def generate_search_request():
-    global stop_generation
-    stop_generation = True
-    model.reset()
-    
-    
-    data = request.get_json()
-    app.logger.info(data)
-    user_query = data.get("query", "")
-    preprompt = data.get("preprompt", "")
-    parameters = data.get("parameters", {})
-    
-    # Extract parameters from the request
-    temperature = parameters.get("temperature", 0.01)
-    truncate = parameters.get("truncate", 1000)
-    max_new_tokens = parameters.get("max_new_tokens", 1024)
-    top_p = parameters.get("top_p", 0.85)
-    repetition_penalty = parameters.get("repetition_penalty", 1.2)
-    top_k = parameters.get("top_k", 30)
-    return_full_text = parameters.get("return_full_text", False)
-
-    
-    
-    tokens = get_system_tokens_for_preprompt(model, preprompt)
-    tokens.append(LINEBREAK_TOKEN)        
-    
-    tokens = get_message_tokens(model=model, role="user", content=user_query[:200]) + [model.token_bos(), BOT_TOKEN, LINEBREAK_TOKEN]
-    stop_generation = False
-    generator = model.generate(
-        tokens,
-        top_k=top_k,
-        top_p=top_p,
-        temp=temperature,
-        repeat_penalty=repetition_penalty
-    )
-
-    # Use Response to stream tokens
-    return Response(generate_tokens(model, generator), content_type='text/plain', status=200, direct_passthrough=True)
 
 response_tokens = bytearray()
 def generate_and_log_tokens(user_request, model, generator):
@@ -245,57 +205,13 @@ def generate_and_log_tokens(user_request, model, generator):
 @app.route('/', methods=['POST'])
 def generate_response():
     global stop_generation
-    stop_generation = True
-    model.reset()
-    
-    data = request.get_json()
-    app.logger.info(data)
-    messages = data.get("messages", [])
-    preprompt = data.get("preprompt", "")
-    parameters = data.get("parameters", {})
-    
-    # Extract parameters from the request
-    temperature = parameters.get("temperature", 0.01)
-    truncate = parameters.get("truncate", 1000)
-    max_new_tokens = parameters.get("max_new_tokens", 1024)
-    top_p = parameters.get("top_p", 0.85)
-    repetition_penalty = parameters.get("repetition_penalty", 1.2)
-    top_k = parameters.get("top_k", 30)
-    return_full_text = parameters.get("return_full_text", False)      
-    
-    tokens = []
-    
-    for message in messages:
-        if message.get("from") == "assistant":
-            message_tokens = get_message_tokens(model=model, role="bot", content=message.get("content", ""))
-        elif message.get("from") == "system":
-            message_tokens = get_message_tokens(model=model, role="system", content=message.get("content", ""))
-        else:
-            message_tokens = get_message_tokens(model=model, role="user", content=message.get("content", ""))
-    
-        tokens.extend(message_tokens)
-            
-    tokens.extend([model.token_bos(), BOT_TOKEN, LINEBREAK_TOKEN])
-
-    
-    app.logger.info('Prompt:')
-    user_request = model.detokenize(tokens[:CONTEXT_SIZE]).decode("utf-8", errors="ignore")
-    app.logger.info(user_request)
-
-    stop_generation = False
-    app.logger.info('Generate started')
+    raw_content = request.data
+    tokens = model.tokenize(raw_content))
     generator = model.generate(
-        tokens[:CONTEXT_SIZE],
-        top_k=top_k,
-        top_p=top_p,
-        temp=temperature,
-        repeat_penalty=repetition_penalty
+        tokens[:CONTEXT_SIZE]
     )
     app.logger.info('Generator created')
 
-
-    
-
     # Use Response to stream tokens
     return Response(generate_and_log_tokens(user_request, model, generator), content_type='text/plain', status=200, direct_passthrough=True)