Use 4bit model

app.py

@@ -46,7 +46,7 @@ def generate_text(prompt, example):
 
 
 parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-parser.add_argument("--model-filename", help="Path to the model file", default="ggml-model-Q8_0.gguf")
+parser.add_argument("--model-filename", help="Path to the model file", default="ggml-model-Q4_0.gguf")
 parser.add_argument("--model-local", help="Path to the model file")
 parser.add_argument("--gpu", help="How many GPU layers to use", default=0, type=int)
 parser.add_argument("--ctx", help="How many context tokens to use", default=1024, type=int)