LLM invocation works

PersonalGPT.xcodeproj/project.pbxproj

@@ -16,6 +16,8 @@
 		48055F802A0E77B40057C3BE /* ToolTip.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48055F7F2A0E77B40057C3BE /* ToolTip.swift */; };
 		48055F852A0E93A70057C3BE /* ggml-RedPajama-INCITE-Chat-3B-v1-q5_1.bin in Resources */ = {isa = PBXBuildFile; fileRef = 48055F842A0E93A60057C3BE /* ggml-RedPajama-INCITE-Chat-3B-v1-q5_1.bin */; };
 		48055F8B2A0E99040057C3BE /* ChatBackend.mm in Sources */ = {isa = PBXBuildFile; fileRef = 48055F8A2A0E99040057C3BE /* ChatBackend.mm */; };
+		48055F8D2A0F96E10057C3BE /* TypingIndicator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48055F8C2A0F96E10057C3BE /* TypingIndicator.swift */; };
+		48055F8F2A0F998F0057C3BE /* ChatBubble.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48055F8E2A0F998F0057C3BE /* ChatBubble.swift */; };
 		48B929A829ED80EA00607E43 /* PersonalGPTApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48B929A729ED80EA00607E43 /* PersonalGPTApp.swift */; };
 		48B929AA29ED80EA00607E43 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48B929A929ED80EA00607E43 /* ContentView.swift */; };
 		48B929AC29ED80EC00607E43 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 48B929AB29ED80EC00607E43 /* Assets.xcassets */; };
@@ -57,6 +59,8 @@
 		48055F842A0E93A60057C3BE /* ggml-RedPajama-INCITE-Chat-3B-v1-q5_1.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; name = "ggml-RedPajama-INCITE-Chat-3B-v1-q5_1.bin"; path = "../../redpajama.cpp/examples/redpajama/models/pythia/ggml-RedPajama-INCITE-Chat-3B-v1-q5_1.bin"; sourceTree = "<group>"; };
 		48055F892A0E97670057C3BE /* PersonalGPT-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "PersonalGPT-Bridging-Header.h"; sourceTree = "<group>"; };
 		48055F8A2A0E99040057C3BE /* ChatBackend.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = ChatBackend.mm; sourceTree = "<group>"; };
+		48055F8C2A0F96E10057C3BE /* TypingIndicator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TypingIndicator.swift; sourceTree = "<group>"; };
+		48055F8E2A0F998F0057C3BE /* ChatBubble.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatBubble.swift; sourceTree = "<group>"; };
 		48B929A429ED80EA00607E43 /* PersonalGPT.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = PersonalGPT.app; sourceTree = BUILT_PRODUCTS_DIR; };
 		48B929A729ED80EA00607E43 /* PersonalGPTApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PersonalGPTApp.swift; sourceTree = "<group>"; };
 		48B929A929ED80EA00607E43 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
@@ -153,6 +157,8 @@
 				48055F7B2A0D42CB0057C3BE /* ChatState.swift */,
 				48055F7D2A0D43620057C3BE /* ThreadWorker.swift */,
 				48055F7F2A0E77B40057C3BE /* ToolTip.swift */,
+				48055F8C2A0F96E10057C3BE /* TypingIndicator.swift */,
+				48055F8E2A0F998F0057C3BE /* ChatBubble.swift */,
 			);
 			path = PersonalGPT;
 			sourceTree = "<group>";
@@ -325,6 +331,8 @@
 				48055F762A0D2B5C0057C3BE /* common-gptneox.cpp in Sources */,
 				48B929AA29ED80EA00607E43 /* ContentView.swift in Sources */,
 				48055F732A0D2B070057C3BE /* ggml.c in Sources */,
+				48055F8F2A0F998F0057C3BE /* ChatBubble.swift in Sources */,
+				48055F8D2A0F96E10057C3BE /* TypingIndicator.swift in Sources */,
 				48B929A829ED80EA00607E43 /* PersonalGPTApp.swift in Sources */,
 				48055F8B2A0E99040057C3BE /* ChatBackend.mm in Sources */,
 				48055F7E2A0D43620057C3BE /* ThreadWorker.swift in Sources */,

PersonalGPT/ChatBubble.swift

@@ -0,0 +1,108 @@
+//
+//  ChatBubble.swift
+//  PrivateAIChat
+//
+//  Created by jrb16 on 13/05/2023.
+//
+
+import SwiftUI
+
+struct ChatBubbleShape: Shape {
+    enum Direction {
+        case left
+        case right
+    }
+
+    let direction: Direction
+
+    func path(in rect: CGRect) -> Path {
+        return (direction == .left) ? leftBubble(in: rect) : rightBubble(in: rect)
+    }
+
+    private func leftBubble(in rect: CGRect) -> Path {
+        let width = rect.width
+        let height = rect.height
+        let path = Path { p in
+            p.move(to: CGPoint(x: 25, y: height))
+            p.addLine(to: CGPoint(x: width - 20, y: height))
+            p.addCurve(to: CGPoint(x: width, y: height - 20),
+                       control1: CGPoint(x: width - 8, y: height),
+                       control2: CGPoint(x: width, y: height - 8))
+            p.addLine(to: CGPoint(x: width, y: 20))
+            p.addCurve(to: CGPoint(x: width - 20, y: 0),
+                       control1: CGPoint(x: width, y: 8),
+                       control2: CGPoint(x: width - 8, y: 0))
+            p.addLine(to: CGPoint(x: 21, y: 0))
+            p.addCurve(to: CGPoint(x: 4, y: 20),
+                       control1: CGPoint(x: 12, y: 0),
+                       control2: CGPoint(x: 4, y: 8))
+            p.addLine(to: CGPoint(x: 4, y: height - 11))
+            p.addCurve(to: CGPoint(x: 0, y: height),
+                       control1: CGPoint(x: 4, y: height - 1),
+                       control2: CGPoint(x: 0, y: height))
+            p.addLine(to: CGPoint(x: -0.05, y: height - 0.01))
+            p.addCurve(to: CGPoint(x: 11.0, y: height - 4.0),
+                       control1: CGPoint(x: 4.0, y: height + 0.5),
+                       control2: CGPoint(x: 8, y: height - 1))
+            p.addCurve(to: CGPoint(x: 25, y: height),
+                       control1: CGPoint(x: 16, y: height),
+                       control2: CGPoint(x: 20, y: height))
+        }
+        return path
+    }
+
+    private func rightBubble(in rect: CGRect) -> Path {
+        let width = rect.width
+        let height = rect.height
+        let path = Path { p in
+            p.move(to: CGPoint(x: 25, y: height))
+            p.addLine(to: CGPoint(x: 20, y: height))
+            p.addCurve(to: CGPoint(x: 0, y: height - 20),
+                       control1: CGPoint(x: 8, y: height),
+                       control2: CGPoint(x: 0, y: height - 8))
+            p.addLine(to: CGPoint(x: 0, y: 20))
+            p.addCurve(to: CGPoint(x: 20, y: 0),
+                       control1: CGPoint(x: 0, y: 8),
+                       control2: CGPoint(x: 8, y: 0))
+            p.addLine(to: CGPoint(x: width - 21, y: 0))
+            p.addCurve(to: CGPoint(x: width - 4, y: 20),
+                       control1: CGPoint(x: width - 12, y: 0),
+                       control2: CGPoint(x: width - 4, y: 8))
+            p.addLine(to: CGPoint(x: width - 4, y: height - 11))
+            p.addCurve(to: CGPoint(x: width, y: height),
+                       control1: CGPoint(x: width - 4, y: height - 1),
+                       control2: CGPoint(x: width, y: height))
+            p.addLine(to: CGPoint(x: width + 0.05, y: height - 0.01))
+            p.addCurve(to: CGPoint(x: width - 11, y: height - 4),
+                       control1: CGPoint(x: width - 4, y: height + 0.5),
+                       control2: CGPoint(x: width - 8, y: height - 1))
+            p.addCurve(to: CGPoint(x: width - 25, y: height),
+                       control1: CGPoint(x: width - 16, y: height),
+                       control2: CGPoint(x: width - 20, y: height))
+        }
+        return path
+    }
+}
+
+struct ChatBubble<Content>: View where Content: View {
+    let direction: ChatBubbleShape.Direction
+    let content: () -> Content
+    init(direction: ChatBubbleShape.Direction, @ViewBuilder content: @escaping () -> Content) {
+        self.content = content
+        self.direction = direction
+    }
+
+    var body: some View {
+        HStack {
+            if direction == .right {
+                Spacer()
+            }
+            content()
+                .clipShape(ChatBubbleShape(direction: direction))
+            if direction == .left {
+                Spacer()
+            }
+        }.padding([(direction == .left) ? .leading : .trailing, .top, .bottom], 20)
+            .padding((direction == .right) ? .leading : .trailing, 50)
+    }
+}

PersonalGPT/ChatState.swift

@@ -10,6 +10,7 @@ import Foundation
 enum MessageRole {
     case user
     case bot
+    case typing
 }
 
 struct MessageData: Hashable, Identifiable {
@@ -80,8 +81,6 @@ class ChatState: ObservableObject {
             backend.initialize()
             if (!backend.initialized) {
                 self.updateReply(role: MessageRole.bot, message: "[System] Could not initialize")
-            } else {
-                self.updateReply(role: MessageRole.bot, message: "[System] Ready to chat")
             }
             self.commitReply()
             self.markFinish()
@@ -147,8 +146,7 @@ class ChatState: ObservableObject {
         }
         inProgress = true
         stopRequested = false
-        // self.backendGenerate(prompt: prompt)
-        dummyGenerate(prompt: prompt)
+        self.backendGenerate(prompt: prompt)
     }
 
     func requestStop() {
@@ -175,12 +173,12 @@ class ChatState: ObservableObject {
         requestedReset = true
 
         threadWorker.push {
-            // self.backend.reset()
+             self.backend.reset()
             DispatchQueue.main.sync {
+                self.inProgress = false
                 self.messages = [MessageData]()
                 self.infoText = ""
                 self.unfinishedRespondMessage = ""
-                self.inProgress = false
                 self.requestedReset = false
             }
         }

PersonalGPT/ContentView.swift

@@ -43,6 +43,9 @@ struct ContentView: View {
                         }
                     }.rotationEffect(.radians(.pi))
                         .scaleEffect(x: -1, y: 1, anchor: .center)
+                    if (state.inProgress && state.unfinishedRespondMessage == "") {
+                        MessageView(role:.typing)
+                    }
                 }
             }
             HStack {

PersonalGPT/MessageView.swift

@@ -7,109 +7,9 @@
 
 import SwiftUI
 
-struct ChatBubbleShape: Shape {
-    enum Direction {
-        case left
-        case right
-    }
-
-    let direction: Direction
-
-    func path(in rect: CGRect) -> Path {
-        return (direction == .left) ? leftBubble(in: rect) : rightBubble(in: rect)
-    }
-
-    private func leftBubble(in rect: CGRect) -> Path {
-        let width = rect.width
-        let height = rect.height
-        let path = Path { p in
-            p.move(to: CGPoint(x: 25, y: height))
-            p.addLine(to: CGPoint(x: width - 20, y: height))
-            p.addCurve(to: CGPoint(x: width, y: height - 20),
-                       control1: CGPoint(x: width - 8, y: height),
-                       control2: CGPoint(x: width, y: height - 8))
-            p.addLine(to: CGPoint(x: width, y: 20))
-            p.addCurve(to: CGPoint(x: width - 20, y: 0),
-                       control1: CGPoint(x: width, y: 8),
-                       control2: CGPoint(x: width - 8, y: 0))
-            p.addLine(to: CGPoint(x: 21, y: 0))
-            p.addCurve(to: CGPoint(x: 4, y: 20),
-                       control1: CGPoint(x: 12, y: 0),
-                       control2: CGPoint(x: 4, y: 8))
-            p.addLine(to: CGPoint(x: 4, y: height - 11))
-            p.addCurve(to: CGPoint(x: 0, y: height),
-                       control1: CGPoint(x: 4, y: height - 1),
-                       control2: CGPoint(x: 0, y: height))
-            p.addLine(to: CGPoint(x: -0.05, y: height - 0.01))
-            p.addCurve(to: CGPoint(x: 11.0, y: height - 4.0),
-                       control1: CGPoint(x: 4.0, y: height + 0.5),
-                       control2: CGPoint(x: 8, y: height - 1))
-            p.addCurve(to: CGPoint(x: 25, y: height),
-                       control1: CGPoint(x: 16, y: height),
-                       control2: CGPoint(x: 20, y: height))
-        }
-        return path
-    }
-
-    private func rightBubble(in rect: CGRect) -> Path {
-        let width = rect.width
-        let height = rect.height
-        let path = Path { p in
-            p.move(to: CGPoint(x: 25, y: height))
-            p.addLine(to: CGPoint(x: 20, y: height))
-            p.addCurve(to: CGPoint(x: 0, y: height - 20),
-                       control1: CGPoint(x: 8, y: height),
-                       control2: CGPoint(x: 0, y: height - 8))
-            p.addLine(to: CGPoint(x: 0, y: 20))
-            p.addCurve(to: CGPoint(x: 20, y: 0),
-                       control1: CGPoint(x: 0, y: 8),
-                       control2: CGPoint(x: 8, y: 0))
-            p.addLine(to: CGPoint(x: width - 21, y: 0))
-            p.addCurve(to: CGPoint(x: width - 4, y: 20),
-                       control1: CGPoint(x: width - 12, y: 0),
-                       control2: CGPoint(x: width - 4, y: 8))
-            p.addLine(to: CGPoint(x: width - 4, y: height - 11))
-            p.addCurve(to: CGPoint(x: width, y: height),
-                       control1: CGPoint(x: width - 4, y: height - 1),
-                       control2: CGPoint(x: width, y: height))
-            p.addLine(to: CGPoint(x: width + 0.05, y: height - 0.01))
-            p.addCurve(to: CGPoint(x: width - 11, y: height - 4),
-                       control1: CGPoint(x: width - 4, y: height + 0.5),
-                       control2: CGPoint(x: width - 8, y: height - 1))
-            p.addCurve(to: CGPoint(x: width - 25, y: height),
-                       control1: CGPoint(x: width - 16, y: height),
-                       control2: CGPoint(x: width - 20, y: height))
-        }
-        return path
-    }
-}
-
-struct ChatBubble<Content>: View where Content: View {
-    let direction: ChatBubbleShape.Direction
-    let content: () -> Content
-    init(direction: ChatBubbleShape.Direction, @ViewBuilder content: @escaping () -> Content) {
-        self.content = content
-        self.direction = direction
-    }
-
-    var body: some View {
-        HStack {
-            if direction == .right {
-                Spacer()
-            }
-            content()
-                .clipShape(ChatBubbleShape(direction: direction))
-            if direction == .left {
-                Spacer()
-            }
-        }.padding([(direction == .left) ? .leading : .trailing, .top, .bottom], 20)
-            .padding((direction == .right) ? .leading : .trailing, 50)
-    }
-}
-
 struct MessageView: View {
     var role: MessageRole
-    var message: String
+    var message: String = ""
 
     var body: some View {
         #if os(macOS)
@@ -128,11 +28,16 @@ struct MessageView: View {
                 Spacer()
             }
             ChatBubble(direction: direction) {
-                Text(message)
-                    .padding(10)
-                    .foregroundColor(textColor)
-                    .background(backgroundColor)
-                    .textSelection(.enabled)
+                if (role != .typing) {
+                    Text(message)
+                        .padding(10)
+                        .foregroundColor(textColor)
+                        .background(backgroundColor)
+                        .textSelection(.enabled)
+                } else {
+                    TypingIndicator()
+                        .frame(width: 70, height: 30)
+                }
             }
             if role != MessageRole.user {
                 Spacer()
@@ -151,7 +56,8 @@ struct MessageView_Previews: PreviewProvider {
                 MessageView(role: MessageRole.bot, message: "Commercial airline flights typically have an altitude between 8,000 and 12,000 feet above sea level.")
                 MessageView(role: MessageRole.user, message: "Write a haiku about the fear of flying")
                 MessageView(role: MessageRole.bot, message: "The fear of flying\nThe fear is real\nThe fear takes my breath away\nThe fear is gone")
+                MessageView(role: MessageRole.typing)
             }
-        }.previewLayout(.fixed(width: 512, height: 512))
+        }.previewLayout(.fixed(width: 512, height: 768))
     }
 }

PersonalGPT/TypingIndicator.swift

@@ -0,0 +1,64 @@
+//
+//  TypingIndicator.swift
+//  PrivateAIChat
+//
+//  Created by jrb16 on 13/05/2023.
+//
+
+import SwiftUI
+
+struct TypingIndicator: View {
+    static let animationDelay = 0.2
+    let  timer = Timer.publish(every: animationDelay, on: .main, in: .common).autoconnect()
+    
+    @State var count: Int = 0
+    @State var slowCount: Int = 0
+    @State var color1: Color = Color(.gray)
+    @State var color2: Color = Color(.gray)
+    @State var color3: Color = Color(.gray)
+    
+    var body: some View {
+        ZStack {
+            HStack(spacing: 5) {
+                Circle()
+                    .offset(y: count == 1 ? 5 : 0)
+                    .foregroundColor(color1)
+                Circle()
+                    .offset(y: count == 2 ? 5 : 0)
+                    .foregroundColor(color2)
+                Circle()
+                    .offset(y: count == 3 ? 5 : 0)
+                    .foregroundColor(color3)
+            }
+            .padding(10)
+            .background(Color.white, in: RoundedRectangle(cornerRadius: 10))
+        }
+        .onReceive(timer, perform:  { _ in
+            withAnimation(.easeInOut(duration: TypingIndicator.animationDelay)) {
+                count = (count + 1) %  4
+                if (count % 2 == 0) {
+                    slowCount = (slowCount + 1) % 4
+                }
+                if slowCount == 1 {
+                    self.color1 = Color(nsColor: .lightGray)
+                    self.color2 = Color(nsColor: .gray)
+                    self.color3 = Color(nsColor: .darkGray)
+                } else if slowCount == 2 {
+                    self.color1 = Color(nsColor: .darkGray)
+                    self.color2 = Color(nsColor: .lightGray)
+                    self.color3 = Color(nsColor: .gray)
+                } else {
+                    self.color1 = Color(nsColor: .gray)
+                    self.color2 = Color(nsColor: .darkGray)
+                    self.color3 = Color(nsColor: .lightGray)
+                }
+            }
+        })
+    }
+}
+
+struct TypingIndicator_Previews: PreviewProvider {
+    static var previews: some View {
+        TypingIndicator()
+    }
+}

redpajama/ChatBackend.mm

@@ -7,10 +7,14 @@
 
 #import <Foundation/Foundation.h>
 #include "PersonalGPT-Bridging-Header.h"
-#include "gptneox.h"
+
+#define GGML_USE_ACCELERATE
 #include "ggml.h"
+#include "gptneox.h"
 #include "common-gptneox.h"
 
+#include <sstream>
+
 gptneox_context **g_ctx;
 
 class CPPChatBackend {
@@ -29,7 +33,7 @@ public:
         return instance;
     }
     
-    int encode(std::string& prompt) {
+    int encode(std::string prompt) {
         // Tokenize prompt with RedPajama special tokens
 
         auto prompt_embd = ::gptneox_tokenize(ctx, prompt, false);
@@ -52,7 +56,6 @@ public:
         past.push_back(embd_inp);
 
         const int n_ctx = gptneox_n_ctx(ctx);
-        const int n_vocab = gptneox_n_vocab(ctx);
 
         // How many tokens to generate - check if there's space in context for at least one token (or batch size tokens?)
         auto inp_size = embd_inp.size();
@@ -84,14 +87,227 @@ public:
             input_i += n_eval;
             n_past += n_eval;
         }
+        
+        // Clear the output text vector
+        output_text_vec.clear();
+        output_enabled = true;
+
+        return 0;
+    }
+    
+    int decode() {
+        const std::vector<std::string>::size_type output_text_vec_size = output_text_vec.size();
+        
+        const int n_ctx = gptneox_n_ctx(ctx);
+        const int n_vocab = gptneox_n_vocab(ctx);
+
+        const float   temp            = params.temp;
+        const int32_t top_k           = params.top_k <= 0 ? gptneox_n_vocab(ctx) : params.top_k;
+        const float   top_p           = params.top_p;
+        const float   tfs_z           = params.tfs_z;
+        const float   typical_p       = params.typical_p;
+        const int32_t repeat_last_n   = params.repeat_last_n < 0 ? n_ctx : params.repeat_last_n;
+        const float   repeat_penalty  = params.repeat_penalty;
+        const float   alpha_presence  = params.presence_penalty;
+        const float   alpha_frequency = params.frequency_penalty;
+        const int     mirostat        = params.mirostat;
+        const float   mirostat_tau    = params.mirostat_tau;
+        const float   mirostat_eta    = params.mirostat_eta;
+        const bool    penalize_nl     = params.penalize_nl;
+
+        // Loop
+        while (output_enabled) {
+            // Get token
+            gptneox_token id = 0;
+
+            {
+                auto logits = gptneox_get_logits(ctx);
+
+                // Apply params.logit_bias map
+                for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) {
+                    logits[it->first] += it->second;
+                }
+
+                // Let's add some custom logit biases that will always help
+                gptneox_token backslash_token = gptneox_str_to_token(ctx, "\\");
+                logits[backslash_token] = -INFINITY;
+
+                std::vector<gptneox_token_data> candidates;
+                candidates.reserve(n_vocab);
+                for (gptneox_token token_id = 0; token_id < n_vocab; token_id++) {
+                    candidates.emplace_back(gptneox_token_data{token_id, logits[token_id], 0.0f});
+                }
+
+                gptneox_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
+
+                // Apply penalties
+                gptneox_token nl_token = gptneox_str_to_token(ctx, "\n");
+                float nl_logit = logits[nl_token];
+                auto last_n_repeat = std::min(std::min((int)last_n_tokens.size(), repeat_last_n), n_ctx);
+                gptneox_sample_repetition_penalty(ctx, &candidates_p,
+                    last_n_tokens.data() + last_n_tokens.size() - last_n_repeat,
+                    last_n_repeat, repeat_penalty);
+                gptneox_sample_frequency_and_presence_penalties(ctx, &candidates_p,
+                    last_n_tokens.data() + last_n_tokens.size() - last_n_repeat,
+                    last_n_repeat, alpha_frequency, alpha_presence);
+                if (!penalize_nl) {
+                    logits[nl_token] = nl_logit;
+                }
+
+                if (temp <= 0) {
+                    // Greedy sampling
+                    id = gptneox_sample_token_greedy(ctx, &candidates_p);
+                } else {
+                    if (mirostat == 1) {
+                        static float mirostat_mu = 2.0f * mirostat_tau;
+                        const int mirostat_m = 100;
+                        gptneox_sample_temperature(ctx, &candidates_p, temp);
+                        id = gptneox_sample_token_mirostat(ctx, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu);
+                    } else if (mirostat == 2) {
+                        static float mirostat_mu = 2.0f * mirostat_tau;
+                        gptneox_sample_temperature(ctx, &candidates_p, temp);
+                        id = gptneox_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);
+                    } else {
+                        // Temperature sampling
+                        gptneox_sample_top_k(ctx, &candidates_p, top_k, 1);
+                        gptneox_sample_tail_free(ctx, &candidates_p, tfs_z, 1);
+                        gptneox_sample_typical(ctx, &candidates_p, typical_p, 1);
+                        gptneox_sample_top_p(ctx, &candidates_p, top_p, 1);
+                        gptneox_sample_temperature(ctx, &candidates_p, temp);
+                        id = gptneox_sample_token(ctx, &candidates_p);
+                    }
+                }
+            }
+
+            // Add output to array
+            output.push_back(id);
+            // Repeat tokens update
+            last_n_tokens.push_back(id);
+            if (last_n_tokens.size() > (std::vector<int>::size_type)params.repeat_last_n) {
+                last_n_tokens.erase(last_n_tokens.begin());
+            }
+            // Redpajama: check if the interactive is done.
+            //std::cout<<" last_n_tokens.size: "<< last_n_tokens[0] <<" "<< last_n_tokens[1] <<" "<< last_n_tokens[2] << std::endl;
+            if (last_n_tokens.size()==3 && last_n_tokens[0]==gptneox_str_to_token(ctx, "<")
+                && last_n_tokens[1]==gptneox_str_to_token(ctx, "human") && last_n_tokens[2]==gptneox_str_to_token(ctx, ">:")) {
+                output_enabled = false;
+                continue;
+            }
+            // Check for eos - end early - check eos before bos in case they are the same
+            if (id == gptneox_token_eos()) {
+                output_enabled = false;
+                continue;
+            }
+            // Check for bos - skip callback if so
+            if (id == gptneox_token_bos()) {
+                continue;
+            }
+            // Convert token to string and display
+            // printf("%s(%d)", gptneox_token_to_str(ctx, id), id);
+
+
+            if (last_n_tokens[2]==gptneox_str_to_token(ctx, "<")){
+                ;
+            }
+            else if (last_n_tokens[2]==gptneox_str_to_token(ctx, "human")){
+                if (last_n_tokens[1]==gptneox_str_to_token(ctx, "<")){
+                    ;
+                }
+                else{
+                    output_text_vec.push_back(gptneox_token_to_str(ctx, id));
+                }
+            }
+            else if (last_n_tokens[1]==gptneox_str_to_token(ctx, "<")){
+                output_text_vec.push_back("<");
+                output_text_vec.push_back(gptneox_token_to_str(ctx, id));
+            }
+            else{
+                output_text_vec.push_back(gptneox_token_to_str(ctx, id));
+            }
+            // Check if we need to run another eval
+            if (output_enabled) {
+                // Send generated token back into model for next generation
+                if (gptneox_eval(ctx, &id, 1, n_past, params.n_threads)) {
+                    fprintf(stderr, "%s : failed to eval\n", __func__);
+                    return -1;
+                }
+                // Increment past count
+                n_past += 1;
+                // Check if we need to forget
+                if (n_past > n_ctx) {
+                    // Not enough room to predict even a single token so purge oldest from past and kv cache
+                    // If nothing in past to purge so simply remove tokens from the beginning of the response
+                    // Remove a batch of 8 or 16 tokens from beginning of response if no past, this helps reduce the frequency of shifts, but will make the model forget quicker if the forget batch size is too high
+                    // In theory, the model can continue to build a response infinitely
+                    int n_forget = 16; //8 //1
+                    if (past.size() > 0) {
+                        n_forget = past.front().size();
+                        past.erase(past.begin());
+                    }
+                    gptneox_shift_kv_cache(ctx, n_forget);
+                    n_past -= n_forget;
+                    //fprintf(stderr, "%s : %d tokens purged from context memory\n", __func__, n_forget);
+                }
+            }
+            
+            // Return if we've added at least one token to the output text vec
+            if (output_text_vec.size() != output_text_vec_size) {
+                return 0;
+            }
+        }
+        // Update past with most recent response
+        past.push_back(output);
 
         return 0;
     }
     
+    std::string getMessage() {
+        std::stringstream ss;
+        for (const auto& str : output_text_vec) {
+            ss << str;
+        }
+        return strip_string(ss.str());
+    }
+    
+    void reset(int random_seed) {
+        past.clear();
+        output.clear();
+        output_text_vec.clear();
+        last_n_tokens.clear();
+        rng.seed(random_seed);
+    }
+    
+    bool stopped() {
+        return !output_enabled;
+    }
+    
 private:
+    gpt_params params;
+    std::mt19937 rng;
+    gptneox_context *ctx;
+    std::vector<std::vector<gptneox_token>> past;
+    int n_past = 0;
+    std::vector<gptneox_token> output;
+    bool output_enabled = false;
+    std::vector<gptneox_token> last_n_tokens;
+    std::vector<std::string> output_text_vec;
+
     CPPChatBackend(const std::string& model_path, int random_seed) {
         params.model = model_path;
         params.n_ctx = 2048;
+        params.n_batch = 128;
+        params.n_predict = 1;
+        params.n_threads = 8;
+        params.mirostat = 2;
+        params.instruct = true;
+        params.top_k = 30;
+        params.top_p = 0.95;
+        params.temp = 0.8;
+        params.repeat_last_n = 3;
+        params.repeat_penalty = 1.1;
+        
+        
+        rng.seed(random_seed);
 
         g_ctx = &ctx;
         
@@ -117,16 +333,26 @@ private:
     ~CPPChatBackend() {
         gptneox_free(ctx);
     }
+    
+    static std::string strip_string(const std::string& s) {
+        size_t start = s.find_first_not_of(" \t\n\r\f\v");
+        size_t end = s.find_last_not_of(" \t\n\r\f\v");
 
-    gpt_params params;
-    std::mt19937 rng;
-    gptneox_context *ctx;
-    std::vector<std::vector<gptneox_token>> past;
-    int n_past = 0;
-    std::vector<gptneox_token> output;
-    bool output_enabled = true;
+        if (start == std::string::npos) { // no non-space characters
+            return "";
+        }
+
+        return s.substr(start, end - start + 1);
+    }
 };
 
+#define RANDOM_SEED 1337
+
+static CPPChatBackend& getBackendInstance() {
+    NSString *path = [[NSBundle mainBundle] pathForResource:@"ggml-RedPajama-INCITE-Chat-3B-v1-q5_1" ofType:@"bin"];
+    return CPPChatBackend::getInstance(path.UTF8String, RANDOM_SEED);
+}
+
 
 @implementation ChatBackendInstance
 
@@ -139,31 +365,37 @@ private:
 }
 
 - (void)initialize {
-    NSString *path = [[NSBundle mainBundle] pathForResource:@"ggml-RedPajama-INCITE-Chat-3B-v1-q5_1" ofType:@"bin"];
-    auto& instance = CPPChatBackend::getInstance(path.UTF8String, 1337);
+    auto& instance = getBackendInstance();
     if (instance.isInitialized()) {
         _initialized = TRUE;
     }
 }
 
-- (void)evaluate {
-}
-
 - (void)encode:(NSString*)prompt {
+    auto& instance = getBackendInstance();
+    auto ret = instance.encode(prompt.UTF8String);
+    NSAssert(ret >= 0, @"Encode failed");
 }
 
 - (void)decode {
+    auto& instance = getBackendInstance();
+    instance.decode();
 }
 
 - (NSString*)getMessage {
-    return @"Nothing to report yet";
+    auto& instance = getBackendInstance();
+    std::string message = instance.getMessage();
+    return [NSString stringWithUTF8String:message.c_str()];
 }
 
 - (bool)stopped {
-    return FALSE;
+    auto& instance = getBackendInstance();
+    return instance.stopped();
 }
 
 - (void)reset {
+    auto& instance = getBackendInstance();
+    instance.reset(RANDOM_SEED);
 }
 
 - (NSString*)runtimeStatsText {

redpajama/PersonalGPT-Bridging-Header.h

@@ -16,13 +16,13 @@
     @property (nonatomic, assign) BOOL initialized;
 
     - (void)initialize;
-    - (void)evaluate;
     - (void)encode:(NSString*)prompt;
     - (void)decode;
     - (void)reset;
     - (NSString*)getMessage;
     - (bool)stopped;
     - (NSString*)runtimeStatsText;
+
 @end
 
 #endif /* PrivateAIChat_Bridging_Header_h */