File size: 14,884 Bytes
50f19fa
 
29078ea
50f19fa
 
 
 
 
 
 
 
 
 
 
 
29078ea
50f19fa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0680f69
29078ea
 
 
 
 
50f19fa
bdf4f51
50f19fa
 
bdf4f51
db28a13
50f19fa
 
 
bdf4f51
 
7769b47
 
bdf4f51
7769b47
 
bdf4f51
 
 
 
 
db28a13
bdf4f51
 
 
db28a13
bdf4f51
 
 
 
db28a13
bdf4f51
 
 
 
 
 
 
db28a13
 
 
bdf4f51
 
 
 
 
 
 
a42de63
bdf4f51
 
 
2db3504
 
 
 
 
 
 
 
 
bdf4f51
50f19fa
0680f69
db28a13
7769b47
 
 
db28a13
7769b47
 
 
8e75c4f
db28a13
7769b47
 
 
9793af4
7769b47
 
50f19fa
db28a13
 
 
50f19fa
7769b47
50f19fa
0e893b5
0680f69
50f19fa
0e893b5
29078ea
50f19fa
 
 
2db3504
 
 
 
 
0e893b5
42b592e
847028b
 
 
7769b47
50f19fa
0680f69
50f19fa
9793af4
4424c49
9793af4
42b592e
2db3504
 
 
 
 
 
 
 
 
db28a13
7769b47
50f19fa
 
db28a13
7769b47
29078ea
 
2db3504
50f19fa
0e893b5
7769b47
9793af4
7769b47
 
50f19fa
db28a13
 
 
7769b47
0ad933c
 
 
 
4e90465
0ad933c
50f19fa
0ad933c
b1abf8e
 
 
 
 
 
 
 
0ad933c
 
 
db28a13
b1abf8e
 
 
db28a13
b1abf8e
 
 
76168c9
db28a13
7769b47
 
9793af4
7769b47
 
0ad933c
db28a13
b1abf8e
db28a13
 
0ad933c
7769b47
0ad933c
50f19fa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7769b47
50f19fa
 
 
 
29078ea
9793af4
29078ea
50f19fa
 
 
 
 
 
0e893b5
7769b47
 
50f19fa
 
 
 
7769b47
fd49c3f
29078ea
 
4e90465
0680f69
50f19fa
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
from gradio.components import Component
import gradio as gr
import pandas as pd
from abc import ABC, abstractclassmethod
import inspect

class BaseTCOModel(ABC):
    # TO DO: Find way to specify which component should be used for computing cost
    def __setattr__(self, name, value):
        if isinstance(value, Component):
            self._components.append(value)
        self.__dict__[name] = value

    def __init__(self):
        super(BaseTCOModel, self).__setattr__("_components", [])
        self.use_case = None  

    def get_components(self) -> list[Component]:
        return self._components
    
    def get_components_for_cost_computing(self):
        return self.components_for_cost_computing
    
    def get_name(self):
        return self.name
    
    def register_components_for_cost_computing(self):
        args = inspect.getfullargspec(self.compute_cost_per_token)[0][1:]
        self.components_for_cost_computing = [self.__getattribute__(arg) for arg in args]
    
    @abstractclassmethod
    def compute_cost_per_token(self):
        pass
    
    @abstractclassmethod
    def render(self):
        pass
    
    def set_name(self, name):
        self.name = name
    
    def set_latency(self, latency):
        self.latency = latency
    
    def get_latency(self):
        return self.latency

class OpenAIModelGPT4(BaseTCOModel):

    def __init__(self):
        self.set_name("(SaaS) OpenAI GPT4")
        self.set_latency("15s") #Default value for GPT4
        super().__init__()

    def render(self):
        def define_cost_per_token(context_length):
            if context_length == "8K":
                cost_per_1k_input_tokens = 0.03
                cost_per_1k_output_tokens = 0.06
            else:
                cost_per_1k_input_tokens = 0.06
                cost_per_1k_output_tokens = 0.12
            return cost_per_1k_input_tokens, cost_per_1k_output_tokens
        
        self.context_length = gr.Dropdown(["8K", "32K"], value="8K", interactive=True,
                                          label="Context size",
                                          visible=False, info="Number of tokens the model considers when processing text")
        self.input_tokens_cost_per_token = gr.Number(0.03, visible=False,
                                           label="($) Price/1K input prompt tokens",
                                           interactive=False
                                           )
        self.output_tokens_cost_per_token = gr.Number(0.06, visible=False,
                                           label="($) Price/1K output prompt tokens",
                                           interactive=False
                                           )
        self.info = gr.Markdown("The cost per input and output tokens values are from OpenAI's [pricing web page](https://openai.com/pricing)", interactive=False, visible=False)
        self.context_length.change(define_cost_per_token, inputs=self.context_length, outputs=[self.input_tokens_cost_per_token, self.output_tokens_cost_per_token])
        
        self.labor = gr.Number(0, visible=False, 
                                label="($) Labor cost per month", 
                                info="This is an estimate of the labor cost of the AI engineer in charge of deploying the model",
                                interactive=True
                                )

    def compute_cost_per_token(self, input_tokens_cost_per_token, output_tokens_cost_per_token, labor):
        cost_per_input_token = (input_tokens_cost_per_token / 1000) 
        cost_per_output_token = (output_tokens_cost_per_token / 1000)

        return cost_per_input_token, cost_per_output_token, labor

class OpenAIModelGPT3_5(BaseTCOModel):

    def __init__(self):
        self.set_name("(SaaS) OpenAI GPT3.5 Turbo")
        self.set_latency("5s") #Average latency value for GPT3.5 Turbo
        super().__init__()

    def render(self):
        def define_cost_per_token(context_length):
            if context_length == "4K":
                cost_per_1k_input_tokens = 0.0015
                cost_per_1k_output_tokens = 0.002
            else:
                cost_per_1k_input_tokens = 0.003
                cost_per_1k_output_tokens = 0.004
            return cost_per_1k_input_tokens, cost_per_1k_output_tokens

        self.context_length = gr.Dropdown(choices=["4K", "16K"], value="4K", interactive=True,
                                          label="Context size",
                                          visible=False, info="Number of tokens the model considers when processing text")
        self.input_tokens_cost_per_token = gr.Number(0.0015, visible=False,
                                           label="($) Price/1K input prompt tokens",
                                           interactive=False
                                           )
        self.output_tokens_cost_per_token = gr.Number(0.002, visible=False,
                                           label="($) Price/1K output prompt tokens",
                                           interactive=False
                                           )
        self.info = gr.Markdown("The cost per input and output tokens values are from OpenAI's [pricing web page](https://openai.com/pricing)", interactive=False, visible=False)
        self.context_length.change(define_cost_per_token, inputs=self.context_length, outputs=[self.input_tokens_cost_per_token, self.output_tokens_cost_per_token])
        
        self.labor = gr.Number(0, visible=False, 
                                label="($) Labor cost per month", 
                                info="This is an estimate of the labor cost of the AI engineer in charge of deploying the model",
                                interactive=True
                                )

    def compute_cost_per_token(self, input_tokens_cost_per_token, output_tokens_cost_per_token, labor):
        cost_per_input_token = (input_tokens_cost_per_token / 1000) 
        cost_per_output_token = (output_tokens_cost_per_token / 1000)

        return cost_per_input_token, cost_per_output_token, labor

class DIYLlama2Model(BaseTCOModel):
    
    def __init__(self):
        self.set_name("(Deploy yourself) Llama 2 70B")
        self.set_latency("27s")
        super().__init__()
    
    def render(self):
        def on_maxed_out_change(maxed_out, input_tokens_cost_per_token, output_tokens_cost_per_token):
            output_tokens_cost_per_token = 0.06656
            input_tokens_cost_per_token = 0.00052
            r = maxed_out / 100
            return input_tokens_cost_per_token * 0.65 / r, output_tokens_cost_per_token * 0.65/ r
        
        self.source = gr.Markdown("""<span style="font-size: 16px; font-weight: 600; color: #212529;">Source</span>""", visible=False)
        self.info = gr.Markdown("The cost per input and output tokens values below are from [these benchmark results](https://www.cursor.so/blog/llama-inference#user-content-fn-llama-paper) that were obtained using the following initial configurations.", 
                                 interactive=False, 
                                 visible=False)
        self.vm = gr.Textbox(value="2x A100 80GB NVLINK", 
                              visible=False,
                              label="Instance of VM with GPU",
                              )
        self.vm_cost_per_hour = gr.Number(4.42, label="Instance cost ($) per hour",
                                      interactive=False, visible=False)
        self.info_vm = gr.Markdown("This price above is from [CoreWeave's pricing web page](https://www.coreweave.com/gpu-cloud-pricing)", interactive=False, visible=False)
        self.maxed_out = gr.Slider(minimum=1, maximum=100, value=65, step=1, label="Maxed out", info="Estimated average percentage of total GPU memory that is used. The instantaneous value can go from very high when many users are using the service to very low when no one does.", visible=False)
        self.info_maxed_out = gr.Markdown(r"""This percentage influences the input and output cost/token values, and more precisely the number of token/s. Here is the formula used:<br>
                                          $CT = \frac{VM_C}{TS}$ where $TS = TS_{max} * \frac{MO}{100}$ <br>
                                          with: <br>
                                          $CT$ = Cost per Token (Input or output), <br>
                                          $VM_C$ = VM Cost per second, <br>
                                          $TS$ = Tokens per second (Input or output), <br>
                                          $TS_{max}$ = Tokens per second when the GPU is maxed out at 100%, <br>
                                          $MO$ = Maxed Out, <br>
                                          """, interactive=False, visible=False)
        self.input_tokens_cost_per_token = gr.Number(0.00052, visible=False,
                                           label="($) Price/1K input prompt tokens",
                                           interactive=False
                                           )
        self.output_tokens_cost_per_token = gr.Number(0.06656, visible=False,
                                           label="($) Price/1K output prompt tokens",
                                           interactive=False
                                           )
        self.maxed_out.change(on_maxed_out_change, inputs=[self.maxed_out, self.input_tokens_cost_per_token, self.output_tokens_cost_per_token], outputs=[self.input_tokens_cost_per_token, self.output_tokens_cost_per_token])
        
        self.labor = gr.Number(5000, visible=False, 
                                label="($) Labor cost per month",
                                info="This is an estimate of the labor cost of the AI engineer in charge of deploying the model",
                                interactive=True
                                )

    def compute_cost_per_token(self, input_tokens_cost_per_token, output_tokens_cost_per_token, labor):
        cost_per_input_token = (input_tokens_cost_per_token / 1000) 
        cost_per_output_token = (output_tokens_cost_per_token / 1000)
        return cost_per_input_token,  cost_per_output_token, labor

class CohereModel(BaseTCOModel):
    def __init__(self):
        self.set_name("(SaaS) Cohere")
        self.set_latency("Not available")
        super().__init__()
    
    def render(self):
        def on_model_change(model):
            if model == "Default":
                cost_per_1M_tokens = 15
            else: 
                cost_per_1M_tokens = 30
            cost_per_1K_tokens = cost_per_1M_tokens / 1000
            return gr.update(value=cost_per_1K_tokens), gr.update(value=cost_per_1K_tokens)
        
        self.model = gr.Dropdown(["Default", "Custom"], value="Default",
                                 label="Model",
                                 interactive=True, visible=False)
        self.input_tokens_cost_per_token = gr.Number(0.015, visible=False,
                                           label="($) Price/1K input prompt tokens",
                                           interactive=False
                                           )
        self.output_tokens_cost_per_token = gr.Number(0.015, visible=False,
                                           label="($) Price/1K output prompt tokens",
                                           interactive=False
                                           )
        self.info = gr.Markdown("The cost per input and output tokens value is from Cohere's [pricing web page](https://cohere.com/pricing?utm_term=&utm_campaign=Cohere+Brand+%26+Industry+Terms&utm_source=adwords&utm_medium=ppc&hsa_acc=4946693046&hsa_cam=20368816223&hsa_grp=154209120409&hsa_ad=666081801359&hsa_src=g&hsa_tgt=dsa-19959388920&hsa_kw=&hsa_mt=&hsa_net=adwords&hsa_ver=3&gad=1&gclid=CjwKCAjww7KmBhAyEiwA5-PUSlyO7pq0zxeVrhViXMd8WuILW6uY-cfP1-SVuUfs-leUAz14xHlOHxoCmfkQAvD_BwE)", interactive=False, visible=False)
        self.model.change(on_model_change, inputs=self.model, outputs=[self.input_tokens_cost_per_token, self.output_tokens_cost_per_token])
        self.labor = gr.Number(0, visible=False, 
                                label="($) Labor cost per month", 
                                info="This is an estimate of the labor cost of the AI engineer in charge of deploying the model",
                                interactive=True
                                )

    def compute_cost_per_token(self, input_tokens_cost_per_token, output_tokens_cost_per_token, labor):
        
        cost_per_input_token = input_tokens_cost_per_token / 1000
        cost_per_output_token = output_tokens_cost_per_token / 1000

        return cost_per_input_token, cost_per_output_token, labor

class ModelPage:
    def __init__(self, Models: BaseTCOModel):
        self.models: list[BaseTCOModel] = []
        for Model in Models:
            model = Model()
            self.models.append(model)

    def render(self):
        for model in self.models:
            model.render()
            model.register_components_for_cost_computing() 

    def get_all_components(self) -> list[Component]:
        output = []
        for model in self.models:
            output += model.get_components()
        return output
    
    def get_all_components_for_cost_computing(self) -> list[Component]:
        output = []
        for model in self.models:
            output += model.get_components_for_cost_computing()
        return output

    def make_model_visible(self, name:str, use_case: gr.Dropdown):
        # First decide which indexes
        output = []
        for model in self.models:
            if model.get_name() == name:
                output+= [gr.update(visible=True)] * len(model.get_components()) 
                # Set use_case value in the model
                model.use_case = use_case
            else:
                output+= [gr.update(visible=False)] * len(model.get_components())
        return output
    
    def compute_cost_per_token(self, *args):
        begin=0
        current_model = args[-3]  
        current_input_tokens = args[-2]
        current_output_tokens = args[-1]
        for model in self.models:
            model_n_args = len(model.get_components_for_cost_computing())
            if current_model == model.get_name():
                model_args = args[begin:begin+model_n_args]
                cost_per_input_token, cost_per_output_token, labor_cost = model.compute_cost_per_token(*model_args)
                model_tco = cost_per_input_token * current_input_tokens + cost_per_output_token * current_output_tokens
                latency = model.get_latency()
                
                return model_tco, latency, labor_cost
            
            begin = begin+model_n_args