Commit
β’
1963f41
1
Parent(s):
5f16d53
added tagging
Browse files
app.py
CHANGED
@@ -17,6 +17,10 @@ INTRODUCTION_TEXT = """
|
|
17 |
- This test is general test for malay grammar.
|
18 |
3. HumanEval, https://github.com/openai/human-eval
|
19 |
- This test is for programming language understanding.
|
|
|
|
|
|
|
|
|
20 |
"""
|
21 |
|
22 |
not_verify = [
|
@@ -33,6 +37,7 @@ not_verify = [
|
|
33 |
|
34 |
close_source = [
|
35 |
{
|
|
|
36 |
'model': 'gpt-4-1106-preview',
|
37 |
'BM-PT3 0-shot': 51.85185185185185,
|
38 |
'BM-PT3 1-shot': 66.66666666666666,
|
@@ -41,7 +46,8 @@ close_source = [
|
|
41 |
'Tatabahasa 1-shot': 73.63896848137536,
|
42 |
'Tatabahasa 3-shots': 75.64469914040114,
|
43 |
},
|
44 |
-
{
|
|
|
45 |
'model': 'gpt-3.5-turbo-0613',
|
46 |
'BM-PT3 0-shot': 36.53846153846153,
|
47 |
'BM-PT3 1-shot': 28.846153846153843,
|
@@ -54,12 +60,14 @@ close_source = [
|
|
54 |
|
55 |
open_source = [
|
56 |
{
|
|
|
57 |
'model': '[llama2-7b](https://huggingface.co/meta-llama/Llama-2-7b-hf)',
|
58 |
'Tatabahasa 0-shot': 24.355300859598856,
|
59 |
'Tatabahasa 1-shot': 28.08022922636103,
|
60 |
'Tatabahasa 3-shots': 24.641833810888254,
|
61 |
},
|
62 |
{
|
|
|
63 |
'model': '[malaysian-llama2-7b-32k](https://huggingface.co/mesolitica/llama-7b-hf-32768-fpf)',
|
64 |
'BM-PT3 0-shot': 20.37037037037037,
|
65 |
'BM-PT3 1-shot': 20.37037037037037,
|
@@ -69,6 +77,7 @@ open_source = [
|
|
69 |
'Tatabahasa 3-shots': 27.507163323782237,
|
70 |
},
|
71 |
{
|
|
|
72 |
'model': '[malaysian-llama2-7b-32k-instructions](https://huggingface.co/mesolitica/malaysian-llama2-7b-32k-instructions-v2)',
|
73 |
'BM-PT3 0-shot': 33.33333333333333,
|
74 |
'BM-PT3 1-shot': 37.03703703703704,
|
@@ -78,6 +87,7 @@ open_source = [
|
|
78 |
'Tatabahasa 3-shots': 41.833810888252145,
|
79 |
},
|
80 |
{
|
|
|
81 |
'model': '[malaysian-llama2-13b-32k](https://huggingface.co/mesolitica/llama-13b-hf-32768-fpf)',
|
82 |
'BM-PT3 0-shot': 33.33333333333333,
|
83 |
'BM-PT3 1-shot': 20.37037037037037,
|
@@ -87,18 +97,21 @@ open_source = [
|
|
87 |
'Tatabahasa 3-shots': 24.355300859598856,
|
88 |
},
|
89 |
{
|
|
|
90 |
'model': '[malaysian-llama2-13b-32k-instructions](https://huggingface.co/mesolitica/malaysian-llama2-13b-32k-instructions)',
|
91 |
'BM-PT3 0-shot': 28.57142857142857,
|
92 |
'BM-PT3 1-shot': 12.244897959183673,
|
93 |
'BM-PT3 3-shots': 17.307692307692307,
|
94 |
},
|
95 |
{
|
|
|
96 |
'model': '[mistral-7b](https://huggingface.co/mistralai/Mistral-7B-v0.1)',
|
97 |
'Tatabahasa 0-shot': 28.939828080229223,
|
98 |
'Tatabahasa 1-shot': 34.38395415472779,
|
99 |
'Tatabahasa 3-shots': 32.95128939828081,
|
100 |
},
|
101 |
{
|
|
|
102 |
'model': '[malaysian-mistral-7b-4k](https://huggingface.co/mesolitica/mistral-7b-4096-fpf)',
|
103 |
'BM-PT3 0-shot': 20.37037037037037,
|
104 |
'BM-PT3 1-shot': 22.22222222222222,
|
@@ -108,6 +121,7 @@ open_source = [
|
|
108 |
'Tatabahasa 3-shots': 24.641833810888254,
|
109 |
},
|
110 |
{
|
|
|
111 |
'model': '[malaysian-mistral-7b-32k](https://huggingface.co/mesolitica/mistral-7b-32768-fpf)',
|
112 |
'BM-PT3 0-shot': 16.666666666666664,
|
113 |
'BM-PT3 1-shot': 16.666666666666664,
|
@@ -117,6 +131,7 @@ open_source = [
|
|
117 |
'Tatabahasa 3-shots': 28.653295128939828,
|
118 |
},
|
119 |
{
|
|
|
120 |
'model': '[malaysian-mistral-7b-32k-instructions](https://huggingface.co/mesolitica/malaysian-mistral-7b-32k-instructions)',
|
121 |
'BM-PT3 0-shot': 40.74074074074074,
|
122 |
'BM-PT3 1-shot': 31.48148148148148,
|
@@ -126,6 +141,7 @@ open_source = [
|
|
126 |
'Tatabahasa 3-shots': 53.86819484240688
|
127 |
},
|
128 |
{
|
|
|
129 |
'model': '[aisingapore/sealion3b](https://huggingface.co/aisingapore/sealion3b)',
|
130 |
'BM-PT3 0-shot': 20.37037037037037,
|
131 |
'BM-PT3 1-shot': 25.925925925925924,
|
@@ -135,6 +151,7 @@ open_source = [
|
|
135 |
'Tatabahasa 3-shots': 24.641833810888254,
|
136 |
},
|
137 |
{
|
|
|
138 |
'model': '[aisingapore/sealion7b](https://huggingface.co/aisingapore/sealion7b)',
|
139 |
'BM-PT3 0-shot': 20.37037037037037,
|
140 |
'BM-PT3 1-shot': 24.074074074074073,
|
|
|
17 |
- This test is general test for malay grammar.
|
18 |
3. HumanEval, https://github.com/openai/human-eval
|
19 |
- This test is for programming language understanding.
|
20 |
+
|
21 |
+
## Tagging
|
22 |
+
|
23 |
+
π’ pretrained β instruction-tuned π¦ close sourced
|
24 |
"""
|
25 |
|
26 |
not_verify = [
|
|
|
37 |
|
38 |
close_source = [
|
39 |
{
|
40 |
+
'T': 'π¦',
|
41 |
'model': 'gpt-4-1106-preview',
|
42 |
'BM-PT3 0-shot': 51.85185185185185,
|
43 |
'BM-PT3 1-shot': 66.66666666666666,
|
|
|
46 |
'Tatabahasa 1-shot': 73.63896848137536,
|
47 |
'Tatabahasa 3-shots': 75.64469914040114,
|
48 |
},
|
49 |
+
{
|
50 |
+
'T': 'π¦',
|
51 |
'model': 'gpt-3.5-turbo-0613',
|
52 |
'BM-PT3 0-shot': 36.53846153846153,
|
53 |
'BM-PT3 1-shot': 28.846153846153843,
|
|
|
60 |
|
61 |
open_source = [
|
62 |
{
|
63 |
+
'T': 'π’',
|
64 |
'model': '[llama2-7b](https://huggingface.co/meta-llama/Llama-2-7b-hf)',
|
65 |
'Tatabahasa 0-shot': 24.355300859598856,
|
66 |
'Tatabahasa 1-shot': 28.08022922636103,
|
67 |
'Tatabahasa 3-shots': 24.641833810888254,
|
68 |
},
|
69 |
{
|
70 |
+
'T': 'π’',
|
71 |
'model': '[malaysian-llama2-7b-32k](https://huggingface.co/mesolitica/llama-7b-hf-32768-fpf)',
|
72 |
'BM-PT3 0-shot': 20.37037037037037,
|
73 |
'BM-PT3 1-shot': 20.37037037037037,
|
|
|
77 |
'Tatabahasa 3-shots': 27.507163323782237,
|
78 |
},
|
79 |
{
|
80 |
+
'T': 'β',
|
81 |
'model': '[malaysian-llama2-7b-32k-instructions](https://huggingface.co/mesolitica/malaysian-llama2-7b-32k-instructions-v2)',
|
82 |
'BM-PT3 0-shot': 33.33333333333333,
|
83 |
'BM-PT3 1-shot': 37.03703703703704,
|
|
|
87 |
'Tatabahasa 3-shots': 41.833810888252145,
|
88 |
},
|
89 |
{
|
90 |
+
'T': 'π’',
|
91 |
'model': '[malaysian-llama2-13b-32k](https://huggingface.co/mesolitica/llama-13b-hf-32768-fpf)',
|
92 |
'BM-PT3 0-shot': 33.33333333333333,
|
93 |
'BM-PT3 1-shot': 20.37037037037037,
|
|
|
97 |
'Tatabahasa 3-shots': 24.355300859598856,
|
98 |
},
|
99 |
{
|
100 |
+
'T': 'β',
|
101 |
'model': '[malaysian-llama2-13b-32k-instructions](https://huggingface.co/mesolitica/malaysian-llama2-13b-32k-instructions)',
|
102 |
'BM-PT3 0-shot': 28.57142857142857,
|
103 |
'BM-PT3 1-shot': 12.244897959183673,
|
104 |
'BM-PT3 3-shots': 17.307692307692307,
|
105 |
},
|
106 |
{
|
107 |
+
'T': 'π’',
|
108 |
'model': '[mistral-7b](https://huggingface.co/mistralai/Mistral-7B-v0.1)',
|
109 |
'Tatabahasa 0-shot': 28.939828080229223,
|
110 |
'Tatabahasa 1-shot': 34.38395415472779,
|
111 |
'Tatabahasa 3-shots': 32.95128939828081,
|
112 |
},
|
113 |
{
|
114 |
+
'T': 'π’',
|
115 |
'model': '[malaysian-mistral-7b-4k](https://huggingface.co/mesolitica/mistral-7b-4096-fpf)',
|
116 |
'BM-PT3 0-shot': 20.37037037037037,
|
117 |
'BM-PT3 1-shot': 22.22222222222222,
|
|
|
121 |
'Tatabahasa 3-shots': 24.641833810888254,
|
122 |
},
|
123 |
{
|
124 |
+
'T': 'π’',
|
125 |
'model': '[malaysian-mistral-7b-32k](https://huggingface.co/mesolitica/mistral-7b-32768-fpf)',
|
126 |
'BM-PT3 0-shot': 16.666666666666664,
|
127 |
'BM-PT3 1-shot': 16.666666666666664,
|
|
|
131 |
'Tatabahasa 3-shots': 28.653295128939828,
|
132 |
},
|
133 |
{
|
134 |
+
'T': 'β',
|
135 |
'model': '[malaysian-mistral-7b-32k-instructions](https://huggingface.co/mesolitica/malaysian-mistral-7b-32k-instructions)',
|
136 |
'BM-PT3 0-shot': 40.74074074074074,
|
137 |
'BM-PT3 1-shot': 31.48148148148148,
|
|
|
141 |
'Tatabahasa 3-shots': 53.86819484240688
|
142 |
},
|
143 |
{
|
144 |
+
'T': 'π’',
|
145 |
'model': '[aisingapore/sealion3b](https://huggingface.co/aisingapore/sealion3b)',
|
146 |
'BM-PT3 0-shot': 20.37037037037037,
|
147 |
'BM-PT3 1-shot': 25.925925925925924,
|
|
|
151 |
'Tatabahasa 3-shots': 24.641833810888254,
|
152 |
},
|
153 |
{
|
154 |
+
'T': 'π’',
|
155 |
'model': '[aisingapore/sealion7b](https://huggingface.co/aisingapore/sealion7b)',
|
156 |
'BM-PT3 0-shot': 20.37037037037037,
|
157 |
'BM-PT3 1-shot': 24.074074074074073,
|