lrl-modelcloud commited on
Commit
28b96cb
1 Parent(s): 6b14ea3

Upload folder using huggingface_hub (#2)

Browse files

- 9768ebe43b80c11e575db1d951572af2bb45474ede807ad71ff41445685fee75 (76b5754299adf4528ce0e3120d823ecd6d54bb53)
- 44e5da5ca204044dbfb00cb3f9fa98a490e23d0f03db129f6b218d6411c04ff7 (7560eb43c3a35d3d8e2d91bac882a1669a005df2)
- f982a90e8b1bd08203b3a06efc3d56cb6f4b9b15597961014c29c8fb3280bbf5 (487be7a867615e33e374668154e834c8dff654f0)
- e6629b0df4bd0297e9c54b115b77ddabda30f8b1e2599a8512ba501e3c02032c (ead3f331632462ecdcee5157c39773b0c5d2b510)
- 319009fdee789e4dc2ef9ade44fdde27bb7f8714faff7ed4bb88fc80a92c255a (fd20877f45176dcce598e85ded3cd50dd4af167a)
- 2038ed65c67f787b0f554d222d5f5071e00077de44bbbcfcabd9a7a8cbf1b61c (8c939d5cc76da5d410b3888e7977e21cf0a18eaf)
- 2db0cbc992f5e8d40ff6fc8b12c12d27c33f086406fe9531978269bf519bb387 (e3113a88b111701211153d5edfb9344bb44f0ae0)
- 47acf2e55b2e182c49564b82c51e9aadc5e7152371113e35bcb706ac38313984 (1c6cfeb1d8f3b4321f55fbb05c60cba38290812b)
- b17a0d2943e6a7e91975da811fc09702a58e9988fb66862aaad53c715448c6da (969b29dcfc0613ff210eedeb6c8bccb0faf470d1)
- 1050c18c71bb26b670f3cb059f49e8a9347bac319ed332127728993246730a53 (f0f2ee6d4a92a68c98d30129dcd9cbf5ecea8c27)
- 2c42a75c50c49f8504ec46b8e6790380cb41013ce05c11773244e67c12f047d5 (78c61d60112c7d7a570f4de52174726a5cb26783)
- 8037d19d50daa5d7c7173a06ed6f210bda60895e66befe7b555f3b5cfda902a9 (c2b98e3036d85820a19c63673a8c04373f0c142f)
- e7bdbe573c94586a7fd098baf980ab87fb0ea21f49c778d7aa384a50ca42a201 (8cf536a415ed937408438ca68f4efdc55250190e)
- 45f2fcb3f7b589ee99dce4b45be692e58ce447677c041bc56ce2d217f36b49a6 (09eb9fa82c499f79100b17dcf6519f1ae98777c9)
- 288baaa26763a63582d9543e11b6760abfbc36291404e07e33ccfb04b5a000f6 (6c539bd49d20c198fbeee68b5143aa4b6d4f92bd)
- 3bea9eb52bedaea7e05a08d10967fa0fcc3aa4c3d03bb89da901704eacfda64e (52d2ba4a2ea88bfe011af7913053d56a3e2ce45f)
- 8f0b6837bc3e80e1fa3c61f794cf4360b744a8867cae235cc553388bfa490ec2 (f4934fe44a9284a0c4f64ad0f72f0591f2e4fe39)
- d14394874f4c761910c5d14684445a287b992c7fe55f9d06293509b78811a22c (8ec6dce1add8cb4c4a03b7a20f8d1a0ffd601a7f)
- cdfff6e0585be6a7194c489f5babeefb2363087256626ef5bfeaf2a0817ecc85 (faa228a20fdd88a27955d46a74826e6f076300ec)
- 087cc7d625a4a9e3fc91faa8c705bfcadbe6dab8548ff4f6cf07750ed02731b2 (a2b5620ba0cea983882be72683e7911034deab46)
- b6cf9a8729b7d32d1d64b4b4e2f48933812a730fbc012d9b1daceeb1f110534d (0de01f73246d9cf4e7ac7de4801d09b281071d5d)
- 632cb309b859bce20c8d3e1b3b105ceba7ce0f41eeaf672562ded32c2b0a07f3 (35fb66cf934708e2e5f74752d97db3d140d36870)
- 295a893c6c2b15075f795741ee52333c0dec32a5b36f736e7dd6bac79bc29c8c (915ed16dc0ac95da92e89acdba471b9fe62f1c31)
- 920f0d7709d09ec3f2b29c4f4f86c098903784994164af03db62a1d440bccb59 (53e65cc88c11ba7e3d9a668c6783c987ad90fb4b)
- 67194ecdd290d6ecd392026855c6740e384a2b1f4ef869a31124a0b81c9ddb54 (9c053e7459374f15ba89ce8300fa76da814e5cc6)
- b33e86e80e0b1c125497a8f9a6ad1eafa0ae70f24d9551d77b1e79ecb919e9bc (ff6dda626eb635f9c60540406ea1478ae8ab6c2e)
- 8917f1518e26013bd41106977befc01e3fa76ab2162643d73faa3e2ac7059a50 (a3db7f20c0a6e986ce765bf79794fcf6bcac2be6)
- b1adc1a4412a577b938d4ebe54231a1ab0458b163f94644fdefd1223809055fd (ee0c0c8a58419b21c68f6a516e6d1cd6791e7754)
- 0964f2c6379e965663237d248b8cc4f5877d61431f3143db933ed750ad810b8f (a5452f721357584562d7e04b9474adf4b7c0b83a)
- 7f8051ffc6cc69e994ff64860de70d170147c300dd492f1e857e038579e9f23b (19a7e948dd9e4cec728f1aed5e7920b2cb9ed7db)
- f49931de7748c7ceb9366c2eae4c1595f85e62d7506d2e789277cabbde0becd1 (6e7115b178c9e1b10e3f2af9eb4da08dd54ccfc4)
- 6b9b2d5e6e979a8c1f85134b209dd071168428e8bc7372cc2352f504e1667786 (bc85fc9c56ca375701b42000d367d21007bf5a5e)
- d4f0b1ab5337758a1b1bd7d24139f5d042d8b0ccfad55e23fc45842576afa526 (a16614ec30827b6df842518b5c6bd0a50cc38698)
- 33bd963adf78dacb049bf13fcc0a4792b5fdec7ba73dcf7815c33ca50df23efa (57d1b7be60ca219ecfcc7cb242dc05a8364685ae)
- 387413fa44067d82d80c1f49f1f446d46c3821838e8baa0dbbe2d209a275b90f (08d06df244629c80436ef4e0bb20f4c8f133ff64)
- a55402b96ab8b068fd7a60c07527a6b9b689f539d60397d55ad9a010287dcabc (6919c47362718a4b5495747da82de7e8a6dd25a9)
- c48ff22578781ed20bc182dfa9c06fd94712a7b8dd7fb1b088af0122bf774221 (d42284e4d99fcd3d0f60d2c94e6636f7f7f87d2b)
- 1abbd1271518e8d61d4f8e62034f5cb4c2a67b96fc7e88c6d837e558f523cb07 (8c75adc420c3b47f0b7beac7d612ddc41e5a295b)
- 7e8735194d382c1e3d5631dcef79a27c472e8bd2bf79276ee559c2147cf02f11 (dc7df262cd7ce534e3917d68e2b48d69abe7b8cc)
- 23ad6e293484ab9b71b6ef9426e700dc6766119fd62c537f961d7eb4738d9684 (d0472e8364333c655ed2f370f0a03f672b177fc3)
- 4c74b0c5a47dbca191eb0bd1877e77d128e8149fb54db9126e9e54ea7ba5c901 (1915de520e67c40fda21454262632ab19a9a4791)
- b9a2193d710d070ccab4271a8f910baedc59074ea85fb6e3ce99fb931512e5e5 (3c3663c60834ff18498648963ccad826a1e000cb)
- 7b730fa7dc0adfe344c7e7f8083cf64580e521cb9328ccc27b40d284441ec10b (a26cdc6697fa787e670320f40ceb3f91f6317977)
- 6b9ec34d5b4ed43aa27592e10e2fb508002e431ea6289e38fdbe88f5ba71d0a6 (59f7a1498689e8bd79c858851e73009182b017b8)
- bbc38fc35a274ef24e7873f38c37e1a5de54906987f95da42d71a87f2975e9d8 (3894fae8bba51ef00ffea4e03a4c6dd58e5c0506)
- 0eb9769927829f93d9e88b51b5d6a91cd99ce6c803f263d589c13a6518985122 (06e84ed140a859fa3d5b537d9763cb844e5d6673)
- 29aa6aede56f3a1a666121806e1bc7a5aae7d40defaa88965c8d9cbac1aa0254 (68fa8da09375b62bf0bf3d4f1f88a1988563a24f)
- 24565d7c067ad1e21c7745e4d7f30d89c77d662d4e9b77c29756d55d83cd4969 (a0342bb57c945e81845f7e53b12e0c268af999a0)
- d6c826650939059066c08cadde4c784df24b41c26b8b13d49a806e991e453f6d (6c8809129fa8670dcec7dd1d3674941443c89e96)
- e225e85722768c1fd988367213eea22198114d9a8db6621144b6a168ddf7561d (1c7caeb8d01c62c7da5f79c8ee77fa31152108c1)
- e6512a4be02c5a30c1a6bfb86f7818d5ae1f9ece4682296906ce3106a6eb4bfe (4c92f9c8c963c38795d7013ff324374a50094176)
- cd78ea3bf8dc2a261a2a254bc4786fae95e2ae5cfdf98225c84c512ae17df5d1 (d18fa2557bba34a0f86db357b0996a2b8724e62f)
- 2bbe8357fe7c21cd3d04a4b1d06f659e501209223b31a99c46a5cfe0a757e830 (cb2bfd2c1d5b1a106c91381e07423b18c9b885ef)
- a62f6115a3ca31ca86d5f6e2279302b2c99510192e3eabf946ffcae1a14ce3c0 (810bcb4301d38577cb66147ea3710e0d4a507d5f)
- 74628101b328c875eabccb8c2535a403af3db7ff85989f4523c238c777222307 (ae044a1694e748f761369eab7da0b2b595e47ac1)
- 1b74bb7a68b8d5a8376e33505cf125f713be199c9f55c648d55af07686924975 (fa6754fe383fb111b09e6dc0f9ec401a026091f1)
- a635328c68b66a235159d525c8701d70a0deeea45eace07b758f0715d4d0ccf0 (e7c7421ecca049a0c49134d0268cfe8a251dc945)
- 3f3fd7890c7aebb3172b13536812af6741204848e11ce4fd101998c21a56ac74 (e2fc1bface4d9ae2cd59809123cc8f9733fc40cd)
- 2eb7edb4742ee0e4924d9f8a2e0feb480f703506cfbc10f2a7190f2d129a0923 (95c414590b595b8c0331c2e9172f545c7173cce3)
- 3e51a202d07500da13fa32bde9c0c1f0a2ed2473cc4d8dfa5ae1bcb2a413d705 (337f50de5e58bc3e3d85a40c6de08a74ad62e392)
- ea89e2716c35e4dc41d4c070f91f9887a896b63d689c1306a0d617adb3e505ec (00e4ebde81f4925c0d23d6c452bc737431efd16d)
- 38d52afe98894b7e91ddd95b616ceb46008c189834b5698652527c83023053e4 (acc23e8403b1ad8a166a21bda56e7302548ba78b)

This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. LICENSE.txt +176 -0
  2. NOTICE.txt +1 -0
  3. README.md +170 -3
  4. added_tokens.json +5 -0
  5. config.json +41 -0
  6. configuration_dbrx.py +264 -0
  7. generation_config.json +8 -0
  8. model-00001-of-00061.safetensors +3 -0
  9. model-00002-of-00061.safetensors +3 -0
  10. model-00003-of-00061.safetensors +3 -0
  11. model-00004-of-00061.safetensors +3 -0
  12. model-00005-of-00061.safetensors +3 -0
  13. model-00006-of-00061.safetensors +3 -0
  14. model-00007-of-00061.safetensors +3 -0
  15. model-00008-of-00061.safetensors +3 -0
  16. model-00009-of-00061.safetensors +3 -0
  17. model-00010-of-00061.safetensors +3 -0
  18. model-00011-of-00061.safetensors +3 -0
  19. model-00012-of-00061.safetensors +3 -0
  20. model-00013-of-00061.safetensors +3 -0
  21. model-00014-of-00061.safetensors +3 -0
  22. model-00015-of-00061.safetensors +3 -0
  23. model-00016-of-00061.safetensors +3 -0
  24. model-00017-of-00061.safetensors +3 -0
  25. model-00018-of-00061.safetensors +3 -0
  26. model-00019-of-00061.safetensors +3 -0
  27. model-00020-of-00061.safetensors +3 -0
  28. model-00021-of-00061.safetensors +3 -0
  29. model-00022-of-00061.safetensors +3 -0
  30. model-00023-of-00061.safetensors +3 -0
  31. model-00024-of-00061.safetensors +3 -0
  32. model-00025-of-00061.safetensors +3 -0
  33. model-00026-of-00061.safetensors +3 -0
  34. model-00027-of-00061.safetensors +3 -0
  35. model-00028-of-00061.safetensors +3 -0
  36. model-00029-of-00061.safetensors +3 -0
  37. model-00030-of-00061.safetensors +3 -0
  38. model-00031-of-00061.safetensors +3 -0
  39. model-00032-of-00061.safetensors +3 -0
  40. model-00033-of-00061.safetensors +3 -0
  41. model-00034-of-00061.safetensors +3 -0
  42. model-00035-of-00061.safetensors +3 -0
  43. model-00036-of-00061.safetensors +3 -0
  44. model-00037-of-00061.safetensors +3 -0
  45. model-00038-of-00061.safetensors +3 -0
  46. model-00039-of-00061.safetensors +3 -0
  47. model-00040-of-00061.safetensors +3 -0
  48. model-00041-of-00061.safetensors +3 -0
  49. model-00042-of-00061.safetensors +3 -0
  50. model-00043-of-00061.safetensors +3 -0
LICENSE.txt ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Databricks Open Model License
2
+
3
+ By using, reproducing, modifying, distributing, performing or displaying
4
+ any portion or element of DBRX or DBRX Derivatives, or otherwise accepting
5
+ the terms of this Agreement, you agree to be bound by this Agreement.
6
+
7
+ Version Release Date: March 27, 2024
8
+
9
+
10
+ Section 1: Definitions
11
+
12
+ “Agreement” means these terms and conditions that govern the use, reproduction,
13
+ modification, distribution, performance or display of DBRX and/or DBRX
14
+ Derivatives and any terms and conditions incorporated by reference.
15
+
16
+ “Databricks” or “we” means Databricks, Inc.
17
+
18
+ “Licensee” or “you” means you, or your employer or any other person or entity
19
+ (if you are entering into this Agreement on such person or entity’s behalf),
20
+ of the age required under applicable laws, rules or regulations to provide
21
+ legal consent and that has legal authority to bind your employer or such other
22
+ person or entity if you are entering in this Agreement on their behalf.
23
+
24
+ “DBRX Derivatives” means all (i) modifications to DBRX, (ii) works based on
25
+ DBRX and (iii) any other derivative works thereof. Outputs are not deemed DBRX
26
+ Derivatives.
27
+
28
+ “DBRX” means the foundational large language models and software and
29
+ algorithms, including machine-learning model code, trained model weights,
30
+ inference-enabling code, training-enabling code, fine-tuning enabling code,
31
+ documentation and other elements of the foregoing identified by Databricks at
32
+ https://github.com/databricks/dbrx, regardless of the source that you obtained
33
+ it from.
34
+
35
+ “Output” means the results of operating DBRX or DBRX Derivatives.
36
+
37
+ As used in this Agreement, “including” means “including without limitation.”
38
+
39
+
40
+ Section 2: License Rights and Conditions on Use and Distribution
41
+
42
+ 2.1 Grant of Rights
43
+
44
+ You are granted a non-exclusive, worldwide, non-transferable and royalty-free
45
+ limited license under Databricks’ intellectual property or other rights owned
46
+ by Databricks embodied in DBRX to use, reproduce, distribute, copy, modify,
47
+ and create derivative works of DBRX in accordance with the terms of this
48
+ Agreement.
49
+
50
+ 2.2 Reproduction and Distribution
51
+
52
+ 1. All distributions of DBRX or DBRX Derivatives must be accompanied by a
53
+ "Notice" text file that contains the following notice: "DBRX is provided
54
+ under and subject to the Databricks Open Model License, Copyright ©
55
+ Databricks, Inc. All rights reserved."
56
+
57
+ 2. If you distribute or make DBRX or DBRX Derivatives available to a third
58
+ party, you must provide a copy of this Agreement to such third party.
59
+
60
+ 3. You must cause any modified files that you distribute to carry prominent
61
+ notices stating that you modified the files.
62
+
63
+ You may add your own intellectual property statement to your modifications of
64
+ DBRX and, except as set forth in this Section, may provide additional or
65
+ different terms and conditions for use, reproduction, or distribution of DBRX
66
+ or DBRX Derivatives as a whole, provided your use, reproduction, modification,
67
+ distribution, performance, and display of DBRX or DBRX Derivatives otherwise
68
+ complies with the terms and conditions of this Agreement. Any additional or
69
+ different terms and conditions you impose must not conflict with the terms of
70
+ this Agreement and in the event of a conflict, the terms and conditions of this
71
+ Agreement shall govern over any such additional or different terms and conditions.
72
+
73
+ 2.3 Use Restrictions
74
+
75
+ You will not use DBRX or DBRX Derivatives or any Output to improve any other
76
+ large language model (excluding DBRX or DBRX Derivatives).
77
+
78
+ You will not use DBRX or DBRX Derivatives:
79
+
80
+ 1. for any restricted use set forth in the Databricks Open Model Acceptable
81
+ Use Policy identified at
82
+ https://www.databricks.com/legal/acceptable-use-policy-open-model
83
+ ("Acceptable Use Policy"), which is hereby incorporated by reference into
84
+ this Agreement; or
85
+
86
+ 2. in violation of applicable laws and regulations.
87
+
88
+ To the maximum extent permitted by law, Databricks reserves the right to
89
+ restrict (remotely or otherwise) usage of DBRX or DBRX Derivatives that
90
+ Databricks reasonably believes are in violation of this Agreement.
91
+
92
+
93
+ Section 3: Additional Commercial Terms
94
+
95
+ If, on the DBRX version release date, the monthly active users of the products
96
+ or services made available by or for Licensee, or Licensee’s affiliates, is
97
+ greater than 700 million monthly active users in the preceding calendar month,
98
+ you must request a license from Databricks, which we may grant to you in our
99
+ sole discretion, and you are not authorized to exercise any of the rights under
100
+ this Agreement unless or until Databricks otherwise expressly grants you such
101
+ rights.
102
+
103
+ If you receive DBRX or DBRX Derivatives from a direct or indirect licensee as
104
+ part of an integrated end user product, then this section (Section 3) of the
105
+ Agreement will not apply to you.
106
+
107
+
108
+ Section 4: Additional Provisions
109
+
110
+ 4.1 Updates
111
+
112
+ Databricks may update DBRX from time to time, and you must make reasonable
113
+ efforts to use the latest version of DBRX.
114
+
115
+ 4.2 Intellectual Property
116
+
117
+ a. No trademark licenses are granted under this Agreement, and in connection
118
+ with DBRX or DBRX Derivatives, neither Databricks nor Licensee may use any name
119
+ or mark owned by or associated with the other or any of its affiliates, except
120
+ as required for reasonable and customary use in describing and redistributing
121
+ DBRX or DBRX Derivatives.
122
+
123
+ b. Subject to Databricks’ ownership of DBRX and DRBX Derivatives made by or for
124
+ Databricks, with respect to any DBRX Derivatives that are made by you, as
125
+ between you and Databricks, you are and will be the owner of such DBRX
126
+ Derivatives.
127
+
128
+ c. Databricks claims no ownership rights in Outputs. You are responsible for
129
+ Outputs and their subsequent uses.
130
+
131
+ d. If you institute litigation or other proceedings against Databricks or any
132
+ entity (including a cross-claim or counterclaim in a lawsuit) alleging that
133
+ DBRX or Outputs or results therefrom, or any portion of any of the foregoing,
134
+ constitutes infringement of intellectual property or other rights owned or
135
+ licensable by you, then any licenses granted to you under this Agreement shall
136
+ terminate as of the date such litigation or claim is filed or instituted. You
137
+ will indemnify and hold harmless Databricks from and against any claim by any
138
+ third party arising out of or related to your use or distribution of DBRX or
139
+ DBRX Derivatives.
140
+
141
+ 4.3 DISCLAIMER OF WARRANTY
142
+
143
+ UNLESS REQUIRED BY APPLICABLE LAW, DBRX AND ANY OUTPUT AND RESULTS THEREFROM
144
+ ARE PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER
145
+ EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE,
146
+ NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU
147
+ ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR
148
+ REDISTRIBUTING DBRX OR DBRX DERIVATIVES AND ANY OUTPUT AND ASSUME ANY RISKS
149
+ ASSOCIATED WITH YOUR USE OF DBRX OR DBRX DERIVATIVES AND ANY OUTPUT AND RESULTS.
150
+
151
+ 4.4 LIMITATION OF LIABILITY
152
+
153
+ IN NO EVENT WILL DATABRICKS OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF
154
+ LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR
155
+ OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY INDIRECT,
156
+ SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF
157
+ DATABRICKS OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE
158
+ FOREGOING.
159
+
160
+ 4.5 Term and Termination
161
+
162
+ The term of this Agreement will commence upon your acceptance of this Agreement
163
+ or access to DBRX or DBRX Derivatives and will continue in full force and
164
+ effect until terminated in accordance with the terms and conditions herein.
165
+ Databricks may terminate this Agreement if you are in breach of any term or
166
+ condition of this Agreement. Upon termination of this Agreement, you shall
167
+ delete and cease use of DBRX or any DBRX Derivatives. Sections 1, 4.2(d), 4.3,
168
+ 4.4, and 4.6 shall survive the termination of this Agreement.
169
+
170
+ 4.6 Governing Law and Jurisdiction
171
+
172
+ This Agreement will be governed and construed under the laws of the State of
173
+ California without regard to choice of law principles, and the UN Convention
174
+ on Contracts for the International Sale of Goods does not apply to this
175
+ Agreement. The courts of California shall have exclusive jurisdiction of any
176
+ dispute arising out of this Agreement.
NOTICE.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ DBRX is provided under and subject to the Databricks Open Model License, Copyright © Databricks, Inc. All rights reserved.
README.md CHANGED
@@ -1,3 +1,170 @@
1
- ---
2
- license: unlicense
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ extra_gated_heading: You need to share contact information with Databricks to access this model
3
+ extra_gated_prompt: >-
4
+
5
+ ### DBRX Terms of Use
6
+
7
+ Use of DBRX is governed by the [Databricks Open Model License](https://www.databricks.com/legal/open-model-license) and the [Databricks Open Model Acceptable Use Policy](https://www.databricks.com/legal/acceptable-use-policy-open-model).
8
+
9
+ extra_gated_fields:
10
+ First Name: text
11
+ Last Name: text
12
+ Organization: text
13
+ By clicking 'Submit' below, I accept the terms of the license and acknowledge that the information I provide will be collected, stored, processed, and shared in accordance with Databricks' Privacy Notice and I understand I can update my preferences at any time: checkbox
14
+ extra_gated_description: >-
15
+ The information you provide will be collected, stored, processed, and shared in accordance with Databricks [Privacy Notice](https://www.databricks.com/legal/privacynotice).
16
+ extra_gated_button_content: Submit
17
+ inference: false
18
+ license: other
19
+ license_name: databricks-open-model-license
20
+ license_link: https://www.databricks.com/legal/open-model-license
21
+ ---
22
+
23
+ # DBRX Instruct
24
+
25
+ * DBRX Instruct is a mixture-of-experts (MoE) large language model trained from scratch by Databricks. DBRX Instruct specializes in few-turn interactions.
26
+ * We are releasing both DBRX Instruct and DBRX Base, the pretrained base model which underlies it, under [an open license](https://www.databricks.com/legal/open-model-license).
27
+ * This is the repository for DBRX Instruct. DBRX Base can be found [here](https://huggingface.co/databricks/dbrx-base).
28
+ * For full details on the DBRX models, please read our [technical blog post](https://www.databricks.com/blog/introducing-dbrx-new-state-art-open-llm).
29
+
30
+
31
+ ## Model Overview
32
+ DBRX is a [transformer-based](https://www.isattentionallyouneed.com/) decoder-only large language model (LLM) that was trained using next-token prediction.
33
+ It uses a *fine-grained* mixture-of-experts (MoE) architecture with 132B total parameters of which 36B parameters are active on any input.
34
+ It was pre-trained on 12T tokens of text and code data.
35
+ Compared to other open MoE models like Mixtral-8x7B and Grok-1, DBRX is fine-grained, meaning it uses a larger number of smaller experts. DBRX has 16 experts and chooses 4, while Mixtral-8x7B and Grok-1 have 8 experts and choose 2.
36
+ This provides 65x more possible combinations of experts and we found that this improves model quality.
37
+ DBRX uses rotary position encodings (RoPE), gated linear units (GLU), and grouped query attention (GQA).
38
+ It uses the GPT-4 tokenizer as provided in the [tiktoken](https://github.com/openai/tiktoken) repository.
39
+ We made these choices based on exhaustive evaluation and scaling experiments.
40
+
41
+ DBRX was pretrained on 12T tokens of carefully curated data and a maximum context length of 32K tokens.
42
+ We estimate that this data is at least 2x better token-for-token than the data we used to pretrain the MPT family of models.
43
+ This new dataset was developed using the full suite of Databricks tools, including Apache Spark™ and Databricks notebooks for data processing, and Unity Catalog for data management and governance.
44
+ We used curriculum learning for pretraining, changing the data mix during training in ways we found to substantially improve model quality.
45
+
46
+ * **Inputs:** DBRX only accepts text-based inputs and accepts a context length of up to 32768 tokens.
47
+ * **Outputs:** DBRX only produces text-based outputs.
48
+ * **Model Architecture:** More detailed information about DBRX Instruct and DBRX Base can be found in our [technical blog post](https://www.databricks.com/blog/introducing-dbrx-new-state-art-open-llm).
49
+ * **License:** [Databricks Open Model License](https://www.databricks.com/legal/open-model-license)
50
+ * **Acceptable Use Policy:** [Databricks Open Model Acceptable Use Policy](https://www.databricks.com/legal/acceptable-use-policy-open-model)
51
+ * **Version:** 1.0
52
+ * **Owner:** Databricks, Inc.
53
+
54
+
55
+ ## Usage
56
+ These are several general ways to use the DBRX models:
57
+ * DBRX Base and DBRX Instruct are available for download on HuggingFace (see our Quickstart guide below). This is the HF repository for DBRX Instruct; DBRX Base can be found [here](https://huggingface.co/databricks/dbrx-base).
58
+ * The DBRX model repository can be found on GitHub [here](https://github.com/databricks/dbrx).
59
+ * DBRX Base and DBRX Instruct are available with [Databricks Foundation Model APIs](https://docs.databricks.com/en/machine-learning/foundation-models/index.html) via both *Pay-per-token* and *Provisioned Throughput* endpoints. These are enterprise-ready deployments.
60
+ * For more information on how to fine-tune using LLM-Foundry, please take a look at our LLM pretraining and fine-tuning [documentation](https://github.com/mosaicml/llm-foundry/blob/main/scripts/train/README.md).
61
+
62
+
63
+ ## Quickstart Guide
64
+ **NOTE: This is DBRX Instruct, and has been instruction finetuned.**
65
+ If you are looking for the base model, please use [DBRX Base](https://huggingface.co/databricks/dbrx-base).
66
+
67
+ Getting started with DBRX models is easy with the `transformers` library. The model requires ~264GB of RAM and the following packages:
68
+
69
+ ```bash
70
+ pip install transformers tiktoken
71
+ ```
72
+
73
+ If you'd like to speed up download time, you can use the `hf_transfer` package as described by Huggingface [here](https://huggingface.co/docs/huggingface_hub/en/guides/download#faster-downloads).
74
+ ```bash
75
+ pip install hf_transfer
76
+ export HF_HUB_ENABLE_HF_TRANSFER=1
77
+ ```
78
+
79
+ ### Run the model on a CPU:
80
+ ```python
81
+ from transformers import AutoTokenizer, AutoModelForCausalLM
82
+ import torch
83
+
84
+ tokenizer = AutoTokenizer.from_pretrained("databricks/dbrx-instruct", trust_remote_code=True)
85
+ model = AutoModelForCausalLM.from_pretrained("databricks/dbrx-instruct", device_map="cpu", torch_dtype=torch.bfloat16, trust_remote_code=True)
86
+
87
+ input_text = "What does it take to build a great LLM?"
88
+ messages = [{"role": "user", "content": input_text}]
89
+ input_ids = tokenizer.apply_chat_template(messages, return_dict=True, tokenize=True, add_generation_prompt=True, return_tensors="pt")
90
+
91
+ outputs = model.generate(**input_ids, max_new_tokens=200)
92
+ print(tokenizer.decode(outputs[0]))
93
+ ```
94
+
95
+ ### Run the model on multiple GPUs:
96
+ ```python
97
+ from transformers import AutoTokenizer, AutoModelForCausalLM
98
+ import torch
99
+
100
+ tokenizer = AutoTokenizer.from_pretrained("databricks/dbrx-instruct", trust_remote_code=True)
101
+ model = AutoModelForCausalLM.from_pretrained("databricks/dbrx-instruct", device_map="auto", torch_dtype=torch.bfloat16, trust_remote_code=True)
102
+
103
+ input_text = "What does it take to build a great LLM?"
104
+ messages = [{"role": "user", "content": input_text}]
105
+ input_ids = tokenizer.apply_chat_template(messages, return_dict=True, tokenize=True, add_generation_prompt=True, return_tensors="pt").to("cuda")
106
+
107
+ outputs = model.generate(**input_ids, max_new_tokens=200)
108
+ print(tokenizer.decode(outputs[0]))
109
+ ```
110
+ If your GPU system supports [FlashAttention2](https://huggingface.co/docs/transformers/perf_infer_gpu_one#flashattention-2), you can add `attn_implementation=”flash_attention_2”` as a keyword to `AutoModelForCausalLM.from_pretrained()` to achieve faster inference.
111
+
112
+
113
+ ## Limitations and Ethical Considerations
114
+ ### Training Dataset Limitations
115
+ The DBRX models were trained on 12T tokens of text, with a knowledge cutoff date of December 2023.
116
+
117
+ The training mix used for DBRX contains both natural-language and code examples. The vast majority of our training data is in the English language. We did not test DBRX for non-English proficiency. Therefore, DBRX should be considered a generalist model for text-based use in the English language.
118
+
119
+ DBRX does not have multimodal capabilities.
120
+
121
+ ### Associated Risks and Recommendations
122
+ All foundation models are novel technologies that carry various risks, and may output information that is inaccurate, incomplete, biased, or offensive.
123
+ Users should exercise judgment and evaluate such output for accuracy and appropriateness for their desired use case before using or sharing it.
124
+ Databricks recommends [using retrieval augmented generation (RAG)](https://www.databricks.com/glossary/retrieval-augmented-generation-rag) in scenarios where accuracy and fidelity are important.
125
+ We also recommend that anyone using or fine-tuning either DBRX Base or DBRX Instruct perform additional testing around safety in the context of their particular application and domain.
126
+
127
+
128
+ ## Intended Uses
129
+ ### Intended Use Cases
130
+ The DBRX models are open, general-purpose LLMs intended and licensed for both commercial and research applications.
131
+ They can be further fine-tuned for various domain-specific natural language and coding tasks.
132
+ DBRX Instruct can be used as an off-the-shelf model for few-turn question answering related to general English-language and coding tasks.
133
+
134
+ Please review the Associated Risks section above, as well as the [Databricks Open Model License](https://www.databricks.com/legal/open-model-license) and [Databricks Open Model Acceptable Use Policy](https://www.databricks.com/legal/acceptable-use-policy-open-model) for further information about permissible uses of DBRX Base and its derivatives.
135
+
136
+ ### Out-of-Scope Use Cases
137
+ DBRX models are not intended to be used out-of-the-box in non-English languages and do not support native code execution, or other forms of function-calling.
138
+ DBRX models should not be used in any manner that violates applicable laws or regulations or in any other way that is prohibited by the [Databricks Open Model License](https://www.databricks.com/legal/open-model-license) and [Databricks Open Model Acceptable Use Policy](https://www.databricks.com/legal/acceptable-use-policy-open-model).
139
+
140
+
141
+ ## Training Stack
142
+ MoE models are complicated to train, and the training of DBRX Base and DBRX Instruct was heavily supported by Databricks’ infrastructure for data processing and large-scale LLM training (e.g., [Composer](https://github.com/mosaicml/composer), [Streaming](https://github.com/mosaicml/streaming), [Megablocks](https://github.com/stanford-futuredata/megablocks), and [LLM Foundry](https://github.com/mosaicml/llm-foundry)).
143
+
144
+ Composer is our core library for large-scale training.
145
+ It provides an optimized training loop, easy [checkpointing](https://docs.mosaicml.com/projects/composer/en/latest/trainer/checkpointing.html) and [logging](https://docs.mosaicml.com/projects/composer/en/latest/trainer/logging.html#wood-logging),
146
+ [FSDP](https://pytorch.org/docs/stable/fsdp.html)-based [model sharding](https://docs.mosaicml.com/projects/composer/en/latest/notes/distributed_training.html#fullyshardeddataparallel-fsdp),
147
+ convenient [abstractions](https://docs.mosaicml.com/projects/composer/en/latest/trainer/time.html), extreme customizability via [callbacks](https://docs.mosaicml.com/projects/composer/en/latest/trainer/callbacks.html), and more.
148
+
149
+ Streaming enables fast, low cost, and scalable training on large datasets from cloud storage. It handles a variety of challenges around deterministic resumption as node counts change, avoiding redundant downloads across devices, high-quality shuffling at scale, sample-level random access, and speed.
150
+
151
+ Megablocks is a lightweight library for MoE training. Crucially, it supports “dropless MoE,” which avoids inefficient padding and is intended to provide deterministic outputs for a given sequence no matter what other sequences are in the batch.
152
+
153
+ LLM Foundry ties all of these libraries together to create a simple LLM pretraining, fine-tuning, and inference experience.
154
+
155
+ DBRX was trained using proprietary optimized versions of the above open source libraries, along with our [LLM training platform](https://www.databricks.com/product/machine-learning/mosaic-ai-training).
156
+
157
+
158
+ ## Evaluation
159
+ We find that DBRX outperforms established open-source and open-weight base models on the [Databricks Model Gauntlet](https://www.databricks.com/blog/llm-evaluation-for-icl), the [Hugging Face Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard), and HumanEval.
160
+ The Databricks Model Gauntlet measures performance on more than 30 tasks across six categories: world knowledge, common sense reasoning, language understanding, reading comprehension, symbolic problem solving, and programming.
161
+ The Hugging Face Open LLM Leaderboard measures the average of ARC-Challenge, HellaSwag, MMLU, TruthfulQA, Winogrande and GSM8k.
162
+ HumanEval measures coding ability.
163
+
164
+ Full evaluation details can be found in our [technical blog post](https://www.databricks.com/blog/introducing-dbrx-new-state-art-open-llm).
165
+
166
+
167
+ ## Acknowledgements
168
+ The DBRX models were made possible thanks in large part to the open-source community, especially:
169
+ * The [MegaBlocks](https://arxiv.org/abs/2211.15841) library, which established a foundation for our MoE implementation.
170
+ * [PyTorch FSDP](https://arxiv.org/abs/2304.11277), which we built on for distributed training.
added_tokens.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "<|im_end|>": 100279,
3
+ "<|im_start|>": 100278,
4
+ "<|pad|>": 100277
5
+ }
config.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "DbrxForCausalLM"
4
+ ],
5
+ "attn_config": {
6
+ "clip_qkv": 8,
7
+ "kv_n_heads": 8,
8
+ "model_type": "",
9
+ "rope_theta": 500000
10
+ },
11
+ "auto_map": {
12
+ "AutoConfig": "configuration_dbrx.DbrxConfig",
13
+ "AutoModelForCausalLM": "modeling_dbrx.DbrxForCausalLM"
14
+ },
15
+ "d_model": 6144,
16
+ "emb_pdrop": 0.0,
17
+ "ffn_config": {
18
+ "ffn_hidden_size": 10752,
19
+ "model_type": "",
20
+ "moe_jitter_eps": 0,
21
+ "moe_loss_weight": 0.05,
22
+ "moe_num_experts": 16,
23
+ "moe_top_k": 4,
24
+ "ffn_act_fn": {
25
+ "name": "silu"
26
+ }
27
+ },
28
+ "initializer_range": 0.02,
29
+ "max_seq_len": 32768,
30
+ "model_type": "dbrx_converted",
31
+ "n_heads": 48,
32
+ "n_layers": 40,
33
+ "output_router_logits": false,
34
+ "resid_pdrop": 0.0,
35
+ "router_aux_loss_coef": 0.05,
36
+ "tie_word_embeddings": false,
37
+ "torch_dtype": "bfloat16",
38
+ "transformers_version": "4.38.2",
39
+ "use_cache": true,
40
+ "vocab_size": 100352
41
+ }
configuration_dbrx.py ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Dbrx configuration."""
2
+ from typing import Any, Optional
3
+
4
+ from transformers.configuration_utils import PretrainedConfig
5
+ from transformers.utils import logging
6
+
7
+ logger = logging.get_logger(__name__)
8
+
9
+ DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
10
+
11
+
12
+ class DbrxAttentionConfig(PretrainedConfig):
13
+ """Configuration class for Dbrx Attention.
14
+
15
+ [`DbrxAttention`] class. It is used to instantiate attention layers
16
+ according to the specified arguments, defining the layers architecture.
17
+
18
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
19
+ documentation from [`PretrainedConfig`] for more information.
20
+
21
+ Args:
22
+ attn_pdrop (`float`, *optional*, defaults to 0.0):
23
+ The dropout probability for the attention layers.
24
+ clip_qkv (`float`, *optional*, defualts to None):
25
+ If not `None`, clip the queries, keys, and values in the attention layer to this value.
26
+ kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads.
27
+ rope_theta (float): The base frequency for rope.
28
+ """
29
+
30
+ def __init__(
31
+ self,
32
+ attn_pdrop: float = 0,
33
+ clip_qkv: Optional[float] = None,
34
+ kv_n_heads: int = 1,
35
+ rope_theta: float = 10000.0,
36
+ **kwargs: Any,
37
+ ):
38
+ super().__init__(**kwargs)
39
+ self.attn_pdrop = attn_pdrop
40
+ self.clip_qkv = clip_qkv
41
+ self.kv_n_heads = kv_n_heads
42
+ self.rope_theta = rope_theta
43
+
44
+ for k in ['model_type']:
45
+ if k in kwargs:
46
+ kwargs.pop(k)
47
+ if len(kwargs) != 0:
48
+ raise ValueError(f'Found unknown {kwargs=}')
49
+
50
+ @classmethod
51
+ def from_pretrained(cls, pretrained_model_name_or_path: str,
52
+ **kwargs: Any) -> 'PretrainedConfig':
53
+ cls._set_token_in_kwargs(kwargs)
54
+
55
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path,
56
+ **kwargs)
57
+
58
+ if config_dict.get('model_type') == 'dbrx':
59
+ config_dict = config_dict['attn_config']
60
+
61
+ if 'model_type' in config_dict and hasattr(
62
+ cls,
63
+ 'model_type') and config_dict['model_type'] != cls.model_type:
64
+ logger.warning(
65
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
66
+ +
67
+ f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'
68
+ )
69
+
70
+ return cls.from_dict(config_dict, **kwargs)
71
+
72
+
73
+ class DbrxFFNConfig(PretrainedConfig):
74
+ """Configuration class for Dbrx FFN.
75
+
76
+ [`DbrxFFN`] class. It is used to instantiate feedforward layers according to
77
+ the specified arguments, defining the layers architecture.
78
+
79
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
80
+ documentation from [`PretrainedConfig`] for more information.
81
+
82
+ Args:
83
+ ffn_act_fn (dict, optional): A dict specifying activation function for the FFN.
84
+ The dict should have a key 'name' with the value being the name of
85
+ the activation function along with any additional keyword arguments.
86
+ ffn_hidden_size (int, optional): The hidden size of the feedforward network.
87
+ moe_num_experts (int, optional): The number of experts in the mixture of experts layer.
88
+ moe_top_k (int, optional): The number of experts to use in the mixture of experts layer.
89
+ moe_jitter_eps (float, optional): The jitter epsilon for the mixture of experts layer.
90
+ moe_loss_weight (float, optional): The loss weight for the mixture of experts layer.
91
+ moe_normalize_expert_weights (float, optional): The normalization factor for the expert weights.
92
+ uniform_expert_assignment (bool, optional): Whether to use uniform expert assignment.
93
+ This should only be used for benchmarking purposes.
94
+ """
95
+
96
+ def __init__(
97
+ self,
98
+ ffn_act_fn: Optional[dict] = None,
99
+ ffn_hidden_size: int = 3584,
100
+ moe_num_experts: int = 4,
101
+ moe_top_k: int = 1,
102
+ moe_jitter_eps: Optional[float] = None,
103
+ moe_loss_weight: float = 0.01,
104
+ moe_normalize_expert_weights: Optional[float] = 1,
105
+ uniform_expert_assignment: bool = False,
106
+ **kwargs: Any,
107
+ ):
108
+ super().__init__()
109
+ if ffn_act_fn is None:
110
+ ffn_act_fn = {'name': 'silu'}
111
+ self.ffn_act_fn = ffn_act_fn
112
+ self.ffn_hidden_size = ffn_hidden_size
113
+ self.moe_num_experts = moe_num_experts
114
+ self.moe_top_k = moe_top_k
115
+ self.moe_jitter_eps = moe_jitter_eps
116
+ self.moe_loss_weight = moe_loss_weight
117
+ self.moe_normalize_expert_weights = moe_normalize_expert_weights
118
+ self.uniform_expert_assignment = uniform_expert_assignment
119
+
120
+ for k in ['model_type']:
121
+ if k in kwargs:
122
+ kwargs.pop(k)
123
+ if len(kwargs) != 0:
124
+ raise ValueError(f'Found unknown {kwargs=}')
125
+
126
+ @classmethod
127
+ def from_pretrained(cls, pretrained_model_name_or_path: str,
128
+ **kwargs: Any) -> 'PretrainedConfig':
129
+ cls._set_token_in_kwargs(kwargs)
130
+
131
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path,
132
+ **kwargs)
133
+
134
+ if config_dict.get('model_type') == 'dbrx':
135
+ config_dict = config_dict['ffn_config']
136
+
137
+ if 'model_type' in config_dict and hasattr(
138
+ cls,
139
+ 'model_type') and config_dict['model_type'] != cls.model_type:
140
+ logger.warning(
141
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
142
+ +
143
+ f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'
144
+ )
145
+
146
+ return cls.from_dict(config_dict, **kwargs)
147
+
148
+
149
+ class DbrxConfig(PretrainedConfig):
150
+ """Configuration class for Dbrx.
151
+
152
+ [`DbrxModel`]. It is used to instantiate a Dbrx model according to the
153
+ specified arguments, defining the model architecture.
154
+
155
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
156
+ documentation from [`PretrainedConfig`] for more information.
157
+
158
+
159
+ Args:
160
+ d_model (`int`, *optional*, defaults to 6144):
161
+ Dimensionality of the embeddings and hidden states.
162
+ n_heads (`int`, *optional*, defaults to 48):
163
+ Number of attention heads for each attention layer in the Transformer encoder.
164
+ n_layers (`int`, *optional*, defaults to 40):
165
+ Number of hidden layers in the Transformer encoder.
166
+ max_seq_len (`int`, *optional*, defaults to 32768):
167
+ The maximum sequence length of the model.
168
+ vocab_size (`int`, *optional*, defaults to 100352):
169
+ Vocabulary size of the Dbrx model. Defines the maximum number of different tokens that can be represented by
170
+ the `inputs_ids` passed when calling [`DbrxModel`].
171
+ resid_pdrop (`float`, *optional*, defaults to 0.0):
172
+ The dropout probability applied to the attention output before combining with residual.
173
+ emb_pdrop (`float`, *optional*, defaults to 0.0):
174
+ The dropout probability for the embedding layer.
175
+ attn_config (`dict`, *optional*):
176
+ A dictionary used to configure the model's attention module.
177
+ ffn_config (`dict`, *optional*):
178
+ A dictionary used to configure the model's FFN module.
179
+ use_cache (`bool`, *optional*, defaults to `False`):
180
+ Whether or not the model should return the last key/values attentions (not used by all models).
181
+ initializer_range (`float`, *optional*, defaults to 0.02):
182
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
183
+ output_router_logits (`bool`, *optional*, defaults to `False`):
184
+ Whether or not the router logits should be returned by the model. Enabling this will also
185
+ allow the model to output the auxiliary loss. See [here]() for more details
186
+ router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
187
+ The aux loss factor for the total loss.
188
+
189
+
190
+ Example:
191
+ ```python
192
+ >>> from transformers import DbrxConfig, DbrxModel
193
+
194
+ >>> # Initializing a Dbrx configuration
195
+ >>> configuration = DbrxConfig()
196
+
197
+ >>> # Initializing a model (with random weights) from the configuration
198
+ >>> model = DbrxModel(configuration)
199
+
200
+ >>> # Accessing the model configuration
201
+ >>> configuration = model.config
202
+ ```
203
+ """
204
+
205
+ model_type = 'dbrx'
206
+ attribute_map = {
207
+ 'num_attention_heads': 'n_heads',
208
+ 'hidden_size': 'd_model',
209
+ 'num_hidden_layers': 'n_layers',
210
+ 'max_position_embeddings': 'max_seq_len'
211
+ }
212
+
213
+ def __init__(
214
+ self,
215
+ d_model: int = 2048,
216
+ n_heads: int = 16,
217
+ n_layers: int = 24,
218
+ max_seq_len: int = 2048,
219
+ vocab_size: int = 32000,
220
+ resid_pdrop: float = 0.0,
221
+ emb_pdrop: float = 0.0,
222
+ attn_config: Optional[DbrxAttentionConfig] = None,
223
+ ffn_config: Optional[DbrxFFNConfig] = None,
224
+ use_cache: bool = True,
225
+ initializer_range: float = 0.02,
226
+ output_router_logits: bool = False,
227
+ router_aux_loss_coef: float = 0.05,
228
+ **kwargs: Any,
229
+ ):
230
+ if attn_config is None:
231
+ self.attn_config = DbrxAttentionConfig()
232
+ elif isinstance(attn_config, dict):
233
+ self.attn_config = DbrxAttentionConfig(**attn_config)
234
+ else:
235
+ self.attn_config = attn_config
236
+
237
+ if ffn_config is None:
238
+ self.ffn_config = DbrxFFNConfig()
239
+ elif isinstance(ffn_config, dict):
240
+ self.ffn_config = DbrxFFNConfig(**ffn_config)
241
+ else:
242
+ self.ffn_config = ffn_config
243
+
244
+ self.d_model = d_model
245
+ self.n_heads = n_heads
246
+ self.n_layers = n_layers
247
+ self.max_seq_len = max_seq_len
248
+ self.vocab_size = vocab_size
249
+ self.resid_pdrop = resid_pdrop
250
+ self.emb_pdrop = emb_pdrop
251
+ self.use_cache = use_cache
252
+ self.initializer_range = initializer_range
253
+ self.output_router_logits = output_router_logits
254
+ self.router_aux_loss_coef = router_aux_loss_coef
255
+
256
+ tie_word_embeddings = kwargs.pop('tie_word_embeddings', False)
257
+ if tie_word_embeddings:
258
+ raise ValueError(
259
+ 'tie_word_embeddings is not supported for Dbrx models.')
260
+
261
+ super().__init__(
262
+ tie_word_embeddings=tie_word_embeddings,
263
+ **kwargs,
264
+ )
generation_config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": [
4
+ 100257,
5
+ 100279
6
+ ],
7
+ "transformers_version": "4.38.2"
8
+ }
model-00001-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5cfd2e4475dbdb8e5225be9fb2c13ff271d6c2523339293eb04bbdef07fa5078
3
+ size 3523439624
model-00002-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:11def40cfb9af754306a54e2edc109866edc84c574d3fcc8b1c394baefa75961
3
+ size 4404245416
model-00003-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e72c1494f4bfa8409e1d0a74fb11b6d63107eabef89309392370ce7900929eb
3
+ size 4227862560
model-00004-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c294ef3f43892d221a12196cd44aac576073babd7a4e463340ee07360b477961
3
+ size 4404245416
model-00005-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cea2c8c07e5f49364e116cf51f7f8dc9e0f4543d6c3456bd127f8d9a244e00d2
3
+ size 4404245416
model-00006-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c1b51cc32f2c535f528d69c43ddc4d48855ec3a8847b9e584e28c68f9efc786
3
+ size 4227862560
model-00007-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a7ba6917cc103162fcf73a706db91d8e02a6dcaff8157127333356ce6b51878
3
+ size 4404245416
model-00008-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ad7178e6f415ccfa44d29ad2f6fbd7ce89fc4d519530cfe493ea33d27ff79c3d
3
+ size 4404245416
model-00009-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7335c39b0e58a2e9e9a88e648effc97bdab49f5be42c7914bde7f039b6924f47
3
+ size 4227862560
model-00010-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a963a521e5fdb4c80a00cc66797cfcf3c02150fcb0ad519de947b27167218fd6
3
+ size 4404245416
model-00011-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51e727ee777189223e273a16507b60b76611db9a20a3a2d8064721d82442005c
3
+ size 4404245416
model-00012-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f50c19c0c494a5bfa78403e4faa431a579f9d9ef353e88c3f8bac123e10e5b2
3
+ size 4227862560
model-00013-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:edcdae75d723df47e628931df522e988d8e32347ef5aaf23196664bbf50bfd5e
3
+ size 4404245416
model-00014-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f576ef3d08ce2450e603d1f9a07e6be0d48af371f25f0f7cb062dd995ce2326b
3
+ size 4404245416
model-00015-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a659155cfb439471ef08e8aceb5dd9010ec1efb8f39e19342e81a9e40a8c9a7
3
+ size 4227862560
model-00016-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc90e6edca1c8206bef1b413647dd1fda65cb76d5bc6b4bfdae43c2cb0697a8d
3
+ size 4404245440
model-00017-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b5058f52c175ebce3409cced0a660d129109d7b257a63f5386fde072f7f8db6
3
+ size 4404245456
model-00018-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78dd6843d4539850acc3e2548bbe52726a50b09d05cc58984a0c11b78af6ed42
3
+ size 4227862592
model-00019-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08e60407654d1206b8061c503be545fa89ec96df5158e6c2efc242644467aacc
3
+ size 4404245456
model-00020-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:518b6178d6a0e7bd49cbb7dc6a291fb0f5c13db61528cc4f5094194568f4f01c
3
+ size 4404245456
model-00021-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4691ae48cb49b454351a8a4fed3ca0d0f1fef3a044d9ceb3ae538632ee22e0fb
3
+ size 4227862592
model-00022-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a76e78c457a6778629f4b124ad2c819a3d748eeb3d9d2db0bce5be04e047936
3
+ size 4404245456
model-00023-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06c89d26a2458d3dfab986b5d0990e5c5e6b22740dbba8982198c9ab00fe7b5e
3
+ size 4404245456
model-00024-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f48f04d4622d45afb74f97837453d8f1eee3d897d83d7f936b2a0ea78acb8bbf
3
+ size 4227862592
model-00025-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bed7741084ba9753eeaefdf57851c75ba0b2fc4efd313a4104543f4f392caeb5
3
+ size 4404245456
model-00026-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f9a82048271b6463de02756d2c6137ad77143974340209eab518f2eb9f87b69
3
+ size 4404245456
model-00027-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3dfa7c1a7243967bdeed47b58e0a6798bfb6fc6debdce2eed1fe7b6a243d41b0
3
+ size 4227862592
model-00028-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ad21eb9485005191f5ad41bf0067b54a168b6b4485df4ce2ab5394f95006884
3
+ size 4404245456
model-00029-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bdda9f1081651d39f5ac0c528117804f1fe8634c01866deebcb4266236b41e7d
3
+ size 4404245456
model-00030-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:34d8d6d24a74ebcd861bed5709bcb8ade5595734bb8f79d1e72e725bdebbc83a
3
+ size 4227862592
model-00031-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:917ebdf9527144865c1a030e9caf4f119a54ecd814088aac1fc3a41dc029a862
3
+ size 4404245456
model-00032-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42e1318b5abb7c2551069ece2733e89a23be885d4b509904b7c0009b811e729a
3
+ size 4404245456
model-00033-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e52a05f9a898caf58749aa641205ce316eefe3d8654c46b2c06271e27093b51
3
+ size 4227862592
model-00034-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbb6c4a96aada8631f017abcbc81fc2156c01be3feedee317fe0e73f99d404f5
3
+ size 4404245456
model-00035-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b78279bf04262ca11571270c1f920eff491a7ef144b5539f0479f380ec5cc77
3
+ size 4404245456
model-00036-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79a15bc8adf90a283a34904d44df5c8631876651e3cc0323db3dafaef65ac6fd
3
+ size 4227862592
model-00037-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fea62a393eefa318c9e3b862936850fce68958eb271a38666ca39f2ad2ee7b48
3
+ size 4404245456
model-00038-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8965ec08934446f3cd1a64373458d22cbfc75b5a3eab7ba0559782a54b8c41ec
3
+ size 4404245456
model-00039-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7bf53ce714b30cf246e5fbfee7c29a8e4a83539439d900984d03254321b9290a
3
+ size 4227862592
model-00040-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1761de629f4b6a779ad431439f2c9114cc3dfbe89a3d177d58cddcfe83831a38
3
+ size 4404245456
model-00041-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b47d80d7eeb65f860b86d6e77ba99d504ae2aeb4babc3b6b21897555d3af5621
3
+ size 4404245456
model-00042-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d244c2975aa94e2827d6bb04854943b27c3c70e115fe8c743a840632c323b0d
3
+ size 4227862592
model-00043-of-00061.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7259b5a8df751cca042f09c6f634d272bb0a53054dec2860621b713268096735
3
+ size 4404245456