JunlinHan commited on
Commit
b579854
1 Parent(s): f84c01f

Upload 61 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -0
  2. CODE_OF_CONDUCT.md +80 -0
  3. CONTRIBUTING.md +31 -0
  4. LICENSE +399 -0
  5. README.md +112 -3
  6. __init__.py +27 -0
  7. assets/40_prompt_images/A 3D scan of AK47, weapon.jpeg +0 -0
  8. assets/40_prompt_images/A DSLR photo of Sydney Opera House.jpg +0 -0
  9. assets/40_prompt_images/A bald eagle carved out of wood.jpg +0 -0
  10. assets/40_prompt_images/A bulldog wearing a black pirate hat.jpeg +0 -0
  11. assets/40_prompt_images/A crab, low poly.jpg +0 -0
  12. assets/40_prompt_images/A photo of a horse walking.jpeg +0 -0
  13. assets/40_prompt_images/A pig wearing a backpack.jpeg +0 -0
  14. assets/40_prompt_images/A product photo of a toy tank.jpg +0 -0
  15. assets/40_prompt_images/A see no evil monkey on a kick drum.jpg +0 -0
  16. assets/40_prompt_images/A statue of angel, blender.jpg +0 -0
  17. assets/40_prompt_images/Corgi riding a rocket.jpeg +0 -0
  18. assets/40_prompt_images/Daenerys Targaryen from game of throne.jpg +0 -0
  19. assets/40_prompt_images/Darth Vader helmet,g highly detailed.jpg +0 -0
  20. assets/40_prompt_images/Dragon armor.jpeg +0 -0
  21. assets/40_prompt_images/Fisherman House, cute, cartoon, blender, stylized.jpg +0 -0
  22. assets/40_prompt_images/Flying Dragon, highly detailed, breathing fire.jpeg +0 -0
  23. assets/40_prompt_images/Handpainted watercolor windmill, hand-painted.jpg +0 -0
  24. assets/40_prompt_images/Katana.jpeg +0 -0
  25. assets/40_prompt_images/Little italian town, hand-painted style.jpg +0 -0
  26. assets/40_prompt_images/Mr Bean Cartoon doing a T Pose.jpg +0 -0
  27. assets/40_prompt_images/Pedestal Fan (White).jpeg +0 -0
  28. assets/40_prompt_images/Pikachu with hat.jpg +0 -0
  29. assets/40_prompt_images/Samurai koala bear.jpg +0 -0
  30. assets/40_prompt_images/TRUMP figure.jpg +0 -0
  31. assets/40_prompt_images/Viking axe, fantasy, weapon, blender, 8k, HD.jpg +0 -0
  32. assets/40_prompt_images/a DSLR photo of a frog wearing a sweater.jpg +0 -0
  33. assets/40_prompt_images/a DSLR photo of a ghost eating a hamburger.jpg +0 -0
  34. assets/40_prompt_images/a DSLR photo of a peacock on a surfboard.jpeg +0 -0
  35. assets/40_prompt_images/a DSLR photo of a squirrel playing guitar.jpg +0 -0
  36. assets/40_prompt_images/a DSLR photo of an eggshell broken in two with an adorable chick standing next to it.jpeg +0 -0
  37. assets/40_prompt_images/an astronaut riding a horse.jpeg +0 -0
  38. assets/40_prompt_images/animal skull pile.jpg +0 -0
  39. assets/40_prompt_images/army Jacket, 3D scan.jpg +0 -0
  40. assets/40_prompt_images/baby yoda in the style of Mormookiee.jpg +0 -0
  41. assets/40_prompt_images/beautiful, intricate butterfly.jpg +0 -0
  42. assets/40_prompt_images/girl riding wolf, cute, cartoon, blender.jpg +0 -0
  43. assets/40_prompt_images/mecha vampire girl chibi.jpg +0 -0
  44. assets/40_prompt_images/military Mech, future, scifi.jpg +0 -0
  45. assets/40_prompt_images/motorcycle, scifi, blender.jpeg +0 -0
  46. assets/40_prompt_images/saber from fate stay night, 3D, girl, anime.jpeg +0 -0
  47. assets/gif1.gif +3 -0
  48. assets/gif2.gif +3 -0
  49. assets/user.png +0 -0
  50. config.json +27 -0
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ assets/gif1.gif filter=lfs diff=lfs merge=lfs -text
37
+ assets/gif2.gif filter=lfs diff=lfs merge=lfs -text
CODE_OF_CONDUCT.md ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Code of Conduct
2
+
3
+ ## Our Pledge
4
+
5
+ In the interest of fostering an open and welcoming environment, we as
6
+ contributors and maintainers pledge to make participation in our project and
7
+ our community a harassment-free experience for everyone, regardless of age, body
8
+ size, disability, ethnicity, sex characteristics, gender identity and expression,
9
+ level of experience, education, socio-economic status, nationality, personal
10
+ appearance, race, religion, or sexual identity and orientation.
11
+
12
+ ## Our Standards
13
+
14
+ Examples of behavior that contributes to creating a positive environment
15
+ include:
16
+
17
+ * Using welcoming and inclusive language
18
+ * Being respectful of differing viewpoints and experiences
19
+ * Gracefully accepting constructive criticism
20
+ * Focusing on what is best for the community
21
+ * Showing empathy towards other community members
22
+
23
+ Examples of unacceptable behavior by participants include:
24
+
25
+ * The use of sexualized language or imagery and unwelcome sexual attention or
26
+ advances
27
+ * Trolling, insulting/derogatory comments, and personal or political attacks
28
+ * Public or private harassment
29
+ * Publishing others' private information, such as a physical or electronic
30
+ address, without explicit permission
31
+ * Other conduct which could reasonably be considered inappropriate in a
32
+ professional setting
33
+
34
+ ## Our Responsibilities
35
+
36
+ Project maintainers are responsible for clarifying the standards of acceptable
37
+ behavior and are expected to take appropriate and fair corrective action in
38
+ response to any instances of unacceptable behavior.
39
+
40
+ Project maintainers have the right and responsibility to remove, edit, or
41
+ reject comments, commits, code, wiki edits, issues, and other contributions
42
+ that are not aligned to this Code of Conduct, or to ban temporarily or
43
+ permanently any contributor for other behaviors that they deem inappropriate,
44
+ threatening, offensive, or harmful.
45
+
46
+ ## Scope
47
+
48
+ This Code of Conduct applies within all project spaces, and it also applies when
49
+ an individual is representing the project or its community in public spaces.
50
+ Examples of representing a project or community include using an official
51
+ project e-mail address, posting via an official social media account, or acting
52
+ as an appointed representative at an online or offline event. Representation of
53
+ a project may be further defined and clarified by project maintainers.
54
+
55
+ This Code of Conduct also applies outside the project spaces when there is a
56
+ reasonable belief that an individual's behavior may have a negative impact on
57
+ the project or its community.
58
+
59
+ ## Enforcement
60
+
61
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
62
+ reported by contacting the project team at <opensource-conduct@fb.com>. All
63
+ complaints will be reviewed and investigated and will result in a response that
64
+ is deemed necessary and appropriate to the circumstances. The project team is
65
+ obligated to maintain confidentiality with regard to the reporter of an incident.
66
+ Further details of specific enforcement policies may be posted separately.
67
+
68
+ Project maintainers who do not follow or enforce the Code of Conduct in good
69
+ faith may face temporary or permanent repercussions as determined by other
70
+ members of the project's leadership.
71
+
72
+ ## Attribution
73
+
74
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
75
+ available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
76
+
77
+ [homepage]: https://www.contributor-covenant.org
78
+
79
+ For answers to common questions about this code of conduct, see
80
+ https://www.contributor-covenant.org/faq
CONTRIBUTING.md ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Contributing to PoseDiffusion
2
+ We want to make contributing to this project as easy and transparent as
3
+ possible.
4
+
5
+ ## Pull Requests
6
+ We actively welcome your pull requests.
7
+
8
+ 1. Fork the repo and create your branch from `main`.
9
+ 2. If you've added code that should be tested, add tests.
10
+ 3. If you've changed APIs, update the documentation.
11
+ 4. Ensure the test suite passes.
12
+ 5. Make sure your code lints.
13
+ 6. If you haven't already, complete the Contributor License Agreement ("CLA").
14
+
15
+ ## Contributor License Agreement ("CLA")
16
+ In order to accept your pull request, we need you to submit a CLA. You only need
17
+ to do this once to work on any of Facebook's open source projects.
18
+
19
+ Complete your CLA here: <https://code.facebook.com/cla>
20
+
21
+ ## Issues
22
+ We use GitHub issues to track public bugs. Please ensure your description is
23
+ clear and has sufficient instructions to be able to reproduce the issue.
24
+
25
+ Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
26
+ disclosure of security bugs. In those cases, please go through the process
27
+ outlined on that page and do not file a public issue.
28
+
29
+ ## License
30
+ By contributing to PoseDiffusion, you agree that your contributions will be licensed
31
+ under the LICENSE file in the root directory of this source tree.
LICENSE ADDED
@@ -0,0 +1,399 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Attribution-NonCommercial 4.0 International
2
+
3
+ =======================================================================
4
+
5
+ Creative Commons Corporation ("Creative Commons") is not a law firm and
6
+ does not provide legal services or legal advice. Distribution of
7
+ Creative Commons public licenses does not create a lawyer-client or
8
+ other relationship. Creative Commons makes its licenses and related
9
+ information available on an "as-is" basis. Creative Commons gives no
10
+ warranties regarding its licenses, any material licensed under their
11
+ terms and conditions, or any related information. Creative Commons
12
+ disclaims all liability for damages resulting from their use to the
13
+ fullest extent possible.
14
+
15
+ Using Creative Commons Public Licenses
16
+
17
+ Creative Commons public licenses provide a standard set of terms and
18
+ conditions that creators and other rights holders may use to share
19
+ original works of authorship and other material subject to copyright
20
+ and certain other rights specified in the public license below. The
21
+ following considerations are for informational purposes only, are not
22
+ exhaustive, and do not form part of our licenses.
23
+
24
+ Considerations for licensors: Our public licenses are
25
+ intended for use by those authorized to give the public
26
+ permission to use material in ways otherwise restricted by
27
+ copyright and certain other rights. Our licenses are
28
+ irrevocable. Licensors should read and understand the terms
29
+ and conditions of the license they choose before applying it.
30
+ Licensors should also secure all rights necessary before
31
+ applying our licenses so that the public can reuse the
32
+ material as expected. Licensors should clearly mark any
33
+ material not subject to the license. This includes other CC-
34
+ licensed material, or material used under an exception or
35
+ limitation to copyright. More considerations for licensors:
36
+ wiki.creativecommons.org/Considerations_for_licensors
37
+
38
+ Considerations for the public: By using one of our public
39
+ licenses, a licensor grants the public permission to use the
40
+ licensed material under specified terms and conditions. If
41
+ the licensor's permission is not necessary for any reason--for
42
+ example, because of any applicable exception or limitation to
43
+ copyright--then that use is not regulated by the license. Our
44
+ licenses grant only permissions under copyright and certain
45
+ other rights that a licensor has authority to grant. Use of
46
+ the licensed material may still be restricted for other
47
+ reasons, including because others have copyright or other
48
+ rights in the material. A licensor may make special requests,
49
+ such as asking that all changes be marked or described.
50
+ Although not required by our licenses, you are encouraged to
51
+ respect those requests where reasonable. More_considerations
52
+ for the public:
53
+ wiki.creativecommons.org/Considerations_for_licensees
54
+
55
+ =======================================================================
56
+
57
+ Creative Commons Attribution-NonCommercial 4.0 International Public
58
+ License
59
+
60
+ By exercising the Licensed Rights (defined below), You accept and agree
61
+ to be bound by the terms and conditions of this Creative Commons
62
+ Attribution-NonCommercial 4.0 International Public License ("Public
63
+ License"). To the extent this Public License may be interpreted as a
64
+ contract, You are granted the Licensed Rights in consideration of Your
65
+ acceptance of these terms and conditions, and the Licensor grants You
66
+ such rights in consideration of benefits the Licensor receives from
67
+ making the Licensed Material available under these terms and
68
+ conditions.
69
+
70
+ Section 1 -- Definitions.
71
+
72
+ a. Adapted Material means material subject to Copyright and Similar
73
+ Rights that is derived from or based upon the Licensed Material
74
+ and in which the Licensed Material is translated, altered,
75
+ arranged, transformed, or otherwise modified in a manner requiring
76
+ permission under the Copyright and Similar Rights held by the
77
+ Licensor. For purposes of this Public License, where the Licensed
78
+ Material is a musical work, performance, or sound recording,
79
+ Adapted Material is always produced where the Licensed Material is
80
+ synched in timed relation with a moving image.
81
+
82
+ b. Adapter's License means the license You apply to Your Copyright
83
+ and Similar Rights in Your contributions to Adapted Material in
84
+ accordance with the terms and conditions of this Public License.
85
+
86
+ c. Copyright and Similar Rights means copyright and/or similar rights
87
+ closely related to copyright including, without limitation,
88
+ performance, broadcast, sound recording, and Sui Generis Database
89
+ Rights, without regard to how the rights are labeled or
90
+ categorized. For purposes of this Public License, the rights
91
+ specified in Section 2(b)(1)-(2) are not Copyright and Similar
92
+ Rights.
93
+ d. Effective Technological Measures means those measures that, in the
94
+ absence of proper authority, may not be circumvented under laws
95
+ fulfilling obligations under Article 11 of the WIPO Copyright
96
+ Treaty adopted on December 20, 1996, and/or similar international
97
+ agreements.
98
+
99
+ e. Exceptions and Limitations means fair use, fair dealing, and/or
100
+ any other exception or limitation to Copyright and Similar Rights
101
+ that applies to Your use of the Licensed Material.
102
+
103
+ f. Licensed Material means the artistic or literary work, database,
104
+ or other material to which the Licensor applied this Public
105
+ License.
106
+
107
+ g. Licensed Rights means the rights granted to You subject to the
108
+ terms and conditions of this Public License, which are limited to
109
+ all Copyright and Similar Rights that apply to Your use of the
110
+ Licensed Material and that the Licensor has authority to license.
111
+
112
+ h. Licensor means the individual(s) or entity(ies) granting rights
113
+ under this Public License.
114
+
115
+ i. NonCommercial means not primarily intended for or directed towards
116
+ commercial advantage or monetary compensation. For purposes of
117
+ this Public License, the exchange of the Licensed Material for
118
+ other material subject to Copyright and Similar Rights by digital
119
+ file-sharing or similar means is NonCommercial provided there is
120
+ no payment of monetary compensation in connection with the
121
+ exchange.
122
+
123
+ j. Share means to provide material to the public by any means or
124
+ process that requires permission under the Licensed Rights, such
125
+ as reproduction, public display, public performance, distribution,
126
+ dissemination, communication, or importation, and to make material
127
+ available to the public including in ways that members of the
128
+ public may access the material from a place and at a time
129
+ individually chosen by them.
130
+
131
+ k. Sui Generis Database Rights means rights other than copyright
132
+ resulting from Directive 96/9/EC of the European Parliament and of
133
+ the Council of 11 March 1996 on the legal protection of databases,
134
+ as amended and/or succeeded, as well as other essentially
135
+ equivalent rights anywhere in the world.
136
+
137
+ l. You means the individual or entity exercising the Licensed Rights
138
+ under this Public License. Your has a corresponding meaning.
139
+
140
+ Section 2 -- Scope.
141
+
142
+ a. License grant.
143
+
144
+ 1. Subject to the terms and conditions of this Public License,
145
+ the Licensor hereby grants You a worldwide, royalty-free,
146
+ non-sublicensable, non-exclusive, irrevocable license to
147
+ exercise the Licensed Rights in the Licensed Material to:
148
+
149
+ a. reproduce and Share the Licensed Material, in whole or
150
+ in part, for NonCommercial purposes only; and
151
+
152
+ b. produce, reproduce, and Share Adapted Material for
153
+ NonCommercial purposes only.
154
+
155
+ 2. Exceptions and Limitations. For the avoidance of doubt, where
156
+ Exceptions and Limitations apply to Your use, this Public
157
+ License does not apply, and You do not need to comply with
158
+ its terms and conditions.
159
+
160
+ 3. Term. The term of this Public License is specified in Section
161
+ 6(a).
162
+
163
+ 4. Media and formats; technical modifications allowed. The
164
+ Licensor authorizes You to exercise the Licensed Rights in
165
+ all media and formats whether now known or hereafter created,
166
+ and to make technical modifications necessary to do so. The
167
+ Licensor waives and/or agrees not to assert any right or
168
+ authority to forbid You from making technical modifications
169
+ necessary to exercise the Licensed Rights, including
170
+ technical modifications necessary to circumvent Effective
171
+ Technological Measures. For purposes of this Public License,
172
+ simply making modifications authorized by this Section 2(a)
173
+ (4) never produces Adapted Material.
174
+
175
+ 5. Downstream recipients.
176
+
177
+ a. Offer from the Licensor -- Licensed Material. Every
178
+ recipient of the Licensed Material automatically
179
+ receives an offer from the Licensor to exercise the
180
+ Licensed Rights under the terms and conditions of this
181
+ Public License.
182
+
183
+ b. No downstream restrictions. You may not offer or impose
184
+ any additional or different terms or conditions on, or
185
+ apply any Effective Technological Measures to, the
186
+ Licensed Material if doing so restricts exercise of the
187
+ Licensed Rights by any recipient of the Licensed
188
+ Material.
189
+
190
+ 6. No endorsement. Nothing in this Public License constitutes or
191
+ may be construed as permission to assert or imply that You
192
+ are, or that Your use of the Licensed Material is, connected
193
+ with, or sponsored, endorsed, or granted official status by,
194
+ the Licensor or others designated to receive attribution as
195
+ provided in Section 3(a)(1)(A)(i).
196
+
197
+ b. Other rights.
198
+
199
+ 1. Moral rights, such as the right of integrity, are not
200
+ licensed under this Public License, nor are publicity,
201
+ privacy, and/or other similar personality rights; however, to
202
+ the extent possible, the Licensor waives and/or agrees not to
203
+ assert any such rights held by the Licensor to the limited
204
+ extent necessary to allow You to exercise the Licensed
205
+ Rights, but not otherwise.
206
+
207
+ 2. Patent and trademark rights are not licensed under this
208
+ Public License.
209
+
210
+ 3. To the extent possible, the Licensor waives any right to
211
+ collect royalties from You for the exercise of the Licensed
212
+ Rights, whether directly or through a collecting society
213
+ under any voluntary or waivable statutory or compulsory
214
+ licensing scheme. In all other cases the Licensor expressly
215
+ reserves any right to collect such royalties, including when
216
+ the Licensed Material is used other than for NonCommercial
217
+ purposes.
218
+
219
+ Section 3 -- License Conditions.
220
+
221
+ Your exercise of the Licensed Rights is expressly made subject to the
222
+ following conditions.
223
+
224
+ a. Attribution.
225
+
226
+ 1. If You Share the Licensed Material (including in modified
227
+ form), You must:
228
+
229
+ a. retain the following if it is supplied by the Licensor
230
+ with the Licensed Material:
231
+
232
+ i. identification of the creator(s) of the Licensed
233
+ Material and any others designated to receive
234
+ attribution, in any reasonable manner requested by
235
+ the Licensor (including by pseudonym if
236
+ designated);
237
+
238
+ ii. a copyright notice;
239
+
240
+ iii. a notice that refers to this Public License;
241
+
242
+ iv. a notice that refers to the disclaimer of
243
+ warranties;
244
+
245
+ v. a URI or hyperlink to the Licensed Material to the
246
+ extent reasonably practicable;
247
+
248
+ b. indicate if You modified the Licensed Material and
249
+ retain an indication of any previous modifications; and
250
+
251
+ c. indicate the Licensed Material is licensed under this
252
+ Public License, and include the text of, or the URI or
253
+ hyperlink to, this Public License.
254
+
255
+ 2. You may satisfy the conditions in Section 3(a)(1) in any
256
+ reasonable manner based on the medium, means, and context in
257
+ which You Share the Licensed Material. For example, it may be
258
+ reasonable to satisfy the conditions by providing a URI or
259
+ hyperlink to a resource that includes the required
260
+ information.
261
+
262
+ 3. If requested by the Licensor, You must remove any of the
263
+ information required by Section 3(a)(1)(A) to the extent
264
+ reasonably practicable.
265
+
266
+ 4. If You Share Adapted Material You produce, the Adapter's
267
+ License You apply must not prevent recipients of the Adapted
268
+ Material from complying with this Public License.
269
+
270
+ Section 4 -- Sui Generis Database Rights.
271
+
272
+ Where the Licensed Rights include Sui Generis Database Rights that
273
+ apply to Your use of the Licensed Material:
274
+
275
+ a. for the avoidance of doubt, Section 2(a)(1) grants You the right
276
+ to extract, reuse, reproduce, and Share all or a substantial
277
+ portion of the contents of the database for NonCommercial purposes
278
+ only;
279
+
280
+ b. if You include all or a substantial portion of the database
281
+ contents in a database in which You have Sui Generis Database
282
+ Rights, then the database in which You have Sui Generis Database
283
+ Rights (but not its individual contents) is Adapted Material; and
284
+
285
+ c. You must comply with the conditions in Section 3(a) if You Share
286
+ all or a substantial portion of the contents of the database.
287
+
288
+ For the avoidance of doubt, this Section 4 supplements and does not
289
+ replace Your obligations under this Public License where the Licensed
290
+ Rights include other Copyright and Similar Rights.
291
+
292
+ Section 5 -- Disclaimer of Warranties and Limitation of Liability.
293
+
294
+ a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
295
+ EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
296
+ AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
297
+ ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
298
+ IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
299
+ WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
300
+ PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
301
+ ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
302
+ KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
303
+ ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
304
+
305
+ b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
306
+ TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
307
+ NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
308
+ INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
309
+ COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
310
+ USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
311
+ ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
312
+ DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
313
+ IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
314
+
315
+ c. The disclaimer of warranties and limitation of liability provided
316
+ above shall be interpreted in a manner that, to the extent
317
+ possible, most closely approximates an absolute disclaimer and
318
+ waiver of all liability.
319
+
320
+ Section 6 -- Term and Termination.
321
+
322
+ a. This Public License applies for the term of the Copyright and
323
+ Similar Rights licensed here. However, if You fail to comply with
324
+ this Public License, then Your rights under this Public License
325
+ terminate automatically.
326
+
327
+ b. Where Your right to use the Licensed Material has terminated under
328
+ Section 6(a), it reinstates:
329
+
330
+ 1. automatically as of the date the violation is cured, provided
331
+ it is cured within 30 days of Your discovery of the
332
+ violation; or
333
+
334
+ 2. upon express reinstatement by the Licensor.
335
+
336
+ For the avoidance of doubt, this Section 6(b) does not affect any
337
+ right the Licensor may have to seek remedies for Your violations
338
+ of this Public License.
339
+
340
+ c. For the avoidance of doubt, the Licensor may also offer the
341
+ Licensed Material under separate terms or conditions or stop
342
+ distributing the Licensed Material at any time; however, doing so
343
+ will not terminate this Public License.
344
+
345
+ d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
346
+ License.
347
+
348
+ Section 7 -- Other Terms and Conditions.
349
+
350
+ a. The Licensor shall not be bound by any additional or different
351
+ terms or conditions communicated by You unless expressly agreed.
352
+
353
+ b. Any arrangements, understandings, or agreements regarding the
354
+ Licensed Material not stated herein are separate from and
355
+ independent of the terms and conditions of this Public License.
356
+
357
+ Section 8 -- Interpretation.
358
+
359
+ a. For the avoidance of doubt, this Public License does not, and
360
+ shall not be interpreted to, reduce, limit, restrict, or impose
361
+ conditions on any use of the Licensed Material that could lawfully
362
+ be made without permission under this Public License.
363
+
364
+ b. To the extent possible, if any provision of this Public License is
365
+ deemed unenforceable, it shall be automatically reformed to the
366
+ minimum extent necessary to make it enforceable. If the provision
367
+ cannot be reformed, it shall be severed from this Public License
368
+ without affecting the enforceability of the remaining terms and
369
+ conditions.
370
+
371
+ c. No term or condition of this Public License will be waived and no
372
+ failure to comply consented to unless expressly agreed to by the
373
+ Licensor.
374
+
375
+ d. Nothing in this Public License constitutes or may be interpreted
376
+ as a limitation upon, or waiver of, any privileges and immunities
377
+ that apply to the Licensor or You, including from the legal
378
+ processes of any jurisdiction or authority.
379
+
380
+ =======================================================================
381
+
382
+ Creative Commons is not a party to its public
383
+ licenses. Notwithstanding, Creative Commons may elect to apply one of
384
+ its public licenses to material it publishes and in those instances
385
+ will be considered the “Licensor.” The text of the Creative Commons
386
+ public licenses is dedicated to the public domain under the CC0 Public
387
+ Domain Dedication. Except for the limited purpose of indicating that
388
+ material is shared under a Creative Commons public license or as
389
+ otherwise permitted by the Creative Commons policies published at
390
+ creativecommons.org/policies, Creative Commons does not authorize the
391
+ use of the trademark "Creative Commons" or any other trademark or logo
392
+ of Creative Commons without its prior written consent including,
393
+ without limitation, in connection with any unauthorized modifications
394
+ to any of its public licenses or any other arrangements,
395
+ understandings, or agreements concerning use of licensed material. For
396
+ the avoidance of doubt, this paragraph does not form part of the
397
+ public licenses.
398
+
399
+ Creative Commons may be contacted at creativecommons.org.
README.md CHANGED
@@ -1,3 +1,112 @@
1
- ---
2
- license: cc-by-nc-nd-4.0
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: cc-by-nc-2.0
3
+ pipeline_tag: image-to-3d
4
+ ---
5
+ # [ECCV 2024] VFusion3D: Learning Scalable 3D Generative Models from Video Diffusion Models
6
+
7
+ [Porject page](https://junlinhan.github.io/projects/vfusion3d.html), [Paper link](https://arxiv.org/abs/2403.12034)
8
+
9
+ VFusion3D is a large, feed-forward 3D generative model trained with a small amount of 3D data and a large volume of synthetic multi-view data. It is the first work exploring scalable 3D generative/reconstruction models as a step towards a 3D foundation.
10
+
11
+ [VFusion3D: Learning Scalable 3D Generative Models from Video Diffusion Models](https://junlinhan.github.io/projects/vfusion3d.html)<br>
12
+ [Junlin Han](https://junlinhan.github.io/), [Filippos Kokkinos](https://www.fkokkinos.com/), [Philip Torr](https://www.robots.ox.ac.uk/~phst/)<br>
13
+ GenAI, Meta and TVG, University of Oxford<br>
14
+ European Conference on Computer Vision (ECCV), 2024
15
+
16
+
17
+ ## News
18
+
19
+ - [08.08.2024] [HF Demo](https://huggingface.co/spaces/facebook/VFusion3D) is available, big thanks to [Jade Choghari](https://github.com/jadechoghari)'s help for making it possible.
20
+ - [25.07.2024] Release weights and inference code for VFusion3D.
21
+
22
+
23
+
24
+ ## Quick Start
25
+
26
+ Getting started with VFusion3D is super easy! 🤗 Here’s how you can use the model with Hugging Face:
27
+
28
+ ### Install Dependencies (Optional)
29
+
30
+ Depending on your needs, you may want to enable specific features like mesh generation or video rendering. We've got you covered with these additional packages:
31
+
32
+ ```bash
33
+ !pip --quiet install imageio[ffmpeg] PyMCubes trimesh rembg[gpu,cli] kiui
34
+ ```
35
+
36
+ ### Load model directly
37
+ ```python
38
+ import torch
39
+ from transformers import AutoModel, AutoProcessor
40
+
41
+ # load the model and processor
42
+ model = AutoModel.from_pretrained("jadechoghari/vfusion3d", trust_remote_code=True)
43
+ processor = AutoProcessor.from_pretrained("jadechoghari/vfusion3d")
44
+
45
+ # download and preprocess the image
46
+ import requests
47
+ from PIL import Image
48
+ from io import BytesIO
49
+
50
+ image_url = 'https://sm.ign.com/ign_nordic/cover/a/avatar-gen/avatar-generations_prsz.jpg'
51
+ response = requests.get(image_url)
52
+ image = Image.open(BytesIO(response.content))
53
+
54
+ # preprocess the image and get the source camera
55
+ image, source_camera = processor(image)
56
+
57
+
58
+ # generate planes (default output)
59
+ output_planes = model(image, source_camera)
60
+ print("Planes shape:", output_planes.shape)
61
+
62
+ # generate a 3D mesh
63
+ output_planes, mesh_path = model(image, source_camera, export_mesh=True)
64
+ print("Planes shape:", output_planes.shape)
65
+ print("Mesh saved at:", mesh_path)
66
+
67
+ # Generate a video
68
+ output_planes, video_path = model(image, source_camera, export_video=True)
69
+ print("Planes shape:", output_planes.shape)
70
+ print("Video saved at:", video_path)
71
+
72
+ ```
73
+ - **Default (Planes):** By default, VFusion3D outputs planes—ideal for further 3D operations.
74
+ - **Export Mesh:** Want a 3D mesh? Just set `export_mesh=True`, and you'll get a `.obj` file ready to roll. You can also customize the mesh resolution by adjusting the `mesh_size` parameter.
75
+ - **Export Video:** Fancy a 3D video? Set `export_video=True`, and you'll receive a beautifully rendered video from multiple angles. You can tweak `render_size` and `fps` to get the video just right.
76
+
77
+ Check out our [demo app](https://huggingface.co/spaces/facebook/VFusion3D) to see VFusion3D in action! 🤗
78
+
79
+ ## Results and Comparisons
80
+
81
+ ### 3D Generation Results
82
+ <img src='assets/gif1.gif' width=950>
83
+
84
+ <img src='assets/gif2.gif' width=950>
85
+
86
+ ### User Study Results
87
+ <img src='assets/user.png' width=950>
88
+
89
+
90
+
91
+ ## Acknowledgement
92
+
93
+ - This inference code of VFusion3D heavily borrows from [OpenLRM](https://github.com/3DTopia/OpenLRM).
94
+
95
+ ## Citation
96
+
97
+ If you find this work useful, please cite us:
98
+
99
+
100
+ ```
101
+ @article{han2024vfusion3d,
102
+ title={VFusion3D: Learning Scalable 3D Generative Models from Video Diffusion Models},
103
+ author={Junlin Han and Filippos Kokkinos and Philip Torr},
104
+ journal={European Conference on Computer Vision (ECCV)},
105
+ year={2024}
106
+ }
107
+ ```
108
+
109
+ ## License
110
+
111
+ - The majority of VFusion3D is licensed under CC-BY-NC, however portions of the project are available under separate license terms: OpenLRM as a whole is licensed under the Apache License, Version 2.0, while certain components are covered by NVIDIA's proprietary license.
112
+ - The model weights of VFusion3D is also licensed under CC-BY-NC.
__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import AutoProcessor
2
+ from .modeling import LRMGenerator, LRMGeneratorConfig
3
+
4
+ import logging
5
+
6
+ # we set up logging
7
+ logging.basicConfig(level=logging.INFO)
8
+ logger = logging.getLogger(__name__)
9
+
10
+ # helper function to install packages
11
+ def install_package(package_name):
12
+ if importlib.util.find_spec(package_name) is None:
13
+ logger.info(f"Package '{package_name}' not found. Installing...")
14
+ print(f"Package '{package_name}' not found. Installing...")
15
+ subprocess.check_call([sys.executable, "-m", "pip", "install", package_name])
16
+ else:
17
+ logger.info(f"Package '{package_name}' is already installed.")
18
+
19
+ # list of packages to ensure are installed
20
+ required_packages = [
21
+ 'imageio[ffmpeg]', 'PyMCubes', 'trimesh', 'rembg[gpu,cli]', 'kiui', 'torchvision', 'Pillow'
22
+ ]
23
+
24
+ for package in required_packages:
25
+ install_package(package)
26
+
27
+ from .processor import LRMImageProcessor
assets/40_prompt_images/A 3D scan of AK47, weapon.jpeg ADDED
assets/40_prompt_images/A DSLR photo of Sydney Opera House.jpg ADDED
assets/40_prompt_images/A bald eagle carved out of wood.jpg ADDED
assets/40_prompt_images/A bulldog wearing a black pirate hat.jpeg ADDED
assets/40_prompt_images/A crab, low poly.jpg ADDED
assets/40_prompt_images/A photo of a horse walking.jpeg ADDED
assets/40_prompt_images/A pig wearing a backpack.jpeg ADDED
assets/40_prompt_images/A product photo of a toy tank.jpg ADDED
assets/40_prompt_images/A see no evil monkey on a kick drum.jpg ADDED
assets/40_prompt_images/A statue of angel, blender.jpg ADDED
assets/40_prompt_images/Corgi riding a rocket.jpeg ADDED
assets/40_prompt_images/Daenerys Targaryen from game of throne.jpg ADDED
assets/40_prompt_images/Darth Vader helmet,g highly detailed.jpg ADDED
assets/40_prompt_images/Dragon armor.jpeg ADDED
assets/40_prompt_images/Fisherman House, cute, cartoon, blender, stylized.jpg ADDED
assets/40_prompt_images/Flying Dragon, highly detailed, breathing fire.jpeg ADDED
assets/40_prompt_images/Handpainted watercolor windmill, hand-painted.jpg ADDED
assets/40_prompt_images/Katana.jpeg ADDED
assets/40_prompt_images/Little italian town, hand-painted style.jpg ADDED
assets/40_prompt_images/Mr Bean Cartoon doing a T Pose.jpg ADDED
assets/40_prompt_images/Pedestal Fan (White).jpeg ADDED
assets/40_prompt_images/Pikachu with hat.jpg ADDED
assets/40_prompt_images/Samurai koala bear.jpg ADDED
assets/40_prompt_images/TRUMP figure.jpg ADDED
assets/40_prompt_images/Viking axe, fantasy, weapon, blender, 8k, HD.jpg ADDED
assets/40_prompt_images/a DSLR photo of a frog wearing a sweater.jpg ADDED
assets/40_prompt_images/a DSLR photo of a ghost eating a hamburger.jpg ADDED
assets/40_prompt_images/a DSLR photo of a peacock on a surfboard.jpeg ADDED
assets/40_prompt_images/a DSLR photo of a squirrel playing guitar.jpg ADDED
assets/40_prompt_images/a DSLR photo of an eggshell broken in two with an adorable chick standing next to it.jpeg ADDED
assets/40_prompt_images/an astronaut riding a horse.jpeg ADDED
assets/40_prompt_images/animal skull pile.jpg ADDED
assets/40_prompt_images/army Jacket, 3D scan.jpg ADDED
assets/40_prompt_images/baby yoda in the style of Mormookiee.jpg ADDED
assets/40_prompt_images/beautiful, intricate butterfly.jpg ADDED
assets/40_prompt_images/girl riding wolf, cute, cartoon, blender.jpg ADDED
assets/40_prompt_images/mecha vampire girl chibi.jpg ADDED
assets/40_prompt_images/military Mech, future, scifi.jpg ADDED
assets/40_prompt_images/motorcycle, scifi, blender.jpeg ADDED
assets/40_prompt_images/saber from fate stay night, 3D, girl, anime.jpeg ADDED
assets/gif1.gif ADDED

Git LFS Details

  • SHA256: 70046163df530bbd5b75b606d53c5aec97cc76de7db1e05e47770319fba3ddb2
  • Pointer size: 133 Bytes
  • Size of remote file: 26.2 MB
assets/gif2.gif ADDED

Git LFS Details

  • SHA256: bddf9ce06c509e08c41082e9772563238e16f21665a397b0b11418ce497793b8
  • Pointer size: 133 Bytes
  • Size of remote file: 15.9 MB
assets/user.png ADDED
config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LRMGenerator"
4
+ ],
5
+ "model_type": "lrm_generator",
6
+ "model_name_or_path": "jadechoghari/vfusion3d",
7
+ "camera_embed_dim": 1024,
8
+ "encoder_feat_dim": 768,
9
+ "encoder_freeze": false,
10
+ "encoder_model_name": "facebook/dinov2-base",
11
+ "model_type": "lrm_generator",
12
+ "rendering_samples_per_ray": 128,
13
+ "torch_dtype": "float32",
14
+ "transformer_dim": 1024,
15
+ "transformer_heads": 16,
16
+ "transformer_layers": 16,
17
+ "transformers_version": "4.42.4",
18
+ "triplane_dim": 80,
19
+ "triplane_high_res": 64,
20
+ "triplane_low_res": 32,
21
+ "auto_map": {
22
+ "AutoConfig": "modeling.LRMGeneratorConfig",
23
+ "AutoModel": "modeling.LRMGenerator",
24
+ "AutoImageProcessor": "processor.LRMImageProcessor"
25
+ },
26
+ "preprocessor_class": "LRMImageProcessor"
27
+ }