{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU", "widgets": { "application/vnd.jupyter.widget-state+json": { "9c6c541edc1347c98dbc452b30dc5284": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_18b244742415432c9a263cdb9339b519", "IPY_MODEL_70f1f8e16654493ea712051e748db3ea", "IPY_MODEL_b3961785070743108bb712abb5f117b9" ], "layout": "IPY_MODEL_c610fbc9163f43a4874cb6f70d2436b2" } }, "18b244742415432c9a263cdb9339b519": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_22c90993784c4017b4ee6cd39447feea", "placeholder": "​", "style": "IPY_MODEL_fba40eab7d62470687a32419dcd120de", "value": "Map: 0%" } }, "70f1f8e16654493ea712051e748db3ea": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_388c884346494beca1ca8e517a6fa436", "max": 15, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_38668c9a32a24a09956302af1a708dbd", "value": 15 } }, "b3961785070743108bb712abb5f117b9": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a71ae75033d449d385a61a70fd17c626", "placeholder": "​", "style": "IPY_MODEL_0ecf42fd5b8648ffaf2d395457604b49", "value": " 0/15 [00:00<?, ? examples/s]" } }, "c610fbc9163f43a4874cb6f70d2436b2": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": "hidden", "width": null } }, "22c90993784c4017b4ee6cd39447feea": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "fba40eab7d62470687a32419dcd120de": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "388c884346494beca1ca8e517a6fa436": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "38668c9a32a24a09956302af1a708dbd": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "a71ae75033d449d385a61a70fd17c626": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0ecf42fd5b8648ffaf2d395457604b49": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "f8a78562909a427d95b4a88edef66c13": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_f34ef24ed4794ae9aece79b2ea5c2dc7", "IPY_MODEL_0000dcfeb965404e91c9e26f299df05c", "IPY_MODEL_bd90641c572e4006b52ea4fa5fc8741e" ], "layout": "IPY_MODEL_8a51eb36c4cc4d238011d67101c18071" } }, "f34ef24ed4794ae9aece79b2ea5c2dc7": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f13a20c5f04848d1ba72aad491824b39", "placeholder": "​", "style": "IPY_MODEL_dad7ec2a9fbe4cd38a652f8205fffe1a", "value": "Pushing dataset shards to the dataset hub: 100%" } }, "0000dcfeb965404e91c9e26f299df05c": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f7033c8a99854a0cb208dec55d709b8a", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_6c350f5aba4b4d559e3a4d6100cfe286", "value": 1 } }, "bd90641c572e4006b52ea4fa5fc8741e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_67dbcb396e384b99afe162e084df4bdd", "placeholder": "​", "style": "IPY_MODEL_73dd4ca5836b4eaaa994c153954f7fed", "value": " 1/1 [00:01<00:00, 1.49s/it]" } }, "8a51eb36c4cc4d238011d67101c18071": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f13a20c5f04848d1ba72aad491824b39": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "dad7ec2a9fbe4cd38a652f8205fffe1a": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "f7033c8a99854a0cb208dec55d709b8a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6c350f5aba4b4d559e3a4d6100cfe286": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "67dbcb396e384b99afe162e084df4bdd": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "73dd4ca5836b4eaaa994c153954f7fed": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "5b3582f209a04a12a0bd0d563e2b9c8c": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_c121b78205c44822af084c9017087124", "IPY_MODEL_52f3bea1142c47778e393359df4217a9", "IPY_MODEL_e21d6578bec148b9b6be4aea45c42bac" ], "layout": "IPY_MODEL_2787d66fc6e94281a4e6bfd631093b91" } }, "c121b78205c44822af084c9017087124": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_821f64ded5034bc99b4c6f1ce16bfce1", "placeholder": "​", "style": "IPY_MODEL_10c5e5accc624c4aba5ce8bd7ecb6f6c", "value": "Creating parquet from Arrow format: 100%" } }, "52f3bea1142c47778e393359df4217a9": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_07eb56d7db104561812e3951bcdf9c9f", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_415799df471b4fc3831bca903b353923", "value": 1 } }, "e21d6578bec148b9b6be4aea45c42bac": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_eced1bfc315c4001b958a870902eb2f9", "placeholder": "​", "style": "IPY_MODEL_eb2eb6f912c34d7a93d7abaef9887138", "value": " 1/1 [00:00<00:00, 7.88ba/s]" } }, "2787d66fc6e94281a4e6bfd631093b91": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "821f64ded5034bc99b4c6f1ce16bfce1": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "10c5e5accc624c4aba5ce8bd7ecb6f6c": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "07eb56d7db104561812e3951bcdf9c9f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "415799df471b4fc3831bca903b353923": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "eced1bfc315c4001b958a870902eb2f9": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "eb2eb6f912c34d7a93d7abaef9887138": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "code", "source": [ "pip install accelerate datasets transformers wandb" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "MHtbuULcrlRj", "outputId": "1c4089ff-8f7a-4fd4-8bdd-9c7111ca2ed8" }, "execution_count": 1, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Collecting accelerate\n", " Downloading accelerate-0.21.0-py3-none-any.whl (244 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m244.2/244.2 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting datasets\n", " Downloading datasets-2.13.1-py3-none-any.whl (486 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m486.2/486.2 kB\u001b[0m \u001b[31m29.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting transformers\n", " Downloading transformers-4.31.0-py3-none-any.whl (7.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.4/7.4 MB\u001b[0m \u001b[31m98.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting wandb\n", " Downloading wandb-0.15.5-py3-none-any.whl (2.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m97.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.22.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.1)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0)\n", "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.0.1+cu118)\n", "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n", "Collecting dill<0.3.7,>=0.3.0 (from datasets)\n", " Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n", "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.27.1)\n", "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.65.0)\n", "Collecting xxhash (from datasets)\n", " Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m28.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting multiprocess (from datasets)\n", " Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m19.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.8.4)\n", "Collecting huggingface-hub<1.0.0,>=0.11.0 (from datasets)\n", " Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m36.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n", "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n", " Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m99.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n", " Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m87.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.4)\n", "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb)\n", " Downloading GitPython-3.1.32-py3-none-any.whl (188 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m188.5/188.5 kB\u001b[0m \u001b[31m26.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting sentry-sdk>=1.0.0 (from wandb)\n", " Downloading sentry_sdk-1.28.1-py2.py3-none-any.whl (214 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.7/214.7 kB\u001b[0m \u001b[31m28.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb)\n", " Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n", "Collecting pathtools (from wandb)\n", " Downloading pathtools-0.1.2.tar.gz (11 kB)\n", " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "Collecting setproctitle (from wandb)\n", " Downloading setproctitle-1.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (67.7.2)\n", "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.4.4)\n", "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n", "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n", "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.0.12)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.2)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.2)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n", "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb)\n", " Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.7.1)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (1.26.16)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.5.7)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.4)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.11.1)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n", "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.0.0)\n", "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (3.25.2)\n", "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (16.0.6)\n", "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2022.7.1)\n", "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb)\n", " Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n", "Building wheels for collected packages: pathtools\n", " Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=a7ff1fd77e0e7782e439608ac936e3696436c08ee76f40f753cb2848eeb32090\n", " Stored in directory: /root/.cache/pip/wheels/e7/f3/22/152153d6eb222ee7a56ff8617d80ee5207207a8c00a7aab794\n", "Successfully built pathtools\n", "Installing collected packages: tokenizers, safetensors, pathtools, xxhash, smmap, setproctitle, sentry-sdk, docker-pycreds, dill, multiprocess, huggingface-hub, gitdb, transformers, GitPython, wandb, datasets, accelerate\n", "Successfully installed GitPython-3.1.32 accelerate-0.21.0 datasets-2.13.1 dill-0.3.6 docker-pycreds-0.4.0 gitdb-4.0.10 huggingface-hub-0.16.4 multiprocess-0.70.14 pathtools-0.1.2 safetensors-0.3.1 sentry-sdk-1.28.1 setproctitle-1.3.2 smmap-5.0.0 tokenizers-0.13.3 transformers-4.31.0 wandb-0.15.5 xxhash-3.2.0\n" ] } ] }, { "cell_type": "code", "source": [ "!git clone https://github.com/huggingface/diffusers" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "zxQ-gSf1z0vG", "outputId": "46a98963-6bd9-4c80-cd66-81d1fa83f41d" }, "execution_count": 2, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Cloning into 'diffusers'...\n", "remote: Enumerating objects: 30774, done.\u001b[K\n", "remote: Counting objects: 100% (525/525), done.\u001b[K\n", "remote: Compressing objects: 100% (292/292), done.\u001b[K\n", "remote: Total 30774 (delta 318), reused 337 (delta 200), pack-reused 30249\u001b[K\n", "Receiving objects: 100% (30774/30774), 20.35 MiB | 23.10 MiB/s, done.\n", "Resolving deltas: 100% (22477/22477), done.\n" ] } ] }, { "cell_type": "code", "source": [ "%cd ./diffusers" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "uXqxb8wr0SnY", "outputId": "9c4d6d2f-ed4e-4aa1-97e3-61045b95d963" }, "execution_count": 3, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "/content/diffusers\n" ] } ] }, { "cell_type": "code", "source": [ "!dir" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "cLEmsYEe0Xns", "outputId": "076c1c05-363f-44fe-cd34-7110b9e22690" }, "execution_count": 4, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "CITATION.cff\t docs MANIFEST.in scripts\t tests\n", "CODE_OF_CONDUCT.md examples PHILOSOPHY.md setup.cfg _typos.toml\n", "CONTRIBUTING.md LICENSE pyproject.toml setup.py\t utils\n", "docker\t\t Makefile README.md src\n" ] } ] }, { "cell_type": "code", "source": [ "!pip install ." ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "6M5wWO-00HDF", "outputId": "8b0a5d62-edb2-422f-e16a-5b464602c6b5" }, "execution_count": 5, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Processing /content/diffusers\n", " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", "Collecting importlib-metadata (from diffusers==0.19.0.dev0)\n", " Downloading importlib_metadata-6.8.0-py3-none-any.whl (22 kB)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from diffusers==0.19.0.dev0) (3.12.2)\n", "Requirement already satisfied: huggingface-hub>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from diffusers==0.19.0.dev0) (0.16.4)\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from diffusers==0.19.0.dev0) (1.22.4)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from diffusers==0.19.0.dev0) (2022.10.31)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from diffusers==0.19.0.dev0) (2.27.1)\n", "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from diffusers==0.19.0.dev0) (8.4.0)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.13.2->diffusers==0.19.0.dev0) (2023.6.0)\n", "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.13.2->diffusers==0.19.0.dev0) (4.65.0)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.13.2->diffusers==0.19.0.dev0) (6.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.13.2->diffusers==0.19.0.dev0) (4.7.1)\n", "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.13.2->diffusers==0.19.0.dev0) (23.1)\n", "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata->diffusers==0.19.0.dev0) (3.16.1)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->diffusers==0.19.0.dev0) (1.26.16)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->diffusers==0.19.0.dev0) (2023.5.7)\n", "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->diffusers==0.19.0.dev0) (2.0.12)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->diffusers==0.19.0.dev0) (3.4)\n", "Building wheels for collected packages: diffusers\n", " Building wheel for diffusers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for diffusers: filename=diffusers-0.19.0.dev0-py3-none-any.whl size=1273866 sha256=6f30a4fc60fdcf32e458e762b5dfdfbd499031eaa0e7485cfa4c8728abf0c02f\n", " Stored in directory: /tmp/pip-ephem-wheel-cache-sy_kn2nd/wheels/95/c5/3b/e1b4269f8a2584de57e75f949a185b48fc4144e9a91fc9965a\n", "Successfully built diffusers\n", "Installing collected packages: importlib-metadata, diffusers\n", "Successfully installed diffusers-0.19.0.dev0 importlib-metadata-6.8.0\n" ] } ] }, { "cell_type": "code", "source": [ "!huggingface-cli login" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "H_C7afM77ryL", "outputId": "e682c9b3-60ff-4efa-f845-eb9fcc25618c" }, "execution_count": 6, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", " _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_|\n", " _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_|\n", " _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_|\n", " \n", " To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n", "Token: \n", "Add token as git credential? (Y/n) y\n", "Token is valid (permission: read).\n", "\u001b[1m\u001b[31mCannot authenticate through git-credential as no helper is defined on your machine.\n", "You might have to re-authenticate when pushing to the Hugging Face Hub.\n", "Run the following command in your terminal in case you want to set the 'store' credential helper as default.\n", "\n", "git config --global credential.helper store\n", "\n", "Read https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more details.\u001b[0m\n", "Token has not been saved to git credential helper.\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ] }, { "cell_type": "code", "source": [ "!export MODEL_NAME=\"runwayml/stable-diffusion-v1-5\"\n", "#!export OUTPUT_DIR=\"/sddata/finetune/lora/pokemon\"\n", "#!export HUB_MODEL_ID=\"pokemon-lora\"\n", "!export DATASET_NAME=\"lambdalabs/pokemon-blip-captions\"" ], "metadata": { "id": "LXICGbQSoy3z" }, "execution_count": 7, "outputs": [] }, { "cell_type": "code", "source": [ "%cd examples/text_to_image" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Ko_QLsgr3nTr", "outputId": "78fc70a4-5c14-4aa4-dab7-d6bcc3d6615f" }, "execution_count": 8, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "/content/diffusers/examples/text_to_image\n" ] } ] }, { "cell_type": "code", "source": [ "!accelerate launch --mixed_precision=\"fp16\" train_text_to_image_lora.py \\\n", " --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 \\\n", " --dataset_name Talha185/tti-tester \\\n", " --dataloader_num_workers=8 \\\n", " --resolution=512 --center_crop --random_flip \\\n", " --train_batch_size=1 \\\n", " --gradient_accumulation_steps=4 \\\n", " --max_train_steps=15000 \\\n", " --learning_rate=1e-04 \\\n", " --max_grad_norm=1 \\\n", " --lr_scheduler=\"cosine\" --lr_warmup_steps=0 \\\n", " #--output_dir Talha185/finetune/lora/andy \\\n", " #--push_to_hub \\\n", " #--hub_model_id pokemon-lora \\\n", " --report_to=wandb \\\n", " --checkpointing_steps=500 \\\n", " --validation_prompt=\"Andy with a hat.\" \\\n", " --seed=1337" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "iMFXmjWFy2op", "outputId": "3dfb74c4-46d0-4ee7-f159-22032d7af2fb" }, "execution_count": 12, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "2023-07-19 13:10:42.146735: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", "The following values were not passed to `accelerate launch` and had defaults used instead:\n", "\t`--num_processes` was set to a value of `1`\n", "\t`--num_machines` was set to a value of `1`\n", "\t`--dynamo_backend` was set to a value of `'no'`\n", "To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\n", "2023-07-19 13:10:51.650408: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", "07/19/2023 13:10:54 - INFO - __main__ - Distributed environment: NO\n", "Num processes: 1\n", "Process index: 0\n", "Local process index: 0\n", "Device: cuda\n", "\n", "Mixed precision type: fp16\n", "\n", "{'thresholding', 'timestep_spacing', 'clip_sample_range', 'variance_type', 'dynamic_thresholding_ratio', 'sample_max_value', 'prediction_type'} was not found in config. Values will be initialized to default values.\n", "{'scaling_factor', 'force_upcast'} was not found in config. Values will be initialized to default values.\n", "{'encoder_hid_dim_type', 'upcast_attention', 'num_class_embeds', 'projection_class_embeddings_input_dim', 'resnet_skip_time_act', 'time_cond_proj_dim', 'class_embeddings_concat', 'conv_out_kernel', 'addition_embed_type', 'mid_block_type', 'num_attention_heads', 'resnet_out_scale_factor', 'only_cross_attention', 'class_embed_type', 'conv_in_kernel', 'dual_cross_attention', 'resnet_time_scale_shift', 'addition_time_embed_dim', 'time_embedding_dim', 'cross_attention_norm', 'encoder_hid_dim', 'mid_block_only_cross_attention', 'time_embedding_act_fn', 'timestep_post_act', 'use_linear_projection', 'addition_embed_type_num_heads', 'time_embedding_type', 'transformer_layers_per_block'} was not found in config. Values will be initialized to default values.\n", "07/19/2023 13:11:18 - WARNING - datasets.builder - Found cached dataset parquet (/root/.cache/huggingface/datasets/Talha185___parquet/Talha185--tti-tester-650c4c5c5377d3c8/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)\n", "100% 1/1 [00:00<00:00, 69.61it/s]\n", "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:560: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n", " warnings.warn(_create_warning_msg(\n", "07/19/2023 13:11:20 - INFO - __main__ - ***** Running training *****\n", "07/19/2023 13:11:20 - INFO - __main__ - Num examples = 15\n", "07/19/2023 13:11:20 - INFO - __main__ - Num Epochs = 3750\n", "07/19/2023 13:11:20 - INFO - __main__ - Instantaneous batch size per device = 1\n", "07/19/2023 13:11:20 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 4\n", "07/19/2023 13:11:20 - INFO - __main__ - Gradient Accumulation steps = 4\n", "07/19/2023 13:11:20 - INFO - __main__ - Total optimization steps = 15000\n", "Steps: 3% 500/15000 [18:32<7:50:27, 1.95s/it, lr=9.97e-5, step_loss=0.0148]07/19/2023 13:29:52 - INFO - accelerate.accelerator - Saving current state to sd-model-finetuned-lora/checkpoint-500\n", "07/19/2023 13:29:52 - INFO - accelerate.checkpointing - Model weights saved in sd-model-finetuned-lora/checkpoint-500/pytorch_model.bin\n", "07/19/2023 13:29:52 - INFO - accelerate.checkpointing - Optimizer state saved in sd-model-finetuned-lora/checkpoint-500/optimizer.bin\n", "07/19/2023 13:29:52 - INFO - accelerate.checkpointing - Scheduler state saved in sd-model-finetuned-lora/checkpoint-500/scheduler.bin\n", "07/19/2023 13:29:52 - INFO - accelerate.checkpointing - Gradient scaler state saved in sd-model-finetuned-lora/checkpoint-500/scaler.pt\n", "07/19/2023 13:29:52 - INFO - accelerate.checkpointing - Random states saved in sd-model-finetuned-lora/checkpoint-500/random_states_0.pkl\n", "07/19/2023 13:29:52 - INFO - __main__ - Saved state to sd-model-finetuned-lora/checkpoint-500\n", "Steps: 6% 855/15000 [31:30<8:27:04, 2.15s/it, lr=9.92e-5, step_loss=0.0107]Traceback (most recent call last):\n", " File \"/usr/lib/python3.10/subprocess.py\", line 1209, in wait\n", " return self._wait(timeout=timeout)\n", " File \"/usr/lib/python3.10/subprocess.py\", line 1959, in _wait\n", " (pid, sts) = self._try_wait(0)\n", " File \"/usr/lib/python3.10/subprocess.py\", line 1917, in _try_wait\n", " (pid, sts) = os.waitpid(self.pid, wait_flags)\n", "KeyboardInterrupt\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/usr/local/bin/accelerate\", line 8, in \n", " sys.exit(main())\n", " File \"/usr/local/lib/python3.10/dist-packages/accelerate/commands/accelerate_cli.py\", line 45, in main\n", " args.func(args)\n", " File \"/usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py\", line 979, in launch_command\n", " simple_launcher(args)\n", " File \"/usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py\", line 625, in simple_launcher\n", " process.wait()\n", " File \"/usr/lib/python3.10/subprocess.py\", line 1222, in wait\n", " self._wait(timeout=sigint_timeout)\n", " File \"/usr/lib/python3.10/subprocess.py\", line 1953, in _wait\n", " time.sleep(delay)\n", "KeyboardInterrupt\n", "Traceback (most recent call last):\n", " File \"/content/diffusers/examples/text_to_image/train_text_to_image_lora.py\", line 949, in \n", " main()\n", " File \"/content/diffusers/examples/text_to_image/train_text_to_image_lora.py\", line 800, in main\n", " accelerator.backward(loss)\n", " File \"/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py\", line 1851, in backward\n", " self.scaler.scale(loss).backward(**kwargs)\n", " File \"/usr/local/lib/python3.10/dist-packages/torch/_tensor.py\", line 487, in backward\n", " torch.autograd.backward(\n", " File \"/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py\", line 200, in backward\n", " Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass\n", "KeyboardInterrupt\n", "Steps: 6% 855/15000 [31:31<8:41:39, 2.21s/it, lr=9.92e-5, step_loss=0.0107]\n", "^C\n" ] } ] }, { "cell_type": "code", "source": [ "from datasets import load_dataset\n", "\n", "dataset = load_dataset(\"andy-fang/andy_portraits\", split = 'train')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "FcF5_SCONQBq", "outputId": "0c0853d4-7bab-4141-cbaf-9cb6b7c0161f" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "WARNING:datasets.builder:Found cached dataset imagefolder (/root/.cache/huggingface/datasets/andy-fang___imagefolder/andy-fang--andy_portraits-02e4c0b66461e199/0.0.0/37fbb85cc714a338bea574ac6c7d0b5be5aff46c1862c1989b20e0771199e93f)\n" ] } ] }, { "cell_type": "code", "source": [ "dataset" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "OCn2CJJfNzH8", "outputId": "ae095d98-a3e6-492b-86b0-41d523d25493" }, "execution_count": null, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "Dataset({\n", " features: ['image'],\n", " num_rows: 15\n", "})" ] }, "metadata": {}, "execution_count": 8 } ] }, { "cell_type": "code", "source": [ "new_column = [\"Andy\"] * len(dataset)\n", "dataset = dataset.add_column(\"text\", new_column)\n", "dataset" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "XAOmVoDaNpoN", "outputId": "d94d1fd0-9827-4243-f43a-4bb5cd3f3d63" }, "execution_count": null, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "Dataset({\n", " features: ['image', 'text'],\n", " num_rows: 15\n", "})" ] }, "metadata": {}, "execution_count": 9 } ] }, { "cell_type": "code", "source": [ "pip install huggingface_hub" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "jdWOezFnOZdc", "outputId": "7efcfa19-efd4-4bdd-d6ec-1ad0e78f86cc" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.10/dist-packages (0.16.4)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (3.12.2)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2023.6.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2.27.1)\n", "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.65.0)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (6.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.7.1)\n", "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (23.1)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (1.26.16)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2023.5.7)\n", "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2.0.12)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.4)\n" ] } ] }, { "cell_type": "code", "source": [ "!huggingface-cli login" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "oFjTbRCnOcfv", "outputId": "88511471-8a0d-49c2-db55-7cae354882ac" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", " _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_|\n", " _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_|\n", " _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_|\n", " \n", " To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n", "Token: \n", "Add token as git credential? (Y/n) y\n", "Token is valid (permission: write).\n", "\u001b[1m\u001b[31mCannot authenticate through git-credential as no helper is defined on your machine.\n", "You might have to re-authenticate when pushing to the Hugging Face Hub.\n", "Run the following command in your terminal in case you want to set the 'store' credential helper as default.\n", "\n", "git config --global credential.helper store\n", "\n", "Read https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more details.\u001b[0m\n", "Token has not been saved to git credential helper.\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ] }, { "cell_type": "code", "source": [ "from datasets import load_dataset\n", "\n", "dataset.push_to_hub(\"Talha185/tti-tester\")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 81, "referenced_widgets": [ "9c6c541edc1347c98dbc452b30dc5284", "18b244742415432c9a263cdb9339b519", "70f1f8e16654493ea712051e748db3ea", "b3961785070743108bb712abb5f117b9", "c610fbc9163f43a4874cb6f70d2436b2", "22c90993784c4017b4ee6cd39447feea", "fba40eab7d62470687a32419dcd120de", "388c884346494beca1ca8e517a6fa436", "38668c9a32a24a09956302af1a708dbd", "a71ae75033d449d385a61a70fd17c626", "0ecf42fd5b8648ffaf2d395457604b49", "f8a78562909a427d95b4a88edef66c13", "f34ef24ed4794ae9aece79b2ea5c2dc7", "0000dcfeb965404e91c9e26f299df05c", "bd90641c572e4006b52ea4fa5fc8741e", "8a51eb36c4cc4d238011d67101c18071", "f13a20c5f04848d1ba72aad491824b39", "dad7ec2a9fbe4cd38a652f8205fffe1a", "f7033c8a99854a0cb208dec55d709b8a", "6c350f5aba4b4d559e3a4d6100cfe286", "67dbcb396e384b99afe162e084df4bdd", "73dd4ca5836b4eaaa994c153954f7fed", "5b3582f209a04a12a0bd0d563e2b9c8c", "c121b78205c44822af084c9017087124", "52f3bea1142c47778e393359df4217a9", "e21d6578bec148b9b6be4aea45c42bac", "2787d66fc6e94281a4e6bfd631093b91", "821f64ded5034bc99b4c6f1ce16bfce1", "10c5e5accc624c4aba5ce8bd7ecb6f6c", "07eb56d7db104561812e3951bcdf9c9f", "415799df471b4fc3831bca903b353923", "eced1bfc315c4001b958a870902eb2f9", "eb2eb6f912c34d7a93d7abaef9887138" ] }, "id": "aEFr8tUNOixA", "outputId": "a43d355a-2ee1-4d35-ab50-d23d94f2313e" }, "execution_count": null, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "Map: 0%| | 0/15 [00:00