Refactor structure

csv/block_items.csv

@@ -1,34 +0,0 @@
-Date,Host,Service,Blocked Cart Items,Message
-"2024-06-25T21:51:35.523Z","""i-0ec7dc300989b0f58""","""cash-commerce-browser""","""[{\""name\"":\""Promotional Email GiftCard $10\"",\""image_url\"":\""https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg\""}, {\""name\"":\""Promotional Email GiftCard $10\"",\""image_url\"":\""https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg\""}]""","Found restricted items: [ShoppingCartProduct{name=Promotional Email GiftCard $10, image_url=https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg}, ShoppingCartProduct{name=Promotional Email GiftCard $10, image_url=https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg}]; {customer_token=C_gqed91mw3, flow_token=4s6ZZrxNlMdUWqbN59nXHURXi, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_3mfvyukp, sup_token=BRAND_57fqf0l740hpnbxev71c6d56e}"
-"2024-06-25T21:43:14.348Z","""i-0ec7dc300989b0f58""","""cash-commerce-browser""","""[{\""name\"":\""$100 Vanilla® Visa® Gift Box Gift Card (plus $5.44 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/55844027\""}]""","Found restricted items: [ShoppingCartProduct{name=$100 Vanilla® Visa® Gift Box Gift Card (plus $5.44 Purchase Fee), url=https://www.walmart.com/ip/seort/55844027}]; {customer_token=C_86dpxgy6b, flow_token=4c546519-bc7c-4ae7-bbf0-5daf158351b1, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=null}"
-"2024-06-25T21:43:05.816Z","""i-044ba56b80a95063d""","""cash-commerce-browser""","""[{\""name\"":\""Promotional Email GiftCard $10\"",\""image_url\"":\""https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg\""}]""","Found restricted items: [ShoppingCartProduct{name=Promotional Email GiftCard $10, image_url=https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg}]; {customer_token=C_fsexdpmbh, flow_token=fRqPMaEwTcPsgLuLQ8mCvf9df, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_3mfvyukp, sup_token=BRAND_57fqf0l740hpnbxev71c6d56e}"
-"2024-06-25T21:35:36.381Z","""i-0ec7dc300989b0f58""","""cash-commerce-browser""","""[{\""name\"":\""GPS7000 GPS Tracker for Vehicles - Hidden Tracking Device for Any Vehicle - Easy Installation on Car's Battery- 10 Days of Service - Subscription Required - Low Cost Subscription Plan Options\"",\""url\"":\""https://www.amazon.com/gp/aw/d/B0BYK99LZC/ref=ox_sc_act_title_delete_4?smid=AWU3BDL6BD5T5&psc=1\"",\""image_url\"":\""https://m.media-amazon.com/images/I/61E6Cex5dUL._AC_AA135_.jpg\""}]""","Found restricted items: [ShoppingCartProduct{name=GPS7000 GPS Tracker for Vehicles - Hidden Tracking Device for Any Vehicle - Easy Installation on Car's Battery- 10 Days of Service - Subscription Required - Low Cost Subscription Plan Options, url=https://www.amazon.com/gp/aw/d/B0BYK99LZC/ref=ox_sc_act_title_delete_4?smid=AWU3BDL6BD5T5&psc=1, image_url=https://m.media-amazon.com/images/I/61E6Cex5dUL._AC_AA135_.jpg}]; {customer_token=C_5cpf4hygz, flow_token=73658682-d99e-4742-b6b4-8ebb7f1543ad, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_ih4h4tzb, sup_token=BRAND_djwngg0n5zluxwp131plevfvw}"
-"2024-06-25T21:35:24.733Z","""i-044ba56b80a95063d""","""cash-commerce-browser""","""[{\""name\"":\""GPS7000 GPS Tracker for Vehicles - Hidden Tracking Device for Any Vehicle - Easy Installation on Car's Battery- 10 Days of Service - Subscription Required - Low Cost Subscription Plan Options\"",\""url\"":\""https://www.amazon.com/gp/aw/d/B0BYK99LZC/ref=ox_sc_act_title_delete_4?smid=AWU3BDL6BD5T5&psc=1\"",\""image_url\"":\""https://m.media-amazon.com/images/I/61E6Cex5dUL._AC_AA135_.jpg\""}]""","Found restricted items: [ShoppingCartProduct{name=GPS7000 GPS Tracker for Vehicles - Hidden Tracking Device for Any Vehicle - Easy Installation on Car's Battery- 10 Days of Service - Subscription Required - Low Cost Subscription Plan Options, url=https://www.amazon.com/gp/aw/d/B0BYK99LZC/ref=ox_sc_act_title_delete_4?smid=AWU3BDL6BD5T5&psc=1, image_url=https://m.media-amazon.com/images/I/61E6Cex5dUL._AC_AA135_.jpg}]; {customer_token=C_5cpf4hygz, flow_token=73658682-d99e-4742-b6b4-8ebb7f1543ad, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_ih4h4tzb, sup_token=BRAND_djwngg0n5zluxwp131plevfvw}"
-"2024-06-25T21:35:20.107Z","""i-003fbf26706b7af3e""","""cash-commerce-browser""","""[{\""name\"":\""GPS7000 GPS Tracker for Vehicles - Hidden Tracking Device for Any Vehicle - Easy Installation on Car's Battery- 10 Days of Service - Subscription Required - Low Cost Subscription Plan Options\"",\""url\"":\""https://www.amazon.com/gp/aw/d/B0BYK99LZC/ref=ox_sc_act_title_delete_4?smid=AWU3BDL6BD5T5&psc=1\"",\""image_url\"":\""https://m.media-amazon.com/images/I/61E6Cex5dUL._AC_AA135_.jpg\""}]""","Found restricted items: [ShoppingCartProduct{name=GPS7000 GPS Tracker for Vehicles - Hidden Tracking Device for Any Vehicle - Easy Installation on Car's Battery- 10 Days of Service - Subscription Required - Low Cost Subscription Plan Options, url=https://www.amazon.com/gp/aw/d/B0BYK99LZC/ref=ox_sc_act_title_delete_4?smid=AWU3BDL6BD5T5&psc=1, image_url=https://m.media-amazon.com/images/I/61E6Cex5dUL._AC_AA135_.jpg}]; {customer_token=C_5cpf4hygz, flow_token=73658682-d99e-4742-b6b4-8ebb7f1543ad, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_ih4h4tzb, sup_token=BRAND_djwngg0n5zluxwp131plevfvw}"
-"2024-06-25T21:35:01.987Z","""i-003fbf26706b7af3e""","""cash-commerce-browser""","""[{\""name\"":\""GPS7000 GPS Tracker for Vehicles - Hidden Tracking Device for Any Vehicle - Easy Installation on Car's Battery- 10 Days of Service - Subscription Required - Low Cost Subscription Plan Options\"",\""url\"":\""https://www.amazon.com/gp/aw/d/B0BYK99LZC/ref=ox_sc_act_title_delete_4?smid=AWU3BDL6BD5T5&psc=1\"",\""image_url\"":\""https://m.media-amazon.com/images/I/61E6Cex5dUL._AC_AA135_.jpg\""}]""","Found restricted items: [ShoppingCartProduct{name=GPS7000 GPS Tracker for Vehicles - Hidden Tracking Device for Any Vehicle - Easy Installation on Car's Battery- 10 Days of Service - Subscription Required - Low Cost Subscription Plan Options, url=https://www.amazon.com/gp/aw/d/B0BYK99LZC/ref=ox_sc_act_title_delete_4?smid=AWU3BDL6BD5T5&psc=1, image_url=https://m.media-amazon.com/images/I/61E6Cex5dUL._AC_AA135_.jpg}]; {customer_token=C_5cpf4hygz, flow_token=73658682-d99e-4742-b6b4-8ebb7f1543ad, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_ih4h4tzb, sup_token=BRAND_djwngg0n5zluxwp131plevfvw}"
-"2024-06-25T21:34:07.013Z","""i-044ba56b80a95063d""","""cash-commerce-browser""","""[{\""name\"":\""$50 Vanilla Visa eGift Card\"",\""url\"":\""https://www.samsclub.com/p/vanilla-e-gift-visa-various-amount/prod25810992\"",\""image_url\"":\""https://scene7.samsclub.com/is/image/samsclub/0079936655659_A?wid=200&hei=200\""}]""","Found restricted items: [ShoppingCartProduct{name=$50 Vanilla Visa eGift Card, url=https://www.samsclub.com/p/vanilla-e-gift-visa-various-amount/prod25810992, image_url=https://scene7.samsclub.com/is/image/samsclub/0079936655659_A?wid=200&hei=200}]; {customer_token=C_haed91mp3, flow_token=Q81SuACF2yWsd6Fn6PRWDsBK4, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_dsjbdob6, sup_token=null}"
-"2024-06-25T21:33:34.552Z","""i-0ec7dc300989b0f58""","""cash-commerce-browser""","""[{\""name\"":\""GPS7000 GPS Tracker for Vehicles - Hidden Tracking Device for Any Vehicle - Easy Installation on Car's Battery- 10 Days of Service - Subscription Required - Low Cost Subscription Plan Options\"",\""url\"":\""https://www.amazon.com/gp/aw/d/B0BYK99LZC/ref=ox_sc_act_title_delete_4?smid=AWU3BDL6BD5T5&psc=1\"",\""image_url\"":\""https://m.media-amazon.com/images/I/61E6Cex5dUL._AC_AA135_.jpg\""}]""","Found restricted items: [ShoppingCartProduct{name=GPS7000 GPS Tracker for Vehicles - Hidden Tracking Device for Any Vehicle - Easy Installation on Car's Battery- 10 Days of Service - Subscription Required - Low Cost Subscription Plan Options, url=https://www.amazon.com/gp/aw/d/B0BYK99LZC/ref=ox_sc_act_title_delete_4?smid=AWU3BDL6BD5T5&psc=1, image_url=https://m.media-amazon.com/images/I/61E6Cex5dUL._AC_AA135_.jpg}]; {customer_token=C_5cpf4hygz, flow_token=73658682-d99e-4742-b6b4-8ebb7f1543ad, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_ih4h4tzb, sup_token=BRAND_djwngg0n5zluxwp131plevfvw}"
-"2024-06-25T21:31:29.788Z","""i-0c92416a905681db6""","""cash-commerce-browser""","""[{\""name\"":\""$200 Vanilla Visa Shiny Bow Gift Card (plus $6.88 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/1042177970\""}]""","Found restricted items: [ShoppingCartProduct{name=$200 Vanilla Visa Shiny Bow Gift Card (plus $6.88 Purchase Fee), url=https://www.walmart.com/ip/seort/1042177970}]; {customer_token=C_smecd0mhm, flow_token=wnHQN78zduV3T55TsN8s0CKHA, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:30:38.963Z","""i-044ba56b80a95063d""","""cash-commerce-browser""","""[{\""name\"":\""$200 Vanilla Visa Shiny Bow Gift Card (plus $6.88 Purchase Fee)\"",\""image_url\"":\""https://i5.walmartimages.com/seo/200-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-6-88-Purchase-Fee_1cc86583-2dcb-4677-bca4-16f685df6170.d43209d754cbb7ab3c09354d6cf906a7.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF\""}]""","Found restricted items: [ShoppingCartProduct{name=$200 Vanilla Visa Shiny Bow Gift Card (plus $6.88 Purchase Fee), image_url=https://i5.walmartimages.com/seo/200-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-6-88-Purchase-Fee_1cc86583-2dcb-4677-bca4-16f685df6170.d43209d754cbb7ab3c09354d6cf906a7.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF}]; {customer_token=C_smecd0mhm, flow_token=CCvc7KfXlUWnhdqtcxORrG4Sp, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:29:54.170Z","""i-0ec7dc300989b0f58""","""cash-commerce-browser""","""[{\""name\"":\""$200 Vanilla Visa Shiny Bow Gift Card (plus $6.88 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/1042177970\""}]""","Found restricted items: [ShoppingCartProduct{name=$200 Vanilla Visa Shiny Bow Gift Card (plus $6.88 Purchase Fee), url=https://www.walmart.com/ip/seort/1042177970}]; {customer_token=C_smecd0mhm, flow_token=oucI3kCcOLy76Hiv8Tb2vxgfa, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:26:19.296Z","""i-0c92416a905681db6""","""cash-commerce-browser""","""[{\""name\"":\""$200 Vanilla® Visa® Gift Box Gift Card (plus $6.88 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/20625678\""}]""","Found restricted items: [ShoppingCartProduct{name=$200 Vanilla® Visa® Gift Box Gift Card (plus $6.88 Purchase Fee), url=https://www.walmart.com/ip/seort/20625678}]; {customer_token=C_002v20mt1, flow_token=iNJAgMr3PvHc07lbZAkdrnEIn, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:20:06.661Z","""i-003fbf26706b7af3e""","""cash-commerce-browser""","""[{\""name\"":\""Promotional Email GiftCard $20\"",\""image_url\"":\""https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg\""}, {\""name\"":\""Promotional Email GiftCard $10\"",\""image_url\"":\""https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg\""}]""","Found restricted items: [ShoppingCartProduct{name=Promotional Email GiftCard $20, image_url=https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg}, ShoppingCartProduct{name=Promotional Email GiftCard $10, image_url=https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg}]; {customer_token=C_ssdwxqyqz, flow_token=roXsSw6mIgIqe5XjF8bO8tH2o, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_3mfvyukp, sup_token=BRAND_57fqf0l740hpnbxev71c6d56e}"
-"2024-06-25T21:18:25.022Z","""i-044ba56b80a95063d""","""cash-commerce-browser""","""[{\""name\"":\""Xbox $100 Gift Card - [Digital]\"",\""url\"":\""https://www.walmart.com/ip/seort/48695417\""}]""","Found restricted items: [ShoppingCartProduct{name=Xbox $100 Gift Card - \[Digital\], url=https://www.walmart.com/ip/seort/48695417}]; {customer_token=C_h9ccehmmq, flow_token=ZichXe9bjvCk2NYseQpasHyxW, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:15:07.437Z","""i-0ec7dc300989b0f58""","""cash-commerce-browser""","""[{\""name\"":\""$25 Vanilla Visa Shiny Bow Gift Card (plus $3.44 Purchase Fee), quantity 2\"",\""image_url\"":\""https://i5.walmartimages.com/seo/25-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-44-Purchase-Fee_852b53b6-7090-407d-a47b-06618b04d244.3c1e4f0aae77e8bf05fdd0660a354377.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF\""}, {\""name\"":\""$50 Vanilla Visa Shiny Bow Gift Card (plus $3.94 Purchase Fee), quantity 2\"",\""image_url\"":\""https://i5.walmartimages.com/seo/50-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-94-Purchase-Fee_97c67191-c411-43a8-8174-5bed3ccfa2ed.bfc91450d0b09abcfbab003e6edf9fcf.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF\""}]""","Found restricted items: [ShoppingCartProduct{name=$25 Vanilla Visa Shiny Bow Gift Card (plus $3.44 Purchase Fee)\, quantity 2, image_url=https://i5.walmartimages.com/seo/25-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-44-Purchase-Fee_852b53b6-7090-407d-a47b-06618b04d244.3c1e4f0aae77e8bf05fdd0660a354377.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF}, ShoppingCartProduct{name=$50 Vanilla Visa Shiny Bow Gift Card (plus $3.94 Purchase Fee)\, quantity 2, image_url=https://i5.walmartimages.com/seo/50-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-94-Purchase-Fee_97c67191-c411-43a8-8174-5bed3ccfa2ed.bfc91450d0b09abcfbab003e6edf9fcf.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF}]; {customer_token=C_gawfj8yrr, flow_token=08e30143-ac4d-428a-963a-f9bbd10cfdb5, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:15:01.385Z","""i-003fbf26706b7af3e""","""cash-commerce-browser""","""[{\""name\"":\""$25 Vanilla Visa Shiny Bow Gift Card (plus $3.44 Purchase Fee)\"",\""image_url\"":\""https://i5.walmartimages.com/seo/25-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-44-Purchase-Fee_852b53b6-7090-407d-a47b-06618b04d244.3c1e4f0aae77e8bf05fdd0660a354377.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF\""}, {\""name\"":\""$50 Vanilla Visa Shiny Bow Gift Card (plus $3.94 Purchase Fee), quantity 2\"",\""image_url\"":\""https://i5.walmartimages.com/seo/50-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-94-Purchase-Fee_97c67191-c411-43a8-8174-5bed3ccfa2ed.bfc91450d0b09abcfbab003e6edf9fcf.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF\""}]""","Found restricted items: [ShoppingCartProduct{name=$25 Vanilla Visa Shiny Bow Gift Card (plus $3.44 Purchase Fee), image_url=https://i5.walmartimages.com/seo/25-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-44-Purchase-Fee_852b53b6-7090-407d-a47b-06618b04d244.3c1e4f0aae77e8bf05fdd0660a354377.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF}, ShoppingCartProduct{name=$50 Vanilla Visa Shiny Bow Gift Card (plus $3.94 Purchase Fee)\, quantity 2, image_url=https://i5.walmartimages.com/seo/50-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-94-Purchase-Fee_97c67191-c411-43a8-8174-5bed3ccfa2ed.bfc91450d0b09abcfbab003e6edf9fcf.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF}]; {customer_token=C_gawfj8yrr, flow_token=08e30143-ac4d-428a-963a-f9bbd10cfdb5, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:14:58.239Z","""i-003fbf26706b7af3e""","""cash-commerce-browser""","""[{\""name\"":\""$25 Vanilla Visa Shiny Bow Gift Card (plus $3.44 Purchase Fee)\"",\""image_url\"":\""https://i5.walmartimages.com/seo/25-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-44-Purchase-Fee_852b53b6-7090-407d-a47b-06618b04d244.3c1e4f0aae77e8bf05fdd0660a354377.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF\""}, {\""name\"":\""$50 Vanilla Visa Shiny Bow Gift Card (plus $3.94 Purchase Fee)\"",\""image_url\"":\""https://i5.walmartimages.com/seo/50-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-94-Purchase-Fee_97c67191-c411-43a8-8174-5bed3ccfa2ed.bfc91450d0b09abcfbab003e6edf9fcf.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF\""}]""","Found restricted items: [ShoppingCartProduct{name=$25 Vanilla Visa Shiny Bow Gift Card (plus $3.44 Purchase Fee), image_url=https://i5.walmartimages.com/seo/25-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-44-Purchase-Fee_852b53b6-7090-407d-a47b-06618b04d244.3c1e4f0aae77e8bf05fdd0660a354377.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF}, ShoppingCartProduct{name=$50 Vanilla Visa Shiny Bow Gift Card (plus $3.94 Purchase Fee), image_url=https://i5.walmartimages.com/seo/50-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-94-Purchase-Fee_97c67191-c411-43a8-8174-5bed3ccfa2ed.bfc91450d0b09abcfbab003e6edf9fcf.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF}]; {customer_token=C_gawfj8yrr, flow_token=08e30143-ac4d-428a-963a-f9bbd10cfdb5, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:13:44.815Z","""i-0c92416a905681db6""","""cash-commerce-browser""","""[{\""name\"":\""Xbox $100 Gift Card - [Digital]\"",\""url\"":\""https://www.walmart.com/ip/seort/48695417\""}]""","Found restricted items: [ShoppingCartProduct{name=Xbox $100 Gift Card - \[Digital\], url=https://www.walmart.com/ip/seort/48695417}]; {customer_token=C_h9ccehmmq, flow_token=W9fV7QLzgEeyuDVEf2101TEnm, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:13:36.779Z","""i-0ec7dc300989b0f58""","""cash-commerce-browser""","""[{\""name\"":\""Home Depot Birthday Cupcake eGift\"",\""url\"":\""https://www.homedepot.com/gift-cards/p/Home-Depot-Birthday-Cupcake-eGift/53L1QLAXWKVF06WJ8DLHBYT6S8/5J0QS6XYWSCJ3H9HG3F6YSRL2C\"",\""image_url\"":\""https://images.thdstatic.com/giftcards/catalog/53L1QLAXWKVF06WJ8DLHBYT6S8/xxlarge/5J0QS6XYWSCJ3H9HG3F6YSRL2C_0608202315:26:21.PNG\""}]""","Found restricted items: [ShoppingCartProduct{name=Home Depot Birthday Cupcake eGift, url=https://www.homedepot.com/gift-cards/p/Home-Depot-Birthday-Cupcake-eGift/53L1QLAXWKVF06WJ8DLHBYT6S8/5J0QS6XYWSCJ3H9HG3F6YSRL2C, image_url=https://images.thdstatic.com/giftcards/catalog/53L1QLAXWKVF06WJ8DLHBYT6S8/xxlarge/5J0QS6XYWSCJ3H9HG3F6YSRL2C_0608202315:26:21.PNG}]; {customer_token=C_ssd2wpm6w, flow_token=V53QOpItzVeaNwXwPC6cewpnt, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_kiqt4xl6, sup_token=BRAND_78qp2zjkqjto3v3xxr5ibueyv}"
-"2024-06-25T21:12:33.161Z","""i-003fbf26706b7af3e""","""cash-commerce-browser""","""[{\""name\"":\""$100 Vanilla® Visa® Gift Box Gift Card (plus $5.44 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/55844027\"",\""image_url\"":\""https://i5.walmartimages.com/seo/100-Vanilla-Visa-Gift-Box-Gift-Card-plus-5-44-Purchase-Fee_d1b5b130-1ae0-4690-a1ee-4e06ff34661a.1e16d3785d608d00f8636a7a91f40182.jpeg?odnHeight=72&odnWidth=72&odnBg=FFFFFF\""}, {\""name\"":\""$50 Vanilla Visa Shiny Bow Gift Card (plus $3.94 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/1529427774\"",\""image_url\"":\""https://i5.walmartimages.com/seo/50-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-94-Purchase-Fee_97c67191-c411-43a8-8174-5bed3ccfa2ed.bfc91450d0b09abcfbab003e6edf9fcf.jpeg?odnHeight=72&odnWidth=72&odnBg=FFFFFF\""}, {\""name\"":\""$25 Vanilla Visa Shiny Bow Gift Card (plus $3.44 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/1314241502\"",\""image_url\"":\""https://i5.walmartimages.com/seo/25-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-44-Purchase-Fee_852b53b6-7090-407d-a47b-06618b04d244.3c1e4f0aae77e8bf05fdd0660a354377.jpeg?odnHeight=72&odnWidth=72&odnBg=FFFFFF\""}]""","Found restricted items: [ShoppingCartProduct{name=$100 Vanilla® Visa® Gift Box Gift Card (plus $5.44 Purchase Fee), url=https://www.walmart.com/ip/seort/55844027, image_url=https://i5.walmartimages.com/seo/100-Vanilla-Visa-Gift-Box-Gift-Card-plus-5-44-Purchase-Fee_d1b5b130-1ae0-4690-a1ee-4e06ff34661a.1e16d3785d608d00f8636a7a91f40182.jpeg?odnHeight=72&odnWidth=72&odnBg=FFFFFF}, ShoppingCartProduct{name=$50 Vanilla Visa Shiny Bow Gift Card (plus $3.94 Purchase Fee), url=https://www.walmart.com/ip/seort/1529427774, image_url=https://i5.walmartimages.com/seo/50-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-94-Purchase-Fee_97c67191-c411-43a8-8174-5bed3ccfa2ed.bfc91450d0b09abcfbab003e6edf9fcf.jpeg?odnHeight=72&odnWidth=72&odnBg=FFFFFF}, ShoppingCartProduct{name=$25 Vanilla Visa Shiny Bow Gift Card (plus $3.44 Purchase Fee), url=https://www.walmart.com/ip/seort/1314241502, image_url=https://i5.walmartimages.com/seo/25-Vanilla-Visa-Shiny-Bow-Gift-Card-plus-3-44-Purchase-Fee_852b53b6-7090-407d-a47b-06618b04d244.3c1e4f0aae77e8bf05fdd0660a354377.jpeg?odnHeight=72&odnWidth=72&odnBg=FFFFFF}]; {customer_token=C_gawfj8yrr, flow_token=08e30143-ac4d-428a-963a-f9bbd10cfdb5, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:09:57.657Z","""i-003fbf26706b7af3e""","""cash-commerce-browser""","""[{\""name\"":\""Birthday Celebration Target Giftcard\"",\""url\"":\""https://www.target.com/p/birthday-celebration-target-giftcard--no-aasa/-/A-81480862\"",\""image_url\"":\""https://target.scene7.com/is/image/Target//GUEST_c2ea8817-d6da-437c-9d85-c39b2e96953d?qlt=80&fmt=pjpeg\""}]""","Found restricted items: [ShoppingCartProduct{name=Birthday Celebration Target Giftcard, url=https://www.target.com/p/birthday-celebration-target-giftcard--no-aasa/-/A-81480862, image_url=https://target.scene7.com/is/image/Target//GUEST_c2ea8817-d6da-437c-9d85-c39b2e96953d?qlt=80&fmt=pjpeg}]; {customer_token=C_8xdwa1m6n, flow_token=9RBhrYZk8g7j2WDYyM3GbfdES, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_3mfvyukp, sup_token=BRAND_57fqf0l740hpnbxev71c6d56e}"
-"2024-06-25T21:09:49.051Z","""i-003fbf26706b7af3e""","""cash-commerce-browser""","""[{\""name\"":\""Birthday Celebration Target Giftcard\"",\""url\"":\""https://www.target.com/p/birthday-celebration-target-giftcard--no-aasa/-/A-81480862\"",\""image_url\"":\""https://target.scene7.com/is/image/Target//GUEST_c2ea8817-d6da-437c-9d85-c39b2e96953d?qlt=80&fmt=pjpeg\""}]""","Found restricted items: [ShoppingCartProduct{name=Birthday Celebration Target Giftcard, url=https://www.target.com/p/birthday-celebration-target-giftcard--no-aasa/-/A-81480862, image_url=https://target.scene7.com/is/image/Target//GUEST_c2ea8817-d6da-437c-9d85-c39b2e96953d?qlt=80&fmt=pjpeg}]; {customer_token=C_8xdwa1m6n, flow_token=J7liXzVSAPSDIaK4CWVLC7bBi, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_3mfvyukp, sup_token=BRAND_57fqf0l740hpnbxev71c6d56e}"
-"2024-06-25T21:09:35.160Z","""i-003fbf26706b7af3e""","""cash-commerce-browser""","""[{\""name\"":\""Promotional Email GiftCard $10\"",\""image_url\"":\""https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg\""}]""","Found restricted items: [ShoppingCartProduct{name=Promotional Email GiftCard $10, image_url=https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg}]; {customer_token=C_fsexdpmbh, flow_token=jVDz28RrrN9tAkim71XsZGZMD, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_3mfvyukp, sup_token=BRAND_57fqf0l740hpnbxev71c6d56e}"
-"2024-06-25T21:09:35.022Z","""i-0ec7dc300989b0f58""","""cash-commerce-browser""","""[{\""name\"":\""Birthday Celebration Target Giftcard\"",\""url\"":\""https://www.target.com/p/birthday-celebration-target-giftcard--no-aasa/-/A-81480862\"",\""image_url\"":\""\""}]""","Found restricted items: [ShoppingCartProduct{name=Birthday Celebration Target Giftcard, url=https://www.target.com/p/birthday-celebration-target-giftcard--no-aasa/-/A-81480862, image_url=}]; {customer_token=C_8xdwa1m6n, flow_token=9pZoMnaqRmVXuWQ9P9pSLefbO, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_3mfvyukp, sup_token=BRAND_57fqf0l740hpnbxev71c6d56e}"
-"2024-06-25T21:09:07.723Z","""i-044ba56b80a95063d""","""cash-commerce-browser""","""[{\""name\"":\""Nintendo Switch Family Online 12 Month Gift Card [Digital]\"",\""url\"":\""https://www.walmart.com/ip/seort/875981752\""}]""","Found restricted items: [ShoppingCartProduct{name=Nintendo Switch Family Online 12 Month Gift Card \[Digital\], url=https://www.walmart.com/ip/seort/875981752}]; {customer_token=C_gq2ekammv, flow_token=03c0b437-4fc5-4967-9692-3efddd21a0d6, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T21:08:46.049Z","""i-044ba56b80a95063d""","""cash-commerce-browser""","""[{\""name\"":\""Promotional Email GiftCard $10\"",\""image_url\"":\""https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg\""}]""","Found restricted items: [ShoppingCartProduct{name=Promotional Email GiftCard $10, image_url=https://target.scene7.com/is/image/Target//GUEST_337c880d-e2c0-4794-9784-962c63cf0646?qlt=80&fmt=pjpeg}]; {customer_token=C_fsexdpmbh, flow_token=e5xlrZ3F45YirfY5AHNFlAvtV, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_3mfvyukp, sup_token=BRAND_57fqf0l740hpnbxev71c6d56e}"
-"2024-06-25T20:55:28.994Z","""i-044ba56b80a95063d""","""cash-commerce-browser""","""[{\""name\"":\""$50 Vanilla® Mastercard® Celebration Dots Gift Card (plus $3.94 Purchase Fee)\"",\""image_url\"":\""https://i5.walmartimages.com/seo/50-Vanilla-Mastercard-Celebration-Dots-Gift-Card-plus-3-94-Purchase-Fee_04e596b9-6d0a-41c9-96ed-7bdf11468cd2.d9453dcf7909a341bf31ee4dce04c833.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF\""}]""","Found restricted items: [ShoppingCartProduct{name=$50 Vanilla® Mastercard® Celebration Dots Gift Card (plus $3.94 Purchase Fee), image_url=https://i5.walmartimages.com/seo/50-Vanilla-Mastercard-Celebration-Dots-Gift-Card-plus-3-94-Purchase-Fee_04e596b9-6d0a-41c9-96ed-7bdf11468cd2.d9453dcf7909a341bf31ee4dce04c833.jpeg?odnHeight=48&odnWidth=48&odnBg=FFFFFF}]; {customer_token=C_nywqkaygt, flow_token=IOAzdzUmK2FLIN1KiCt8xQZQd, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T20:55:26.016Z","""i-0c92416a905681db6""","""cash-commerce-browser""","""[{\""name\"":\""$50 Vanilla® Mastercard® Celebration Dots Gift Card (plus $3.94 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/46518449\""}]""","Found restricted items: [ShoppingCartProduct{name=$50 Vanilla® Mastercard® Celebration Dots Gift Card (plus $3.94 Purchase Fee), url=https://www.walmart.com/ip/seort/46518449}]; {customer_token=C_nywqkaygt, flow_token=dEIHJrVnTIH6xsmTYspEfBcle, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T20:55:13.996Z","""i-0c92416a905681db6""","""cash-commerce-browser""","""[{\""name\"":\""$50 Vanilla® Mastercard® Celebration Dots Gift Card (plus $3.94 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/46518449\""}, {\""name\"":\""$50 Vanilla® Visa® eGift Card (plus $3.94 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/761861368\""}]""","Found restricted items: [ShoppingCartProduct{name=$50 Vanilla® Mastercard® Celebration Dots Gift Card (plus $3.94 Purchase Fee), url=https://www.walmart.com/ip/seort/46518449}, ShoppingCartProduct{name=$50 Vanilla® Visa® eGift Card (plus $3.94 Purchase Fee), url=https://www.walmart.com/ip/seort/761861368}]; {customer_token=C_nywqkaygt, flow_token=MJgyeF7Eew00jGOPMLCG2rHyD, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T20:52:39.628Z","""i-0c92416a905681db6""","""cash-commerce-browser""","""[{\""name\"":\""$50 Vanilla® Visa® eGift Card (plus $3.94 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/761861368\""}]""","Found restricted items: [ShoppingCartProduct{name=$50 Vanilla® Visa® eGift Card (plus $3.94 Purchase Fee), url=https://www.walmart.com/ip/seort/761861368}]; {customer_token=C_nywqkaygt, flow_token=WJm0vCQG5TcMjVjTvpMqrfOGs, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T20:52:36.059Z","""i-0ec7dc300989b0f58""","""cash-commerce-browser""","""[{\""name\"":\""$50 Vanilla Visa Shiny Bow Gift Card (plus $3.94 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/1529427774\""}, {\""name\"":\""$50 Vanilla® Visa® eGift Card (plus $3.94 Purchase Fee)\"",\""url\"":\""https://www.walmart.com/ip/seort/761861368\""}]""","Found restricted items: [ShoppingCartProduct{name=$50 Vanilla Visa Shiny Bow Gift Card (plus $3.94 Purchase Fee), url=https://www.walmart.com/ip/seort/1529427774}, ShoppingCartProduct{name=$50 Vanilla® Visa® eGift Card (plus $3.94 Purchase Fee), url=https://www.walmart.com/ip/seort/761861368}]; {customer_token=C_nywqkaygt, flow_token=a5YVHg3kkZn8fb3e3TT1b9GlZ, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_fopbedhc, sup_token=BRAND_4sxo1et5hf8sug8lh500413rj}"
-"2024-06-25T20:52:02.686Z","""i-0ec7dc300989b0f58""","""cash-commerce-browser""","""[{\""name\"":\""Delta Airlines Wedding $250 Gift Card (Email Delivery)\"",\""url\"":\""https://www.target.com/p/delta-airlines-wedding--250-gift-card--email-delivery---no-aasa/-/A-89537719\"",\""image_url\"":\""https://target.scene7.com/is/image/Target//GUEST_0de9a4fb-7f0e-4d85-806f-e841d7664a59?qlt=80&fmt=pjpeg\""}]""","Found restricted items: [ShoppingCartProduct{name=Delta Airlines Wedding $250 Gift Card (Email Delivery), url=https://www.target.com/p/delta-airlines-wedding--250-gift-card--email-delivery---no-aasa/-/A-89537719, image_url=https://target.scene7.com/is/image/Target//GUEST_0de9a4fb-7f0e-4d85-806f-e841d7664a59?qlt=80&fmt=pjpeg}]; {customer_token=C_80exppmwa, flow_token=pnOyvjetUVkuZfhIOQwOQCINH, payment_method=PAYMENT_METHOD_SINGLE_USE_PAYMENT, merchant_token=M_3mfvyukp, sup_token=BRAND_57fqf0l740hpnbxev71c6d56e}"

genai_SDK/Seq2Seq.py

@@ -1,250 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-import torch
-import torch.nn as nn
-import torch.optim as optim
-from torch.utils.data import DataLoader, random_split
-import pandas as pd
-from tqdm import tqdm
-import time
-from .Utilities import LanguageDataset
-
-class Seq2Seq():
-    """
-    Base class for Seq2Seq (text-generation models). This class will be inherited by wrappers of transformers like GPT2
-    and T5.
-
-    Attributes:
-
-    Methods:
-
-    """
-
-    def __init__(self, gpu=0, max_length=0, model_path=None):
-
-        # Load Seq2Seq to device based on available hardware
-        if torch.cuda.is_available():
-            self.device = torch.device('cuda')
-        else:
-            try:
-                self.device = torch.device('mps') # Apple Silicon
-            except Exception:
-                self.device = torch.device('cpu')
-
-        # GPU that model will run on
-        self.gpu = gpu
-
-        # Model specs
-        if model_path: self.model = torch.load(model_path).to(self.device)
-        else: self.model = None
-        self.model_name = ""
-        self.tokenizer = None
-        self.max_length = max_length
-
-        # Training specs
-        self.train_loader = None
-        self.valid_loader = None
-        self.results = pd.DataFrame(columns=['epoch', 'model_arch', 'batch_size', 'gpu', 'training_loss', 'validation_loss', 'epoch_duration_sec'])
-
-    def load_data(self, df, batch_size, train_ratio=0.8):
-        self.batch_size = batch_size
-        dataset = LanguageDataset(df, self.tokenizer)
-        train_size = int(0.8*len(dataset))
-        valid_size = len(dataset) - train_size
-        train_data, valid_data = random_split(dataset, [train_size, valid_size])
-        self.max_length = dataset.max_length
-        self.train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True)
-        self.valid_loader = DataLoader(valid_data, batch_size=self.batch_size)
-
-    """ Return training results """
-    def summary(self):
-        return self.results
-
-    """ Save model to path """
-    def to_pt(self, path):
-        torch.save(self.model, path)
-
-
-class GPT2(Seq2Seq):
-    """
-    This is the GPT2 implementation of Seq2Seq.
-    """
-
-    def __init__(self, gpu, model_name, batch_size=16):
-        super().__init__(gpu, max_length=0)
-        from transformers import GPT2Tokenizer, GPT2LMHeadModel
-        self.model_name = model_name
-        self.model = GPT2LMHeadModel.from_pretrained(self.model_name).to(self.device)
-        self.tokenizer = GPT2Tokenizer.from_pretrained(self.model_name)
-        self.tokenizer.pad_token = self.tokenizer.eos_token
-
-    def train(self, num_epochs=3, train_ratio=0.8):
-        criterion = nn.CrossEntropyLoss(ignore_index=self.tokenizer.pad_token_id)
-        optimizer = optim.Adam(self.model.parameters(), lr=5e-4)
-
-        # Init a results dataframe
-
-        results = pd.DataFrame(columns=['epoch', 'transformer', 'batch_size', 'gpu',
-                                        'training_loss', 'validation_loss', 'epoch_duration_sec'])
-        # The training loop
-        for epoch in range(num_epochs):
-            start_time = time.time()  # Start the timer for the epoch
-
-            # Training
-            ## This line tells the self.model we're in 'learning mode'
-            self.model.train()
-            epoch_training_loss = 0
-            train_iterator = tqdm(self.train_loader,
-                                  desc=f"Training Epoch {epoch + 1}/{num_epochs} Batch Size: {self.batch_size}, Transformer: {self.model_name}")
-            for batch in train_iterator:
-                optimizer.zero_grad()
-                inputs = batch['input_ids'].squeeze(1).to(self.device)
-                targets = inputs.clone()
-                outputs = self.model(input_ids=inputs, labels=targets)
-                loss = outputs.loss
-                loss.backward()
-                optimizer.step()
-                train_iterator.set_postfix({'Training Loss': loss.item()})
-                epoch_training_loss += loss.item()
-            avg_epoch_training_loss = epoch_training_loss / len(train_iterator)
-
-            # Validation
-            ## This line below tells the self.model to 'stop learning'
-            self.model.eval()
-            epoch_validation_loss = 0
-            total_loss = 0
-            valid_iterator = tqdm(self.valid_loader, desc=f"Validation Epoch {epoch + 1}/{num_epochs}")
-            with torch.no_grad():
-                for batch in valid_iterator:
-                    inputs = batch['input_ids'].squeeze(1).to(self.device)
-                    targets = inputs.clone()
-                    outputs = self.model(input_ids=inputs, labels=targets)
-                    loss = outputs.loss
-                    total_loss += loss
-                    valid_iterator.set_postfix({'Validation Loss': loss.item()})
-                    epoch_validation_loss += loss.item()
-
-            avg_epoch_validation_loss = epoch_validation_loss / len(self.valid_loader)
-
-            end_time = time.time()  # End the timer for the epoch
-            epoch_duration_sec = end_time - start_time  # Calculate the duration in seconds
-
-            new_row = {'transformer': self.model_name,
-                       'batch_size': self.batch_size,
-                       'gpu': self.gpu,
-                       'epoch': epoch + 1,
-                       'training_loss': avg_epoch_training_loss,
-                       'validation_loss': avg_epoch_validation_loss,
-                       'epoch_duration_sec': epoch_duration_sec}  # Add epoch_duration to the dataframe
-
-            self.results.loc[len(self.results)] = new_row
-            print(f"Epoch: {epoch + 1}, Validation Loss: {total_loss / len(self.valid_loader)}")
-
-    def generate_text(self, input_str, top_k=16, top_p=0.95, temperature=1.0, repetition_penalty=1.2):
-        # Encode string to tokens
-        input_ids= self.tokenizer.encode(input_str, return_tensors='pt').to(self.device)
-
-        # Feed tokens to model and get outcome tokens
-        output = self.model.generate(
-            input_ids,
-            max_length=self.max_length,
-            num_return_sequences=1,
-            do_sample=True,
-            top_k=top_k,
-            top_p=top_p,
-            temperature=temperature,
-            repetition_penalty=repetition_penalty
-        )
-
-        # Decode tokens to string
-        decoded_output = self.tokenizer.decode(output[0], skip_special_tokens=True)
-        return decoded_output
-
-class FlanT5(Seq2Seq):
-    """
-    This is the T5 implementation of Seq2Seq - it is designed to support T5 models of various sizes.
-    """
-    def __init__(self, gpu, model_name, batch_size=16):
-        super().__init__(gpu, max_length=0)
-        from transformers import T5ForConditionalGeneration, T5Tokenizer
-        self.model_name = model_name
-        self.model = T5ForConditionalGeneration.from_pretrained(self.model_name).to(self.device)
-        self.tokenizer = T5Tokenizer.from_pretrained(self.model_name)
-        self.tokenizer.pad_token = self.tokenizer.eos_token
-
-    def train(self, num_epochs=3, train_ratio=0.8):
-        criterion = nn.CrossEntropyLoss(ignore_index=self.tokenizer.pad_token_id)
-        optimizer = optim.Adam(self.model.parameters(), lr=5e-4)
-
-        # Init a results dataframe
-
-        self.results = pd.DataFrame(columns=['epoch', 'transformer', 'batch_size', 'gpu',
-                                        'training_loss', 'validation_loss', 'epoch_duration_sec'])
-        # The training loop
-        for epoch in range(num_epochs):
-            start_time = time.time()  # Start the timer for the epoch
-
-            # Training
-            ## This line tells the model we're in 'learning mode'
-            self.model.train()
-            epoch_training_loss = 0
-            train_iterator = tqdm(self.train_loader,
-                                  desc=f"Training Epoch {epoch + 1}/{num_epochs} Batch Size: {self.batch_size}, Transformer: {self.model_name}")
-            for batch in train_iterator:
-                optimizer.zero_grad()
-                inputs = batch['input_ids'].squeeze(1).to(self.device)
-                targets = batch['labels'].squeeze(1).to(self.device)
-                outputs = self.model(input_ids=inputs, labels=targets)
-                loss = outputs.loss
-                loss.backward()
-                optimizer.step()
-                train_iterator.set_postfix({'Training Loss': loss.item()})
-                epoch_training_loss += loss.item()
-            avg_epoch_training_loss = epoch_training_loss / len(train_iterator)
-
-            # Validation
-            ## This line below tells the model to 'stop learning'
-            self.model.eval()
-            epoch_validation_loss = 0
-            total_loss = 0
-            valid_iterator = tqdm(self.valid_loader, desc=f"Validation Epoch {epoch + 1}/{num_epochs}")
-            with torch.no_grad():
-                for batch in valid_iterator:
-                    inputs = batch['input_ids'].squeeze(1).to(self.device)
-                    targets = batch['labels'].squeeze(1).to(self.device)
-                    outputs = self.model(input_ids=inputs, labels=targets)
-                    loss = outputs.loss
-                    total_loss += loss
-                    valid_iterator.set_postfix({'Validation Loss': loss.item()})
-                    epoch_validation_loss += loss.item()
-
-            avg_epoch_validation_loss = epoch_validation_loss / len(self.valid_loader)
-
-            end_time = time.time()  # End the timer for the epoch
-            epoch_duration_sec = end_time - start_time  # Calculate the duration in seconds
-
-            new_row = {'transformer': self.model_name,
-                       'batch_size': self.batch_size,
-                       'gpu': self.gpu,
-                       'epoch': epoch + 1,
-                       'training_loss': avg_epoch_training_loss,
-                       'validation_loss': avg_epoch_validation_loss,
-                       'epoch_duration_sec': epoch_duration_sec}  # Add epoch_duration to the dataframe
-
-            self.results.loc[len(self.results)] = new_row
-            print(f"Epoch: {epoch + 1}, Validation Loss: {total_loss / len(self.valid_loader)}")
-
-    def generate_text(self, input_str, top_k=16, top_p=0.95, temperature=1.0, repetition_penalty=1.2):
-        # Encode input string into tensors via the FlanT5 tokenizer
-        input_ids = self.tokenizer.encode(input_str, return_tensors='pt', max_length=self.max_length, truncation=True).to(self.device)
-        # Run tensors through model to get output tensor values
-        output_ids = self.model.generate(input_ids,
-            max_length=self.max_length,
-            do_sample=True,
-            top_k=top_k,
-            top_p=top_p,
-            temperature=temperature,
-            repetition_penalty=repetition_penalty)
-        # Decode output tensors to text vi
-        output_str = self.tokenizer.decode(output_ids[0], skip_special_tokens=True)
-        return output_str

genai_SDK/Utilities.py

@@ -1,563 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-import pandas as pd
-from torch.utils.data import Dataset
-
-class LanguageDataset(Dataset):
-    def __init__(self, df, tokenizer):
-        # Make sure data is compatible
-        if len(df.columns) !=2:
-            raise Exception("Dataset can only have two columns!")
-
-        self.data = df.to_dict(orient='records')
-        self.tokenizer = tokenizer
-
-        # set the length of smallest square needed
-        self.max_length = smallest_square_length(df)
-        self.labels = df.columns
-
-    def __len__(self):
-        return len(self.data)
-    def __getitem__(self, i):
-        X = self.data[i][self.labels[0]]
-        Y = self.data[i][self.labels[1]]
-        if str(type(self.tokenizer)) == "<class 'transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer'>":
-            return self.tokenizer.encode_plus(X + ' | ' + Y,
-                                                return_tensors='pt',
-                                                max_length = self.max_length,
-                                                padding='max_length',
-                                                truncation=True)
-        elif str(type(self.tokenizer)) == "<class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>":
-            input_tokens = self.tokenizer.encode_plus(
-                X,
-                max_length=self.max_length,
-                padding='max_length',
-                truncation=True,
-                return_tensors='pt'
-            )
-            target_tokens = self.tokenizer.encode_plus(
-                Y,
-                max_length=self.max_length,
-                padding='max_length',
-                truncation=True,
-                return_tensors='pt'
-            )
-            return {
-                'input_ids': input_tokens['input_ids'].squeeze(),
-                # 'attention_mask': input_tokens['attention_mask'].squeeze(),
-                'labels': target_tokens['input_ids'].squeeze()
-            }
-
-
-
-def smallest_square_length(df):
-    col1 = df[df.columns[0]].astype(str).apply(lambda x: len(x)).max()
-    col2 = df[df.columns[1]].astype(str).apply(lambda x: len(x)).max()
-
-    max_length = max(col1, col2)
-
-    x = 2
-    while x < max_length:
-        x = x * 2
-
-    return x
-
-def levenshtein_distance(str1, str2):
-    """
-    Computes the Levenshtein distance between two strings.
-    Parameters:
-        str1 (str): The first string.
-        str2 (str): The second string.
-    Returns:
-        int: The Levenshtein distance between the two strings.
-    """
-    m, n = len(str1), len(str2)
-    dp = [[0] * (n + 1) for _ in range(m + 1)]
-
-    for i in range(m + 1):
-        dp[i][0] = i
-
-    for j in range(n + 1):
-        dp[0][j] = j
-
-    for i in range(1, m + 1):
-        for j in range(1, n + 1):
-            if str1[i - 1] == str2[j - 1]:
-                dp[i][j] = dp[i - 1][j - 1]
-            else:
-                dp[i][j] = 1 + min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1])
-
-    return dp[m][n]
-
-def grid_search(model, tokenizer, input_str, topK_values, topP_values, temperature_values, repetition_penalty_values, expected_output):
-    """
-     Conducts a grid search over specified hyperparameters to find the best text generation settings (GPT series).
-     Parameters:
-     - model: The pre-trained model used for text generation.
-     - tokenizer: The tokenizer associated with the model.
-     - input_str: The input string to the model for text generation.
-     - topK_values: A list of integer values for the topK sampling hyperparameter.
-     - topP_values: A list of float values for the topP (nucleus) sampling hyperparameter.
-     - temperature_values: A list of float values for the temperature setting of the model.
-     - repetition_penalty_values: A list of float values for penalizing repetitions in the generated text.
-     - expected_output: The expected output string against which generated texts are evaluated using the Levenshtein distance.
-     Returns:
-     - results: A pandas DataFrame containing the combination of hyperparameters, the generated output for each combination, and its Levenshtein distance from the expected output.
-     Notes:
-     - The function prints out the best hyperparameters found during the search, based on the smallest Levenshtein distance.
-     - Levenshtein distance measures the number of edits required to transform one string into another.
-     """
-    results = pd.DataFrame(columns=['topK', 'topP', 'temperature', 'repetition_penalty', 'generated_output', 'levenshtein_distance'])
-    min_distance = 9999999
-    for topK in topK_values:
-        for topP in topP_values:
-            for temperature in temperature_values:
-                for repetition_penalty in repetition_penalty_values:
-                    # try:
-                    generated_output = model.generate_text(input_str, topK, topP, temperature, repetition_penalty)
-                    # print(generated_output)
-                    distance = levenshtein_distance(generated_output, expected_output)
-                    if distance < min_distance:
-                        print(f'topK={topK}, topP={topP}, temperature={temperature}, repetition_penalty={repetition_penalty}, levenshtein_distance={distance}')
-                        min_distance = distance
-
-                    new_row = {'topK': topK,
-                               'topP': topP,
-                               'temperature': temperature,
-                               'repetition_penalty': repetition_penalty,
-                               'generated_output': generated_output,
-                               'levenshtein_distance': distance
-                               }
-                    results.loc[len(results)] = new_row
-
-    return results.sort_values(by='levenshtein_distance', ascending=True)
-
-
-def to_coreml(gpt_model, path=''):
-    import torch
-
-    device = torch.device('mps')
-
-    if torch.cuda.is_available():
-        device = torch.device('cuda')
-    else:
-        try:
-            device = torch.device('mps')  # Apple Silicon
-        except Exception:
-            device = torch.device('cpu')
-    if path != '': lm_head_model = torch.load(path, map_location=device)
-    else: lm_head_model = gpt_model.model
-
-    """
-    Recreate the Core ML model from scratch using
-    coremltools' neural_network.NeuralNetworkBuilder
-    """
-    import coremltools
-    import coremltools.models.datatypes as datatypes
-    from coremltools.models import neural_network as neural_network
-    from coremltools.models.utils import save_spec
-    import numpy as np
-    import torch
-    model_name = 'model'
-
-    model = lm_head_model.transformer
-
-    wte = model.wte.weight.data.cpu().numpy().transpose() # shape (768, 50257) /!\ i hate this
-    wpe = model.wpe.weight.data.cpu().numpy().transpose() # shape (768, 1024)
-
-    sequence_length = 128
-    steps = model.config.n_layer
-
-    # build model
-    input_features = [
-        ('input_ids', datatypes.Array(sequence_length)),
-        ('position_ids', datatypes.Array(sequence_length)),
-    ]
-    output_features = [('output_logits', None)]
-
-    builder = neural_network.NeuralNetworkBuilder(
-        input_features,
-        output_features,
-        mode=None,
-        disable_rank5_shape_mapping=True,
-    )
-    builder.add_expand_dims(
-        name='input_ids_expanded_to_rank5',
-        input_name='input_ids',
-        output_name='input_ids_expanded_to_rank5',
-        axes=(1, 2, 3, 4)
-    )
-    builder.add_expand_dims(
-        name='position_ids_expanded_to_rank5',
-        input_name='position_ids',
-        output_name='position_ids_expanded_to_rank5',
-        axes=(1, 2, 3, 4)
-    )
-    builder.add_embedding(
-        name='token_embeddings',
-        input_name='input_ids_expanded_to_rank5',
-        output_name='token_embeddings',
-        W=wte,
-        b=None,
-        input_dim=50257,
-        output_channels=768,
-        has_bias=False,
-    )
-    builder.add_embedding(
-        name='positional_embeddings',
-        input_name='position_ids_expanded_to_rank5',
-        output_name='positional_embeddings',
-        W=wpe,
-        b=None,
-        input_dim=1024,
-        output_channels=768,
-        has_bias=False,
-    )
-
-    # Input:, Output: (seq, 1, 768, 1, 1)
-    builder.add_add_broadcastable(
-        name='embeddings_addition',
-        input_names=['token_embeddings', 'positional_embeddings'],
-        output_name=f'{0}_previous_block'
-    )
-
-    for i in range(steps):
-        print(i)
-        ln_weight = model.h[i].ln_1.weight.data.cpu().numpy().reshape((1, 1, 768, 1, 1))
-        ln_bias = model.h[i].ln_1.bias.data.cpu().numpy().reshape((1, 1, 768, 1, 1))
-        ln_epsilon = model.h[i].ln_1.eps
-
-        builder.add_mvn(
-            name=f"{i}_block_ln_1",
-            input_name=f"{i}_previous_block",
-            # output_name=f"{i}_block_ln_1_output",
-            output_name=f"{i}_block_ln_1",
-            across_channels=True,
-            normalize_variance=True,
-            epsilon=ln_epsilon
-        )
-
-        builder.add_scale(
-            name=f"{i}_block_ln_1_scaled",
-            input_name=f"{i}_block_ln_1",
-            output_name=f"{i}_block_ln_1_scaled",
-            W=ln_weight,
-            b=ln_bias,
-            has_bias=True,
-            shape_scale=[768],
-            shape_bias=[768]
-        )
-
-        builder.add_transpose(
-            name=f"{i}_block_ln_1_reshape",
-            input_name=f"{i}_block_ln_1_scaled",
-            output_name=f"{i}_block_ln_1_scaled_transposed",
-            axes=(1, 0, 2, 3, 4)
-        )
-
-
-        conv_1D_bias = model.h[i].attn.c_attn.bias.data.cpu().numpy().reshape((1, 1, 2304, 1, 1))
-        conv_1D_weights = model.h[i].attn.c_attn.weight.cpu().data.numpy().transpose().reshape((1, 768, 2304, 1, 1))
-
-        builder.add_inner_product(
-            name=f"{i}_block_attn_conv",
-            input_name=f"{i}_block_ln_1_scaled_transposed",
-            output_name=f"{i}_block_attn_conv",
-            input_channels=768,
-            output_channels=2304,
-            W=conv_1D_weights,
-            b=conv_1D_bias,
-            has_bias=True
-        )
-
-        builder.add_split(
-            name=f"{i}_block_attn_qkv_split",
-            input_name=f"{i}_block_attn_conv",
-            output_names=[f"{i}_block_attn_q", f"{i}_block_attn_k", f"{i}_block_attn_v"]
-        )
-
-        builder.add_rank_preserving_reshape(
-            name=f"{i}_block_attn_q_reshape",
-            input_name=f"{i}_block_attn_q",
-            output_name=f"{i}_block_attn_q_reshape",
-            output_shape=(1, 1, sequence_length, 12, 64)
-        )
-
-        builder.add_transpose(
-            name=f"{i}_block_attn_q_reshape_permuted",
-            input_name=f"{i}_block_attn_q_reshape",
-            output_name=f"{i}_block_attn_q_reshape_permuted",
-            axes=(0, 1, 3, 2, 4)
-        )
-
-        builder.add_rank_preserving_reshape(
-            name=f"{i}_block_attn_k_reshape",
-            input_name=f"{i}_block_attn_k",
-            output_name=f"{i}_block_attn_k_reshape",
-            output_shape=(1, 1, sequence_length, 12, 64)
-        )
-
-        builder.add_transpose(
-            name=f"{i}_block_attn_k_reshape_permuted",
-            input_name=f"{i}_block_attn_k_reshape",
-            output_name=f"{i}_block_attn_k_reshape_permuted",
-            axes=(0, 1, 3, 4, 2)
-        )
-
-        builder.add_rank_preserving_reshape(
-            name=f"{i}_block_attn_v_reshape",
-            input_name=f"{i}_block_attn_v",
-            output_name=f"{i}_block_attn_v_reshape",
-            output_shape=(1, 1, sequence_length, 12, 64)
-        )
-
-        builder.add_transpose(
-            name=f"{i}_block_attn_v_reshape_permuted",
-            input_name=f"{i}_block_attn_v_reshape",
-            output_name=f"{i}_block_attn_v_reshape_permuted",
-            axes=(0, 1, 3, 2, 4)
-        )
-
-        builder.add_batched_mat_mul(
-            name=f"{i}_block_attn_qv_matmul",
-            input_names=[f"{i}_block_attn_q_reshape_permuted", f"{i}_block_attn_k_reshape_permuted"],
-            output_name=f"{i}_block_attn_qv_matmul"
-        )
-
-        builder.add_scale(
-            name=f"{i}_block_attn_qv_matmul_scaled",
-            input_name=f"{i}_block_attn_qv_matmul",
-            output_name=f"{i}_block_attn_qv_matmul_scaled",
-            W=np.array(1/8),
-            b=0,
-            has_bias=False
-        )
-
-        bias_0 = model.h[i].attn.bias
-        nd = ns = sequence_length
-        b = (model.h[i].attn.bias[:, :, ns-nd:ns, :ns]).unsqueeze(0)
-
-        builder.add_scale(
-            name=f"{i}_block_attn_bias",
-            input_name=f"{i}_block_attn_qv_matmul_scaled",
-            output_name=f"{i}_block_attn_bias",
-            W=b,
-            b=None,
-            has_bias=False,
-            shape_scale=[1, sequence_length, sequence_length]
-        )
-
-        bias_constant_0 = -1e4 * torch.logical_not(b)
-
-        builder.add_bias(
-            name=f"{i}_block_attn_afterbias",
-            input_name=f"{i}_block_attn_bias",
-            output_name=f"{i}_block_attn_afterbias",
-            # output_name=f"output_logits",
-            b=bias_constant_0,
-            shape_bias=[1, sequence_length, sequence_length],
-        )
-
-        builder.add_squeeze(
-            name=f"{i}_squeezit",
-            input_name=f"{i}_block_attn_afterbias",
-            output_name=f"{i}_squeezit",
-            axes=[0, 1]
-        )
-
-        builder.add_softmax(
-            name=f"{i}_block_attn_softmax",
-            input_name=f"{i}_squeezit",
-            output_name=f"{i}_block_attn_softmax",
-        )
-
-        builder.add_expand_dims(
-            name=f"{i}_expandit",
-            input_name=f"{i}_block_attn_softmax",
-            output_name=f"{i}_expandit",
-            axes=[0, 1]
-        )
-
-        builder.add_batched_mat_mul(
-            name=f"{i}_block_full_attention",
-            input_names=[f"{i}_expandit", f"{i}_block_attn_v_reshape_permuted"],
-            output_name=f"{i}_block_full_attention"
-        )
-
-        builder.add_transpose(
-            name=f"{i}_block_full_attention_merged_t",
-            input_name=f"{i}_block_full_attention",
-            output_name=f"{i}_block_full_attention_merged_t",
-            axes=[0, 1, 3, 2, 4]
-        )
-
-        builder.add_rank_preserving_reshape(
-            name=f"{i}_block_full_attention_merged",
-            input_name=f"{i}_block_full_attention_merged_t",
-            output_name=f"{i}_block_full_attention_merged",
-            output_shape=[1, 1, 1, sequence_length, 768]
-        )
-
-        builder.add_transpose(
-            name=f"{i}_block_attn_conv_proj_t",
-            input_name=f"{i}_block_full_attention_merged",
-            output_name=f"{i}_block_attn_conv_proj_t",
-            axes=[0, 3, 4, 1, 2]
-        )
-
-        conv_1D_proj_bias = model.h[i].attn.c_proj.bias.data.cpu().numpy().reshape((1, 1, 768, 1, 1))
-        conv_1D_proj_weights = model.h[i].attn.c_proj.weight.data.cpu().numpy().transpose().reshape((1, 768, 768, 1, 1))
-
-        # Input:, Output: (1, 3, 768, 1, 1)
-        builder.add_inner_product(
-            name=f"{i}_block_attn_conv_proj",
-            input_name=f"{i}_block_attn_conv_proj_t",
-            output_name=f"{i}_block_attn_conv_proj",
-            input_channels=768,
-            output_channels=768,
-            W=conv_1D_proj_weights,
-            b=conv_1D_proj_bias,
-            has_bias=True
-        )
-
-        # Input: (seq, 1, 768, 1, 1), Output: (1, seq, 768, 1, 1)
-        builder.add_transpose(
-            name=f"{i}_previous_block_t",
-            input_name=f'{i}_previous_block',
-            output_name=f"{i}_previous_block_t",
-            axes=[1, 0, 2, 3, 4]
-        )
-
-        # Input: [(1, seq, 768, 1, 1), (1, seq, 768, 1, 1)], Output: (1, seq, 768, 1, 1)
-        builder.add_add_broadcastable(
-            name=f"{i}_block_xa_sum",
-            input_names=[f"{i}_previous_block_t", f"{i}_block_attn_conv_proj"],
-            output_name=f"{i}_block_xa_sum",
-            # output_name=f"output_logits"
-        )
-
-        ln_2_weight = model.h[i].ln_2.weight.data.cpu().numpy().reshape((1, 1, 768, 1, 1))
-        ln_2_bias = model.h[i].ln_2.bias.data.cpu().numpy().reshape((1, 1, 768, 1, 1))
-        ln_2_epsilon = model.h[i].ln_2.eps
-
-        # Input: (1, seq, 768, 1, 1), Output:
-        builder.add_mvn(
-            name=f"{i}_block_ln_2",
-            input_name=f"{i}_block_xa_sum",
-            output_name=f"{i}_block_ln_2",
-            across_channels=True,
-            normalize_variance=True,
-            epsilon=ln_2_epsilon
-        )
-
-        builder.add_scale(
-            name=f"{i}_block_ln_2_scaled",
-            input_name=f"{i}_block_ln_2",
-            # output_name=f"output_logits",
-            output_name=f"{i}_block_ln_2_scaled",
-            W=ln_2_weight,
-            b=ln_2_bias,
-            has_bias=True,
-            shape_scale=[768],
-            shape_bias=[768]
-        )
-
-        mlp_conv_1D_fc_bias = model.h[i].mlp.c_fc.bias.data.cpu().numpy().reshape((1, 1, 3072, 1, 1))
-        mlp_conv_1D_fc_weights = model.h[i].mlp.c_fc.weight.data.cpu().numpy().transpose().reshape((1, 768, 3072, 1, 1))
-
-        # Input:, Output: (1, 3, 3072, 1, 1)
-        builder.add_inner_product(
-            name=f"{i}_block_mlp_conv_fc",
-            input_name=f"{i}_block_ln_2_scaled",
-            output_name=f"{i}_block_mlp_conv_fc",
-            # output_name=f"output_logits",
-            input_channels=768,
-            output_channels=3072,
-            W=mlp_conv_1D_fc_weights,
-            b=mlp_conv_1D_fc_bias,
-            has_bias=True
-        )
-
-        builder.add_gelu(
-            name=f"{i}_block_mlp_gelu",
-            input_name=f"{i}_block_mlp_conv_fc",
-            output_name=f"{i}_block_mlp_gelu",
-            # output_name=f"output_logits",
-            mode='TANH_APPROXIMATION'
-        )
-
-        mlp_conv_1D_proj_bias = model.h[i].mlp.c_proj.bias.data.cpu().numpy().reshape((1, 1, 768, 1, 1))
-        mlp_conv_1D_proj_weights = model.h[i].mlp.c_proj.weight.data.cpu().numpy().transpose().reshape((1, 3072, 768, 1, 1))
-
-        # Input:, Output: (1, 3, 3072, 1, 1)
-        builder.add_inner_product(
-            name=f"{i}_block_mlp_conv_proj",
-            input_name=f"{i}_block_mlp_gelu",
-            output_name=f"{i}_block_mlp_conv_proj",
-            # output_name=f"output_logits",
-            input_channels=3072,
-            output_channels=768,
-            W=mlp_conv_1D_proj_weights,
-            b=mlp_conv_1D_proj_bias,
-            has_bias=True
-        )
-
-        builder.add_add_broadcastable(
-            name=f"{i}_block_xm_sum",
-            input_names=[f"{i}_block_xa_sum", f"{i}_block_mlp_conv_proj"],
-            # output_name=f"output_logits"
-            output_name=f"{i + 1}_previous_block_final"
-        )
-
-        builder.add_transpose(
-            name=f"{i}_block_xm_sum_t",
-            input_name=f"{i + 1}_previous_block_final",
-            output_name=f"{i + 1}_previous_block",
-            axes=[1, 0, 2, 3, 4]
-        )
-
-
-    ln_f_weight = model.ln_f.weight.data.cpu().numpy().reshape((1, 1, 768, 1, 1))
-    ln_f_bias = model.ln_f.bias.data.cpu().numpy().reshape((1, 1, 768, 1, 1))
-    ln_f_epsilon = model.ln_f.eps
-
-    # Input: (1, seq, 768, 1, 1), Output:
-    builder.add_mvn(
-        name=f"ln_f",
-        input_name=f"{steps}_previous_block_final",
-        output_name=f"ln_f",
-        # output_name=f"output_logits",
-        across_channels=True,
-        normalize_variance=True,
-        epsilon=ln_f_epsilon
-    )
-
-    builder.add_scale(
-        name=f"ln_f_scaled",
-        input_name=f"ln_f",
-        output_name=f"ln_f_scaled",
-        # output_name=f"output_logits",
-        W=ln_f_weight,
-        b=ln_f_bias,
-        has_bias=True,
-        shape_scale=[768],
-        shape_bias=[768]
-    )
-
-    lm_head_weights = lm_head_model.lm_head.weight.data.cpu().numpy().reshape((1, 50257, 768, 1, 1))
-
-    builder.add_inner_product(
-        name="lm_head",
-        input_name="ln_f_scaled",
-        output_name="output_logits",
-        input_channels=768,
-        output_channels=50257,
-        W=lm_head_weights,
-        b=None,
-        has_bias=False
-    )
-
-    # compile spec to model
-    mlmodel = coremltools.models.MLModel(builder.spec)
-
-    save_spec(builder.spec, f'{model_name}-{sequence_length}-{steps}.mlmodel')

genai_SDK/__init__.py

@@ -1,2 +0,0 @@
-from .Utilities import *
-from .Seq2Seq import *

main.py

@@ -1,24 +0,0 @@
-from datasets import load_dataset, DatasetDict, Dataset
-import pandas as pd
-import ast
-
-import genai_SDK
-from genai_SDK.Seq2Seq import GPT2
-
-# Load dataset from huggingface
-dataset = load_dataset("erwanlc/cocktails_recipe_no_brand")
-
-# Convert to a pandas dataframe
-data = [{'title': item['title'], 'raw_ingredients': item['raw_ingredients']} for item in dataset['train']]
-df = pd.DataFrame(data)
-
-# Just extract the ingredient names, nothing else
-df.raw_ingredients = df.raw_ingredients.apply(lambda x: ', '.join([y[1] for y in ast.literal_eval(x)]))
-#display(df.head())
-
-model = GPT2(gpu=0, model_name="distilgpt2")
-model.load_data(df=df, batch_size=8)
-
-model.train(num_epochs=2)
-
-print(model.generate_text("Annual Planning"))

pyproject.toml

@@ -1,23 +0,0 @@
-[tool.poetry]
-name = "hackweek2024-sup-genai-tools"
-version = "0.1.0"
-description = ""
-authors = ["Your Name <you@example.com>"]
-readme = "README.md"
-
-[tool.poetry.dependencies]
-python = "^3.12"
-torch = "^2.3.1"
-torchtext = "^0.18.0"
-transformers = "^4.41.2"
-sentencepiece = "^0.2.0"
-pandas = "^2.2.2"
-tqdm = "^4.66.4"
-datasets = "^2.20.0"
-scikit-learn = "^1.5.0"
-accelerate = "^0.31.0"
-
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"

restrictedItems/parse.py

@@ -1,11 +0,0 @@
-import csv, json
-
-with open('csv/block_items.csv', newline='') as csvfile:
-  reader = csv.reader(csvfile, delimiter=',', quotechar='"')
-  for i, row in enumerate(reader):
-    if i == 0:
-      continue
-    items = json.loads(json.loads(row[3]))
-
-    for item in items:
-      print(item['name'])

restrictedItems/predict.py

@@ -1,40 +0,0 @@
-from transformers import BertTokenizer, BertForSequenceClassification
-import torch
-
-# Load the trained model and tokenizer
-model = BertForSequenceClassification.from_pretrained("/Users/slei/hackweek2024-sup-genai-tools/hackweek2024-sup-genai-tools/trained_model")
-tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-
-
-# Function to predict the class of a single input text
-def predict(text):
-  # Preprocess the input text
-  inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True)
-
-  # Make predictions
-  with torch.no_grad():
-    outputs = model(**inputs)
-
-  # Get the predicted class
-  logits = outputs.logits
-  predicted_class = torch.argmax(logits, dim=1).item()
-
-  return predicted_class
-
-label_map = {0: 'Allowed Item', 1: 'Restricted Item'}
-
-def main():
-  while True:
-    # Prompting the user for input
-    user_input = input("Enter something: ")
-
-    predicted_class = predict(user_input)
-
-    # Map the predicted class to a human-readable label
-    predicted_label = label_map[predicted_class]
-
-    # Displaying the user input
-    print(f'The item "{user_input}" is classified as: "{predicted_label}"')
-
-if __name__ == "__main__":
-  main()

restrictedItems/train.py

@@ -1,132 +0,0 @@
-from transformers import BertTokenizer, BertForSequenceClassification, Trainer, TrainingArguments
-from sklearn.model_selection import train_test_split
-import torch
-
-# Load dataset
-# Assume dataset is a list of tuples (text, label)
-# dataset = [("item description 1", 0), ("item description 2", 1), ...]
-
-restricted_dataset = [
-  ("Promotional Email GiftCard $10", 1),
-  ("$100 Vanilla® Visa", 1),
-  ("Promotional Email GiftCard $10", 1),
-  ("GPS7000 GPS Tracker for", 1),
-  ("$50 Vanilla Visa eGift", 1),
-  ("GPS7000 GPS Tracker for", 1),
-  ("$200 Vanilla Visa Shiny", 1),
-  ("$200 Vanilla Visa Shiny", 1),
-  ("$200 Vanilla Visa Shiny", 1),
-  ("$200 Vanilla® Visa", 1),
-  ("Promotional Email GiftCard $20", 1),
-  ("Xbox $100 Gift Card", 1),
-  ("$25 Vanilla Visa Shiny", 1),
-  ("$25 Vanilla Visa Shiny", 1),
-  ("$25 Vanilla Visa Shiny", 1),
-  ("Xbox $100 Gift Card", 1),
-  ("Home Depot Birthday Cupcake", 1),
-  ("$100 Vanilla® Visa", 1),
-  ("Birthday Celebration Target Giftcard", 1),
-  ("Birthday Celebration Target Giftcard", 1),
-  ("Promotional Email GiftCard $10", 1),
-  ("Birthday Celebration Target Giftcard", 1),
-  ("Nintendo Switch Family Online", 1),
-  ("Promotional Email GiftCard $10", 1),
-  ("$50 Vanilla® Mastercard", 1),
-  ("$50 Vanilla® Mastercard", 1),
-  ("$50 Vanilla® Mastercard", 1),
-  ("$50 Vanilla® Visa", 1),
-  ("$50 Vanilla Visa Shiny", 1),
-  ("Delta Airlines Wedding $250", 1)
-]
-
-
-normal_dataset =[
-  ("Kerrygold Grass-Fed Pure Irish Garlic & Herb Butter Stick, 3", 0),
-   ("bettergoods Garlic, Parmesan, & Basil Butter, 3 oz", 0),
-    ("Birds Eye Savory Herb Riced Cauliflower, 10 oz (Frozen)", 0),
-     ("Great Value Root Blend, Beets, Carrots, Parsnips and Sweet Potatoes", 0),
-      ("Fresh Blueberries, 18 oz Container", 0),
-       ("Mixpresso 3 Piece Black Canisters Sets For The Kitchen, Kitchen Jars With", 0),
-        ("Freshness Guaranteed Chicken Breast Tenderloins, 2.25 - 3.2", 0),
-         ("Kiolbassa Smoked Meats Beef Hickory Smoked Sausage, 4 links - 13oz", 0),
-          ("Hot Pockets Frozen Snacks, Pepperoni Pizza Buttery Crust, 5 Sandwiches", 0),
-           ("Kool Aid Jammers Tropical Punch Kids Drink 0% Juice Box Pouches, 10", 0),
-            ("Frito-Lay Flavor Mix Variety Pack Snack Chips, 1oz Bags, 18 Count", 0),
-             ("State Fair Classic Corn Dogs, 42.7 oz, 16 Count", 0),
-              ("ASURION 2 Year Sporting Goods Protection Plan ($175 - $199.99)", 0),
-               ("6% Incline Walking Pad Treadmill 320+ lb Capacity, Under The Desk", 0),
-                ("Renpure Biotin & Collagen Thickening Conditioner for All Hair Types, 32 fl", 0),
-                 ("Renpure Biotin & Collagen Thickening Hair Shampoo for All Hair Types, 32", 0),
-                  ("eos Shea Better Body Lotion for Dry Skin, Vanilla Cashmere, 16 fl", 0),
-                   ("Degree Ultra Clear Long Lasting Men's Antiperspirant Deodorant Dry Spray,", 0),
-                    ("Tide PODS Liquid Laundry Detergent, Original Scent, HE Compatible, 42 Count", 0),
-                     ("DEER PARK Brand 100% Natural Spring Water, 16.9-ounce", 0),
-                      ("Great Value Milk Whole Vitamin D Gallon Plastic Jug", 0),
-                       ("Jumbo Russet Potatoes Whole Fresh, 8 lb Bag", 0),
-                        ("Great Value Butter Pecan Flavored Ice Cream, 16 fl oz", 0),
-                         ("Beef Lean Stew Meat, 1.0 - 1.5 lb Tray", 0),
-                          ("Great Value Spaghetti 16oz", 0),
-                           ("Great Value Flavored with Meat Pasta Sauce, 24 oz", 0),
-                            ("Kentucky Kernel Original Seasoned Flour, Coating Mix for Frying, Value Size", 0)
-  ]
-
-dataset = restricted_dataset + normal_dataset
-
-# Split dataset
-train_texts, val_texts, train_labels, val_labels = train_test_split([item[0] for item in dataset], [item[1] for item in dataset], test_size=0.2)
-import pdb; pdb.set_trace()
-
-# Load pre-trained BERT tokenizer
-tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-
-# Tokenize data
-train_encodings = tokenizer(train_texts, truncation=True, padding=True)
-val_encodings = tokenizer(val_texts, truncation=True, padding=True)
-
-# Convert to torch Dataset
-class ShoppingCartDataset(torch.utils.data.Dataset):
-  def __init__(self, encodings, labels):
-    self.encodings = encodings
-    self.labels = labels
-
-  def __getitem__(self, idx):
-    item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
-    item['labels'] = torch.tensor(self.labels[idx])
-    return item
-
-  def __len__(self):
-    return len(self.labels)
-
-train_dataset = ShoppingCartDataset(train_encodings, train_labels)
-val_dataset = ShoppingCartDataset(val_encodings, val_labels)
-
-# Load pre-trained BERT model
-model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
-
-# Training arguments
-training_args = TrainingArguments(
-  output_dir='../results',
-  num_train_epochs=3,
-  per_device_train_batch_size=16,
-  per_device_eval_batch_size=16,
-  warmup_steps=500,
-  weight_decay=0.01,
-  logging_dir='./logs',
-  logging_steps=10,
-)
-
-# Trainer
-trainer = Trainer(
-  model=model,
-  args=training_args,
-  train_dataset=train_dataset,
-  eval_dataset=val_dataset,
-)
-
-# Train model
-trainer.train()
-
-# Evaluate model
-trainer.evaluate()
-
-model.save_pretrained('trained_model')