{ "metadata": { "kernelspec": { "language": "python", "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python", "version": "3.7.12", "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "file_extension": ".py" } }, "nbformat_minor": 4, "nbformat": 4, "cells": [ { "cell_type": "markdown", "source": "# Download data from Wikipedia", "metadata": {} }, { "cell_type": "code", "source": "# install wikipedia API python wrapper\n! pip install wikipedia", "metadata": { "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "execution": { "iopub.status.busy": "2022-08-20T21:43:59.293655Z", "iopub.execute_input": "2022-08-20T21:43:59.294792Z", "iopub.status.idle": "2022-08-20T21:44:15.263363Z", "shell.execute_reply.started": "2022-08-20T21:43:59.294746Z", "shell.execute_reply": "2022-08-20T21:44:15.262171Z" }, "trusted": true }, "execution_count": 3, "outputs": [] }, { "cell_type": "code", "source": "import wikipedia\nimport json\nimport traceback", "metadata": { "execution": { "iopub.status.busy": "2022-08-20T21:44:15.265341Z", "iopub.execute_input": "2022-08-20T21:44:15.265753Z", "iopub.status.idle": "2022-08-20T21:44:15.470330Z", "shell.execute_reply.started": "2022-08-20T21:44:15.265709Z", "shell.execute_reply": "2022-08-20T21:44:15.468665Z" }, "trusted": true }, "execution_count": 4, "outputs": [] }, { "cell_type": "code", "source": [ "# titles to download, from https://en.wikipedia.org/wiki/List_of_mainstream_rock_performers\n", "\n", "pages_titles = \"\"\"10cc\n", "10_Years_(band)\n", "3_Doors_Down\n", "311_(band)\n", "38_Special_(band)\n", "Accept_(band)\n", "AC/DC\n", "Bryan_Adams\n", "Aerosmith\n", "AFI_(band)\n", "Air_Supply\n", "The_Alan_Parsons_Project\n", "Alice_in_Chains\n", "The_All-American_Rejects\n", "The_Allman_Brothers_Band\n", "Alter_Bridge\n", "Ambrosia_(band)\n", "America_(band)\n", "The_Animals\n", "Adam_Ant\n", "Anthrax_(American_band)\n", "April_Wine\n", "Arcade_Fire\n", "Arctic_Monkeys\n", "Asia_(band)\n", "Audioslave\n", "Avenged_Sevenfold\n", "Awolnation\n", "The_B-52's\n", "Bachman–Turner_Overdrive\n", "Bad_Company\n", "Badfinger\n", "The_Band\n", "The_Bangles\n", "Barenaked_Ladies\n", "Bay_City_Rollers\n", "The_Beach_Boys\n", "The_Beatles\n", "Beck\n", "Ben_Folds_Five\n", "Pat_Benatar\n", "Chuck_Berry\n", "The_Big_Bopper\n", "Billy_Talent\n", "The_Black_Crowes\n", "The_Black_Keys\n", "Black_Sabbath\n", "Black_Stone_Cherry\n", "Black_Veil_Brides\n", "Blink-182\n", "Bloodhound_Gang\n", "Blue_October\n", "Blue_Öyster_Cult\n", "Blues_Traveler\n", "James_Blunt\n", "Blur_(band)\n", "Bon_Jovi\n", "Boston_(band)\n", "David_Bowie\n", "Bowling_for_Soup\n", "Boys_Like_Girls\n", "Bread_(band)\n", "Breaking_Benjamin\n", "Bring_Me_the_Horizon\n", "Jackson_Browne\n", "Buckcherry\n", "Jeff_Buckley\n", "Bullet_for_My_Valentine\n", "Bush_(British_band)\n", "The_Byrds\n", "Cage_the_Elephant\n", "Cake_(band)\n", "Canned_Heat\n", "The_Cab\n", "The_Cardigans\n", "The_Cars\n", "Catfish_and_the_Bottlemen\n", "Harry_Chapin\n", "Tracy_Chapman\n", "Cheap_Trick\n", "Chevelle_(band)\n", "Chicago_(band)\n", "Chubby_Checker\n", "Cinderella_(band)\n", "Dallas_Green_(musician)\n", "Eric_Clapton\n", "The_Clash\n", "Eddie_Cochran\n", "Joe_Cocker\n", "Coheed_and_Cambria\n", "Cold_Chisel\n", "Coldplay\n", "Collective_Soul\n", "Phil_Collins\n", "Alice_Cooper\n", "Chris_Cornell\n", "Elvis_Costello\n", "Counting_Crows\n", "The_Cranberries\n", "Crash_Test_Dummies\n", "Cream_(band)\n", "Creed_(band)\n", "Creedence_Clearwater_Revival\n", "Jim_Croce\n", "Crosby,_Stills,_Nash_&_Young\n", "Christopher_Cross\n", "Sheryl_Crow\n", "Crowded_House\n", "The_Cult\n", "The_Cure\n", "Damn_Yankees_(band)\n", "Dashboard_Confessional\n", "Daughtry_(band)\n", "The_Dave_Clark_Five\n", "Dave_Matthews_Band\n", "Days_of_the_New\n", "Death_Cab_for_Cutie\n", "Deep_Purple\n", "Def_Leppard\n", "Deftones\n", "Depeche_Mode\n", "Bo_Diddley\n", "Dio_(band)\n", "Dire_Straits\n", "Disturbed_(band)\n", "Fats_Domino\n", "Donovan\n", "The_Doobie_Brothers\n", "The_Doors\n", "Dr._Hook_&_the_Medicine_Show\n", "Dropkick_Murphys\n", "Drowning_Pool\n", "Duran_Duran\n", "Ian_Dury\n", "Bob_Dylan\n", "Eagles_(band)\n", "Echo_&_the_Bunnymen\n", "Duane_Eddy\n", "Edgar_Winter\n", "Electric_Light_Orchestra\n", "Emerson,_Lake_&_Palmer\n", "England_Dan_&_John_Ford_Coley\n", "Melissa_Etheridge\n", "Europe_(band)\n", "Evanescence\n", "Everclear_(band)\n", "Everlast\n", "The_Everly_Brothers\n", "Extreme_(band)\n", "Faces_(band)\n", "Faith_No_More\n", "Fall_Out_Boy\n", "Bryan_Ferry\n", "Filter_(band)\n", "Finger_Eleven\n", "FireHouse\n", "Five_Finger_Death_Punch\n", "Five_for_Fighting\n", "The_Fixx\n", "The_Flaming_Lips\n", "Fleetwood_Mac\n", "Flogging_Molly\n", "Florence_and_the_Machine\n", "Flyleaf_(band)\n", "Foals_(band)\n", "Dan_Fogelberg\n", "John_Fogerty\n", "Foo_Fighters\n", "Foreigner_(band)\n", "Foster_the_People\n", "The_Four_Seasons_(band)\n", "Peter_Frampton\n", "Franz_Ferdinand_(band)\n", "The_Fray\n", "Glenn_Frey\n", "Fuel_(band)\n", "Fun_(band)\n", "Peter_Gabriel\n", "Garbage_(band)\n", "Genesis_(band)\n", "Ghost_(Swedish_band)\n", "Gin_Blossoms\n", "Gary_Glitter\n", "The_Go-Go's\n", "Godsmack\n", "Golden_Earring\n", "Goo_Goo_Dolls\n", "Good_Charlotte\n", "Grand_Funk_Railroad\n", "Grateful_Dead\n", "Great_White\n", "Green_Day\n", "Greta_Van_Fleet\n", "The_Guess_Who\n", "Guns_N'_Roses\n", "Halestorm\n", "Bill_Haley_&_His_Comets\n", "Hall_&_Oates\n", "George_Harrison\n", "Heart_(band)\n", "Jimi_Hendrix\n", "Don_Henley\n", "Herman's_Hermits\n", "Highly_Suspect\n", "Hinder\n", "The_Hives\n", "Hole_(band)\n", "The_Hollies\n", "Buddy_Holly\n", "Hoobastank\n", "Hootie_&_the_Blowfish\n", "Icehouse_(band)\n", "Billy_Idol\n", "Imagine_Dragons\n", "Incubus_(band)\n", "Interpol_(band)\n", "INXS\n", "Iron_Maiden\n", "The_J._Geils_Band\n", "The_Jam\n", "Tommy_James_and_the_Shondells\n", "Jane's_Addiction\n", "Jefferson_Airplane\n", "Jefferson_Starship\n", "The_Jesus_and_Mary_Chain\n", "Jet_(Australian_band)\n", "Jethro_Tull_(band)\n", "Joan_Jett\n", "Jimmy_Eat_World\n", "Billy_Joel\n", "Elton_John\n", "Janis_Joplin\n", "Journey_(band)\n", "Joy_Division\n", "Judas_Priest\n", "Kaiser_Chiefs\n", "Kaleo_(band)\n", "Kansas_(band)\n", "Keane_(band)\n", "Kid_Rock\n", "The_Killers\n", "Killswitch_Engage\n", "Kings_of_Leon\n", "The_Kinks\n", "Kiss_(band)\n", "Korn\n", "Lenny_Kravitz\n", "Lacuna_Coil\n", "Lamb_of_God_(band)\n", "Avril_Lavigne\n", "Led_Zeppelin\n", "John_Lennon\n", "Huey_Lewis_and_the_News\n", "Jerry_Lee_Lewis\n", "Lifehouse_(band)\n", "Limp_Bizkit\n", "Linkin_Park\n", "Little_Richard\n", "Little_River_Band\n", "Live_(band)\n", "Living_Colour\n", "Kenny_Loggins\n", "Loverboy\n", "The_Lovin'_Spoonful\n", "The_Lumineers\n", "Lynyrd_Skynyrd\n", "The_Mamas_&_the_Papas\n", "Marilyn_Manson\n", "The_Marshall_Tucker_Band\n", "Matchbox_Twenty\n", "John_Mayer\n", "Paul_McCartney\n", "Meat_Loaf\n", "Megadeth\n", "John_Mellencamp\n", "Men_at_Work\n", "Metallica\n", "Midnight_Oil\n", "Mike_and_the_Mechanics\n", "Modest_Mouse\n", "Eddie_Money\n", "The_Monkees\n", "The_Moody_Blues\n", "Alanis_Morissette\n", "Van_Morrison\n", "Morrissey\n", "Mötley_Crüe\n", "Motörhead\n", "Mudvayne\n", "Mumford_&_Sons\n", "Muse_(band)\n", "My_Chemical_Romance\n", "Nickelback\n", "Stevie_Nicks\n", "Harry_Nilsson\n", "Nine_Inch_Nails\n", "Nirvana_(band)\n", "No_Doubt\n", "Ted_Nugent\n", "Oasis_(band)\n", "The_Offspring\n", "Roy_Orbison\n", "Ozzy_Osbourne\n", "Our_Lady_Peace\n", "The_Outfield\n", "P.O.D.\n", "Panic!_at_the_Disco\n", "Pantera\n", "Papa_Roach\n", "Paramore\n", "Pearl_Jam\n", "A_Perfect_Circle\n", "Tom_Petty_and_the_Heartbreakers\n", "Pink_Floyd\n", "Pixies_(band)\n", "Robert_Plant\n", "Poison_(American_band)\n", "The_Police\n", "Iggy_Pop\n", "Pop_Evil\n", "The_Presidents_of_the_United_States_of_America_(band)\n", "The_Pretenders\n", "Elvis_Presley\n", "The_Pretty_Reckless\n", "Primus_(band)\n", "Puddle_of_Mudd\n", "Queen_(band)\n", "Queens_of_the_Stone_Age\n", "Queensrÿche\n", "Quiet_Riot\n", "R.E.M.\n", "Radiohead\n", "Rage_Against_the_Machine\n", "Rainbow_(rock_band)\n", "Rammstein\n", "Ramones\n", "Red_Hot_Chili_Peppers\n", "Lou_Reed\n", "REO_Speedwagon\n", "Rise_Against\n", "The_Rolling_Stones\n", "Linda_Ronstadt\n", "Roxy_Music\n", "Royal_Blood_(band)\n", "Rush_(band)\n", "Saliva_(band)\n", "Sam_Fender\n", "Santana_(band)\n", "Joe_Satriani\n", "Saving_Abel\n", "Scorpions_(band)\n", "The_Script\n", "Seether\n", "Bob_Seger\n", "Sepultura\n", "Sex_Pistols\n", "Shakin'_Stevens\n", "Shinedown\n", "Silverchair\n", "Simon_&_Garfunkel\n", "Simple_Minds\n", "Simple_Plan\n", "Skid_Row_(American_band)\n", "Skillet_(band)\n", "Slade\n", "Slayer\n", "Slipknot_(band)\n", "Small_Faces\n", "Smash_Mouth\n", "The_Smashing_Pumpkins\n", "The_Smiths\n", "Smokie_(band)\n", "Snow_Patrol\n", "Social_Distortion\n", "Soundgarden\n", "Bruce_Springsteen\n", "Billy_Squier\n", "Staind\n", "Ringo_Starr\n", "Starset\n", "Starship_(band)\n", "Status_Quo_(band)\n", "Steely_Dan\n", "Steppenwolf_(band)\n", "Steve_Miller_Band\n", "Rod_Stewart\n", "Sting_(musician)\n", "The_Stone_Roses\n", "Stone_Sour\n", "Stone_Temple_Pilots\n", "The_Strokes\n", "Styx_(band)\n", "Sublime_(band)\n", "Sum_41\n", "Supertramp\n", "Survivor_(band)\n", "The_Sweet\n", "System_of_a_Down\n", "T._Rex_(band)\n", "Talking_Heads\n", "James_Taylor\n", "Tenacious_D\n", "Tesla_(band)\n", "Theory_of_a_Deadman\n", "Thin_Lizzy\n", "Third_Eye_Blind\n", "Thirty_Seconds_to_Mars\n", "George_Thorogood\n", "Thousand_Foot_Krutch\n", "Three_Days_Grace\n", "Three_Dog_Night\n", "Tool_(band)\n", "Toto_(band)\n", "Traffic_(band)\n", "The_Tragically_Hip\n", "Train_(band)\n", "Traveling_Wilburys\n", "Travis_(band)\n", "Trivium_(band)\n", "Twenty_One_Pilots\n", "Twisted_Sister\n", "U2\n", "Uriah_Heep_(band)\n", "The_Used\n", "Steve_Vai\n", "Ritchie_Valens\n", "Vampire_Weekend\n", "Van_Halen\n", "Stevie_Ray_Vaughan\n", "Velvet_Revolver\n", "The_Velvet_Underground\n", "The_Verve\n", "Volbeat\n", "Joe_Walsh\n", "Warrant_(American_band)\n", "Weezer\n", "Jack_White\n", "The_White_Stripes\n", "White_Zombie_(band)\n", "Whitesnake\n", "The_Who\n", "Paul_McCartney_and_Wings\n", "Steve_Winwood\n", "The_Yardbirds\n", "Yes_(band)\n", "Neil_Young\n", "Frank_Zappa\n", "Rob_Zombie\n", "The_Zombies\n", "ZZ_Top\"\"\".split(\n", " \"\\n\"\n", ")" ], "metadata": { "execution": { "iopub.status.busy": "2022-08-20T23:34:24.681697Z", "iopub.execute_input": "2022-08-20T23:34:24.682223Z", "iopub.status.idle": "2022-08-20T23:34:24.693942Z", "shell.execute_reply.started": "2022-08-20T23:34:24.682178Z", "shell.execute_reply": "2022-08-20T23:34:24.693004Z" }, "trusted": true }, "execution_count": 54, "outputs": [] }, { "cell_type": "code", "source": [ "for i, raw_title in enumerate(pages_titles):\n", " if i % 10 == 0:\n", " print(i / len(pages_titles) * 100)\n", " try:\n", " page = wikipedia.page(title=raw_title.replace(\"_\", \" \"), auto_suggest=False)\n", " id_ = page.pageid\n", " url = page.url\n", " dic = {\"content\": page.content, \"meta\": {\"name\": page.title, \"url\": url}}\n", "\n", " with open(f\"/kaggle/working/rock_wiki/{id_}.json\", \"w\") as fo:\n", " json.dump(dic, fo)\n", " except Exception as e:\n", " traceback.print_exc()\n", " print(raw_title)" ], "metadata": { "execution": { "iopub.status.busy": "2022-08-20T23:34:49.157641Z", "iopub.execute_input": "2022-08-20T23:34:49.158086Z", "iopub.status.idle": "2022-08-20T23:44:29.346317Z", "shell.execute_reply.started": "2022-08-20T23:34:49.158047Z", "shell.execute_reply": "2022-08-20T23:44:29.345032Z" }, "trusted": true }, "execution_count": 57, "outputs": [] }, { "cell_type": "code", "source": "! tar -czvf rock_wiki.tar.gz ./rock_wiki", "metadata": { "execution": { "iopub.status.busy": "2022-08-20T23:50:44.643851Z", "iopub.execute_input": "2022-08-20T23:50:44.644378Z", "iopub.status.idle": "2022-08-20T23:50:44.650366Z", "shell.execute_reply.started": "2022-08-20T23:50:44.644328Z", "shell.execute_reply": "2022-08-20T23:50:44.649169Z" }, "trusted": true }, "execution_count": 60, "outputs": [] } ] }