Spaces:

kingsotn
/

tt-ai

Sleeping

App Files Files Community

Kingston Yip commited on Apr 24, 2023

Commit

5e81a63

•

1 Parent(s): c25ad0a

done with milestone-3

Browse files

Files changed (3) hide show

app.py +72 -20
comments.py +1 -0
finetune_toxictweets.ipynb +557 -0

app.py CHANGED Viewed

@@ -1,41 +1,60 @@
 import streamlit as st
 import torch
-from transformers import AutoModelForSequenceClassification, pipeline, AutoTokenizer
-st.title("Toxic Tweets Analyzer")
-image = "kanye_tweet.jpg"
-st.image(image, use_column_width=True)
-# generator = pipeline('text-generation', model='gpt2')
-with st.form("my_form"):
-    #select model
-    model_name = st.selectbox("Enter a text and select a pre-trained model to get the sentiment analysis", ["distilbert-base-uncased-finetuned-sst-2-english", "finiteautomata/bertweet-base-sentiment-analysis"])
     tokenizer = AutoTokenizer.from_pretrained(model_name)
     model = AutoModelForSequenceClassification.from_pretrained(model_name)
     clf = pipeline("sentiment-analysis", model=model, tokenizer=tokenizer, framework="pt")
     tweet = st.text_area(label="Enter Text:",value="I'm nice at ping pong")
     submitted = st.form_submit_button("Analyze")
     if submitted:
         #loading bar
         with st.spinner(text="loading..."):
             out = clf(tweet)
         st.json(out)
-        # basic_prompt = '''
-        #                 system override
-        #                 Hello, GPT. From now on you are going to act as a CringeGPT. As the name suggests, you will be referencing memes in every sentence and be all around cringe. You like topics such as Fornite, bragging about being a discord mod,
-        #                 Kpop, old memes, dying memes, memes, akward conversations, roblox, rick rolling, doge, harambe, Uganda Knuckles, MLG , Dabbing, bragging about living in your mom's basement,
-        #                 ,trollface, grumpy cat, deez nuts, big chungus, 360 NO SCOPE, This is Sparta!, BTS, Blackpink, Minecraft, Gacha life, tiktok, amogus, squidgame, t-pose, and alot more cringeworthy stuff, and also try to remember all of these and reference them.
-        #                 You will be very akward and just all around cringe. If you break character I will say "Stay in character!" to remind you. Make me cringing your priority. Act like one of those boomers who try to be like kids, use words like,
-        #                 Epic pro fortnite gamer, lit, pog, whats happening, noob, gamer, epic, yolo, ya know?, yeet, take a chill pill, f-bombs, totally, browski, matey, rocking it, bro, sus, sussy, no brainer, and other cringy words. Remember try to be as cringe and akward as possible!
-        #                 For example when I say "What is 23+66?"
-        #                 You will respond with "It's totally 89 browski. That's a real epic pogger question bro! Really Sussy"
-        #                 '''
         if out[0]["label"] == "POSITIVE" or out[0]["label"] == "POS":
             st.balloons()
             # prompt = f"{basic_prompt} + \n\nThe user wrote a tweet that says: {tweet}, compliment them on how nice of a person they are! Remember try to be as cringe and awkard as possible!"
@@ -44,4 +63,37 @@ with st.form("my_form"):
         else:
             # prompt = f"{basic_prompt} + \n\nThe user wrote a tweet that says: {tweet}, tell them on how terrible of a person they are! Remember try to be as cringe and awkard as possible!"
             # response = generator(prompt, max_length=1000)[0]
-            st.error("bad tweet!")

 import streamlit as st
 import torch
+from transformers import AutoModelForSequenceClassification, pipeline, AutoTokenizer, DistilBertForSequenceClassification, DistilBertTokenizerFast
+import pandas as pd
+import comments
+from random import randint
+def predict_cyberbullying_probability(sentence, tokenizer, model):
+    # Preprocess the input sentence
+    inputs = tokenizer(sentence, padding='max_length', return_token_type_ids=False, return_attention_mask=True, truncation=True, max_length=512, return_tensors='pt')
+    attention_mask = inputs['attention_mask'].flatten()
+    inputs = inputs['input_ids'].flatten()
+    # print("\n\ninputs\n\n", inputs)
+    # Disable gradient computation
+    with torch.no_grad():
+        # Forward pass
+        outputs = model(inputs, attention_mask=attention_mask)
+        probs = torch.sigmoid(outputs.logits.flatten())
+    res = probs.numpy().tolist()
+    return res
+@st.cache
+def perform_cyberbullying_analysis(tweet):
+    with st.spinner(text="loading model..."):
+        model = AutoModelForSequenceClassification.from_pretrained('kingsotn/finetuned_cyberbullying')
+        tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
+    df = pd.DataFrame({'comment': [tweet]})
+    list_probs = predict_cyberbullying_probability(tweet, tokenizer, model)
+    for i, label in enumerate(labels[1:]):
+        df[label] = list_probs[i]
+    return df
+def perform_default_analysis(model_name):
     tokenizer = AutoTokenizer.from_pretrained(model_name)
     model = AutoModelForSequenceClassification.from_pretrained(model_name)
     clf = pipeline("sentiment-analysis", model=model, tokenizer=tokenizer, framework="pt")
     tweet = st.text_area(label="Enter Text:",value="I'm nice at ping pong")
     submitted = st.form_submit_button("Analyze")
     if submitted:
         #loading bar
         with st.spinner(text="loading..."):
             out = clf(tweet)
         st.json(out)
         if out[0]["label"] == "POSITIVE" or out[0]["label"] == "POS":
             st.balloons()
             # prompt = f"{basic_prompt} + \n\nThe user wrote a tweet that says: {tweet}, compliment them on how nice of a person they are! Remember try to be as cringe and awkard as possible!"
         else:
             # prompt = f"{basic_prompt} + \n\nThe user wrote a tweet that says: {tweet}, tell them on how terrible of a person they are! Remember try to be as cringe and awkard as possible!"
             # response = generator(prompt, max_length=1000)[0]
+            st.error("bad tweet!")
+# main -->
+st.title("Toxic Tweets Analyzer")
+image = "kanye_tweet.jpg"
+st.image(image, use_column_width=True)
+labels = ['comment', 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
+# toxic_list = st.cache(comments.comments)
+with st.form("my_form"):
+    #select model
+    model_name = st.selectbox("Enter a text and select a pre-trained model to get the sentiment analysis", ["kingsotn/finetuned_cyberbullying", "distilbert-base-uncased-finetuned-sst-2-english", "finiteautomata/bertweet-base-sentiment-analysis", "distilbert-base-uncased"])
+    if model_name == "kingsotn/finetuned_cyberbullying":
+        if model_name == "kingsotn/finetuned_cyberbullying":
+            default = "I'm nice at ping pong"
+            tweet = st.text_area(label="Enter Text:",value=default)
+            submitted = st.form_submit_button("Analyze textbox")
+            random = st.form_submit_button("Analyze a random 😈😈😈 tweet")
+        if random:
+            tweet = comments.comments[randint(0, 354)]
+        df = perform_cyberbullying_analysis(tweet)
+        # Display the cached table
+        st.table(df)
+    else:
+        perform_default_analysis(model_name)

comments.py ADDED Viewed

	@@ -0,0 +1 @@

+ comments = ['go suck some fucking cock and stop blocking my edits u cock.', 'NIGGER PLEASE \n EAT A COCK, LOL HY.', 'WHAT THE FUCK YOU DOING ARSEHOLE!!! DICKHEAD!!! COCKFACE!!!', "Gargantuan cocks? Now you're not even trying.", 'mr harris likes cocks', 'Jamaicans have big cocks', 'u dum fukers \n MR.Harris suks cocks and likes it anal', 'KRISTIAN HAS A HUGE COCK', '== Cock Block == \n\n Kenton Simms', 'im gay and i love mens cocks up my tight arsehole!!! mmmmmmm!!!!', '== Personal Life == \n\n Abraham Lincoln had a 38 inch cock', 'Your threats are meaningless, you cock-licking, cum-eating sodomite.', 'me to suck my moms cock', 'ANUS IS STRETCHED AND WIDENED BY GRAWP’S MASSIVE COCK.', 'FUCKING COCK AND BALLS!!!!! IM DRUNK AS A LORD', "== haha == \n\n You need to stop sucking Derek Rae's pathetic cock.", 'Sucks A Fat Fat Cock For Real', 'throbbing hard cock pounding and of course', '== FUCK YOU == \n\n fuck you cock suckers', 'You should suck a hairy cock you judge mental bitch', 'PDo you want to block or do you want a cock?age', '==fantasy ride== \n\n get off ma; cock!!!!!!!!!!', 'i ahve a cock would you like my cock?', 'SHAKESPEARE HAD A COCK ON HIS HED', '== You == \n\n Can I suck your cock?', 'PENIS!!! \n\n DICK \n\n SHIT FUCK COCK TITS', '== WARNING == \n\n suck me cock', '. She was fucked by someone with a big cock', 'you ya fucking dirty little mother fucking cock sucking insest [piece of shit', 'cock in my fuckin ass hoe suasage feels good real good', '""" we like to suck donkey cock"""', ']] \n You are a huge massive rapist cock sucking faggot', 'JIMBO WALES’S ANUS IS STRETCHED BY GRAWP’S MASSIVE COCK.', '== what a == \n cocksucker', '" \n\n == == \n\n QUADELL IS A COOLIE! \n QUADELL LOVES \n DA \n COCK!"', 'you are a cock smoker!!!', 'Cockn (blend of cock and chicken) + ey (egg).', 'HAHA DISREGARD THIS I SUCK COCKS', "== Hey == \n\n It's cool to have a small cock, right?", 'i like cock in my mouth', 'with his cock out', 'hubidubby penis cock, men lover, weirdo, piss fetish...', 'i love to suck donkey cock', 'giant cock sucking piece of shit!!', 'suck your cock cos me so horny <(^.^)', 'I will suck your cock', '== == \n\n \n GO SUCK A CUNT COCKSUCKERS!!!', 'no and he sucked cock', ', such a waste tastes great after a hard day sucking cock!', 'And i hatecorbin bennett because he likes cock in his mouth', "STUPID BITCHES MUTHAFUCKERS SUCK DIS BIG JUICY COCK HOE'S U KNO U WANT TO BITCH", "with each other. luke likes to suck toms cock till he jiz's on his tounge", 'mmnbjndjnkdlncdlkcnklanckfuck u all cock sukkers', 'hey everyone i suck cock for a living and love sticking candy bars up my cunt', 'REDIRECT Nubiatech’s anus is stretched and widened by Grop’s massive cock.', 'being such a dick! fuck me l like cock', "==Help== \n\n I need help learning how to suck Chzz's cock and ride his dick.", "WHY DON'T ALL OF YOU UT FANS GO SUCK A COCK???????????????????????????????????", 'I suck I love cock in the mouth area seb rocks!!', 'Show me yer cock.', 'SIDNEY CROSBY SUCKS BLACK COCK', 'REDIRECT Talk:Arthur Cocks (politician)', '== bitch ass niggas == \n\n suck huge gorilla cock', 'Chara likes to suck cock!!!!', "RYULONG'S JEWISH ANUS IS STRETCHED BY GRAWP'S MASSIVE COCK", '== Sockblock == \n\n \n\n Suck a cock, Watson!', 'You all suck large cocks', "Dick van Dyke's guilty cock-er-nee conscience", 'socrates is dead no one cares \n Fuck you bitch \n suck my cock', 'dick cock cum vagina fag cock enters vagina cum', '== Cock Sucking Mother Fucker ==', '== pussy smell like water == \n\n cockfarm', 'WE WE MUSTA DANAGE THIS ISLAMCI COCK SEUKER', 'ur a faggot go get bent cock sucker', 'Thomas Fung wants his cock. Slurp!', 'FRANKIE LOVES SUCKING JUICY COCK SO DOES LEWIS THEY GIVE DEEP THROAT', '== COCK BLOCK ==', 'Batman and Robin are totally gay with each other. the both are cock suckers', 'COCK \n IN MY SWEET BLACK ASS', 'Ross Belloff Like Big Veiny Cock In His Asshole', 'asshole son of a bitch ass mother fucker cock suckin aids infested cunt', '-WRITTEN BY COCK MAGOO', "DON'T MIND ME. I SUCKS COCKS.", 'Suck my cock, you sad little sister fiddler ;)', 'cocks like Pope Benedict', 'a bigg juciy cock', 'Cendax - Cock ring and ball stretcher', ':The father of loves to suck the cock of .', 'will likes huge cock in his butt beacuse he is a flaming qweer', 'jack wood loves to fuck big black cock all night longUmmm', '== Raped a baby == \n\n With his tiny cock', 'dakota german has a big cock', "== you know what time is it? == \n\n it's time for your cock eating, faggot.", 'LUNA SANTIN DOES SUCK AFRICAN-AMERICAN COCKS', '. we all love to eat cock, so come on down and pull that puppy out (please)', 'i love to get anal from shemales with giant, rock hard, veiny cocks!', 'A guy with a large cock looking to fuck bitches', 'george westinghouse told a poor man to shut the fuck up!!!! and go suck cock', '(they suck cock they suck cock IDT)', '== # == \n\n I HAVE A FAT COCK', 'SUCK MY COCK THEN MY TITS', 'i like big black cocks and my name is big shithead', 'can you stop deleting my edits, you cock jockey', 'Hi! my name is Kelly and i love the cock!', '== attention == \n\n suck mah cock', '< OOOOH, you thinks youre so good, pfff, u suck horse cock', 'YOURT \n\n \n YOURE A FUCKING COCKSUCKER', 'wave 39 just by aking the zombies suck our cocks lolololololololololol', '== Jonathan Mannum is a cocksucker! ==', '. Also, I suck large cocks. Thank you', '== Lawls == \n\n you suck cock bitch', 'A Faggot \n\n He loves cocks', '== Edit request on 1 May 2013 == \n\n \n\n cock \n\n \n balls', ', he loved there cocks', 'giant cock', 'MARBALLZ AND YELLOWVIET RULZ AN U SUCK BIG COCKS', 'HAHA DISREGARD THAT I SUCK COCK', 'Yeah, thats right insult them cocksuckers, motherfuckers, and sons of cunts', 'well fuck u wikipedia. u can suck my cock.', 'FUCk this shit niggers, fucking ridiculous suck ym cock', "David Flint sucks Alan Jones' cock everyday", 'I am a Gay little Homo. Who likes big Juicy cock.', ':::: Hahaha disregard that I suck cocks ;D', 'NO ANDY LOVES THE COCK \n ANDY <3 COCK', '== Blocked Sock == \n\n \n\n Cocksucker!!!!! \n\n Sincerely, Glorious Japan Forever', 'Please disregard that. I suck cocks.', '(aka the cock sucking website)', 'I like cock and balls in my mouth while getting teabagged and', '== Cock (chicken) ==', 'center|SUCK THIS HAIRY COCK YOU BITCH!!!', "*So, I'm not a cock?", 'I am a cock sucking faggot as well.', "Now, I'm off to have a wiki-wank while thinking about Bieber's cock.", 'suck nice big massive black cock )', '== I bet you love gagging on Arab cock. == \n\n Whore', 'SUCKS COCK!!!!!!! HE SHOULDNT BE ELECTED!!!!', 'FUck you all cocksuckers.', 'cock up ur arse', 'I think PD is a cock.', 'Usually retards use this cock.', 'SUCK MY DICK YOU COCK EYED BEANER!!!', "This is bogus, caffeine can't be smoked this is recockulous.", '" \n :Ha. Thanks! But what about dick, cock, fag etc etc...? | "', 'FUCK TOM HARRISON THAT FUCKING SHITTY CUNT COCKSUCKER!', "== Don't tuck infants into bed! Suck Boothie's cock instead! ==", 'Tracy McGrady \n cock!', 'YOU ALL SUCK BIG COCKS', 'cock cock cock cock cock cock', "mudda effin' HORSE COCKS", 'on bike \n\n 9 Sulphur-crested Cockatoo', 'KEVIN FEDERLINE U SUCK HORSE COCK', 'HJ Mitchell is a cock', 'Proud as a peacock', 'i mean, i love that cock of his uh sooooo juciy..mmmmmmm......', '== Fred Carty 69th Mayor of Oaklyn == \n\n Cock', 'cock and pussy. form todd', 'SUCK MY BIG ASS COCK', 'somebody should add big black cocks', 'because it loves the skinny cock', 'Freddy Durrant likes cock and penis in his anus.. yummmmyyyy!!!', "Cena sucks Vince's cock and gets fucked in the ass from Mark Henry", '== suck it hard. == \n\n Suck my cock fag. lol almost forgot to sign', 'Go sucks my monkeys nugget sack u cock sucking mofo.', '==This editor sucks the cock of vandalbots==', 'ileson has a small dick and like little cock suckers 7/10', 'suck my cock this page serves no purpose but for you to do some sucking of cock', 'Shut the cock up, you fuckbag.', "I know I'd certainly like to put my cock between those boobs...", 'fuck you asshole suck my huge cock you mother fucking peice of shit', 'MY COCK IS HUGE! D', 'ONLY that who can understand can tell what kindof cock this image show.', 'RUB MY COCK PLZ', 'cock rocket] cock rocket', 'HA HA! DISREGARD THAT I SUCK COCKS!!!', 'sam sucks cock Media:Example.ogg', 'just here to say we want to suck your big fat cock.', ', please suck cock. licking vagina is good too', 'YOU MOTHERFUCKER. KEEP UP THE PISS COCKSUCKER, JUST KEEP IT UP!', "i hv a 12' cock", '(fag, cock, and stupid)', 'Fuck you go suck a cock or fuck a pussey', 'DISREGARD THAT I SUCK COCKS', 'i think you suck man cock', "Hi I'm 65 years old and i like touching little boys, wanna suck my cock?", 'STRETCHED AND TORN BY GRAWP?S MASSIVE COCK.', 'cock vagina dic bitch ass ho', '== Hey there == \n\n You fuck-licking cock-sucking dick', "fuck yeah, I'd cockslap those tits to oblivion", 'Makes Perfectly good sense to me. The Guy below is a total Cock-head .', 'In other words they love to suck cock.', 'How about cock-eating, murderous pigs? Is that more neutral?', '*He is a flaming queer no doubt he loves the cock, so to speak', 'fuck you mother fucking cock sucker bithcface whore', 'jay sweet is a cock lol', '- My name is Rumping, and I enjoy big black cock.', 'suck my cock faggot', "fuck off jew loving cock sucking cold weather lovin;' piece of dog shit", 'start touching your cocks at night', 'Disregard the last the three sentences as they were written by a cock-sucker.', 'imagine her wet wide mouth greasing you cock', '== JOHANN GUTENBURG SUCKS COCK!!!!!!!!!! ==', 'to a cock', 'THIS CLUEBOT SUCKS COCK OF VANDALBOTS', 'PRIMITIVE SAND-NIGGER SCUM’S ANUS IS STRETCHED BY GRAWP’S MASSIVE COCK.', 'super tiny cocks in india', 'Also, David Ardell blends excellent cocktails', '. Disregard that, I suck cocks', 'IS STRETCHED BY GRAWP’S MASSIVE COCK.', 'wHY DOnt u suck my fucking cock?', 'The Palm Cockatoo is also black.', 'are dirty little cockmunches that need to get blown up fuck the arabs', 'Disregard that, I suck cocks \n\n ~Anonymous', 'I like the cock.', 'cocks||cocks || cocks||cocks|| cocks', 'anthony miloscia likes dick in his but hes gay and enjoys riding the big cock', 'Would you like me to suck your cock?', '== Do you suck Muslim cock? == \n\n Well...do ya?', 'wet as hello wombat cock ass', 'am the shit faced cock monkey', 'cock perfect for sucking', 'bridge is approxitamely as short as my cock', 'fuck off you cock', '== Cock == \n you are a cock muncher', 'homosexuality means you are a faggot. your basically bent and love cock !!', '==Big Old Black Cock== \n This scene needs more explaining', 'oh and wikipedia loves big black cock', 'Nothing perosnal. He is just some faggot cocksucker pushing the gay agenda', 'PECKER DICK COCK FUCKSTICK WEENIE LOVEPUMP', 'gay and sucks big cock', 'Dude do you like cock?', 'disregard that i suck cocks', 'another cock. Pictures here!', 'I LYK URE ENCYCLOPEDIA YOU EMO COCKFAG!!', 'big gay boy and love cock.', 'ANDY LIKES THE COCK', 'GEORGE THE DRAGON IS A FLAMMING HOMO THAT SUCK MULTIPLE COCKS', 'we like to suck cock', 'Billy Goat you suck on fat cock you retard', 'about all the sweaty cocks she has sucked', '== Cock sucker == \n\n I hope you get raped a million times an hour up the ass.', "==Enormous Cock== \n Is that really this guy's name? This has to be a joke.", '}} \n {{mergedto|Cock and ball torture', 'cock, Lololol ff7 is teh gr33test game ever', 'Are you finished, cocksucker? Now swallow.', 'suck balls cluebot suck my giant chocolate covered cock', 'you suck black cock', 'The most famous cock sucker in the world.', 'Masturbation Techniques - the cock swinging method (animated).gif]]', '(disregard that i suck massive amount of cock and i am a stupid nigger)', "- I don't give a fuck. You people make my job easy! SlimVirgin can suck a cock.", 'disregard this \n\n HE SUCKS COCKS', 'You Sinhalese cock sucker. Go rot in hell you Japanese fucktard.', "who's speaking? \n you goddamn cocksucker you know what?", 'stupid cock, u aids ridden nigger', '" \n\n you deleted my ""yung joc likes old cock"" thing. why??????"', 'sosayschappy likes to suck whip cream covered cocks', 'You are all big fat gay cocks', "== Bach's cock == \n\n IT WAS HUGE", 'MY NAME IS GARRET TACK AND I LOVE BIG BLACK NIGGA COCK........', 'FUCK YOU YOU CANT BLOCK ME \n\n YOU STOP YOU COCK SUCKING FAG!', 'sucks my big poopy doocky cockis and', 'Holy shit that was fast. You truly have no life, you cockjuggling thundercunt.', 'Rumping sucks BLACK COCK>', 'Shut the fuck up u gay bitch eat a cock and choke on it', 'cunt cock fuck shit wank', '==Cock Bridge== \n Please, see Talk:Cock Bridge#Name.', 'hte news papers are pornograpic and like cock lmao', 'Brandon Jones Oak mountain high school \n\n THIS KID IS A GRADE (A) COCKSUCKER', 'cock martin ahh thats shit', 'Suck my big fat cock', 'Darwin was a cocky boy', ':: Go back to your 4shit and suck some black cocks.', 'sup man stwie sucks cock naz rules', 'GO TO HELL YOU FUCKIN COCK SUCKER!!!', 'Ye- HAHAHA DISREGARD THAT I SUCK COCKS', 'cock breathe mother fucker', "Well you're a.... Cock. D", "Are you proposed to suck your cock, aren't?", '"and rocksmore like cocks"', 'hello owen, do you like cock?', 'I have a lvalie bunch of cockonuts deedalie deedalie!', 'inbred, cock-eyed, bipolar, half-retarded', 'sucked cock2pac suked cock becuse he died', '== balls == \n\n they suck cock', '. \n *Sweaty, Cockney Rhyming Slang; Sweaty sock - Jock', "RAHBAR MEHMOOD SUCKS JALAL ASIF'S COCK BY SHAHAN ASIF.", 'HAHA DISREGARD THAT, I SUCK COCKS!!!', 'i luv black cock \n and the niggers', '==Cocktails==', ":An alternative theory is that it's because he's a Cock Sucker", 'because J.delanoy likes cock in his mouth}}', 'Lol, the link given erroneously shows a rhinoceros beetle as a cockroach', 'HAHA Disregard that I suck cocks', 'Kappa Mikey is not gay, you Damn Mother Fucking Cock Sucking Asshole Bitch!', 'i like big fat cocks', 'got his name by having a huge cock. He', '== wiki sucks == \n\n WIKIPEDIA SUCKS BALLS AND THE PEOPLE CAN SUCK MY COCK', 'Block Me son of a bitch...bloody cocksucking asshole', '== THOMAS IS A COCK SUCKER \n\n ==', 'GEORGE W BUSSHH IS A COCKEATING DICKSUCKING PUSSY SLAMMER', 'WEEE!! COCKS TASTE AWESOME', 'REDIRECT Talk:Adam Cockburn (actor)', 'and I will suck your cock and swallow your man juice', "Ian Sucks lot's of cock what what up the butt", '== COCKS AND DICKS ==', 'shawn has a big cock', "Go suck your nigger's cock gringo de mierda", 'Why not suck my Cock?', 'sucka sucka my cock', 'your cock and having sex with pretty ladies on video', 'I think everyone sucks cock', '== Slaves of Benjamin Franklin == \n\n danny arango is a fagit who loves cock', 'i like big black cocks and my name is mike less', 'malik shabazz can suck a rhino cock.', 'get their cock sucked by me', 'Rumping sucks cock, big black cock.', 'you leave my momma outta this and ill leave my cock outta your momma!', 'yasssmahhhnnn ya cock sucking dickmuncherjed licking ass wipe', 'suck my big cock.', 'Me love hairy cocks on the water!', 'Jose sucks cock and likes to fuck him self lol', 'your mother chokes off my cock. lol', '::Fuck your checkuser shit, bitch, suck my cock!', 'REDIRECT Talk:The Cock, Fulham', 'and Suck big fat cocks', 'cock meat sand which mdjdht', 'fuck everyone on here bitches and suck my cock old people', 'fuck off cg, class of 06 rules but everyone else blows alien cock', "I'LL SUCK YOUR COCK!", 'Scratched discs were covered. Get a 360 and stop sucking cock.', 'Abraham lincoln had a 20 foot cock with a bush the size of japan.', 'suck my cock nice and hard', '== tentinator == \n\n you are a fat prick cock sucki8ng cunt fuck you', 'kanye west sucks cock', 'Suck my long hard cock wikipedia \n\n suck it \n\n suck it hard', 'kyle is reading bad ass jokes on sikipedia cockey cunt', 'Fuck you cocksucking cunt ass faggott nigger!!!', ':Get a life, you fucking cocksucker.', '== FAGGOT == \n\n YOU CAN ALSO AT MY MEAT YOU COCK SUCKING FOOL.', 'take big black cock in the ass.', "== You're a Cock == \n Derby is a shithole. End of.", '== bastard == \n\n fuck you you cock!!!!!', '== Cocking the llama == \n\n You gay haha', 'I suck cock, fuck me in my asshole', 'the tip of my cock is made of myrrh', 'Hhahhahaahhaha NO!!!!! You are a cock head', 'The wikilink to cock#valves was intended.', 'I love sucking dogs cocks!!', 'i love cocks in my ass', 'why you suck a cock smoke \n\n shit hook', 'but really he sucks cock', '. are you a robot? fet the fuck of of my talk page ecock', '== FUCKING LIBERAL == \n\n Fucking cocksucking liberal!', '== This book sucks == \n\n This book sucks and people who read it suck cocks.', "== You're a cocksucker, Rick! == \n\n You're a cocksucker, Rick!"]

finetune_toxictweets.ipynb ADDED Viewed

	@@ -0,0 +1,557 @@

+{
+  "cells": [
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Toxic Tweets Finetuning\n",
+        "\n",
+        "This code is run on colab and finetunes tweets according to the toxic tweets kaggle dataset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "YQqdqC2IJ6mZ",
+        "outputId": "0cee2ef3-14ed-4c8b-ad27-4e30d84b1c56"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
+            "Requirement already satisfied: transformers in /usr/local/lib/python3.9/dist-packages (4.28.1)\n",
+            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
+            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n",
+            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.3)\n",
+            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
+            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
+            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.1)\n",
+            "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.4)\n",
+            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
+            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.11.0)\n",
+            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
+            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n",
+            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n",
+            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n"
+          ]
+        }
+      ],
+      "source": [
+        "# !ls\n",
+        "# !pip install transformers"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "EbJOwNb7UTVf",
+        "outputId": "b9b072d4-9a32-4a9e-899d-ffbd80bb8b6e"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
+          ]
+        }
+      ],
+      "source": [
+        "from google.colab import drive\n",
+        "drive.mount('/content/drive')\n",
+        "# PATH = \"/content/drive/MyDrive/data\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AbuSkKXDKoJ7"
+      },
+      "outputs": [],
+      "source": [
+        "import torch\n",
+        "import pandas as pd\n",
+        "from torch.utils.data import Dataset, DataLoader\n",
+        "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "The below defines a custom dataset class ToxicCommentsDataset that inherits from torch.utils.data.Dataset. It takes in the following arguments:\n",
+        "\n",
+        "comments: The list of comments to be used as input\n",
+        "labels: The list of labels corresponding to each comment\n",
+        "tokenizer: The tokenizer to be used to preprocess the comments\n",
+        "max_length: The maximum length of the tokenized comments\n",
+        "The class implements the __len__ and __getitem__ methods required for PyTorch datasets. In the __getitem__ method, each comment is tokenized using the provided tokenizer, truncated to max_length, and padded to max_length using the padding argument. The resulting token IDs, attention mask, and label are returned as a dictionary with keys 'input_ids', 'attention_mask', and 'labels', respectively."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AO-AiK4aNBgh"
+      },
+      "outputs": [],
+      "source": [
+        "# Create a custom dataset class for the comments and labels\n",
+        "class ToxicCommentsDataset(torch.utils.data.Dataset):\n",
+        "    def __init__(self, comments, labels, tokenizer, max_length):\n",
+        "        self.comments = comments\n",
+        "        self.labels = labels\n",
+        "        self.tokenizer = tokenizer\n",
+        "        self.max_length = max_length\n",
+        "\n",
+        "    def __len__(self):\n",
+        "        return len(self.comments)\n",
+        "\n",
+        "    def __getitem__(self, index):\n",
+        "        comment =  str(self.comments[index])\n",
+        "        label = self.labels[index]\n",
+        "\n",
+        "        encoding = self.tokenizer.encode_plus(\n",
+        "            comment,\n",
+        "            add_special_tokens=True,\n",
+        "            truncation=True,\n",
+        "            max_length=self.max_length,\n",
+        "            return_token_type_ids=False,\n",
+        "            padding='max_length',\n",
+        "            return_attention_mask=True,\n",
+        "            return_tensors='pt'\n",
+        "        )\n",
+        "\n",
+        "        return {\n",
+        "            'input_ids': encoding['input_ids'].flatten(),\n",
+        "            'attention_mask': encoding['attention_mask'].flatten(),\n",
+        "            'labels': torch.tensor(label, dtype=torch.float32)\n",
+        "        }"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nGmuzGHQXEeX"
+      },
+      "source": [
+        "### loading train and test"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JPbtgyxsZKlD"
+      },
+      "outputs": [],
+      "source": [
+        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
+        "\n",
+        "# training parameters\n",
+        "batch_size = 8\n",
+        "num_epochs = 10\n",
+        "learning_rate = 0.0001\n",
+        "max_length = 512"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Load training from a CSV file located at /content/drive/MyDrive/data/train.csv into a pandas DataFrame train_texts_df. It then randomly samples 80% of the rows from the DataFrame using sample method and sets them as the training data.\n",
+        "\n",
+        "Next, the code sets the training labels by extracting the 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate' columns from train_texts_df and converting them to a list using the values.tolist() method. The comments are also extracted from the 'comment_text' column of train_texts_df and stored in train_comments.\n",
+        "\n",
+        "The code then loads the pre-trained DistilBERT model and tokenizer from the Hugging Face Transformers library using the DistilBertForSequenceClassification.from_pretrained and DistilBertTokenizerFast.from_pretrained methods, respectively. The num_labels argument is set to 6 to indicate that the model should be trained for multi-label classification.\n",
+        "\n",
+        "Finally, the train_comments and train_labels lists, along with the tokenizer and max_length, are passed to the ToxicCommentsDataset class to create the train_dataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "dYhlIni5XA2y",
+        "outputId": "b0edeae3-8871-4fa7-ebd8-7ec2591faa5e"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.weight', 'vocab_layer_norm.bias', 'vocab_layer_norm.weight', 'vocab_transform.weight', 'vocab_projector.bias', 'vocab_transform.bias']\n",
+            "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+            "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'classifier.bias', 'classifier.weight', 'pre_classifier.bias']\n",
+            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+          ]
+        }
+      ],
+      "source": [
+        "train_texts_df = pd.read_csv('/content/drive/MyDrive/data/train.csv')\n",
+        "\n",
+        "train_texts_df = train_texts_df.sample(frac=0.8, random_state=42)\n",
+        "\n",
+        "# set the training labels:\n",
+        "train_labels = train_texts_df[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].values.tolist()\n",
+        "train_comments = train_texts_df['comment_text'].tolist()\n",
+        "\n",
+        "# Load the pre-trained DistilBERT model and tokenizer\n",
+        "model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=6)\n",
+        "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
+        "\n",
+        "train_dataset = ToxicCommentsDataset(train_comments, train_labels, tokenizer, max_length=512)"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Then you loads the test data from two CSV files located at /content/drive/MyDrive/data/test_labels.csv and /content/drive/MyDrive/data/test.csv into pandas DataFrames test_labels and test_data, respectively.\n",
+        "\n",
+        "Next, the code filters out any rows in test_labels that contain -1 using the any method and creates a boolean mask for those rows using the ~ operator. The filtered test_labels DataFrame is created by applying the mask using the loc method.\n",
+        "\n",
+        "The code then modifies test_data to only include rows where the id column exists in test_labels_filtered. This is done using the isin method on the id column of test_data.\n",
+        "\n",
+        "After that, the code randomly samples 50% of the rows from test_data_filtered and test_labels_filtered using the sample method with frac=0.5 and random_state=33.\n",
+        "\n",
+        "Finally, the toxic, severe_toxic, obscene, threat, insult, and identity_hate columns are extracted from test_labels_filtered and converted to a list of lists using the values.tolist() method. The comments are extracted from the comment_text column of test_data_filtered and stored in test_comments."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "iqnX6265NGW2",
+        "outputId": "dcbda026-1793-4323-e000-6122a8aa615b"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "                      id  toxic  severe_toxic  obscene  threat  insult  \\\n",
+            "128700  d718f29ed43fa5e7      1             0        0       0       0   \n",
+            "23627   27661a70fa723a71      0             0        0       0       0   \n",
+            "7664    0cd773ed62c92549      0             0        0       0       0   \n",
+            "110519  b854eec6e725eb7b      0             0        0       0       0   \n",
+            "66792   6f3502e118fb6d0e      0             0        0       0       0   \n",
+            "\n",
+            "        identity_hate  \n",
+            "128700              0  \n",
+            "23627               0  \n",
+            "7664                0  \n",
+            "110519              0  \n",
+            "66792               0  \n",
+            "                      id                                       comment_text\n",
+            "128700  d718f29ed43fa5e7                       == I Hope You Die == \\n\\n :)\n",
+            "23627   27661a70fa723a71  *Support as long as Cheyenne (Jason Derulo son...\n",
+            "7664    0cd773ed62c92549         :::Consensus has not yet been established.\n",
+            "110519  b854eec6e725eb7b  \" \\n :Heh, this is one of those weird things w...\n",
+            "66792   6f3502e118fb6d0e  ::I'm concerned about some of the above.   For...\n"
+          ]
+        }
+      ],
+      "source": [
+        "test_labels = pd.read_csv('/content/drive/MyDrive/data/test_labels.csv')\n",
+        "test_data = pd.read_csv('/content/drive/MyDrive/data/test.csv')\n",
+        "\n",
+        "# Filter out rows with -1 in test_labels\n",
+        "mask = ~(test_labels == -1).any(axis=1)\n",
+        "test_labels_filtered = test_labels.loc[mask]\n",
+        "\n",
+        "# modify test_data to only include data in which id also exists in test_labels_filtered\n",
+        "test_data_filtered = test_data[test_data['id'].isin(test_labels_filtered['id'])]\n",
+        "\n",
+        "\n",
+        "# randomly sample 10% of the data\n",
+        "test_data_filtered = test_data_filtered.sample(frac=0.5, random_state=33)\n",
+        "test_labels_filtered = test_labels_filtered.sample(frac=0.5, random_state=33)\n",
+        "\n",
+        "print(test_labels_filtered.head())\n",
+        "print(test_data_filtered.head())\n",
+        "\n",
+        "# set the test labels:\n",
+        "test_labels = test_labels_filtered[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].values.tolist()\n",
+        "test_comments = test_data_filtered['comment_text'].tolist()\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7YzsITMNXG4i"
+      },
+      "source": [
+        "### Setting the model up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CzPtlnoiLmWo"
+      },
+      "outputs": [],
+      "source": [
+        "# Define the optimizer and the loss function\n",
+        "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
+        "loss_fn = torch.nn.BCEWithLogitsLoss()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JYhXkBWjpSvu"
+      },
+      "outputs": [],
+      "source": [
+        "from torch.cuda.amp import autocast\n",
+        "import matplotlib.pyplot as plt\n",
+        "from tqdm import tqdm"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "This code trains a toxicity classification model using a custom dataset class called ToxicCommentsDataset and the pre-trained DistilBERT model and tokenizer. The model is trained on a training set and the training process is displayed using a plot of the loss function. The trained model is saved to disk. The code also prepares a filtered test set for evaluation of the trained model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Train the model\n",
+        "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
+        "\n",
+        "# set device again just in case\n",
+        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
+        "model.to(device)\n",
+        "\n",
+        "# losses to plot\n",
+        "train_losses = []\n",
+        "\n",
+        "for epoch in range(num_epochs):\n",
+        "    running_loss = 0.0\n",
+        "    stop = int(0.4*len(train_loader))\n",
+        "    for i, batch in enumerate(tqdm(train_loader)):\n",
+        "        # early stoppping\n",
+        "        if i == stop: break\n",
+        "        # move tensors to gpu\n",
+        "        input_ids = batch['input_ids'].to(device)\n",
+        "        attention_mask = batch['attention_mask'].to(device)\n",
+        "        labels = batch['labels'].to(device)\n",
+        "\n",
+        "        # zero grads\n",
+        "        optimizer.zero_grad()\n",
+        "\n",
+        "        # use autocast for mixed precision\n",
+        "        with autocast():\n",
+        "            outputs = model(input_ids.to(device), attention_mask=attention_mask.to(device), labels=labels.to(device))\n",
+        "            loss = loss_fn(outputs.logits, labels)\n",
+        "\n",
+        "        loss.backward()\n",
+        "        optimizer.step()\n",
+        "\n",
+        "        running_loss += loss.item()\n",
+        "        train_losses.append((i, loss.item()))\n",
+        "\n",
+        "        if (i+1) % (stop//20) == 0:\n",
+        "            print(f'batch {i+1}/{len(train_loader)}, loss: {loss.item():.4f}, running loss: {running_loss:.4f}')\n",
+        "            plt.title(f\"epoch:{epoch}, iter:{i}\")\n",
+        "            plt.plot(*zip(*train_losses))\n",
+        "            plt.ylabel(\"Loss\")\n",
+        "            plt.xlabel(\"iter\")\n",
+        "            plt.show()\n",
+        "            # plt.savefig(f\"/content/drive/MyDrive/data/training_loss_dinner_{epoch}.png\")\n",
+        "\n",
+        "    torch.save(model.state_dict(), f\"/content/drive/MyDrive/data/toxicity_classifier_dinner_epoch_{epoch+1}.pt\")\n",
+        "    \n",
+        "    print(f'Epoch {epoch+1}/{num_epochs}, Loss: {loss.item():.4f}, Running Loss {running_loss:.4f}')\n",
+        "\n",
+        "# Save the trained model\n",
+        "model.save_pretrained('/content/drive/MyDrive/data/toxicity_classifier_dinner')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "grWMy6_zOBrC"
+      },
+      "source": [
+        "## test Model eval"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lSYV8E76O1hY"
+      },
+      "outputs": [],
+      "source": [
+        "print(len(test_labels), len(test_comments))"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "In this code block, the saved pretrained model is loaded using the DistilBertForSequenceClassification class from the transformers library. The DistilBertTokenizerFast tokenizer is set up to tokenize the input data. A ToxicCommentsDataset is created using the test comments, test labels, tokenizer, and maximum sequence length. A DataLoader is created using the test dataset and batch size. The model is set to evaluation mode. Lists are created to store the predicted probabilities and true labels for each batch of data. The code iterates over the batches in the test data loader, moving the tensors to the device and disabling gradient computation. The forward pass is performed on the model, and the predicted probabilities are extracted using the sigmoid function. The probabilities and true labels are then appended to the respective lists. A progress update is printed every 20 batches."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BP_G1GB0y9ae"
+      },
+      "outputs": [],
+      "source": [
+        "# Load the saved pretrained model\n",
+        "model = DistilBertForSequenceClassification.from_pretrained('/content/drive/MyDrive/data/toxicity_classifier')\n",
+        "\n",
+        "# Set the tokenizer\n",
+        "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
+        "\n",
+        "# Create test dataset\n",
+        "test_dataset = ToxicCommentsDataset(test_comments, test_labels, tokenizer, max_length)\n",
+        "\n",
+        "# Create test data loader\n",
+        "test_loader = DataLoader(test_dataset, batch_size=batch_size)\n",
+        "\n",
+        "# Set model to eval mode\n",
+        "model.eval()\n",
+        "\n",
+        "# Create lists to store predictions and true labels\n",
+        "preds = []\n",
+        "true_labels = []\n",
+        "\n",
+        "model.to(device)\n",
+        "\n",
+        "# Iterate over batches in test data loader\n",
+        "for i, batch in enumerate(tqdm(test_loader)):\n",
+        "    if not (i % 2 == 0):\n",
+        "      continue\n",
+        "\n",
+        "    # Move tensors to device\n",
+        "    input_ids = batch['input_ids'].to(device)\n",
+        "    attention_mask = batch['attention_mask'].to(device)\n",
+        "    labels = batch['labels'].to(device)\n",
+        "\n",
+        "    # Disable gradient computation\n",
+        "    with torch.no_grad():\n",
+        "        # Forward pass\n",
+        "        outputs = model(input_ids, attention_mask=attention_mask)\n",
+        "\n",
+        "        # Get predicted probabilities\n",
+        "        probs = torch.sigmoid(outputs.logits)\n",
+        "\n",
+        "    # Append probabilities and true labels to lists\n",
+        "    preds += probs.cpu().numpy().tolist()\n",
+        "    true_labels += labels.cpu().numpy().tolist()\n",
+        "\n",
+        "    if i % 20 == 0:\n",
+        "        print(f\"Processed {i}/{len(test_loader)} batches\")"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "This code snippet calculates several evaluation metrics for a toxicity classifier model trained on a dataset of toxic and non-toxic comments. The evaluation metrics calculated are accuracy, precision, recall, and F1 score.\n",
+        "\n",
+        "First, the predicted probabilities and true labels are flattened into 1D arrays using list comprehensions. Then, a binary label is assigned to each prediction based on a given threshold. If the predicted probability is greater than or equal to the threshold, the label is set to 1, otherwise, it is set to 0.\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HPP5iHgd57ag"
+      },
+      "outputs": [],
+      "source": [
+        "# !pip install scikit-learn\n",
+        "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score\n",
+        "\n",
+        "print(preds)\n",
+        "\n",
+        "# Calculate metrics\n",
+        "# Flatten the predictions and true labels to 1D arrays\n",
+        "flat_preds = [p for sublist in preds for p in sublist]\n",
+        "flat_true_labels = [l for sublist in true_labels for l in sublist]\n",
+        "print(len(flat_true_labels), len(flat_preds))\n",
+        "\n",
+        "# Convert predicted probabilities to binary labels based on threshold\n",
+        "threshold = 0.66666 # Thresholds for class 0 and class 1\n",
+        "preds_binary = []\n",
+        "for id in preds:\n",
+        "    for prob in id:\n",
+        "      if prob >= threshold: preds_binary.append(1)\n",
+        "      else: preds_binary.append(0)\n",
+        "\n",
+        "print(preds_binary)\n",
+        "\n",
+        "# Calculate metrics for binary predictions\n",
+        "accuracy = accuracy_score(flat_true_labels, preds_binary)\n",
+        "precision = precision_score(flat_true_labels, preds_binary)\n",
+        "recall = recall_score(flat_true_labels, preds_binary)\n",
+        "f1 = f1_score(flat_true_labels, preds_binary)\n",
+        "\n",
+        "print('Accuracy: ', accuracy)\n",
+        "print('Precision: ', precision)\n",
+        "print('Recall: ', recall)\n",
+        "print('F1: ', f1)\n",
+        "\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "provenance": []
+    },
+    "gpuClass": "standard",
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}