utils.py

import cleantext
import ftfy
import json
import hashlib
from io import StringIO
from datetime import datetime
    
def clean_text(input):

    text = ftfy.fix_text(input)
    text = cleantext.clean(
        text,
        extra_spaces=True,
        stemming=False,
        stopwords=False,
        lowercase=True,
        numbers=False,
        punct=False
    )
    return(text)

def df_to_csv(df):
    csv = StringIO()
    df.to_csv(csv, index=False)
    csv.seek(0)
    csv_data = csv.getvalue()
    return(csv_data)

def serialize_data(data):
    
    def converter(o):
        if isinstance(o, datetime):
            return o.__str__()

    return json.dumps(data, default=converter)

def hash(input):
    sha1 = hashlib.sha1()
    encoded = json.dumps([input], sort_keys=True).encode()
    sha1.update(encoded)
    return sha1.hexdigest()