Package: cmtt
CMTT is a wrapper library that makes code-mixed text processing more efficient than ever. More documentation incoming!Installationpip install cmttGetting StartedHow to use this library:fromcmtt.dataimport*fromcmtt.preprocessingimport*# Loading json filesresult_json=load_url('https://world.openfoodfacts.org/api/v0/product/5060292302201.json')# Loading csv filesresult_csv=load_url('https://gist.githubusercontent.com/rnirmal/e01acfdaf54a6f9b24e91ba4cae63518/raw/b589a5c5a851711e20c5eb28f9d54742d1fe2dc/datasets.csv')# List the key properties available for the datasets provided by the cmtt librarykeys=list_dataset_keys()# List all datasets provided by cmtt based on search_key and search_termdata=list_cmtt_datasets(search_key="task",search_term="ner",isPrint=True)# Download multiple datasets provided by cmtt, returning a list of paths where the datasets get downloaded# The Datasets are downloaded into a new 'cmtt' directory inside the user profile directory of the operating systemlst=download_cmtt_datasets(["linc_ner_hineng","L3Cube_HingLID_all","linc_lid_spaeng"])# Download a dataset from a url, returning the path where the dataset gets downloaded# The Dataset is downloaded into a new directory 'datasets' inside the current working directorypath=download_dataset_url('https://world.openfoodfacts.org/api/v0/product/5060292302201.json')# Whitespace Tokenizertext="Hello world! This is a python code. Adding random words activate code decrease wastage."WhitespaceT=WhitespaceTokenizer()tokenized_text_whitespace=WhitespaceT.tokenize(text)# Word TokenizerWordT=WordTokenizer(do_lower_case=False)tokenized_text_word=WordT.tokenize(text)# Wordpiece TokenizerWordpieceT=Wordpiece_tokenizer()tokenized_text_wordpiece=WordpieceT.tokenize(text)# Devanagari Tokenizerdevanagari_text="मैं इनदोनों श्रेणियों के बीच कुछ भी० सामान्य नहीं देखता। मैं कुछ नहीं, ट ट॥"DevanagariT=DevanagariTokenizer()tokenized_text_devanagari_words=DevanagariT.word_tokenize(devanagari_text)tokenized_text_devanagari_characters=DevanagariT.character_tokenize(devanagari_text)# DeTokenizerswhitespace_text=WhitespaceT.detokenize(tokenized_text_whitespace)word_text=WordT.detokenize(tokenized_text_word)wordpiece_text=WordpieceT.detokenize(tokenized_text_wordpiece)devanagari_text=DevanagariT.word_detokenize(tokenized_text_devanagari_words)# Search functionalityinstances,list_instances=search_word(text,'this',tokenize=True,width=3)# Sentence piece based tokenizers for Hindi, Hinglish, English and Devnagari Hindi and Roman English Text# Download the models for the tokenizers. If already downloaded then cmtt does not download it again.download_model('hi')download_model('hi-en')download_model('en')download_model('hinDev_engRom')# Sentence piece based Tokenizer for English_en=" This is a sentence-piece based tokenizer, supporting the english language."Spm_en=Sentencepiece_tokenizer('en')lst=Spm_en.tokenize(_en)withopen(r"test_en.txt",'w',encoding="utf-8")asf:foriinlst:f.write(i+"\n")# Sentence piece based Tokenizer for Hindi_hi=" मैं इनदोनों श्रेणियों के बीच कुछ भी० सामान्य नहीं देखता।"Spm_hi=Sentencepiece_tokenizer('hi')lst=Spm_hi.tokenize(_hi)withopen(r"test_hi.txt",'w',encoding="utf-8")asf:foriinlst:f.write(i+"\n")# Sentence piece based Tokenizer for Hinglish_hien=" hi kya haal chaal? hum cmtt naamkaran ki python library develop kar rahe hain"Spm_hien=Sentencepiece_tokenizer('hi-en')lst=Spm_hien.tokenize(_hien)withopen(r"test_hien.txt",'w',encoding="utf-8")asf:foriinlst:f.write(i+"\n")# Sentence piece based Tokenizer for Devnagari Hindi and Roman English Mixed Text_hinDev_engRom=" कैसे हो मित्र? How are you? I am good."Spm_hien=Sentencepiece_tokenizer('hinDev_engRom')lst=Spm_hien.tokenize(_hinDev_engRom)withopen(r"test_hinDev_engRom.txt",'w',encoding="utf-8")asf:foriinlst:f.write(i+"\n")# Sentence Piece detokenizerpath=os.path.dirname(os.path.realpath(__file__))f=open(os.path.join(path,"test_hien.txt"),encoding="utf-8")tokens=[]withfasreader:whileTrue:token=reader.readline()ifnottoken:breaktoken=token.strip()tokens.append(token)detokenized_text=Spm_hien.detokenize(tokens)# Stemmer for English wordsstemmer=PorterStemmer()stemming=stemmer.stem("activate")ContributorsParas GuptaTarun SharmaReuben Devanesan