I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Fetch saved model weights and load the weights file
Can you write Python code for it?

#! wget https://github.com/rahulremanan/python_tutorial/raw/master/NLP/10-Sentiment_analysis/weights/RT_LSTM.h5

try:
  model4.load_weights(WEIGHTS_FILE)
  print ('Loaded model weights from: {} ...'.format(WEIGHTS_FILE))
except:
  print ('No model weights file: {} found ...'.format(WEIGHTS_FILE))


! wget https://github.com/rahulremanan/python_tutorial/raw/master/NLP/10-Sentiment_analysis/weights/RT_LSTM.h5

try:
  model1.load_weights(WEIGHTS_FILE)
  print ('Loaded model weights from: {} ...'.format(WEIGHTS_FILE))
except:
  print ('No model weights file: {} found ...'.format(WEIGHTS_FILE))


#! wget https://github.com/rahulremanan/python_tutorial/raw/master/NLP/10-Sentiment_analysis/weights/RT_LSTM.h5

try:
  model3.load_weights(WEIGHTS_FILE)
  print ('Loaded model weights from: {} ...'.format(WEIGHTS_FILE))
except:
  print ('No model weights file: {} found ...'.format(WEIGHTS_FILE))


#! wget https://github.com/rahulremanan/python_tutorial/raw/master/NLP/10-Sentiment_analysis/weights/RT_LSTM.h5

try:
  model2.load_weights(WEIGHTS_FILE)
  print ('Loaded model weights from: {} ...'.format(WEIGHTS_FILE))
except:
  print ('No model weights file: {} found ...'.format(WEIGHTS_FILE))


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Finding the maximum length of the review in the training corpus
Can you write Python code for it?

r_len=[]
for text in X_train_text:
    word=word_tokenize(text)
    l=len(word)
    r_len.append(l)

MAX_REVIEW_LEN=np.max(r_len)
MAX_REVIEW_LEN

max_features = num_unique_word
max_words = MAX_REVIEW_LEN
batch_size = 128
epochs = 3
num_classes=y.shape[1]
print ('Total number of sentiment classes: {} ...'.format(num_classes))


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Glove word embedding
Can you write Python code for it?

def get_coefs(word, *arr):
    return word, np.asarray(arr, dtype='float32')

def get_embed_mat(EMBEDDING_FILE, max_features,embed_dim):
    # word vectors
    embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE, encoding='utf8'))
    print('Found %s word vectors.' % len(embeddings_index))

    # embedding matrix
    word_index = tokenizer.word_index
    num_words = min(max_features, len(word_index) + 1)
    all_embs = np.stack(embeddings_index.values()) #for random init
    embedding_matrix = np.random.normal(all_embs.mean(), all_embs.std(),
                                        (num_words, embed_dim))
    for word, i in word_index.items():
        if i >= max_features:
            continue
        embedding_vector = embeddings_index.get(word)
        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector
    max_features = embedding_matrix.shape[0]

    return embedding_matrix

# embedding matrix
EMBEDDING_FILE = '/content/glove.6B.100d.txt'
embed_dim = 100 #word vector dim
embedding_matrix = get_embed_mat(EMBEDDING_FILE,max_features,embed_dim)
print(embedding_matrix.shape)


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Import dependencies
Can you write Python code for it?

import nltk
import os
import gc
import warnings
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats

from keras.preprocessing import sequence,text
from keras.preprocessing.text import Tokenizer
from keras.models import Sequential
from keras.layers import Dense,Dropout,Embedding,LSTM,Conv1D,GlobalMaxPooling1D,Flatten,MaxPooling1D,GRU,SpatialDropout1D,Bidirectional
from keras.callbacks import EarlyStopping
from keras.utils import to_categorical
from keras.losses import categorical_crossentropy
from keras.optimizers import Adam
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score,confusion_matrix,classification_report,f1_score
warnings.filterwarnings("ignore")
#pd.set_option('display.max_colwidth',100)
pd.set_option('display.max_colwidth', -1)


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Load model weights from weights file
Can you write Python code for it?

try:
  model3.load_weights(WEIGHTS_FILE)
  print ('Loaded model weights from: {} ...'.format(WEIGHTS_FILE))
except:
  print ('No model weights file: {} found ...'.format(WEIGHTS_FILE))


try:
  model4.load_weights(WEIGHTS_FILE)
  print ('Loaded model weights from: {} ...'.format(WEIGHTS_FILE))
except:
  print ('No model weights file: {} found ...'.format(WEIGHTS_FILE))


try:
  model1.load_weights(WEIGHTS_FILE)
  print ('Loaded model weights from: {} ...'.format(WEIGHTS_FILE))
except:
  print ('No model weights file: {} found ...'.format(WEIGHTS_FILE))


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Padding the input text for a fixed input length
Can you write Python code for it?

X_train = sequence.pad_sequences(X_train, maxlen=max_words)
X_val = sequence.pad_sequences(X_val, maxlen=max_words)
X_test = sequence.pad_sequences(X_test, maxlen=max_words)
print(X_train.shape,X_val.shape,X_test.shape)


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Run inference for custom user input
Can you write Python code for it?

input_string = ['This movie was horrible']
input_text = tokenizer.texts_to_sequences(input_string)
input_sequence = sequence.pad_sequences(input_text, maxlen=max_words)
y_pred_LSTM=model1.predict(input_sequence,verbose=1)
print ('Input string: {} ...'.format(input_string))
print ('Sentiment for the input string: {} ...'.format(Sent_dic[np.argmax(y_pred_LSTM)]))

input_string = ['this movie was great']
input_text = tokenizer.texts_to_sequences(input_string)
input_sequence = sequence.pad_sequences(input_text, maxlen=max_words)
y_pred_LSTM=model1.predict(input_sequence,verbose=1)
print ('Input string: {} ...'.format(input_string))
print ('Sentiment for the input string: {} ...'.format(Sent_dic[np.argmax(y_pred_LSTM)]))


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Running model inference on the test data
Can you write Python code for it?

test_element=5
input_sequence = np.asarray([list(X_test[test_element])])
y_pred=model5.predict(input_sequence,verbose=1)
print ('Input string: {} ...'.format(test_text[test_element]))
print ('Sentiment for the input string: {} ...'.format(Sent_dic[np.argmax(y_pred)]))


test_element=5
input_sequence = np.asarray([list(X_test[test_element])])
y_pred=model2.predict(input_sequence,verbose=1)
print ('Input string: {} ...'.format(test_text[test_element]))
print ('Sentiment for the input string: {} ...'.format(Sent_dic[np.argmax(y_pred)]))


test_element=5
input_sequence = np.asarray([list(X_test[test_element])])
y_pred=model3.predict(input_sequence,verbose=1)
print ('Input string: {} ...'.format(test_text[test_element]))
print ('Sentiment for the input string: {} ...'.format(Sent_dic[np.argmax(y_pred)]))


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Save the model weights
Can you write Python code for it?


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Summarize the distribution of the sentiment classes
Can you write Python code for it?

train.groupby('Sentiment')['PhraseId'].nunique()

import seaborn as sns

sns.countplot(data=train,x='Sentiment',)


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Summarize the training data
Get the unqiue label values in the training data
The sentiment labels are:

0 - negative
1 - somewhat negative
2 - neutral
3 - somewhat positive
4 - positive
Can you write Python code for it?

train['Sentiment'].unique()

 Sent_dic={0:'negative', 1:'somewhat negative', 2:'neutral', 3:'somewhat positive', 4:'positive'}


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Tokenize the input text
Tokenizing using Keras text pre-processor. This class allows to vectorize a text corpus, by turning each text into either a sequence of integers (each integer being the index of a token in a dictionary) or into a vector where the coefficient for each token could be binary, based on word count, based on tf-idf...
Can you write Python code for it?

tokenizer = Tokenizer(num_words=max_features)
tokenizer.fit_on_texts(list(X_train_text))
X_train = tokenizer.texts_to_sequences(X_train_text)
X_val = tokenizer.texts_to_sequences(X_val_text)
X_test = tokenizer.texts_to_sequences(test_text)


I am working on the file "NLP/10-Sentiment_analysis/notebook/Rotten_Tomatoes_movie_reviews_classifier.ipynb".
The first task is:
Train the model
Can you write Python code for it?

%%time
history3=model3.fit(X_train,
                    y_train,
                    validation_data=(X_val, y_val),
                    epochs=epochs,
                    batch_size=batch_size,
                    verbose=1)


%%time
history2=model2.fit(X_train,
                    y_train,
                    validation_data=(X_val, y_val),
                    epochs=epochs,
                    batch_size=batch_size,
                    verbose=1)


%%time
history4=model4.fit(X_train,
                    y_train,
                    validation_data=(X_val, y_val),
                    epochs=epochs,
                    batch_size=batch_size,
                    verbose=1)


I am working on the file "NLP/NLP - Character-based LSTM Generator.ipynb".
The first task is:
Create the Training set
Build a training and test dataset. Take 40 characters and then save the 41st character. We will teach the model that a certain 40 char sequence should generate the 41st char. Use a step size of 3 so there is overlap in the training set and we get a lot more 40/41 samples.
Can you write Python code for it?

maxlen = 40
step = 3
sentences = []
next_chars = []

for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i: i+maxlen])
    next_chars.append(text[i + maxlen])

print("sequences: ", len(sentences))

print(sentences[0])
print(sentences[1])

print(next_chars[0])


I am working on the file "NLP/NLP - Character-based LSTM Generator.ipynb".
The first task is:
Generate new sequence
Can you write Python code for it?

import random

def sample(preds, temperature=1.0):
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

import sys
start_index = random.randint(0, len(text) - maxlen - 1)
for diversity in [0.2, 0.5, 1.0]:
    print()
    print('----- diversity:', diversity)
    generated = ''
    sentence = text[start_index: start_index + maxlen]
    generated += sentence
    print('----- Generating with seed: "' + sentence + '"')
    sys.stdout.write(generated)
    for i in range(400):
        x = np.zeros((1, maxlen, len(chars)))
        for t, char in enumerate(sentence):
            x[0, t, char_indices[char]] = 1.
        preds = model.predict(x, verbose=0)[0]
        next_index = sample(preds, diversity)
        next_char = indices_char[next_index]
        generated += next_char
        sentence = sentence[1:] + next_char
        sys.stdout.write(next_char)
        sys.stdout.flush()
    print()


I am working on the file "NLP/NLP - Character-based LSTM Generator.ipynb".
The first task is:
Train the Model
Can you write Python code for it?

epochs = 2
batch_size = 128

model.fit(X, y, batch_size=batch_size, epochs=epochs)


I am working on the file "NLP/NLP - Harry Potter Generator.ipynb".
The first task is:
Create the Model
Can you write Python code for it?

from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers import LSTM
from keras.optimizers import RMSprop

model = Sequential()
model.add(LSTM(256, recurrent_dropout=0.0, input_shape=(maxlen, len(chars)), return_sequences=True))
model.add(LSTM(256, recurrent_dropout=0.0, input_shape=(maxlen, len(chars)), return_sequences=True))
model.add(LSTM(256, recurrent_dropout=0.0,  input_shape=(maxlen, len(chars))))
model.add(Dense(2*len(chars)))
model.add(Dense(len(chars)))
model.add(Activation('softmax'))
optimizer = RMSprop()
model.compile(loss='categorical_crossentropy', optimizer=optimizer)

model.summary()


I am working on the file "NLP/NLP - Harry Potter Generator.ipynb".
The first task is:
Create the Training set
Build a training and test dataset. Take 40 characters and then save the 41st character. We will teach the model that a certain 40 char sequence should generate the 41st char. Use a step size of 3 so there is overlap in the training set and we get a lot more 40/41 samples.
Can you write Python code for it?


I am working on the file "NLP/NLP - Harry Potter Generator.ipynb".
The first task is:
Generate new sequence
Can you write Python code for it?

model.load_weights("potter_lstm_weights_0568.h5")

import random

def sample(preds, temperature=1.0):
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

import sys
start_index = random.randint(0, len(text) - maxlen - 1)
for diversity in [0.2, 0.5, 1.0]:
    print()
    print('----- diversity:', diversity)
    generated = ''
    sentence = text[start_index: start_index + maxlen]
    generated += sentence
    print('----- Generating with seed: "' + sentence + '"')
    sys.stdout.write(generated)
    for i in range(400):
        x = np.zeros((1, maxlen, len(chars)))
        for t, char in enumerate(sentence):
            x[0, t, char_indices[char]] = 1.
        preds = model.predict(x, verbose=0)[0]
        next_index = sample(preds, diversity)
        next_char = indices_char[next_index]
        generated += next_char
        sentence = sentence[1:] + next_char
        sys.stdout.write(next_char)
        sys.stdout.flush()
    print()


I am working on the file "NLP/NLP - Harry Potter Generator.ipynb".
The first task is:
One-hot encode
Can you write Python code for it?

import numpy as np

X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1


I am working on the file "NLP/NLP.ipynb".
The first task is:
Add your own dictionary
Can you write Python code for it?

# Dict objects can also be used to check words against a custom list of correctly-spelled words
# known as a Personal Word List. This is simply a file listing the words to be considered, one word per line.
# The following example creates a Dict object for the personal word list stored in “mywords.txt”:
pwl = enchant.request_pwl_dict("../../Data_nlp/mywords.txt")

pwl.check('pappapero'), pwl.suggest('cittin'), pwl.check('altro')

# PyEnchant also provides the class DictWithPWL which can be used to combine a language dictionary
# and a personal word list file:
d2 = enchant.DictWithPWL("it_IT", "../../Data_nlp/mywords.txt")

d2.check('altro') & d2.check('pappapero'), d2.suggest('cittin')

%%timeit
d2.suggest('poliza')


I am working on the file "NLP/NLP.ipynb".
The first task is:
Other modules:
- CmdLineChecker
The module enchant.checker.CmdLineChecker provides the class CmdLineChecker which can be used to interactively check the spelling of some text. It uses standard input and standard output to interact with the user through a command-line interface. The code below shows how to create and use this class from within a python application, along with a short sample checking session:

wxSpellCheckerDialog

The module enchant.checker.wxSpellCheckerDialog provides the class wxSpellCheckerDialog which can be used to interactively check the spelling of some text. The code below shows how to create and use such a dialog from within a wxPython application.
Word2vec

pip install gensim
pip install pyemd
https://radimrehurek.com/gensim/models/word2vec.html
Can you write Python code for it?

import gensim, logging
from  gensim.models import Word2Vec

model = gensim.models.KeyedVectors.load_word2vec_format(
    '../../Data_nlp/GoogleNews-vectors-negative300.bin.gz', binary=True)

model.doesnt_match("breakfast brian dinner lunch".split())

# give text with w1 w2 your_distance to check if model and w1-w2 have give the same distance
model.evaluate_word_pairs()

len(model.index2word)

# check accuracy against a premade grouped words
questions_words = model.accuracy('../../Data_nlp/word2vec/trunk/questions-words.txt')
phrases_words = model.accuracy('../../Data_nlp/word2vec/trunk/questions-phrases.txt')

questions_words[4]['incorrect']

print( model.n_similarity(['pasta'], ['spaghetti']) )
print( model.n_similarity(['pasta'], ['tomato']) )
print( model.n_similarity(['pasta'], ['car']) )
print( model.n_similarity(['cat'], ['dog']) )

model.similar_by_vector( model.word_vec('welcome') )

model.similar_by_word('welcome')

model.syn0[4,]

model.index2word[4]

model.word_vec('is')

model.syn0norm[4,]

model.vector_size

import numpy as np
model.similar_by_vector( (model.word_vec('Goofy') + model.word_vec('Minni'))/2 )

import pyemd
# This method only works if `pyemd` is installed (can be installed via pip, but requires a C compiler).

sentence_obama = 'Obama speaks to the media in Illinois'.lower().split()
sentence_president = 'The president greets the press in Chicago'.lower().split()

# Remove their stopwords.
import nltk
stopwords = nltk.corpus.stopwords.words('english')
sentence_obama = [w for w in sentence_obama if w not in stopwords]
sentence_president = [w for w in sentence_president if w not in stopwords]

# Compute WMD.
distance = model.wmdistance(sentence_obama, sentence_president)
print(distance)

import nltk
stopwords = nltk.corpus.stopwords.words('english')

def sentence_distance(s1, s2):
    sentence_obama = [w for w in s1.split() if w not in stopwords]
    sentence_president = [w for w in s2.split() if w not in stopwords]
    print(sentence_obama, sentence_president, sep='\t')
   ...


I am working on the file "NLP/NLP.ipynb".
The first task is:
conclusion:
- distance work well
- the order of the words is not taken into account
Translate using google translate

https://github.com/ssut/py-googletrans
should be free and unlimted, interned connection required
pip install googletrans
Can you write Python code for it?

from googletrans import Translator

o = open("../../AliceNelPaeseDelleMeraviglie.txt")
all = ''
for l in o: all += l

translator = Translator()

for i in range(42, 43, 1):
    print(all[i * 1000:i * 1000 + 1000], end='\n\n')
    print(translator.translate(all[i * 1000:i * 1000 + 1000], dest='en').text)

## if language is not passed it is guessed, so it can detect a language
frase = "Ciao Giulia, ti va un gelato?"
det = translator.detect(frase)
print("Languge:", det.lang, " with confidence:", det.confidence)

# command line usage, but it seems to don't work to me
!translate "veritas lux mea" -s la -d en

translations = translator.translate(
    ['The quick brown fox', 'jumps over', 'the lazy dog'], dest='ko')
for translation in translations:
    print(translation.origin, ' -> ', translation.text)

phrase = translator.translate(frase, 'en')
phrase.origin, phrase.text, phrase.src, phrase.pronunciation, phrase.dest


I am working on the file "NLP/NLP.ipynb".
The first task is:
tokenization
As explained above, the module enchant.tokenize provides the ability to split text into its component words. The current implementation is based only on the rules for the English language, and so might not be completely suitable for your language of choice. Fortunately, it is straightforward to extend the functionality of this module.
To implement a new tokenization routine for the language TAG, simply create a class/function “tokenize” within the module “enchant.tokenize.TAG”. This function will automatically be detected by the module’s get_tokenizer function and used when appropriate. The easiest way to accomplish this is to copy the module “enchant.tokenize.en” and modify it to suit your needs.
Can you write Python code for it?

from enchant.tokenize import get_tokenizer
tknzr = get_tokenizer("en_US") # not tak for it_IT up to now
[w for w in tknzr("this is some simple text")]

from enchant.tokenize import get_tokenizer, HTMLChunker
tknzr = get_tokenizer("en_US")
[w for w in tknzr("this is really important text")]

tknzr = get_tokenizer("en_US",chunkers=(HTMLChunker,))
[w for w in tknzr("this is really important text")]

from enchant.tokenize import get_tokenizer, EmailFilter
tknzr = get_tokenizer("en_US")
[w for w in tknzr("send an email to fake@example.com please")]

tknzr = get_tokenizer("en_US", filters = [EmailFilter])
[w for w in tknzr("send an email to fake@example.com please")]


I am working on the file "NLP/NLP101 - Tokenization, Sentiment.ipynb".
The first task is:
Stop-words
Can you write Python code for it?

import nltk
nltk.download("stopwords")
stop_words = nltk.corpus.stopwords.words("english")

stop_words[:10]


I am working on the file "NLP/NLP101 - Tokenization, Sentiment.ipynb".
The first task is:
Tokenizing Social Media
Can you write Python code for it?

from nltk.tokenize.casual import casual_tokenize

tweet = "OMG @twitterguy that was sooooooooo cool :D :D :D!!!!"
print(casual_tokenize(tweet))

casual_tokenize(tweet, reduce_len=True, strip_handles=True)


I am working on the file "NLP/NLTK_in_a_Notebook.ipynb".
The first task is:
Lemmization:
riduzione al lemma: considera anche il contesto della parola
Can you write Python code for it?

# http://wordnet.princeton.edu/

from nltk.stem import WordNetLemmatizer

wordnet_lemmatizer = WordNetLemmatizer()

wordnet_lemmatizer.lemmatize('are')

wordnet_lemmatizer.lemmatize('is', pos='v')
# risulta quindi importante fare pos tagging prima della lemmization


I am working on the file "NLP/NLTK_in_a_Notebook.ipynb".
The first task is:
Named Entity Recognizer (NER)
Can you write Python code for it?

from nltk.tag.stanford import StanfordNERTagger

english_nertagger = StanfordNERTagger('C:\\stanford-ner-2014-08-27\\classifiers\\english.all.3class.distsim.crf.ser.gz',
                              'C:\\stanford-ner-2014-08-27\\stanford-ner.jar')

english_nertagger.tag('Pincopallino is working at StarWars in Montevarchi'.split())


I am working on the file "NLP/NLTK_in_a_Notebook.ipynb".
The first task is:
POS Tagging
Can you write Python code for it?

# POS tagging
nltk.help.upenn_tagset('RB')
nltk.help.upenn_tagset('NN.*')

# Non c'è un solo pos tagger...quello di default è il maxent treebank pos tagger, ma ci sono anche
# il crf,hmm,brill,tnt

# QUALI SONO LE DIFFERENZE??

# Trainare un tagger:
from nltk.corpus import treebank
train_data = treebank.tagged_sents()[:3000]
test_data = treebank.tagged_sents()[3000:]

from nltk.tag import tnt
tnt_pos_tagger = tnt.TnT()
tnt_pos_tagger.train(train_data)
tnt_pos_tagger.evaluate(test_data)

import pickle
f = open(‘tnt_treebank_pos_tagger.pickle’, ‘w’)
pickle.dump(tnt_pos_tagger, f)
f.close()

tnt_tagger.tag(nltk.word_tokenize(“this is a tnt treebank tnt tagger”))


I am working on the file "NLP/NLTK_in_a_Notebook.ipynb".
The first task is:
Parser: analisi logica
Can you write Python code for it?

from nltk.parse.stanford import StanfordParser

english_parser = StanfordParser('C:\\stanford-parser-full-2014-08-27\\stanford-parser.jar',
                                'C:\\stanford-parser-full-2014-08-27\\stanford-parser-3.4.1-models.jar')

analisi = english_parser.parse_sents('Francesco is a good guy')


I am working on the file "NLP/nlp eda.ipynb".
The first task is:
Save intermediate import dictionaries and results
Can you write Python code for it?

save_data = df_extract.to_csv(index=False)
s3_res = boto3.resource('s3')
s3_res.Bucket('dogfaces').put_object(Key='reviews/extract_breed_review.csv', Body=save_data)

# save breed_lookup
# save breed_dict
with open('breed_lookup.pickle', 'wb') as handle:
    pickle.dump(breed_lookup, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('breed_dict.pickle', 'wb') as handle:
    pickle.dump(breed_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)

# source 2: classified dog names
breed_classes = pd.read_csv("s3://dogfaces/tensor_model/output_labels_20170907.txt",names=['breed'])
breed_classes.head()


I am working on the file "NLP/nlp eda.ipynb".
The first task is:
for poodles:create each type of poodle a separate look up item and delete the original poodle one.
to add: american bulldog, merge with bulldog here
to add: bull mastiff = bullmstiffs
to add english springer = 'english springer spaniels'
german shorhaired = 'german shorthaired pointers'
german shepherd = 'german shepherd dog'
to add basset and merge ['basset hound', 'petits bassets griffons vendeens']
Can you write Python code for it?

# poodles:
for breed in not_found_breed:
    if breed.endswith('poodle') or breed=='wheaten terrier':
        breed_dict[breed] = set(breed.split())|set([snowball.stem(w) for w in breed.split()])
    breed_dict.pop('poodle', None)

# bullmastiff
if 'bull mastiff' in not_found_breed:
    breed_dict['bull mastiff'] = breed_dict['bullmastiffs']
    breed_dict.pop('bullmastiffs', None)

# english springer
if 'english springer' in not_found_breed:
    breed_dict['english springer'] = breed_dict['english springer spaniels']
    breed_dict.pop('english springer spaniels', None)

# german short haired, german shepherd and 'american bulldog'
name = 'american bulldog'
if name in not_found_breed:
    breed_dict[name]  = breed_dict['bulldog'] | set(name.split()) | set([snowball.stem(w) for w in name.split()])
    breed_dict.pop('bulldog', None)

name = 'german shorthaired'
if name in not_found_breed:
    breed_dict[name] = breed_dict['german shorthaired pointers']
    breed_dict.pop('german shorthaired pointers', None)
name = 'german shepherd'
if name in not_found_breed:
    breed_dict[name] = breed_dict['german shepherd dog']
    breed_dict.pop('german shepherd dog', None)

# basset dog
breed_dict['basset'] = breed_dict['basset hound']|breed_dict['petits bassets griffons vendeens']

'basset' in base_breeds

sorted(breed_dict.keys())

ind = np.random.randint(df_reviews.shape[0])
text_review = df_reviews['review_content'][ind].lower()
print text_review
puncs = string.punctuation
reduced_set = set([snowball.stem(x) for x in (set(filter(lambda x: x not in puncs, word_tokenize(text_review)))
 - stopword_set)])
po_breeds = []
for w in reduced_set:
    if w in breed_lookup:
        po_breeds.extend(breed_lookup[w])
print po_breeds

df_reviews.columns

def getReviewBreed(text):
    ntext = text.decode('utf-8')
    reduced_set = set([snowball.stem(x) for x in
                       (set(filter(lambda x: x not in string.punctuation,
                                   word_tokenize(ntext.lower()))) - stopword_set)])
   ...


I am working on the file "NLP/regex.ipynb".
The first task is:
Some useful examples
Can you write Python code for it?

# get a number
s = "Your number is <b>123</b>"
m = re.search(r"\d+", s)
print(m.group())

regex = re.compile(r'([a-z][a-z-\']+[a-z])')
print(regex.findall("HELLO W-O-R-L-D"))  # this has uppercase
print(regex.findall("HELLO W-O-R-L-D".lower()))  # lets lowercase
print(regex.findall("123hello456world789"))

r = re.compile("Sent from my (iPhone|iPod)")

print(r.match('Sent from my Ipad'))


I am working on the file "NLP/regex.ipynb".
The first task is:
using re module
Can you write Python code for it?

# to perform case-insensitive matching
print(re.IGNORECASE)
re.IGNORECASE =True
print(re.IGNORECASE)

# perform searches in multiline mode, each \n is consider to be a new string (only for some commands)
re.MULTILINE

# split the string by the occurrences of pattern or return the entire string
re.split(pattern=' ', string=phrase)

# return all non-overlapping matches
re.findall(pattern="ma*", string=phrase)

# return iterator yielding match pattern in string
for i in re.finditer(pattern="ma*", string=phrase):
    print(i)
print(
    i.end(),
    i.endpos,
    i.pos,
    i.start(),
    i.span(),
    i.string,
    i.string[i.start():i.end()],
    sep='\n')

# replace pattern with repl only for the first occurrence
re.sub(pattern="ma*", repl='_MA_', string = phrase)

# same as re.sub but return the number of replacements
re.subn(pattern='ma*', repl='_MA_',string=phrase)

# MatchObject always have a boolean value of True
if mysearch:
    print('ok')

# return one or more subgroups, number of the group=0 return the entire match,
# if the group as a name you can pass the name of the group
mysearch.group(0)

# return all the groups found
mysearch.groups()
# return all groups with name into a group
mysearch.groupdict()

# return start and end of the group number
mysearch.span(0), mysearch.start(0), mysearch.end(0)


I am working on the file "NLP/review_breed_analysis.ipynb".
The first task is:
Model version 1: average
Can you write Python code for it?

# calculating each toy's score
#df_scored = scored_df.copy()
df_scored = scored_df.copy()
df_scored.pop('review_id')
df_scored.pop('rating')
def non_zero_count(x):
    return np.sum(x[x>0])
df_breed_count = df_scored.groupby('toy_id').agg(non_zero_count).reset_index()
df_breed_count.head()

breed_columns = [x for x in scored_df.columns if x not in ['toy_id', 'rating', 'review_id']]
mat_scored2 = scored_df[breed_columns].copy().values

mat_scored2 = scored_df['rating'].values.reshape((61202,1))*mat_scored2

df_scored_sum = pd.DataFrame(data=mat_scored2, columns=breed_columns)
df_scored_sum = pd.concat([scored_df['toy_id'].copy(), df_scored_sum], axis=1)

df_breed_wet_sum = df_scored_sum.groupby('toy_id').sum().reset_index()
df_breed_wet_sum.head()

df_breed_wet_sum.sort_values(by='toy_id', axis=0, inplace=True)
df_breed_count.sort_values(by='toy_id', axis=0, inplace=True)

weighted_mat = df_breed_count[breed_columns].values
weighted_sum = df_breed_wet_sum[breed_columns].values
with np.errstate(divide='ignore', invalid='ignore'):
    res_mat = np.true_divide(weighted_sum, weighted_mat)
    res_mat[res_mat==np.inf]=0
    res_mat = np.nan_to_num(res_mat)

df_scored_finalscore = pd.DataFrame(data=res_mat, columns=breed_columns)
df_scored_finalscore = pd.concat([df_breed_count['toy_id'].copy(), df_scored_finalscore], axis=1)

df_scored_finalscore.head()

df_toy = pd.read_csv("s3://dogfaces/reviews/toys.csv")
df_toy.head(3)

# make recommendations:
def getRecommendations(probs, score_df, toy_df, k, add_info=None):
    # probs is a dictionary
    keys = probs.keys()
    D = score_df.shape[1]-1
    prob_v = np.array(probs.values()).reshape((D,1))
    score_mat = score_df[keys].values
    fscore_mat = score_mat.dot(prob_v)
    top_ind = np.argsort(-fscore_mat[:,0])[:k]
    top_toy = score_df['toy_id'].values[top_ind]
    likely_ratings = pd.DataFrame({"likely rating":fscore_mat[:,0][top_ind]}, index=None)
    if not add_info:
        toy_info =...


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq.ipynb".
The first task is:
1. Project Objectives

In this notebook we are going to develop a project that will incorporate the most relevant elements of natural language processing that we will study in the subject.
The objective is to analyze the main themes of a corpus of documents, as well as the structure of the data corpus itself using different visualizations to facilitate the design and adjustment of the tools used in the process.
The following figure summarizes the steps and  libraries that we will follow during this project.

Corpus Acquisition: We will ingest a collection of documents for further processing in python. Import techniques will differ based on the kind of documents we need to work with, and their original format (i.e., PDF, XML, etc). It some cases, we will need to use crawling techniques.

Corpus Processing: Here, we will work with unstructured text, i.e., the original text as it was written by a person. In its original format, the data cannot be efficiently used by most machine learning algorithms, so it is necessary to apply a series of transformation to make data usable and fit the characteristics of machine learning algorithms.

Semantic Analysis: We will rely on Topic Modeling and, in particular, Latent Dirichlet Allocation, to find out the leading topics in the dataset as well as the main topics of each document.

Graph Analysis: Finally, a graph will be calculated based on the semantic characterization of all documents. Some relevant statistics of the graph will be calculated, and relevant documents will be identified from this perspective. For graph visualization we will rely on a external application (gephi) that offers interactivity and better scalability properties.

After completing the project, students will have acquired the following skills:

Ability to work with different sources of textual data

Knowledge of the most used text preprocessing steps for learning topics

Ability to handle different topic modeling tools, knowing which are the hyperparameters that must be adjusted, and the...
Can you write Python code for it?

## Read files from web. You can comment this block if you are reading files
## from your local drive
import requests
from io import BytesIO #Useful for presenting data as a byte stream

url_base = 'http://www.tsc.uc3m.es/~jarenas/NSFdata/'
web_read = True

if web_read:
    project_url = url_base + '1600057.xml'
    project_xml = requests.get(project_url).text
else:
    pass #Implement here function to read XML file

print(project_xml)


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq.ipynb".
The first task is:
2.1.2. Parsing the XML file
XML is an inherently hierarchical data format, and the most natural way to represent it is with a tree. The ElementTree module has two classes for this purpose:

ElementTree represents the whole XML document as a tree
Element represents a single node in this tree

We can parse the XML structure directly from a string, or by reading an XML file
Can you write Python code for it?

import xml.etree.ElementTree as ET

root = ET.fromstring(project_xml)

# Use the following fragment if you want to read data from a local file
# tree = ET.parse(xmlfile)
# root = tree.getroot


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq.ipynb".
The first task is:
2.2. Building the dataset
Now, we will use the function you just implemented, to create a database that we will use throughout the project.
For simplicity, in this section we will simply store the dataset as a list containing the dictionaries associated to each of the considered projects in a time interval.
Can you write Python code for it?

# Construct an iterator (or a list) for the years you want to work with
years = [2017, 2018, 2019]
NSF_data = []

for year in years:
    print('\nDownloading file for year', year, '...')
    if web_read:
        url = url_base + str(year) + '.zip'
        binary_file = BytesIO(requests.get(url).content)
        zpobj = zp.ZipFile(binary_file)
    else:
        pass #Read zpobj directly from local file

    for count, fileinzip in enumerate(zpobj.namelist()):
        if not count%1000:
            print('\rProcessing file', str(count), end='')
        if fileinzip.endswith('xml'):
            #Some files seem to be incorrectly parsed
            try:
                project_dictio = parse_xmlproject(zpobj.read(fileinzip))
                if project_dictio['abstract']: #We keep only the projects with abstract
                    NSF_data.append(project_dictio)
            except:
                pass


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq.ipynb".
The first task is:
Exercise 1: Using standard string processing functions, implement a function to extract from the previous string the following elements:

The project abstract
The project identifier
Can you write Python code for it?

def proj_abstr_id(xmlstring):
    #<SOL>
    # Your solution goes here
    abstract = #
    prj_id = xmlstring.split('<AwardID>')[1].split('</AwardID>')[0]
    #</SOL>
    return abstract, prj_id

abstract, prj_id = proj_abstr_id(project_xml)
print('Project Identifier:', prj_id)
print('This is the abstract:', abstract)


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq.ipynb".
The first task is:
Exercise 3: In the previous code cell we obtained the histogram iterating over all projects, which is very inefficient for long lists. Rewrite a more efficient implementation using the Counter class provided by collections library.
Can you write Python code for it?

from collections import Counter

#<SOL>
all_y = [project['year'] project in NSF_data]
hist_values = Counter(all_y)
#</SOL>


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq.ipynb".
The first task is:
Have a look at the table, and try to understand why some elements of the tree have empty content and why some have no children at all whereas other can have many children.
(By the way, note that creating a dataframe is an easy way to display lists of lists or lists of tuples in fancy manner.)
The presented classes and functions are all you need to solve the following exercise. However, there are many other interesting functions that can probably make it easier for you to work with XML files. For more information, please refer to the ElementTree API.
2.1.3. Parsing the XML project files
Exercise 2: Implement a function that parses an XML string and provides as its output a dictionary with fields:
project_code      (string)
title             (string)
abstract          (string)
budget            (float)
year              (string)
institution       (tuple with elements: name, zipcode, and statecode)
Can you write Python code for it?

def parse_xmlproject(xml_string):
    """This function processess the specified XML field,
    and outputs a dictionary with the desired project information

    :xml_string: String with XML content
    :Returns: Dictionary with indicated files
    """

    #<SOL>
    root = ET.fromstring(xml_string)
    dictio = {}

    for child in root[0]:
        if child.tag.lower() == 'awardtitle':
            dictio['title'] = child.text
        elif child.tag.lower() == 'awardeffectivedate':
            dictio['year'] = str(child.text[-4:])
        elif child.tag.lower() == 'awardamount':
            dictio['budget'] = float(child.text)
        elif child.tag.lower() == 'abstractnarration':
            dictio['abstract'] = child.text
        elif child.tag.lower() == 'awardid':
            dictio['project_code'] = child.text
        elif child.tag.lower() == 'institution':
            #For the institution we have to access the children elements
            #and search for the name, zipcode, and statecode only
            name = ''
            zipcode = ''
            statecode = ''
            for child2 in child:
                if child2.tag.lower() == 'name':
                    name = child2.text
                elif child2.tag.lower() == 'zipcode':
                    zipcode = child2.text
                elif child2.tag.lower() == 'statecode':
                    statecode = child2.text
            dictio['institution'] = (name, zipcode, statecode)

    return dictio
    #</SOL>

#Test the function and print the result
parse_xmlproject(project_xml)


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq.ipynb".
The first task is:
fromstring() parses XML from a string directly into an Element, which is the root element of the parsed tree. Other parsing functions may create an ElementTree, but we will not cover them here.
As an Element, root has a tag and a dictionary of attributes:
Can you write Python code for it?


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq_students.ipynb".
The first task is:
1. Project Objectives

In this notebook we are going to develop a project that will incorporate the most relevant elements of natural language processing that we will study in the subject.
The objective is to analyze the main themes of a corpus of documents, as well as the structure of the data corpus itself using different visualizations to facilitate the design and adjustment of the tools used in the process.
The following figure summarizes the steps and  libraries that we will follow during this project.

Corpus Acquisition: We will ingest a collection of documents for further processing in python. Import techniques will differ based on the kind of documents we need to work with, and their original format (i.e., PDF, XML, etc). It some cases, we will need to use crawling techniques.

Corpus Processing: Here, we will work with unstructured text, i.e., the original text as it was written by a person. In its original format, the data cannot be efficiently used by most machine learning algorithms, so it is necessary to apply a series of transformation to make data usable and fit the characteristics of machine learning algorithms.

Semantic Analysis: We will rely on Topic Modeling and, in particular, Latent Dirichlet Allocation, to find out the leading topics in the dataset as well as the main topics of each document.

Graph Analysis: Finally, a graph will be calculated based on the semantic characterization of all documents. Some relevant statistics of the graph will be calculated, and relevant documents will be identified from this perspective. For graph visualization we will rely on a external application (gephi) that offers interactivity and better scalability properties.

After completing the project, students will have acquired the following skills:

Ability to work with different sources of textual data

Knowledge of the most used text preprocessing steps for learning topics

Ability to handle different topic modeling tools, knowing which are the hyperparameters that must be adjusted, and the...
Can you write Python code for it?


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq_students.ipynb".
The first task is:
2.1.2. Parsing the XML file
XML is an inherently hierarchical data format, and the most natural way to represent it is with a tree. The ElementTree module has two classes for this purpose:

ElementTree represents the whole XML document as a tree
Element represents a single node in this tree

We can parse the XML structure directly from a string, or by reading an XML file
Can you write Python code for it?


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq_students.ipynb".
The first task is:
2.2. Building the dataset
Now, we will use the function you just implemented, to create a database that we will use throughout the project.
For simplicity, in this section we will simply store the dataset as a list containing the dictionaries associated to each of the considered projects in a time interval.
Can you write Python code for it?


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq_students.ipynb".
The first task is:
Children are nested, and we can access specific child nodes by index. We can also access the text of specified elements. For instance:
Can you write Python code for it?

all_elements = []
for child in root[0]:
    n_children = len(child)
    all_elements.append([child.tag, child.attrib, child.text, n_children])

elements_df = pd.DataFrame(all_elements, columns=['Tag', 'Attributes', 'Content', '#Children'])
elements_df


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq_students.ipynb".
The first task is:
Exercise 1: Using standard string processing functions, implement a function to extract from the previous string the following elements:

The project abstract
The project identifier
Can you write Python code for it?

def proj_abstr_id(xmlstring):
    #<SOL>
    # Your solution goes here
    abstract = #
    prj_id = #
    #</SOL>
    return abstract, prj_id

abstract, prj_id = proj_abstr_id(project_xml)
print('Project Identifier:', prj_id)
print('This is the abstract:', abstract)


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq_students.ipynb".
The first task is:
Exercise 4: Compute the average length of the abstracts of all projects in the dataset
Can you write Python code for it?


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq_students.ipynb".
The first task is:
Have a look at the table, and try to understand why some elements of the tree have empty content and why some have no children at all whereas other can have many children.
(By the way, note that creating a dataframe is an easy way to display lists of lists or lists of tuples in fancy manner.)
The presented classes and functions are all you need to solve the following exercise. However, there are many other interesting functions that can probably make it easier for you to work with XML files. For more information, please refer to the ElementTree API.
2.1.3. Parsing the XML project files
Exercise 2: Implement a function that parses an XML string and provides as its output a dictionary with fields:
project_code      (string)
title             (string)
abstract          (string)
budget            (float)
year              (string)
institution       (tuple with elements: name, zipcode, and statecode)
Can you write Python code for it?

def parse_xmlproject(xml_string):
    """This function processess the specified XML field,
    and outputs a dictionary with the desired project information

    :xml_string: String with XML content
    :Returns: Dictionary with indicated files
    """

    root = ET.fromstring(xml_string)
    dictio = {}
    #<SOL>

    #</SOL>
    return dictio

#Test the function and print the result
parse_xmlproject(project_xml)


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq_students.ipynb".
The first task is:
It also has children nodes over which we can iterate:
Can you write Python code for it?

for child in root:
    print(child.tag, child.attrib)


I am working on the file "NLP1.CorpusAcquisition/NSF_CorpusAcq_students.ipynb".
The first task is:
We will extract some characteristics of the constructed dataset:
Can you write Python code for it?

print('Number of projects in dataset:', len(NSF_data))

####
budget_data = list(map(lambda x: x['budget'], NSF_data))
print('Average budget of projects in dataset (USD):', np.mean(budget_data))

####
insti_data = list(map(lambda x: x['institution'] if 'institution' in x else None, NSF_data))
print('Number of unique institutions in dataset:', len(set(insti_data)))

#### Breakdown of projects by starting year
counts = dict()
for project in NSF_data:
    counts[project['year']] = counts.get(project['year'],0) + 1

keys = list(counts.keys())
# get values in the same order as keys, and parse percentage values
vals = [counts[k] for k in keys]
keys = [int(k) for k in keys]
fig, axes = plt.subplots(1, 1, figsize=(8, 4), sharex=True)
fig.suptitle('Breakdown of projects by starting year')
sns.barplot(ax = axes, x=keys, y=vals)
plt.show()


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
2.2. Obtaining Model Info
You can retrieve the most relevant information about available language models using the following command
spacy.info('model_name')
Note that you can only apply this command on models that have already been downloades. Otherwise, an exception is thrown.
Exercise 1: Run the following command and find the information related to
   - Components included in the pipeline
   - Are all components enabled?
   - How many types of entities can be recognized by the corresponding component?
   - What Part-of-Speech elements can you recognize?
   - What is the dimension of the word-embeddings incorporated in the model?
Detailed information about some specific components of the pipeline, as well as how they can be used, will be studied in the next sections.
Can you write Python code for it?


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
3.3. Usual Pipelines Components and Annotations

4. Linguistic Features

In this Section we will review a set of liguistic features provided by most spaCy pretrained pipelines. We will focus mainly on pipeline components that are relevant to build Bag of Words (BoW) representations of text.
4.1. Tokenizer
4.1.1. Word Tokenization
The Tokenizer is always the first component of the spaCy pretrained pipelines.

It has the important role of producing a Doc object out of a text string
It first splits the string using blank spaces
Then tokens are processed sequentially from left to right performing two operations
First, language-specific rules and exceptions are applied (e.g., in English "don't" is splitted into two separate tokens, but U.K. is kept as one token)
Second, prefixes or suffixes are identified. This is relevant to separate punctuation marks from the main tokens

It is important to note that tokenization rules, as well as exceptions are language specific. This means you need to make sure that the languages of the text and the selected Tokenizer match, otherwise you could get unexpected results.

Once the Doc object has been created, you can easily iterate over the identified tokens. Note also that the original text is preserved. You can access the string representation of Doc, Span, Token and even Lexeme objects by using the text attribute.
Can you write Python code for it?

shortext = 'Natural Language Processing is a key component of many relevant Artificial Intelligence Applications.' \
           ' Libraries such as spaCy v3 make it simple to benefit from statistical NLP models based on neural networks.' \
           ' It is estimated that NLP market in the U.S. will grow to around 30000 MUSD during the next five years.' \
           ' I don\'t know how accurate this is, but a solid growth is guaranteed'
shortdoc = nlp(shortext)

print(colored('============= The original text information is still kept in the Doc object =============', 'blue'))
print(shortdoc)

print(colored('\n============= Identified Tokens =============', 'red'))
for token in shortdoc:
    print(token.text, end='\t\t')
#print('\t\t'.join([token.text for token in shortdoc]))


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
4.2. Part of Speech Tagging
The next component available in most spaCy pipelines is a POS tagger. The role of this component is to predict the part of speech of every detected token. This is where the statistical models come in. These components have been trained using machine learning methods over a large set of annotated data. If necessary, the models can be fine-tuned providing additional training data. This can be sometimes helpful when working in specialized domains.

The attributes calculated by all pipeline components after the Tokenizer are added as additional attributes to each of the patterns.
POS tags can be accessed as Token.pos_ (POS strings are hashed, and the underscore facilitates accessing the readable format of the variable).
Finer tags can be accessed as Token.tag_

The following code fragment allows us to represent the calculated POS for each token in the provided text.
Can you write Python code for it?

text = 'Modern condensed matter physics research has produced novel materials with fundamental properties that underpin a remarkable number of cutting-edge technologies. It is now generally accepted that novel materials are necessary for critical advances in technologies and whoever discovers novel materials generally controls the science and technology of the future. Transition metal oxides have attracted enormous interest within both the basic and applied science communities.  However, for many decades, the overwhelming balance of effort was focused on the 3d-elements (such as iron, copper, etc.) and their compounds; the heavier 4d- and 5d-elements (such as ruthenium, iridium, etc., which constitute two thirds of the d-elements listed in the Periodic Table) and their compounds have been largely ignored until recently. The principal investigator seeks to discover novel materials containing 4d- and/or 5d-elements and understand how they offer wide-ranging opportunities for the discovery of new physics and, ultimately, new device paradigms. This project also provides rigorous training to all students involved, focusing on synthesis and characterization techniques covering a broad spectrum of materials and experimental probes available in the principal investigator\'s laboratory. Technical Abstract: Physics driven by spin-orbit interactions is among the most important topics in contemporary condensed matter physics. Since the spin-orbit interaction is comparable to the on-site Coulomb and other relevant interactions, it creates a unique balance between competing interactions that drive complex behaviors and exotic states not observed in other materials. The project encompasses a systematic effort to elucidate physics of novel phenomena in spin-orbit-coupled and correlated materials and a rigorous search for new materials having exotic ground states. This project focuses on the following areas: (1) Novel phenomena at high pressures and high magnetic fields, (2) Unusual correlations between the insulating gap and...


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
4.4. Named Entity Recognition
According to spaCy documentation:
spaCy features an extremely fast statistical entity recognition system, that assigns labels to contiguous spans of tokens. The default trained pipelines can indentify a variety of named and numeric entities, including companies, locations, organizations and products. You can add arbitrary classes to the entity recognition system, and update the model with new examples.
A named entity is a “real-world object” that’s assigned a name – for example, a person, a country, a product or a book title. spaCy can recognize various types of named entities in a document, by asking the model for a prediction.
Because models are statistical and strongly depend on the examples they were trained on, this doesn’t always work perfectly and might need some tuning later, depending on your use case
You can iterate over the entities found in the text using doc.ents, as illustrated in the following example that displays also the types of the entities found.
Can you write Python code for it?

doc = nlp(english_text)
df_ents = pd.DataFrame([[ent.text, ent.label_, spacy.explain(ent.label_)] for ent in doc.ents], columns=['Entity', 'Type', 'Description'])
df_ents


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
4.5. Lemmatization
English lemmatizer in spaCy consists of the following elements:

Lookup tables
Rule-based lemmatizer, that exploits POS information
List-based exceptions aquired from WordNet

The annotation attribute can be easily accessed as Token.lemma_
Can you write Python code for it?

doc = nlp(text)
print(colored('============= Original text =============', 'blue'))
print(doc.text)
print(colored('\n============= Lemmas =============', 'red'))
print(' '.join([token.lemma_ for token in doc]))


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
Discussion: You can check that the NER algorithm is not always very accurate, both with respect to detection of entities and entity type classification. Note, however, that for some other texts performance can be much better. The following example contains an excerpt from the Albert Einstein wikipedia web page.

How does NER accuracy in this example compares to the previous case?
What do you believe is the reason for this?
Can you write Python code for it?

from spacy import displacy

wiki_text = 'Albert Einstein (14 March 1879 – 18 April 1955) was a German-born theoretical physicist, widely acknowledged to be one of the greatest physicists of all time.' \
            ' Einstein is known widely for developing the theory of relativity, but he also made important contributions to the development of the theory of quantum mechanics.' \
            ' He received the 1921 Nobel Prize in Physics "for his services to theoretical physics, and especially for his discovery of the law of the photoelectric effect".' \
            ' Einstein was born in the German Empire, but moved to Switzerland in 1895, forsaking his German citizenship the following year.' \
            ' Einstein was awarded a PhD by the University of Zürich.' \
            ' On the eve of World War II, he endorsed a letter to President Franklin D. Roosevelt.'

wiki_doc = nlp(wiki_text)
displacy.render(wiki_doc, style="ent", jupyter=True, options={'distance': 90})

entypes = set([ent.label_ for ent in wiki_doc.ents])
df_ent = pd.DataFrame([[enttyp, spacy.explain(enttyp)] for enttyp in entypes], columns=['Entity type', 'Description'])
df_ent


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
Example: The following code fragment adapts the example provided in the documenation for spacy-langdetect to construct a new pipeline that concatenates xx_sent_ud_sm and spacy-langdetect.
The new pipeline is then used to calculate variables english_text and spanish_text
Can you write Python code for it?

from spacy.language import Language
from spacy_langdetect import LanguageDetector

# Add LanguageDetector and assign it a string name
@Language.factory("language_detector")
def create_language_detector(nlp, name):
    return LanguageDetector(language_detection_function=None)

mult_nlp = spacy.load('xx_sent_ud_sm')
mult_nlp.add_pipe('language_detector', last=True)
mult_doc = mult_nlp(multilingualtext)
# document level language detection. Think of it like average language of the document!
print(colored('============= Document level language detection =============', 'blue'))
print(mult_doc._.language)
# sentence level language detection
print(colored('\n============= Sentence level language detection =============', 'red'))
for sent in mult_doc.sents:
    print(sent, sent._.language)
# English and Spanish Texts
print(colored('\n============= English sentences =============', 'green'))
english_text = ' '.join([sent.text for sent in mult_doc.sents if sent._.language['language']=='en'])
print(english_text)

print(colored('\n============= Spanish sentences =============', 'green'))
spanish_text = ' '.join([sent.text for sent in mult_doc.sents if sent._.language['language']=='es'])
print(spanish_text)


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
Exercise 3: You can access all vocab elements as nlp.vocab. Each element of the vocabulary is known as a Lexeme
   - Use command dir to examine what are the different methods and attributes of Lexeme objects.
   - For each element in the vocabulary, print the text representation, the hash representation, and whether the term should be considered as a stopword or not.
   - Find all stopwords in the Vocabulary
   - Which is the current size of your vocabulary? Create an additional doc object from a text with words that have not been previously used, and check the new size of the vocabulary after processing the new document.
--Hint: For displaying the vocabulary in a convenient format, you can store the requested information in a Pandas DataFrame, and print the DataFrame instead
Can you write Python code for it?

#<SOL>
vocab = []
for wd in nlp.vocab:
    vocab.append([wd.text, wd.orth, wd.is_stop])

vocab = pd.DataFrame(vocab, columns=['Term', 'Hash', 'Stopword'])
vocab
#</SOL>

#<SOL>
stopw = vocab[vocab['Stopword']==True]
stopw
#</SOL>

#<SOL>
print(len(nlp.vocab))
doc_aux = nlp('This is just a catalogue of some weird nomenclatures to increase the vocabulary size: USA, OMG, teeth, tooth. We can also include words which are in a different language or just typos: mecachis, guau, hause')
print(len(nlp.vocab))
#</SOL>


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
Exercise 5: Use spaCy command spacy.explain() to obtain the descriptions of all POS and TAG values that you got for the previous text fragment. Avoid repetitions.
Can you write Python code for it?

# Descriptions for POS values
#<SOL>
all_pos = set(df.POS.values.tolist())
pos_df = pd.DataFrame([[el, spacy.explain(el)] for el in all_pos], columns=['POS', 'Description'])
pos_df
#</SOL>

# Descriptions for TAGS values
#<SOL>
all_tag = set(df.TAG.values.tolist())
pos_df = pd.DataFrame([[el, spacy.explain(el)] for el in all_tag], columns=['TAG', 'Description'])
pos_df
#</SOL>


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
However, be aware that sentences are not identified by the Tokenizer element we have just described, but sentence tokenization is carried out instead as a subproduct of the dependency extraction component, that we will shortly review.
This can be a problem for multilingual documents, since all components of the previously used pipeline assumes an input in English language. In this case, what we normally do is:
   - Split the document in sentences using a multilingual sentence tokenizer
   - Detect the language of each sentence
   - Use the appropriate pipeline for each sentence depending on its language
Exercise 4: Split the following paragraph into two variables english_text and spanish_text using multilingual sentence tokenizers and language detection libraries.

Sentence tokenization: If you opt to use spaCy, you can use the multilingual pipeline xx_sent_ud_sm which provides just a basic (rule-based) sentence tokenizer. You may also use NLTK library (from nltk.tokenize import sent_tokenize)

Language detection: You can use python library langdetect, or the pipeline component spacy-langdetect for spaCy, which is just a wrapper for the previous library
Can you write Python code for it?

!python -m spacy download xx_sent_ud_sm
!pip install --upgrade spacy_langdetect

multilingualtext = 'Natural Language Processing is a key component of many relevant Artificial Intelligence Applications.' \
           ' El Procesamiento de Lenguaje Natural es un componente de gran importancia en multitud de aplicaciones de la Inteligencia Artificial.' \
           ' Libraries such as spaCy v3 make it simple to benefit from statistical NLP models based on neural networks.' \
           ' SpaCy v3 y otras librerías similares hacen posible emplear métodos de NLP basados en redes neuronales de manera sencilla.' \
           ' It is estimated that NLP market in the U.S. will grow to around 30000 MUSD during the next five years.' \
           ' Se estima que el mercado del NLP en USA será de alrededor de 30.000 millones de dolares en cinco años.'

#<SOL>

english_text =
spanish_text =
#</SOL>


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
In order to use a specific model you need to download it first. If working locally, you will need to download the model just once; however, in the cloud your environment resets and you will need to download the model on each session.
For this tutorial we will use an English model of medium size, the smallest model that incorporates word embeddings. For a complete list of available models, please refer to the spaCy website.
Can you write Python code for it?

!python -m spacy download en_core_web_md


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial.ipynb".
The first task is:
Unlike other spaCy components, the Tokenizer is not a statistical model. A finite set of rules and exceptions are encoded. If you wish to modify its behavior, you cannot retrain the component using labeled data. Instead, you would need to extend the list of rules and exceptions.
The following example adds an exception to expand word MUSD into tokens MUSD. Newly added exceptions are always applied after previous rules. Note also that exceptions must preserve the original text. Otherwise, an exception will be raised.
Can you write Python code for it?

# Add special case rule
from spacy.symbols import ORTH
special_case = [{ORTH: "M"}, {ORTH: "USD"}]
nlp.tokenizer.add_special_case("MUSD", special_case)

shortdoc = nlp(shortext)

print(colored('============= The original text information is still kept in the Doc object =============', 'blue'))
print(shortdoc)

print(colored('\n============= Identified Tokens =============', 'red'))
for token in shortdoc:
    print(token.text, end='\t\t')
#print('\t\t'.join([token.text for token in shortdoc]))


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
2.2. Obtaining Model Info
You can retrieve the most relevant information about available language models using the following command
spacy.info('model_name')
Note that you can only apply this command on models that have already been downloades. Otherwise, an exception is thrown.
Exercise 1: Run the following command and find the information related to
   - Components included in the pipeline
   - Are all components enabled?
   - How many types of entities can be recognized by the corresponding component?
   - What Part-of-Speech elements can you recognize?
   - What is the dimension of the word-embeddings incorporated in the model?
Detailed information about some specific components of the pipeline, as well as how they can be used, will be studied in the next sections.
Can you write Python code for it?


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
3.3. Usual Pipelines Components and Annotations

4. Linguistic Features

In this Section we will review a set of liguistic features provided by most spaCy pretrained pipelines. We will focus mainly on pipeline components that are relevant to build Bag of Words (BoW) representations of text.
4.1. Tokenizer
4.1.1. Word Tokenization
The Tokenizer is always the first component of the spaCy pretrained pipelines.

It has the important role of producing a Doc object out of a text string
It first splits the string using blank spaces
Then tokens are processed sequentially from left to right performing two operations
First, language-specific rules and exceptions are applied (e.g., in English "don't" is splitted into two separate tokens, but U.K. is kept as one token)
Second, prefixes or suffixes are identified. This is relevant to separate punctuation marks from the main tokens

It is important to note that tokenization rules, as well as exceptions are language specific. This means you need to make sure that the languages of the text and the selected Tokenizer match, otherwise you could get unexpected results.

Once the Doc object has been created, you can easily iterate over the identified tokens. Note also that the original text is preserved. You can access the string representation of Doc, Span, Token and even Lexeme objects by using the text attribute.
Can you write Python code for it?


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
4.1.2. Sentence Tokenization
Note that with the Doc object you can also iterate over sentences
Can you write Python code for it?

for sentence in shortdoc.sents:
    print(sentence.text)


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
4.5. Lemmatization
English lemmatizer in spaCy consists of the following elements:

Lookup tables
Rule-based lemmatizer, that exploits POS information
List-based exceptions aquired from WordNet

The annotation attribute can be easily accessed as Token.lemma_
Can you write Python code for it?


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
4.6. Other Annotations
Exercise 6: Have a look at the available attributes and functions of spaCy tokens using Python dir command.
Find out the significance of the following attributes: is_stop, is_alpha, is_digit, like_url, like_email, like_num, vector, and test your findings using text examples of your own.
5. Final Implementation of a pre-processing pipeline
Exercise 7: Implement a function that takes a string object and outputs the lemmatized text, ready for calculating BoW representation. The function should carry out the following steps:

Sentence tokenization and filtering of non-English sentences
Tokenization
POS
Lemmatization
Keep only alphanumeric tokens
Keep nouns, verbs, and adjectives
Generic Stopword removal
Specific Stopword removal
Can you write Python code for it?

mult_nlp = spacy.load('xx_sent_ud_sm')
mult_nlp.add_pipe('language_detector', last=True)
nlp = spacy.load('en_core_web_md')
nlp.disable_pipe('parser')
nlp.disable_pipe('ner')

valid_POS = set(['VERB', 'NOUN', 'ADJ', 'PROPN'])
specific_stw = set(['relevant', 'simple', 'base'])

def text_preprocessing(rawtext):
    #<SOL>

    #</SOL>

print(colored('============= Original text =============', 'blue'))
print(multilingualtext)
print(colored('\n============= Lemmatized text =============', 'red'))
print(text_preprocessing(multilingualtext))


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
Discussion: You can check that the NER algorithm is not always very accurate, both with respect to detection of entities and entity type classification. Note, however, that for some other texts performance can be much better. The following example contains an excerpt from the Albert Einstein wikipedia web page.

How does NER accuracy in this example compares to the previous case?
What do you believe is the reason for this?
Can you write Python code for it?


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
Example: The following code fragment adapts the example provided in the documenation for spacy-langdetect to construct a new pipeline that concatenates xx_sent_ud_sm and spacy-langdetect.
The new pipeline is then used to calculate variables english_text and spanish_text
Can you write Python code for it?


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
Exercise 5: Use spaCy command spacy.explain() to obtain the descriptions of all POS and TAG values that you got for the previous text fragment. Avoid repetitions.
Can you write Python code for it?

# Descriptions for POS values
#<SOL>

#</SOL>

# Descriptions for TAGS values
#<SOL>

#</SOL>


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
However, be aware that sentences are not identified by the Tokenizer element we have just described, but sentence tokenization is carried out instead as a subproduct of the dependency extraction component, that we will shortly review.
This can be a problem for multilingual documents, since all components of the previously used pipeline assumes an input in English language. In this case, what we normally do is:
   - Split the document in sentences using a multilingual sentence tokenizer
   - Detect the language of each sentence
   - Use the appropriate pipeline for each sentence depending on its language
Exercise 4: Split the following paragraph into two variables english_text and spanish_text using multilingual sentence tokenizers and language detection libraries.

Sentence tokenization: If you opt to use spaCy, you can use the multilingual pipeline xx_sent_ud_sm which provides just a basic (rule-based) sentence tokenizer. You may also use NLTK library (from nltk.tokenize import sent_tokenize)

Language detection: You can use python library langdetect, or the pipeline component spacy-langdetect for spaCy, which is just a wrapper for the previous library
Can you write Python code for it?

!python -m spacy download xx_sent_ud_sm
!pip install --upgrade spacy_langdetect

multilingualtext = 'Natural Language Processing is a key component of many relevant Artificial Intelligence Applications.' \
           ' El Procesamiento de Lenguaje Natural es un componente de gran importancia en multitud de aplicaciones de la Inteligencia Artificial.' \
           ' Libraries such as spaCy v3 make it simple to benefit from statistical NLP models based on neural networks.' \
           ' SpaCy v3 y otras librerías similares hacen posible emplear métodos de NLP basados en redes neuronales de manera sencilla.' \
           ' It is estimated that NLP market in the U.S. will grow to around 30000 MUSD during the next five years.' \
           ' Se estima que el mercado del NLP en USA será de alrededor de 30.000 millones de dolares en cinco años.'

#<SOL>

#</SOL>

print(colored('\n============= English sentences =============', 'green'))
print(english_text)

print(colored('\n============= Spanish sentences =============', 'green'))
print(spanish_text)


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
In order to use a specific model you need to download it first. If working locally, you will need to download the model just once; however, in the cloud your environment resets and you will need to download the model on each session.
For this tutorial we will use an English model of medium size, the smallest model that incorporates word embeddings. For a complete list of available models, please refer to the spaCy website.
Can you write Python code for it?


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
Note how in the example we could easily access all lemmas and entities found by iterating over the document (variable doc) itself or over its entitities (doc.ents)
3.2. Architecture
Central data structures:

Language: is instantiated when loading the model, and contains the pipeline. Transforms text into spaCy documents.
Doc: Sequence of tokens with annotations. We can iterate over tokens, access individual tokens (doc[3]) or a span of tokens (doc[5:15]).
Vocab: Unique vocabulary associated to the language. Vocabulary is composed of Lexemes that are hashed and stored in the vocabulary with word vectors and attributes. This is memory efficient and assures a unique ground truth.

Source: https://spacy.io/architecture-415624fc7d149ec03f2736c4aa8b8f3c.svg

The Tokenizer component of the pipeline is special, since this is where the Doc object is generated from the text. Subsequent pipeline components perform operations in place, obtanining new attributes that are stored as annotations in the tokens.

Pipeline components can be fine-tuned using annotated data
New components can be easily implemented and added to the Pipeline

Exercise 2:
   - Find the Spans associated to the following text fragments contained in the original text:
      * structural, transport, magnetic, thermal and dielectric properties
      * temperature, magnetic field, pressure and doping
      * This project also provides rigorous training to all students involved
   - Use command dir to examine what are the different methods and attributes of the Span object
   - Recover the vector representation associated to each of the previous strings
   - Compute the Euclidean distances between the selected Spans
--Hint: To compute Euclidean distances at this point, it can be convenient to use numpy function np.linalg.norm. Later in the notebook you will find that spaCy provides functions to carry...
Can you write Python code for it?


I am working on the file "NLP2.Spacy_Tutorial (Data Preprocessing)/spaCy_tutorial_students.ipynb".
The first task is:
Unlike other spaCy components, the Tokenizer is not a statistical model. A finite set of rules and exceptions are encoded. If you wish to modify its behavior, you cannot retrain the component using labeled data. Instead, you would need to extend the list of rules and exceptions.
The following example adds an exception to expand word MUSD into tokens MUSD. Newly added exceptions are always applied after previous rules. Note also that exceptions must preserve the original text. Otherwise, an exception will be raised.
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch10.ipynb".
The first task is:
First-Order Logic
first-order logic由以下幾種元素組成:

terms: 個體變數、個體常數兩種
predicates: 相當於function，可以接受任意參數，並決定某種動作或狀態

下面會介紹Typed-Logic，有e(代表entity/term)、t(代表formula)兩種basic type。多個basic type可以組成complex type，例如<e,t>或<e,<e,t>>等等。
Can you write Python code for it?

from nltk.sem.logic import LogicParser
tlp = LogicParser(True)

# function本身是一種complex type <e,?>
#   之所以有 ? 是因為我們沒有指定walk的type
# John是一個entity，所以屬於basic type e
john_walk = tlp.parse('walk(John)')
print john_walk.function, john_walk.function.type, john_walk.argument, john_walk.argument.type

# 要指定function的type，可以用signature來指定
walk_sig = {'walk': '<e, t>'}
john_walk = tlp.parse('walk(John)', walk_sig)
print john_walk.function, john_walk.function.type, john_walk.argument, john_walk.argument.type

# existential quantifier
tlp.parse('exists x.(dog(x) & disappear(x))')

# universal quantifier
tlp.parse('all x.(dog(x) -> disappear(x))')

# 在formula中的變數，如果只有一個字元，會被當成free variable
tlp.parse('dog(Cyril)').free()

tlp.parse('dog(x)').free()

tlp.parse('exists x.own(y, x)').free()


I am working on the file "NLP_With_Python/Ch10.ipynb".
The first task is:
Model
一個model M，如果用在first-order logic language L上，則定義為M = <D, Val>。其中D稱為domain，Val稱為valuation function。
Val的用途是將L中的individual constant及predicate symbol都賦予一個屬於domain的值。
Can you write Python code for it?

from nltk.sem import Valuation, Model
v = [('adam', 'b1'), ('betty', 'g1'), ('fido', 'd1'),
     ('girl', set(['g1', 'g2'])), ('boy', set(['b1', 'b2'])),
     ('dog', set(['d1'])),
     ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1')]))]
val = Valuation(v)
dom = val.domain
m = Model(dom, val)

dom

m.evaluate('love(fido, betty)', nltk.Assignment(dom))

m.evaluate('love(adam, betty)', nltk.Assignment(dom))

m.evaluate('love(john, betty)', nltk.Assignment(dom))


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
A Pronouncing Dictionary
nltk有包含CMU Pronouncing Dictionary for US English，專門為語音合成的程式而設計。裡面所使用的音標可以參考: https://en.wikipedia.org/wiki/Arpabet
Can you write Python code for it?

entry = nltk.corpus.cmudict.entries()
len(entry)

entry[10000:10010]

[w for w,pron in entry if pron[-5:] == [u'V', u'IH2', u'ZH', u'AH0', u'N']]


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
Brown Corpus
Brown Corpus在1961年由布朗大學所建立，包含500種來源的文字，這些來源分類為news, editorial等等。由file id可以看出類別，例如"ca"開頭的是新聞、"cd"開頭的是宗教、"cm"開頭的是科幻。
Can you write Python code for it?

from nltk.corpus import brown as br
br.categories()

br.fileids(categories='news')[:5]

br.fileids(categories='science_fiction')[:5]


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
ConditionalFreqDist的常用函式

cfd = ConditionalFreqDist(pairs): 將pairs=(c,s)轉換為頻率分布
cfd.conditions(): 列出pairs[0]的所有可能
cfd[cond]: 用cond作為條件查詢，也就是pairs[0]==cond的清單
cfd[cond][sample]: pairs[0]==cond && pairs[1]==sample的清單
cfd.tabulate(conditions=cond, samples=samp): 將多個cond與多個sample的對應關係畫成表格
cfd.plot(): 畫出頻率分布
cfd.plot(samples, conditions): 畫出頻率分布

詞彙分析資源 (Lexical Resources)
單字或句子的附加資訊，都屬於詞彙分析的一部分，例如單字的詞性、字義、頻率等。
Wordlist Corpora
第一種分析方式，是拿字典作比較，找出不存在於字典的單字。
Can you write Python code for it?

vocabulary = set(w.lower() for w in nltk.corpus.words.words())
austen = set(w.lower() for w in nltk.corpus.gutenberg.words('austen-sense.txt'))
list(austen.difference(vocabulary))[:10]


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
Corpus共同的API

fileids(): 檔案清單
fileids([categories]): 特定分類下的檔案清單
categories(): 分類清單
categories([fileids]): 檔案所歸屬的分類清單
raw(): 原始資料
raw(fileids=[f1,f2,f3]): 取得指定檔案的原始資料
raw(categories=[c1,c2]): 取得指定分類的原始資料
words(): 單字陣列
words(fileids=[f1,f2,f3]): 取得指定檔案的單字陣列
words(categories=[c1,c2]): 取得指定分類的單字陣列
sents(): 單句陣列
sents(fileids=[f1,f2,f3]): 取得指定檔案的單句陣列
sents(categories=[c1,c2]): 取得指定分類的單句陣列
abspath(fileid): 檔案在硬碟中的位置
encoding(fileid): 檔案的編碼(如果已知)
open(fileid): 將檔案打開成stream的格式
root(): corpus的根目錄
readme(): 關於這個corpus的資訊

使用自己的Corpus
如果你有純文字檔想當成Corpus，可以使用ntlk.corpus.PlaintextCorpusReader。
```python

from nltk.corpus import PlaintextCorpusReader
corpus_root = '/usr/share/dict'
wordlists = PlaintextCorpusReader(corpus_root, '.*')
wordlists.fileids()
['README', 'connectives', 'propernames', 'web2', 'web2a', 'words']
wordlists.words('connectives')
['the', 'of', 'and', 'to', 'a', 'in', 'that', 'is', ...]
```

如果你有Penn Treebank樹狀結構的資料，可以使用ntlk.corpus.BracketParseCorpusReader。
```python

from nltk.corpus import BracketParseCorpusReader
corpus_root = r"C:\corpora\penntreebank\parsed\mrg\wsj"
file_pattern = r"./wsj_..mrg"
ptb = BracketParseCorpusReader(corpus_root, file_pattern)
ptb.fileids()
['00/wsj_0001.mrg', '00/wsj_0002.mrg', '00/wsj_0003.mrg', '00/wsj_0004.mrg', ...]
len(ptb.sents())
49208
ptb.sents(fileids='20/wsj_2013.mrg')[19]
['The', '55-year-old', 'Mr.', 'Noriega', 'is', "n't", 'as', 'smooth', 'as', 'the',
'shah', 'of', 'Iran', ',', 'as', 'well-born', 'as', 'Nicaragua', "'s", 'Anastasio',
'Somoza', ',', 'as', 'imperial', 'as', 'Ferdinand', 'Marcos', 'of', 'the', 'Philippines',
'or', 'as', 'bloody', 'as', 'Haiti', "'s", 'Baby', Doc', 'Duvalier', '.']
```

2.2 條件式頻率分布
在統計條件分布時，我們需要將文字轉換成(condition, event)的格式，例如Brown中，有15個condition(每個分類一個)，及1161192個event(每個單字一個)。
FreqDist()是單純的頻率分布，以list為參數。ConditionalFreqDist()則是條件式頻率分布，以list of pairs為參數。
Can you write Python code for it?

import nltk
from nltk.corpus import gutenberg as gut
emma = gut.words(gut.fileids()[0])
freq_uni = nltk.FreqDist(emma)
freq_bi = nltk.FreqDist(nltk.bigrams(emma))

freq_uni.most_common()[:5]

freq_bi.most_common()[:5]


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
Inaugural Address Corpus
這個Corpus是歷任美國總統的就職演說，檔名包含年份及總統名，例如"2009-Obama.txt"。
Can you write Python code for it?

from nltk.corpus import inaugural as ina
ina.sents('2009-Obama.txt')

cfd = nltk.ConditionalFreqDist((target, f[:4]) for f in ina.fileids() for w in ina.words(f)
                        for target in ['america', 'right'] if w.lower().startswith(target))

%matplotlib inline
cfd.plot()


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
Reuters Corpus
Reuters Corpus包含10788則新聞，總共有130萬字，分成90個主題。為了方便machine learning，所有的file id都以"training/"或"test/"開頭，例如"test/14826"表示在test set中的一則新聞。
Reuters Corpus與Brown不同的地方在於分類可能重疊，一篇新聞可以同時屬於多個分類。新聞的分類可由categories(file_id)來查詢，也可以由fileid(category)找出分類對應的file id。
Can you write Python code for it?

from nltk.corpus import reuters as rt
rt.fileids(['wheat', 'corn'])[:3]

rt.categories([u'test/14832', u'test/14841', u'test/14858'])


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
Shoebox and Toolbox Lexicons
Toolbox是語言學家最常用的工具，在第11章會再詳細介紹。
Can you write Python code for it?

from nltk.corpus import toolbox
toolbox.entries('rotokas.dic')[0]


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
Web and Chat Text
nltk.corpus.webtext包含了Firefox討論區、神鬼奇航的劇本、還有一些路邊聽到的對話等等。
Can you write Python code for it?

from nltk.corpus import webtext as web
for f in web.fileids():
    print '{0:15}{1}'.format(f, web.words(f)[:7])


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
Wordnet
WordNet是一個語意字典，也可以說是給電腦看的字典。當兩個字的字義相同，例如motocar和automobile，就稱這兩個字為synonyms，而這兩個字會組成一個synset。一個synset永遠只包含一個字義，但可以有很多單字。
Can you write Python code for it?

from nltk.corpus import wordnet as wn
wn.synsets('sleep')
# sleep共有6種不同的字義，分布在6個synsets中

# 每個synset可能由多個字組成
[syn.lemma_names() for syn in wn.synsets('sleep')]

# 查詢synset所代表的意義
[syn.definition() for syn in wn.synsets('sleep')]

# synset的例句
[syn.examples() for syn in wn.synsets('sleep')]


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
antonymy是相反的關係。
Can you write Python code for it?

lemma = wn.synsets('vertical')[2].lemmas()[0]
lemma.antonyms()


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
ntlk.corpus.nps_char包含一萬筆即時通訊的記錄，所有對話的姓名都以"UserNNN"代替。nps_char分為15個file id，對應不同的chat room，例如file id中間有"20s"表示是20歲的聊天室，有"teen"表示是青少年專用。file id前面的日期都是2006年，而檔名中的"???posts"代表裡面有幾則訊息。
Can you write Python code for it?

from nltk.corpus import nps_chat as nps
for f in nps.fileids():
    print '{0:30}{1}'.format(f, nps.words(f)[:5])


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
使用Brown Corpus時，所有函式都可以加上categories = ['xxx','yyy']或是fileid = ['xxx','yyy']來限制
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
例如我們想要讀取Jane Austen所著的"Emma"，可以用words()這個函式取得。
Can you write Python code for it?

emma = nltk.corpus.gutenberg.words('austen-emma.txt')
emma[:10]  # 看作品的前10個字


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
在發音中的數字代表重音，例如1表示重音，2表示次重音，0表示輕音。
Can you write Python code for it?

def stress(pron):
    return [c[-1] for c in pron if c[-1].isdigit()]
[w for w, pron in entry if stress(pron) == ['0','1','0','2','0','0']]


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
如果遇到字典中沒有的字，可以手動新增，但新增的結果並不會存檔，下次讀出來還是會缺字。
Can you write Python code for it?

pdict['blog']

pdict['blog'] = [['B','L','AA1','G']]

pdict['blog']


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
我們常利用Brown Corpus來研究不同類別的文字中，用詞會有什麼不一樣。例如我想知道不同文字中，"can", "could", "may", "might", "must", "will"的比例。
Can you write Python code for it?

modals = ["can", "could", "may", "might", "must", "will"]
print '{0:15} {1:>6} {2:>6} {3:>6} {4:>6} {5:>6} {6:>6}'.format \
            ('category', modals[0], modals[1], modals[2], modals[3], modals[4], modals[5])
for cat in br.categories():
    text = br.words(categories=cat)
    dist = nltk.FreqDist([w.lower() for w in text])
    num = len(text)
    freq = [float(dist[m]) / num * 10000 for m in modals]
    print '{0:15} {1:6.2f} {2:6.2f} {3:6.2f} {4:6.2f} {5:6.2f} {6:6.2f}'.format \
            (cat, freq[0], freq[1], freq[2], freq[3], freq[4], freq[5])


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
要比較兩個synsets接近的程度，可以看它們共同的祖先有多高。如果共同的祖先是entity，表示幾乎沒有關係，因為entity已經是root了。例如我們要找left和right的字義中最接近的。
Can you write Python code for it?

l = wn.synsets('left', wn.NOUN)
r = wn.synsets('right', wn.NOUN)
for left in l:
    for right in r:
        ancestor = left.common_hypernyms(right)[0]
        sim = left.path_similarity(right)
        if sim > 0.1:
            print '{0:18}{1:18}{2:25}{3}'.format(left.name(), right.name(), ancestor.name(), sim)


I am working on the file "NLP_With_Python/Ch2.ipynb".
The first task is:
邏輯關係: entailment。如果A動作是由B,C,D...組成的，則我們說A entails B,C,D...
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
Accessing Text from the Web and from Disk
Electronic Books
Gutenberg上每一本電子書都有編號，只要知道編號就可以下載電子書，例如"Crime and Punishment"是第2554號，可以用下面的方法取得。
Can you write Python code for it?

from urllib import urlopen
url = "http://www.gutenberg.org/files/2554/2554.txt"
raw = urlopen(url).read()
type(raw)


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
Dealing with HTML
Can you write Python code for it?

url = "http://news.bbc.co.uk/2/hi/health/2284783.stm"
html = urlopen(url).read()
html[:60]

from bs4 import BeautifulSoup
bs = BeautifulSoup(html, "lxml")

tokens = nltk.word_tokenize(bs.get_text())
tokens[:5]


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
Formatting: From Lists to Strings
Can you write Python code for it?

s = ['We', 'called', 'him', 'Tortoise', 'because', 'he', 'taught', 'us', '.']
' '.join(s)

';'.join(s)


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
Lemmatization
Can you write Python code for it?

s = ['women', 'are', 'living']
wnl = nltk.WordNetLemmatizer()
zip(s, [wnl.lemmatize(t, 'v') for t in s], [wnl.lemmatize(t, 'n') for t in s])


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
Normalize Text
正規化是指將文字變成統一的格式，例如轉成小寫、取字根。
Can you write Python code for it?

porter = nltk.PorterStemmer()
lanc = nltk.LancasterStemmer()
w = ['I','was','playing','television','in','the','painted','garden']
[(a, porter.stem(a), lanc.stem(a)) for a in w]


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
Regular Expressions

.: 對應任意一個字元
^abc: 字串開頭為abc
abc$: 字串結尾為abc
[abc]: 對應[]內出現的字元，例如a或b或c
[A-Z0-9]: 對應範圍內的字元，例如A到Z以及0到9
ed|ing|s: 對應數種指定的字串
*: 前一個符號重複0次以上
+: 前一個符號重複1次以上
?: 前一個符號重複0次或1次
{n}: 前一個符號重複恰好n次
{n,}: 前一個符號重複至少n次
{,n}: 前一個符號重複最多n次
{m,n}: 前一個符號重複最少m次、最多n次
a(b|c)+: 括號表示|運算的範圍
Can you write Python code for it?

import re

# 一般re.search傳回的是一個 _sre.SRE_Match 物件
# 將物件轉換成 bool 值，如果 True 代表有找到符合的字串
bool(re.search('ed$', 'played')), bool(re.search('ed$', 'happy'))

# re.findall 可以取出部分的字串，要取出的部分由()的範圍決定
re.findall('\[\[(.+?)[\]|\|]+', 'My Name is [[Bany]] Hung, this is my [[Dog|Animal]]')

# re.sub 會取代字串
re.sub('\[\[.+?[\]|\|]+', '###', 'My Name is [[Bany]] Hung, this is my [[Dog|Animal]]')


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
Strings: Lowest Level Text
字串是一種immutable的資料形態，設定值之後就不能修改。因此所有字串函式都會產生複製後的字串，而不會修改原始的字串。
用三個引號"""來定義字串，可以將換行一起放進去。
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
str支援+和*兩個operator。+代表連接兩個字串，*代表重複一個字串。
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
unicodedata可以印出unicode中對字元的描述
Can you write Python code for it?

import unicodedata
lines = codecs.open(path, encoding='latin2').readlines()
line = lines[2]
print line.encode('unicode_escape')

for c in line:
    if ord(c) > 127:
        print '%s U+%04x %s' % (c.encode('utf8'), ord(c), unicodedata.name(c))


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
作stemming之前，要將原始資料存在dict中，以方便使用。
Can you write Python code for it?

class IndexedText(object):
    def __init__(self, stemmer, text):
        self._text = text
        self._stemmer = stemmer
        self._index = nltk.Index((self._stem(word), i) for (i, word) in enumerate(text))

    def concordance(self, word, width=40):
        key = self._stem(word)
        wc = int(width/4)                # words of context
        for i in self._index[key]:
            lcontext = ' '.join(self._text[i-wc:i])
            rcontext = ' '.join(self._text[i:i+wc])
            ldisplay = '%*s'  % (width, lcontext[-width:])
            rdisplay = '%-*s' % (width, rcontext[:width])
            print ldisplay, rdisplay

    def _stem(self, word):
        return self._stemmer.stem(word).lower()

grail = nltk.corpus.webtext.words('grail.txt')
text = IndexedText(porter, grail)
text.concordance('lie')


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
產生的token可以進一步轉換成nltk text形態，以便進一步處理。
Can you write Python code for it?

text = nltk.Text(tokens)
text

text.collocations()


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
用[i]可以存取第i+1個字元，如果字串長度是n，則i的範圍為0到n-1。
用[-i]可以存取後面數過來第i個字元，如果字串長度是n，則i的範圍為-1到-n。
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch3.ipynb".
The first task is:
註：如果要使用proxy下載，可以用
proxy = {'http': 'http://www.someproxy.com:3128'}
raw = urlopen(url, proxies=proxy).read()
剛下載的raw是字串形態，包含許多不必要的字，所以需要tokenization，使單字分離。
Can you write Python code for it?

tokens = nltk.word_tokenize(raw)
tokens[:10]


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
Conditions
將list放在if中，會直接判斷list是否為空，相當於if len(list) > 0:。
Can you write Python code for it?

e = []
if e: print e, " is not empty"

e = []
if not e: print e, " is empty"


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
Equality
用==是比較兩個元素值是否相同。
用is是比較兩個元素是否參考同一個物件。
Can you write Python code for it?

a is b[0], a is b[1]

b = a[:]
# 因為用複製的，所以值相同但物件不同
a is b, a == b


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
Function Inputs and Outputs
在設計function時，要注意，如果會修改輸入參數，最好不要有輸出，否則會讓使用者混淆。
def sort1(a):   # OK, 會修改輸入但沒有輸出
    a.sort()
def sort2(a):   # OK, 不會修改輸入, 有輸出
    return sorted(a)
def sort3(a):   # BAD, 有修改輸入又有輸出, 一定會有人搞錯
    a.sort()
    return a
所有function的參數都是call-by-value，但要注意，如果參數是一個list，list傳入的value是物件id，傳到function內部後變成可修改的list。
Can you write Python code for it?

def func1(a):
    a[0] = 'modified'
s = ['hello', 'world']
func1(s)
s


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
Lambda Expression
lambda是用來產生臨時性function的方法。
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
Letter Trie
Can you write Python code for it?

def insert(trie, key, value):
    if key:
        first, rest = key[0], key[1:]
        if first not in trie:
            trie[first] = {}  # empty dict
        insert(trie[first], rest, value)  # key[1:] is new key
    else:
        trie['value'] = value

trie = nltk.defaultdict(dict)

insert(trie, 'chat', 100)
insert(trie, 'chair', 2000)
insert(trie, 'chien', 150)
trie

trie['c']['h']['a']['t']['value']


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
Matplotlib
Can you write Python code for it?

%matplotlib inline
import matplotlib

from matplotlib import pylab

import nltk

nltk.ngrams()


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
Named Arguments
Can you write Python code for it?

def generic(*a, **b):
    print a   # 集中所有 unnamed arguments
    print b   # 集中所有 names arguments

generic(1, 3.5, 'money', zzz='maybe', ggg='good')

def func(*a, z):
    print a, z   # 因為有指定 *a 收集所有 unnamed arguments，造成 z 出錯
func('hi', 'this')


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
Structure of a Python Module
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
Variable Scope
Python遵守LGB Rule，先找local，再找global，再找built-in。
function可以透過global關鍵字創造global變數，但實際上越少用越好，這會影響function的可用性。
Check Variable Type
一般用assert(cond)配合isinstance來完成。assert當參數為False時，會出現AssertionError。
Can you write Python code for it?

a = 'hello'
assert(isinstance(a, basestring))  # 沒問題

a = 3
assert(isinstance(a, basestring))  # 錯誤


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
any()判斷一個list是否存在True的元素，all()判斷一個list是否全為True，in用來判斷值是否存在list中。
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
利用tuple可以同時進行多個元素的取代。
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
用zip可以將多個list結合成tuple。
Can you write Python code for it?

a = range(5)
b = range(5, 10)
zip(a, b, a, b)

list(enumerate(b))  # enumerate 會傳回 (index, a[index])

a = [5, 3, 2, 4, 1]
a.sort()  # .sort() 會直接修改原始list
a

a = [5, 3, 2, 4, 1]
sorted(a), a  # 用sorted()不會影響原始list


I am working on the file "NLP_With_Python/Ch4.ipynb".
The first task is:
重覆元素的方法
Can you write Python code for it?


I am working on the file "NLP_With_Python/Ch5.ipynb".
The first task is:
Confusion Matrix
Can you write Python code for it?

brown_sents = brown.sents()
brown_tagged_sents = brown.tagged_sents(tagset = 'universal')
default_tagger = nltk.DefaultTagger('NOUN')
unigram_tagger = nltk.UnigramTagger(brown_tagged_sents[:4000], backoff=default_tagger)
bigram_tagger = nltk.BigramTagger(brown_tagged_sents[:4000], backoff=unigram_tagger)

unigram_tagger.tag(nltk.word_tokenize('I like your mother'))

test = [tag for sent in brown_sents[4000:] for (word, tag) in bigram_tagger.tag(sent)]
gold = [tag for sent in brown_tagged_sents[4000:] for (word, tag) in sent]
print nltk.ConfusionMatrix(gold, test)


I am working on the file "NLP_With_Python/Ch5.ipynb".
The first task is:
Default Dictionary
如果試著存取不存在的key，會造成錯誤。defaultdict可以使存取不存在的key時，自動新增一個預設值。
Can you write Python code for it?

f = nltk.defaultdict(int)
f['color'] = 4
f

f['dream']  # dream不存在，但查詢時會自動新增

f  # 查詢dream後，就直接新增了一個dream

f = nltk.defaultdict(lambda: 'xxx')
f['hello'] = 'world'
f

f['here'] = f['here'] + 'comment'
f


I am working on the file "NLP_With_Python/Ch5.ipynb".
The first task is:
Default Tagger
第一步，先準備一個預設的tagger，統計所有part-of-speech中，那一種詞性出現頻率最高，就視為預設值。
Can you write Python code for it?

tags = [tag for (word, tag) in brown.tagged_words(categories='news')]
nltk.FreqDist(tags).max()

default_tagger = nltk.DefaultTagger('NN')  # 因為NN頻率最高，所以未知詞性的情況一律當成NN
default_tagger.tag(nltk.word_tokenize('i like my mother and dog'))

# 當然預測的準確率很差，因為只有13%是真的NN
default_tagger.evaluate(brown_tagged_sents)


I am working on the file "NLP_With_Python/Ch5.ipynb".
The first task is:
Inverting a Dictionary
dict的設計是用key來查詢value，如果你希望用value來查詢key，速度會很慢。一種簡單的解法是用(value,key)重新產生一個dict，就可以用value查詢key。
Can you write Python code for it?

old = dict(nltk.corpus.brown.tagged_words()[:100])
new = dict((value, key) for (key, value) in old.items())

new['JJ']  # 雖然成功的反相，但只能查出最後輸入的字

new2 = nltk.defaultdict(list)  # 當key不存在時，都視為empty list
for (key, value) in old.items():
    new2[value].append(key)

new2['JJ']


I am working on the file "NLP_With_Python/Ch5.ipynb".
The first task is:
Mapping Words to Properties
要儲存mapping資料，最自然的方法就是用dictionary，又稱為associative array或hash array。一般的陣列是用整數作為index，但dictionary中則是用任何hasable的資料作為index，例如字串、tuple等等。
mapping資料的應用有:

書的index: 將單字mapping到頁數
thesaurus: 將字義mapping到一組同義字
字典: 將單字mapping到單字的解釋
比較字集: 將單字mapping到多國語言的單字
Can you write Python code for it?

pos = {}  # 在python中定義dictionary最簡單的方法
pos['hello'] = 'world'
pos['right'] = 'here'
pos

[w for w in pos]  # 用for的時候會找出key

pos.keys()

pos.items()

pos.values()

pos = dict(hello = 'world', right = 'here')  # 另一種定義方式
pos


I am working on the file "NLP_With_Python/Ch5.ipynb".
The first task is:
Regular Expression Tagger
用經驗判斷那些字尾可能是什麼詞性，然後用regular expression寫成條件。
Can you write Python code for it?

patterns = [
    (r'.*ing$', 'VBG'),
    (r'.*ed$', 'VBD'),
    (r'.*es$', 'VBZ'),
    (r'.*ould$', 'MD'),
    (r'.*\'s$', 'NN$'),
    (r'.*s$', 'NNS'),
    (r'^-?[0-9]+(.[0-9]+)?$', 'CD'),
    (r'.*', 'NN')
]

regexp_tagger = nltk.RegexpTagger(patterns)
regexp_tagger.tag(nltk.word_tokenize('i could be sleeping in 9 AM'))

regexp_tagger.evaluate(brown_tagged_sents)


I am working on the file "NLP_With_Python/Ch5.ipynb".
The first task is:
Unigram Tagger
Can you write Python code for it?

unigram_tagger = nltk.UnigramTagger(brown.tagged_sents(categories='news')[:500])

unigram_tagger.tag(nltk.word_tokenize('i could be sleeping in 9 AM'))


I am working on the file "NLP_With_Python/Ch5.ipynb".
The first task is:
corpus中也有tagged sentences:
Can you write Python code for it?