NLTK

##Library
```python
import nltk
nltk.download('punkt')

from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize

nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer

```

##Example 1
```python
nltk.download('state_union')
from nltk.corpus import state_union

train_text = state_union.raw("2005-GWBush.txt")
sample_text = state_union.raw("2006-GWBush.txt")

word_tokens = word_tokenize(train_text)
sentence_tokens = sent_tokenize(train_text)

print (len(word_tokens))
print (word_tokens[:5])
print ()
print (len(sentence_tokens))
print (sentence_tokens[:2])
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/08a8d4c4-a21f-e1c9-e0ae-cdeac8350343.png)


```python
stop_words = stopwords.words('english')
symbol = ["'", '"', ':', ';', '.', ',', '-', '!', '?', "'s", '(', ')', '_']

word_selected = [w.lower() for w in word_tokens if w.lower() not in stop_words + symbol]
print (len(word_selected))
print (word_selected[:10])
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/d98dfc47-6e5a-fcb3-bed4-a678f8495666.png)


```python
frequency = nltk.FreqDist(word_selected)

print (frequency['president'])
print (frequency['union'])
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/7759c670-0d17-13f6-aec2-01a0947d7d63.png)

```python
plt.figure(figsize=(10, 5))
frequency.plot(30,cumulative=True)
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/07e988a4-dfaf-3f7f-1a7d-7c4948200129.png)


```python
plt.figure(figsize=(10, 5))
frequency.plot(30,cumulative=False)
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/b9845c8c-214e-f7a7-9def-68202938702c.png)


##Example 2 (Movie reviews)
```python
import numpy as np
import nltk
nltk.download('punkt')

from nltk.tokenize import sent_tokenize, word_tokenize

from nltk.corpus import stopwords
nltk.download('stopwords')

from nltk.corpus import movie_reviews
nltk.download('movie_reviews')

import random
```

```python
print ('# of pos: ', len(movie_reviews.fileids('pos')))
print ('# of neg: ', len(movie_reviews.fileids('neg')))
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/aa674c77-3443-2eee-9af7-46e3859d6e4c.png)

```python
%%time

stop_words = stopwords.words('english')
symbol = ["'", '"', ':', ';', '.', ',', '-', '!', '?', "'s", '(', ')', '_']

all_words = []
for w in movie_reviews.words():
  if w.lower() not in stop_words + symbol:
    all_words.append(w.lower())

all_words = nltk.FreqDist(all_words)

print ('# of words: ', len(all_words))
print (all_words.most_common(5))
print (all_words['like'])
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/e099e3aa-4369-c543-caa5-84b1a61d23b7.png)

```python
#word_features = list(all_words.keys())[:3000]

word_features = []
for w in dict(all_words.most_common(3000)).keys():
  word_features.append(w)

print (word_features[:10])
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/5f1cada7-1521-e5dd-e319-53662a73c56a.png)

```python
def find_features(document):
  words = set(document)
  features = {}
  for w in word_features:
    features[w] = (w in words)

  return features

documents = [(list(movie_reviews.words(fileid)), category)
             for category in movie_reviews.categories()
             for fileid in movie_reviews.fileids(category)]

random.shuffle(documents)

data = [(find_features(doc), category) for (doc, category) in documents]

print (np.array(data).shape)
print (data[0][0])
print (data[0][1])
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/c91f3839-fa45-ae75-24b3-28cfbb02c1b8.png)

```python
data_train = data[:1800]
data_test = data[1800:]

nb = nltk.NaiveBayesClassifier.train(data_train)

print ('NaiveBayes accuracy: ', nltk.classify.accuracy(nb, data_test))
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/cf84bf69-a5d8-0564-4506-06b364c05fe1.png)

```python
nb.show_most_informative_features(15)
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/9026e667-b4ad-0ac4-32d9-14a89526eb08.png)


```python
#saving
import pickle

path = '/content/gdrive/My Drive/MILIZE/'

save_nb = open(path+'naivebayes.pickle', 'wb')
pickle.dump(nb, save_nb)
save_nb.close()

saved_nb = open(path+'naivebayes.pickle', "rb")
nb_2 = pickle.load(saved_nb)
saved_nb.close()
```

```python
#sklearn
from nltk.classify.scikitlearn import SklearnClassifier
from sklearn.naive_bayes import MultinomialNB,BernoulliNB

from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.svm import SVC, LinearSVC, NuSVC

mnb = SklearnClassifier(MultinomialNB())
mnb.train(data_train)
print("MultinomialNB accuracy:", nltk.classify.accuracy(mnb, data_test))

bnb = SklearnClassifier(BernoulliNB())
bnb.train(data_train)
print("BernoulliNB accuracy percent:",nltk.classify.accuracy(bnb, data_test))

lr = SklearnClassifier(LogisticRegression())
lr.train(data_train)
print("Logistic Regression accuracy:", nltk.classify.accuracy(lr, data_test))

svc = SklearnClassifier(SVC())
svc.train(data_train)
print("SVC accuracy percent:",nltk.classify.accuracy(svc, data_test))

```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/0b1d5d15-329f-e5fb-b59b-1071eb2be6a5.png)

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/17cc2d8a-e870-6e79-b417-d16609b9e94d.png)

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/4259491a-66e6-12b1-2508-242693b0cfef.png)