Create Main

Main

@@ -0,0 +1,34 @@
+import string
+import gensim
+import matplotlib.pyplot as plt
+import seaborn as sns
+import folium
+from collections import defaultdict
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+from gensim import corpora
+from wordcloud import WordCloud
+
+url = "http://www.freepatentsonline.com/download/us-patents-granted-2016.zip"
+dataset_zip = pd.io.zipcode.ZipFile(url)
+patent_data = pd.read_csv(dataset_zip.open('us-patents-granted-2016.csv'), encoding='ISO-8859-1')
+
+patent_data.dropna(inplace=True)
+
+patent_data['Title'] = patent_data['Title'].str.lower()
+patent_data['Abstract'] = patent_data['Abstract'].str.lower()
+
+patent_data['TitleTokens'] = patent_data['Title'].apply(nltk.word_tokenize)
+patent_data['AbstractTokens'] = patent_data['Abstract'].apply(nltk.word_tokenize)
+
+stop_words = set(stopwords.words('english'))
+lemmatizer = WordNetLemmatizer()
+
+patent_data['TitleClean'] = patent_data['TitleTokens'].apply(lambda x: ' '.join([w for w in x if not w in stop_words]))
+patent_data['AbstractClean'] = patent_data['AbstractTokens'].apply(lambda x: ' '.join([w for w in x if not w in stop_words]))
+
+patent_data['TitleLemma'] = patent_data['TitleClean'].apply(lambda x: ' '.join([lemmatizer.lemmatize(w) for w in x.split()]))
+patent_data['AbstractLemma'] = patent_data['AbstractClean'].apply(lambda x: ' '.join([lemmatizer.lemmatize(w) for w in x.split()]))
+
+patent_data['Text'] = patent_data['TitleLemma'] + ' ' + patent_data['AbstractLemma']
+documents = list(patent_data['Text'])