Files changed (1) hide show
  1. liBotGradio.py +290 -0
liBotGradio.py ADDED
@@ -0,0 +1,290 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from docx import Document
3
+ import pandas as pd
4
+ from sklearn.feature_extraction.text import TfidfVectorizer
5
+ from sklearn.metrics.pairwise import cosine_similarity
6
+ import os
7
+ import csv
8
+ import time
9
+ import pickle
10
+ import logging
11
+ from nltk.tokenize import word_tokenize
12
+ from nltk.corpus import stopwords
13
+ import string
14
+ from selenium import webdriver
15
+ from selenium.webdriver.common.by import By
16
+ from selenium.webdriver.support.ui import WebDriverWait
17
+ from selenium.webdriver.support import expected_conditions as EC
18
+ from selenium.common.exceptions import NoSuchElementException, TimeoutException
19
+
20
+ class LinkedInBot:
21
+ def __init__(self, delay=5):
22
+ if not os.path.exists("data"):
23
+ os.makedirs("data")
24
+ self.delay = delay
25
+ self.driver = webdriver.Chrome()
26
+
27
+ def login(self, email, password):
28
+ """Go to LinkedIn and login"""
29
+ self.driver.maximize_window()
30
+ self.driver.get('https://www.linkedin.com/login')
31
+ self.driver.find_element(By.ID, 'username').send_keys(email)
32
+ self.driver.find_element(By.ID, 'password').send_keys(password)
33
+ self.driver.find_element(By.XPATH, "//button[@type='submit']").click()
34
+
35
+ def save_cookie(self, path):
36
+ with open(path, 'wb') as filehandler:
37
+ pickle.dump(self.driver.get_cookies(), filehandler)
38
+
39
+ def load_cookie(self, path):
40
+ with open(path, 'rb') as cookiesfile:
41
+ cookies = pickle.load(cookiesfile)
42
+ for cookie in cookies:
43
+ self.driver.add_cookie(cookie)
44
+
45
+ def search_linkedin(self, keywords, location, date_posted):
46
+ """Enter keywords into the search bar"""
47
+ self.driver.get("https://www.linkedin.com/jobs/")
48
+ self.driver.get(f"https://www.linkedin.com/jobs/search/?keywords={keywords}&location={location}&f_TPR={date_posted}")
49
+
50
+ def wait(self, by=By.ID, text=None, t_delay=None, max_retries=3):
51
+ """Wait until a specific element is present on the page."""
52
+ delay = self.delay if t_delay is None else t_delay
53
+ retries = 0
54
+ while retries < max_retries:
55
+ try:
56
+ WebDriverWait(self.driver, delay).until(EC.presence_of_element_located((by, text)))
57
+ return # Element found, exit the loop
58
+ except TimeoutException:
59
+ retries += 1
60
+ logging.warning(f"Element not found, retrying... ({retries}/{max_retries})")
61
+ time.sleep(delay) # Wait before retrying
62
+ logging.error("Element not found after retries.")
63
+
64
+ def scroll_to(self, job_list_item):
65
+ """Scroll to the list item in the column and click on it."""
66
+ self.driver.execute_script("arguments[0].scrollIntoView();", job_list_item)
67
+ job_list_item.click()
68
+
69
+ def extract_additional_details(self, job):
70
+ """Extracts additional details like company size, position level, salary, job type, industry, and skills if available."""
71
+ company_size = None
72
+ position_level = None
73
+ salary = None
74
+ job_type = None
75
+ industry = None
76
+ skills = None
77
+
78
+ try:
79
+ additional_info = job.find_element(By.CLASS_NAME, "job-details-jobs-unified-top-card__job-insight")
80
+
81
+ # Extract salary
82
+ salary_element = additional_info.find_element(By.XPATH, ".//span[contains(@class, 'job-details-jobs-unified-top-card__job-insight-view-model-secondary')]")
83
+ salary = salary_element.text.strip()
84
+
85
+ # Extract job type, position level, and industry
86
+ for span in additional_info.find_elements(By.XPATH, ".//span[contains(@class, 'job-details-jobs-unified-top-card__job-insight-view-model-secondary')]"):
87
+ text = span.text.strip()
88
+ if "Hybrid" in text:
89
+ job_type = text
90
+ elif "Full-time" in text:
91
+ job_type = text
92
+ elif "Mid-Senior level" in text:
93
+ position_level = text
94
+ else:
95
+ industry = text
96
+
97
+ # Extract company size and industry
98
+ company_info = additional_info.find_element(By.XPATH, ".//span")
99
+ company_info_text = company_info.text.strip()
100
+ if "employees" in company_info_text:
101
+ company_size = company_info_text.split(" · ")[0]
102
+ industry = company_info_text.split(" · ")[1]
103
+ else:
104
+ industry = company_info_text
105
+
106
+ # Extract skills
107
+ skills_button = additional_info.find_element(By.CLASS_NAME, "job-details-jobs-unified-top-card__job-insight-text-button")
108
+ skills_link = skills_button.find_element(By.TAG_NAME, "a")
109
+ skills = skills_link.text.split(": ")[1]
110
+
111
+ except NoSuchElementException:
112
+ pass
113
+
114
+ return company_size, position_level, salary, job_type, industry, skills
115
+
116
+ def get_position_data(self, job):
117
+ """Gets the position data for a posting."""
118
+ job_info = job.text.split('\n')
119
+ if len(job_info) < 3:
120
+ logging.warning("Incomplete job information, skipping...")
121
+ return None
122
+
123
+ position, company, *details = job_info
124
+ location = details[0] if details else None
125
+ description = self.get_job_description(job)
126
+
127
+ return [position, company, location, description]
128
+
129
+
130
+ def extract_additional_details(self, job):
131
+ """Extracts additional details like company size, position level, salary, and job type if available."""
132
+ company_size = None
133
+ position_level = None
134
+ salary = None
135
+ job_type = None
136
+
137
+ try:
138
+ additional_info = job.find_element(By.CLASS_NAME, "job-card-search__company-size").text
139
+ if "employees" in additional_info:
140
+ company_size = additional_info.strip()
141
+ except NoSuchElementException:
142
+ pass
143
+
144
+ try:
145
+ position_level = job.find_element(By.CLASS_NAME, "job-card-search__badge").text
146
+ except NoSuchElementException:
147
+ pass
148
+
149
+ try:
150
+ salary = job.find_element(By.CLASS_NAME, "job-card-search__salary").text
151
+ except NoSuchElementException:
152
+ pass
153
+
154
+ try:
155
+ job_type = job.find_element(By.CLASS_NAME, "job-card-search__job-type").text
156
+ except NoSuchElementException:
157
+ pass
158
+
159
+ return company_size, position_level, salary, job_type
160
+
161
+ def get_job_description(self, job):
162
+ """Gets the job description."""
163
+ self.scroll_to(job)
164
+ try:
165
+ description_element = self.driver.find_element(By.CLASS_NAME, "jobs-description")
166
+ description = description_element.text
167
+ except NoSuchElementException:
168
+ description = None
169
+ return description
170
+
171
+ def get_application_link(self, job):
172
+ """Gets the job application link."""
173
+ try:
174
+ application_link_element = job.find_element(By.CLASS_NAME, "job-card-search__apply-button-container").find_element(By.TAG_NAME, "a")
175
+ application_link = application_link_element.get_attribute("href")
176
+ except NoSuchElementException:
177
+ application_link = None
178
+ return application_link
179
+
180
+ def run(self, email, password, keywords, location, date_posted):
181
+ if os.path.exists("data/cookies.txt"):
182
+ self.driver.get("https://www.linkedin.com/")
183
+ self.load_cookie("data/cookies.txt")
184
+ self.driver.get("https://www.linkedin.com/")
185
+ else:
186
+ self.login(email=email, password=password)
187
+ self.save_cookie("data/cookies.txt")
188
+
189
+ logging.info("Begin LinkedIn keyword search")
190
+ self.search_linkedin(keywords, location, date_posted)
191
+ self.wait()
192
+
193
+ csv_file_path = os.path.join("data", "data.csv")
194
+ with open(csv_file_path, "w", newline="", encoding="utf-8") as csvfile:
195
+ writer = csv.writer(csvfile)
196
+ writer.writerow(["Position", "Company", "Location", "Description"])
197
+
198
+ page = 1
199
+ while True:
200
+ jobs = self.driver.find_elements(By.CLASS_NAME, "occludable-update")
201
+ for job in jobs:
202
+ job_data = self.get_position_data(job)
203
+ if job_data:
204
+ position, company, location, description = job_data
205
+ writer.writerow([position, company, location, description])
206
+
207
+ next_button_xpath = f"//button[@aria-label='Page {page + 1}']"
208
+ next_button = self.driver.find_elements(By.XPATH, next_button_xpath)
209
+ if next_button:
210
+ next_button[0].click()
211
+ self.wait()
212
+ page += 1
213
+ else:
214
+ break
215
+
216
+ logging.info("Done scraping.")
217
+ logging.info("Closing session.")
218
+ self.close_session()
219
+
220
+ def close_session(self):
221
+ """Close the actual session"""
222
+ logging.info("Closing session")
223
+ self.driver.close()
224
+
225
+ # Function to extract keywords from text
226
+ def extract_keywords(text):
227
+ # Tokenize the text
228
+ tokens = word_tokenize(text.lower())
229
+ # Remove stopwords and punctuation
230
+ stopwords_list = set(stopwords.words("english"))
231
+ tokens = [token for token in tokens if token not in stopwords_list and token not in string.punctuation]
232
+ return tokens
233
+
234
+ # Function to process uploaded resume
235
+ def process_resume(uploaded_file):
236
+ docx = Document(uploaded_file.name)
237
+ resume_text = ""
238
+ for paragraph in docx.paragraphs:
239
+ resume_text += paragraph.text + "\n"
240
+ return resume_text
241
+
242
+ def keyword_similarity_check(resume_text, df, keywords):
243
+ vectorizer = TfidfVectorizer()
244
+ job_descriptions = df["Description"].fillna("")
245
+ tfidf_matrix = vectorizer.fit_transform(job_descriptions)
246
+
247
+ # Extract keywords from the resume and job descriptions
248
+ resume_keywords = extract_keywords(resume_text)
249
+ job_description_keywords = [extract_keywords(desc) for desc in job_descriptions]
250
+
251
+ # Calculate the number of common keywords
252
+ common_keywords_count = sum(1 for keyword in resume_keywords if keyword in keywords)
253
+ job_common_keywords_counts = [sum(1 for keyword in job_keywords if keyword in keywords) for job_keywords in job_description_keywords]
254
+
255
+ # Calculate similarity scores based on the number of common keywords
256
+ similarity_scores = [count / len(keywords) * 100 for count in job_common_keywords_counts]
257
+ df["Similarity (%)"] = similarity_scores
258
+ df.to_csv("data/data.csv", index=False)
259
+ return df
260
+
261
+ def cosine_similarity_check(resume_text, df):
262
+ vectorizer = TfidfVectorizer()
263
+ job_descriptions = df["Description"].fillna("")
264
+ tfidf_matrix = vectorizer.fit_transform(job_descriptions)
265
+ resume_tfidf = vectorizer.transform([resume_text])
266
+ similarity_scores = cosine_similarity(resume_tfidf, tfidf_matrix)[0]
267
+ df["Similarity (%)"] = similarity_scores * 100
268
+ df.to_csv("data/data.csv", index=False)
269
+ return df
270
+
271
+ def main(email, password, keywords, location, date_posted, resume_file):
272
+ bot = LinkedInBot()
273
+ bot.run(email, password, keywords, location, date_posted)
274
+
275
+ df = pd.read_csv("data/data.csv")
276
+
277
+ if resume_file:
278
+ resume_text = process_resume(resume_file)
279
+ keywords = extract_keywords(resume_text)
280
+ df = keyword_similarity_check(resume_text, df, keywords)
281
+ df = cosine_similarity_check(resume_text, df)
282
+
283
+ return df
284
+
285
+ iface = gr.Interface(fn=main,
286
+ inputs=["text", "text", "text", "text", "text", "file"],
287
+ outputs="csv",
288
+ title="LinkedIn Job Analysis",
289
+ description="Enter your LinkedIn credentials and search criteria to scrape job postings. Upload a resume to check for job similarity.")
290
+ iface.launch()