Upload 5 files

app.py

@@ -0,0 +1,32 @@
+import gradio as gr
+import os
+from functools import partial
+from full_scraping_script_v3 import run_scrapping
+import gdown
+from config import json_url_id
+download_url = f'https://drive.google.com/uc?id={json_url_id}'
+output = 'secret_google_service_account.json'
+gdown.download(download_url, output, quiet=False)
+
+title = ""
+with gr.Blocks(title=title,theme='nota-ai/theme',css="footer {visibility: hidden}") as demo:
+    gr.Markdown(f"## {title}")
+            
+    with gr.Row():
+        with gr.Column(scale=6):
+            with gr.Row():
+                with gr.Column(scale=8):
+                    password = gr.Textbox( lines=1, label="Tatooine password")
+                    cookie = gr.Textbox( lines=1, label="Cookie")
+                with gr.Column(scale=1):
+                    chat_submit_button = gr.Button(value="Submit ▶")
+            with gr.Row():
+                listing = gr.Markdown("Waiting for password...") 
+
+                    
+    fn_chat = run_scrapping
+    
+    chat_submit_button.click(fn=fn_chat, inputs=[password, cookie], outputs=[listing])
+
+
+demo.launch(max_threads=40)

config.py

@@ -0,0 +1,21 @@
+import os
+
+openai_api_key = os.environ.get("openai_api_key")
+proxycurl_api_key = os.environ.get("proxycurl_api_key")
+
+ftp_host = os.environ.get("ftp_host")
+ftp_user = os.environ.get("ftp_user")
+ftp_pass = os.environ.get("ftp_pass")
+csv_export_name = os.environ.get("csv_export_name")
+password = os.environ.get("password")
+
+folder_id = os.environ.get("folder_id")
+json_url_id = os.environ.get("json_url_id")
+
+places_morocco = [["Casablanca, Morocco"], ["Tanger, Morocco"],["Rabat, Morocco"],["Marrakech, Morocco"],["Agadir, Morocco"],["Kenitra, Morocco"]]
+job_title_morocco = "software engineer"
+amount_morocco = 70
+
+places_world = [["United Kingdom"], ["France"],["Canada"],["Belgium"],["Germany"],["Saudi Arabia"], ["United arab emirates"]]
+job_title_world = "software engineer remote"
+amount_world = 70

full_scraping_script_v3.py

@@ -0,0 +1,305 @@
+
+
+import os
+import pandas as pd
+import ftplib
+import requests
+import requests
+import logging
+
+from openai import OpenAI
+from linkedin_jobs_scraper import LinkedinScraper
+from linkedin_jobs_scraper.events import Events, EventData, EventMetrics
+from linkedin_jobs_scraper.query import Query, QueryOptions, QueryFilters
+from linkedin_jobs_scraper.filters import RelevanceFilters, TimeFilters, TypeFilters, ExperienceLevelFilters, \
+    OnSiteOrRemoteFilters, SalaryBaseFilters
+
+from config import *
+from logs import save_logs
+
+client = OpenAI(api_key=openai_api_key)
+
+# Replace 'your_li_at_cookie_value_here' with the actual value you copied
+
+# Change root logger level (default is WARN)
+logging.basicConfig(level=logging.INFO)
+
+
+
+
+def scrape(location = ["Morocco"], job_title = "Développeur java", page_offset = 2, amount = 30, remote = False, horizon = "month"):
+
+  if remote:
+    onsite = [OnSiteOrRemoteFilters.REMOTE]
+  else:
+    onsite = [OnSiteOrRemoteFilters.REMOTE, OnSiteOrRemoteFilters.HYBRID, OnSiteOrRemoteFilters.ON_SITE]
+
+  if horizon in "month":
+    timespan = TimeFilters.MONTH
+  elif horizon in "day":
+    timespan = TimeFilters.DAY
+  else:
+    timespan = TimeFilters.MONTH
+  result = []
+  # Fired once for each successfully processed job
+  def on_data(data: EventData):
+      print('[ON_DATA]', data.title, data.company, data.company_link, data.date, data.link, data.insights,
+            len(data.description))
+      result.append(data)
+
+  # Fired once for each page (25 jobs)
+  def on_metrics(metrics: EventMetrics):
+      print('[ON_METRICS]', str(metrics))
+
+
+  def on_error(error):
+      print('[ON_ERROR]', error)
+
+
+  def on_end():
+      print('[ON_END]')
+
+
+  scraper = LinkedinScraper(
+      chrome_executable_path=None,  # Custom Chrome executable path (e.g. /foo/bar/bin/chromedriver)
+      chrome_binary_location=None,  # Custom path to Chrome/Chromium binary (e.g. /foo/bar/chrome-mac/Chromium.app/Contents/MacOS/Chromium)
+      chrome_options=None,  # Custom Chrome options here
+      headless=True,  # Overrides headless mode only if chrome_options is None
+      max_workers=1,  # How many threads will be spawned to run queries concurrently (one Chrome driver for each thread)
+      slow_mo=0.5,  # Slow down the scraper to avoid 'Too many requests 429' errors (in seconds)
+      page_load_timeout=40  # Page load timeout (in seconds)
+  )
+
+  # Add event listeners
+  scraper.on(Events.DATA, on_data)
+  scraper.on(Events.ERROR, on_error)
+  scraper.on(Events.END, on_end)
+
+  queries = [
+      Query(
+          query=job_title,
+          options=QueryOptions(
+              locations=location,
+              apply_link=False,  # Try to extract apply link (easy applies are skipped). If set to True, scraping is slower because an additional page must be navigated. Default to False.
+              skip_promoted_jobs=True,  # Skip promoted jobs. Default to False.
+              page_offset=page_offset,  # How many pages to skip
+              limit=amount,
+              filters=QueryFilters(
+                  # company_jobs_url='https://www.linkedin.com/jobs/search/?f_C=1441%2C17876832%2C791962%2C2374003%2C18950635%2C16140%2C10440912&geoId=92000000',  # Filter by companies.
+                  relevance=RelevanceFilters.RECENT,
+                  time=TimeFilters.MONTH,
+                  # type=[TypeFilters.FULL_TIME, TypeFilters.INTERNSHIP],
+                  on_site_or_remote=onsite,
+              )
+          )
+      ),
+  ]
+
+  scraper.run(queries)
+  return result
+
+def upload_pic(image_url, image_filename):
+  image_filename = str(image_filename) +".png"
+  # URL of the image to download
+
+  # Download the image
+  response = requests.get(image_url)
+  image_data = response.content
+
+  # Save the image locally
+  with open("img.png", "wb") as file:
+      file.write(image_data)
+
+  # Upload the image to the FTP server
+  with ftplib.FTP(ftp_host) as ftp:
+      ftp.login(ftp_user, ftp_pass)
+      with open("img.png", "rb") as file:
+          ftp.storbinary(f"STOR {image_filename}", file)
+
+  print(f"{image_filename} uploaded successfully to {ftp_host}")
+
+def upload_ftp(filename):
+  # Upload the image to the FTP server
+  with ftplib.FTP(ftp_host) as ftp:
+      ftp.login(ftp_user, ftp_pass)
+      with open(filename, "rb") as file:
+          ftp.storbinary(f"STOR {filename}", file)
+
+  print(f"{filename} uploaded successfully to {ftp_host}")
+
+def run_scrapping(input_password, cookie = ""):
+  if len(cookie) > 10:
+    os.environ['LI_AT_COOKIE'] = cookie
+    
+  if input_password != password:
+    return "Wrong password"
+
+  jobs = []
+  for place in places_morocco:
+    job_unit = scrape(place,job_title_morocco, 0, amount_morocco,  horizon = "day")
+    jobs += job_unit
+    # break
+
+  jobs_remote = []
+  for place in places_world:
+    job_unit = scrape(place,job_title, 0, amount_world,remote = True,  horizon = "day")
+    jobs_remote += job_unit
+
+  df = pd.DataFrame(jobs + jobs_remote)
+  df_tatooine = pd.DataFrame(columns=["titre", "slug", "description", "job_type", "type_contrat", "experience", "pays", "ville", "salaire", "url", "entreprise_info", "entreprise_logo", "seo_description"])
+  df_tatooine["titre"] = df["title"]
+  df_tatooine["description"] = df["description"]
+  # df_tatooine["pays"] = df["place"]
+  # df_tatooine["ville"] = df["place"].str.split(",")[:]
+  df_tatooine[['ville', 'pays']] = df['location'].str.rsplit(', ', n=1, expand=True)
+  df_tatooine["url"] = df["link"]
+  df_tatooine = df_tatooine.fillna('')
+  df_tatooine["job_id"] = df["job_id"]
+  raw_content = []
+  titles = df_tatooine["titre"].to_list()
+  descs = df_tatooine["description"].to_list()
+  for i in range(len(df_tatooine["titre"].to_list())):
+    raw_content.append(titles[i] + descs[i])
+
+  seo_descs = []
+  job_types = []
+  type_contrats = []
+  response_rythmes = []
+  for i, content in enumerate(raw_content):
+    response = client.chat.completions.create(
+      model="gpt-4o-mini",
+      messages=[
+        {
+          "role": "system",
+          "content": [
+            {
+              "type": "text",
+              "text": "Your task will be to convert a job offer into a short description for SEO as it will be posted on a job board called tatooinejobs. You don't answer anything else than the SEO description, make sure it is highly converting and convincing. Make sure it's not too heavy and easy to read. \nDig into any human psychological biases you can find to make sure people will click on it. Make sure it's mainly oriented to promote the job board TatooineJobs."
+            }
+          ]
+        },
+        {
+          "role": "user",
+          "content": [
+            {
+              "type": "text",
+              "text": content
+            }
+          ]
+        }
+      ],
+      temperature=1,
+      max_tokens=3656,
+      top_p=1,
+      frequency_penalty=0,
+      presence_penalty=0,
+      response_format={
+        "type": "text"
+      }
+    ).choices[0].message.content
+    response_types = client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[
+      {
+        "role": "system",
+        "content": [
+          {
+            "type": "text",
+            "text": "Your task is to determine from a job offer its type of contract and work. You need to output a json with the following fields:\n\n{\n\"job_type\": [ ...] # to be chosen from [CDI, CDD, Freelance, Stage],\n\"rythme_travail\" : [...] # to be chosen from [Full-time, Part-time] ,\n\"type_contrat\" : [...] # to be chosen from [Hybrid, Full-remote, On-site] \n }\n\nFill in the most likely to be"
+          }
+        ]
+      },
+      {
+        "role": "user",
+        "content": [
+          {
+            "type": "text",
+            "text": content
+          }
+        ]
+      }
+    ],
+    temperature=1,
+    max_tokens=1567,
+    top_p=1,
+    frequency_penalty=0,
+    presence_penalty=0,
+    response_format={
+      "type": "json_object"
+    }
+    ).choices[0].message.content
+
+    response_job_type = eval(response_types)["job_type"]
+    response_type_contrat = eval(response_types)["type_contrat"]
+    response_rythme = eval(response_types)["rythme_travail"]
+    seo_descs.append(response)
+    job_types.append(str(response_job_type))
+    type_contrats.append(str(response_type_contrat))
+    response_rythmes.append(str(response_rythme))
+
+    if i % 10 == 0:
+      print(f"Processed {i+1} job offer out of {len(raw_content)}")
+
+  df_tatooine["seo_description"] = seo_descs
+  df_tatooine['job_type'] = type_contrats
+  df_tatooine['type_contrat'] = job_types
+  df_tatooine['rythme_travail'] = response_rythmes
+  df_tatooine["entreprise_info"] = df["company"]
+
+  profile_pic = []
+  headers = {'Authorization': 'Bearer ' + proxycurl_api_key}
+
+  api_endpoint = 'https://nubela.co/proxycurl/api/linkedin/company/resolve'
+
+  for i, company in enumerate(df_tatooine["entreprise_info"].to_list()):
+    params = {
+    'company_name': company,
+    'enrich_profile': 'enrich',
+    }
+
+    response = requests.get(api_endpoint,
+                            params=params,
+                            headers=headers)
+    content = response.json()
+    if i == 0:
+      print(content)
+
+    if i % 5 == 0:
+      print(f"Processed {i+1} link out of {len(df_tatooine)}")
+    profile_pic.append(content["profile"]["profile_pic_url"])
+    # profile_pic.append("")
+
+
+  df_tatooine["entreprise_logo"] = profile_pic
+  for rec in df_tatooine.to_dict(orient="records"):
+    job_id = rec["job_id"]
+    url = rec["entreprise_logo"]
+    try:
+      upload_pic(url, job_id)
+    except:
+      pass
+
+
+  df_tatooine["job_type"] = df_tatooine.job_type.str.replace("['", "").str.replace("']", "").str.replace("'", "")
+  df_tatooine["type_contrat"] = df_tatooine.type_contrat.str.replace("['", "").str.replace("']", "").str.replace("'", "")
+  df_tatooine["rythme_travail"] = df_tatooine.rythme_travail.str.replace("['", "").str.replace("']", "").str.replace("'", "")
+
+  df_not_morocco = df_tatooine[~df_tatooine['pays'].isin(["Morocco"])]
+  df_morocco = df_tatooine[df_tatooine['pays'].isin(["Morocco"])]
+  df_not_morocco['pays'] = df_not_morocco['ville']
+  df_not_morocco['ville'] = None
+  new_df = pd.concat([df_morocco, df_not_morocco])
+  mapper = {
+      "Morocco" : "Maroc",
+      "United Kingdom" : "Royaume-Uni",
+      "Belgium" : "Belgique",
+      "Germany" : "Allemagne",
+      "Saudi Arabia" : "Arabie saoudite",
+      "United arab emirates" : "Émirats arabes unis",
+  }
+  new_df['pays'] = new_df.pays.replace(mapper)
+
+  # new_df.to_csv(csv_export_name, sep=";")
+  upload_ftp(csv_export_name)
+  save_logs(csv_export_name)
+  return "Scrapping done!"

logs.py

@@ -0,0 +1,49 @@
+import os
+from google.oauth2 import service_account
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
+from datetime import datetime
+
+def save_logs(filename, folder_id = ""):
+
+    # Get the current date and time
+    now = datetime.now()
+    # filename = str(now).replace(":","").replace(" ","").replace("-","").replace(".","")+".txt"
+    # with open(filename, 'w') as file:
+    #     file.write(to_save)
+    # Path to the service account key file
+    SERVICE_ACCOUNT_FILE = 'secret_google_service_account.json'
+
+    # Define the required scopes
+    SCOPES = ['https://www.googleapis.com/auth/drive.file']
+
+    # Authenticate using the service account key file
+    credentials = service_account.Credentials.from_service_account_file(
+        SERVICE_ACCOUNT_FILE, scopes=SCOPES)
+
+    # Build the Google Drive API client
+    service = build('drive', 'v3', credentials=credentials)
+
+    # Specify the folder ID where you want to upload the file
+
+    # Metadata of the file to be uploaded
+    file_metadata = {
+        'name': filename,  # Name of the file to be uploaded
+        'parents': [folder_id]  # Folder ID
+    }
+
+    # Path to the file you want to upload
+    file_path = filename
+
+    # Create a MediaFileUpload object to upload the file
+    media = MediaFileUpload(file_path)
+
+    # Use the Drive API to upload the file
+    file = service.files().create(
+        body=file_metadata,
+        media_body=media,
+        fields='id'
+    ).execute()
+
+    # Print the file ID of the uploaded file
+    print('Saved in Google Drive - File ID: %s' % file.get('id'))

requirements.txt

@@ -0,0 +1,17 @@
+linkedin-jobs-scraper
+openai
+gradio
+requests
+bs4
+openai
+pandas
+numpy
+bs4
+nltk
+tiktoken
+pdf2image
+gdown
+google-auth
+google-auth-oauthlib
+google-auth-httplib2
+google-api-python-client