Upload 4 files

app.py

@@ -0,0 +1,131 @@
+import gradio as gr 
+from module import generate_resume_urls
+
+import os
+import tempfile
+import shutil
+
+def write_result_to_file(result, filename="result.txt"):
+    # Create temp folder
+    result = "\n\n".join(result)
+    temp_dir = tempfile.mkdtemp()
+
+    # Write result to file
+    with open(os.path.join(temp_dir, filename), "w") as f:
+        f.write(result)
+        f.close()
+        
+    return os.path.join(temp_dir, filename)
+
+def get_links(job_search_status_enable,gender_enable,salary_enable,education_level_enable,experience_enable,age_enable,
+                             links,use_filters,use_inc_values,job_search_status,gender,age_from,age_to,salary_from,salary_to,education_level,experience):
+    result_links = []
+    
+    cout_slogan = "Кол-во ссылок: "
+    links_list = links.split("\n")
+    # Remove empy elemts from links_list
+    links_list = [link for link in links_list if link]
+
+    if not use_filters:
+        for link in links_list:
+            result = generate_resume_urls(link)
+            result_links += result
+            
+        cout_slogan = cout_slogan + str(len(result_links))
+        return write_result_to_file(result_links),cout_slogan
+
+    # When use_filters is True
+    else:
+        selected_filters = []
+        filter_values = {}
+        
+        for link in links_list:
+            if age_enable:
+                selected_filters.append("age")
+                # print(age_from,age_to)
+                filter_values["age"] = list(range(age_from,age_to))
+                diffrence = age_to - age_from
+
+                if diffrence <= 0:
+                    filter_values["age"].append(age_from)
+                else:
+                    filter_values["age"].append(age_to)
+
+            if job_search_status_enable:
+                selected_filters.append("job_search_status")
+                print(job_search_status)
+                filter_values["job_search_status"] = job_search_status
+
+            if gender_enable:
+                selected_filters.append("gender")
+                filter_values["gender"] = gender
+
+            if salary_enable:
+                selected_filters.append("salary")
+                filter_values["salary_from"] = salary_from
+                filter_values["salary_to"] = salary_to
+
+            if education_level_enable:
+                selected_filters.append("education_level")
+                filter_values["education_level"] = education_level
+
+            if experience_enable:
+                selected_filters.append("experience")
+                filter_values["experience"] = experience
+
+            for link in links_list:
+                result = generate_resume_urls(link, selected_filters, filter_values, use_inc_values)
+                result_links += result
+
+        cout_slogan = cout_slogan + str(len(result_links))
+        return write_result_to_file(result_links),cout_slogan
+
+with gr.Blocks() as demo:
+    gr.Markdown("# Модуль для увеличения кол-ва ссылок за счет добавления фильтра")
+    
+    with gr.Row():
+        links = gr.TextArea(label="Введите ссылки",placeholder="https://hh.ru/search/resume?text=Региональный+визуальный+мерчендайзер")
+    
+    use_filters = gr.Checkbox(label="Использовать Фильтры ( если выключенно, создается максимально кол-во ссылок )")
+    with gr.Accordion("Фильтры",open=True):
+        # Набор фильтров
+        use_inc_values = gr.Checkbox(label="Добавить к списку резюме, список без указанного возраста и зарплаты")
+        with gr.Row():
+            with gr.Row():
+                job_search_status_enable = gr.Checkbox(label="Статус поиска работы",value=True)
+                gender_enable = gr.Checkbox(label="Пол",value=True)
+            with gr.Row():
+                age_enable = gr.Checkbox(label="Возраст",value=True)
+            with gr.Row():
+                salary_enable = gr.Checkbox(label="Зарплата",value=True)
+
+            education_level_enable = gr.Checkbox(label="Образование",value=True)
+            experience_enable =  gr.Checkbox(label="Опыт работы",value=True)
+        
+    with gr.Accordion("Настройка фильтров",open=False):
+        with gr.Row():
+            job_search_status = gr.CheckboxGroup(["unknown", "not_looking_for_job", "looking_for_offers", "active_search", "has_job_offer", "accepted_job_offer"], label="Статус поиска работы")
+            gender = gr.Radio(["male", "female","both"], label="Пол")
+        with gr.Row():
+            age_from = gr.Slider(18, 70,value=18, step=1, label="Возраст от")
+            age_to = gr.Slider(18, 70,value=71,step=1, label="Возраст до")
+        with gr.Row():
+            salary_from = gr.Slider(0, 1000000000,step=1,value=0, label="Зарплата от")
+            salary_to = gr.Slider(0, 1000000000,step=1,value=1000000000,label="Зарплата до")
+            
+        education_level = gr.CheckboxGroup(["secondary", "special_secondary", "unfinished_higher", "bachelor", "master", "doctor", "candidate", "higher"], label="Уровень образования")
+        experience = gr.CheckboxGroup(["noExperience", "between1And3", "between3And6", "moreThan6"], label="Опыт работы")
+
+    with gr.Row():
+        get_result_btn = gr.Button("Получить")
+    with gr.Row():
+        result_field = gr.File(label="Результат", interactive=False)
+    with gr.Row():
+        result_count_field = gr.Label(value="Кол-во ссылок")
+
+    get_result_btn.click(fn=get_links,inputs=
+                         [   job_search_status_enable,gender_enable,salary_enable,education_level_enable,experience_enable,age_enable,
+                             links,use_filters,use_inc_values,job_search_status,gender,age_from,age_to,salary_from,salary_to,education_level,experience],
+                         outputs=[result_field,result_count_field])
+    
+demo.launch()

funcs.py

@@ -0,0 +1,88 @@
+import urllib.parse
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from bs4 import BeautifulSoup
+import json
+
+from module_1_pre.funcs import generate_resume_urls, generate_specific_urls
+
+
+class ResumeSpider(scrapy.Spider):
+    name = "resume_spider"
+
+    custom_settings = {
+        'RETRY_HTTP_CODES': [400],
+        'RETRY_TIMES': 5,
+        'LOG_ENABLED': False
+    }
+    
+    def __init__(self, base_url=None,*args, **kwargs):
+        super(ResumeSpider, self).__init__(*args, **kwargs)
+        self.base_url = base_url
+
+    def start_requests(self):
+        urls = generate_resume_urls(self.base_url)
+        for url in urls:
+            yield scrapy.Request(url=url, callback=self.parse_general)
+
+    def parse_general(self, response):
+        soup = BeautifulSoup(response.text, 'html.parser')
+        main_element = soup.find('main', class_='resume-serp-content')
+
+        if main_element and main_element.find('div', attrs={'data-resume-id': True}):
+            page_count = self.get_page_count(soup)
+
+            if page_count < 250:
+                result = {
+                    'url': response.url,
+                    'pages': page_count
+                }
+                with open('resume_urls.json', 'a') as f:
+                    json.dump(result, f, ensure_ascii=False, indent=4)
+                    f.write('\n')
+            else:
+                self.base_url = response.url
+                specific_urls = generate_specific_urls(self.base_url)
+                for url in specific_urls:
+                    yield scrapy.Request(url=url, callback=self.parse_specific)
+        else:
+            print(f'No resumes found on page: {response.url}')
+
+    def parse_specific(self, response):
+        soup = BeautifulSoup(response.text, 'html.parser')
+        page_count = self.get_page_count(soup)
+
+        result = {
+            'url': response.url,
+            'pages': page_count
+        }
+        with open('resume_urls.json', 'a', encoding='utf-8') as f:
+                json.dump(result, f, ensure_ascii=False, indent=4, separators=(',', ': '))
+                f.write(',\n')  # Add comma and newline for valid JSON array
+                
+    def get_page_count(self, soup):
+        pager = soup.find('div', class_='pager')
+        if pager:
+            last_page_link = pager.find_all('a', attrs={'data-qa': 'pager-page'})[-1]
+            last_page = int(last_page_link.text.strip())
+            return last_page
+        return 1
+
+
+def stage1(base_url):
+    process = CrawlerProcess()
+    process.crawl(ResumeSpider,base_url)
+    process.start()
+
+
+
+
+# for url in urls:
+#     result = generate_resume_urls(url)
+#     print(result)
+#     print("\n")
+
+# lens = len(['https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&job_search_status=unknown&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&job_search_status=not_looking_for_job&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&job_search_status=looking_for_offers&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&job_search_status=active_search&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&job_search_status=has_job_offer&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&job_search_status=accepted_job_offer&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&experience=moreThan6&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&experience=between3And6&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&experience=between1And3&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&experience=noExperience&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&gender=male&label=only_with_gender&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&gender=female&label=only_with_gender&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&education_level=higher&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&education_level=unfinished_higher&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&education_level=master&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&education_level=bachelor&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80', 'https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&logic=normal&pos=full_text&exp_period=all_time&education_level=special_secondary&text=%D0%9A%D0%B0%D1%81%D1%81%D0%B8%D1%80'])
+# print(lens)
+
+

module.py

@@ -0,0 +1,143 @@
+import urllib.parse
+
+import urllib.parse
+
+def generate_resume_urls(base_url, selected_filters=None, filter_values=None, use_incredible_values=True):
+    parsed_url = urllib.parse.urlparse(base_url)
+    query_params = urllib.parse.parse_qs(parsed_url.query)
+
+    if "/resumes/" in parsed_url.path:
+        search_text = parsed_url.path.split("/")[-1].replace("_", " ")
+    else:
+        search_text = query_params.get("text", [""])[0]
+
+    encoded_text = urllib.parse.quote(search_text)
+
+    filter_options = {
+        "job_search_status": ["unknown", "not_looking_for_job", "looking_for_offers", "active_search", "has_job_offer", "accepted_job_offer"],
+        "gender": ["male", "female"],
+        "age": list(range(18, 71)),
+        "salary" : False,
+        "salary_from": 0,
+        "salary_to": 1000000000,
+        "education_level": ["secondary", "special_secondary", "unfinished_higher", "bachelor", "master", "doctor", "candidate", "higher"],
+        "experience": ["noExperience", "between1And3", "between3And6", "moreThan6"]
+    }
+
+    if filter_values:
+        for key in filter_values.keys():
+            # print(key)
+            if key == "salary":
+                filter_options[key] = filter_values[key]
+                continue
+            if key == "salary_from":
+                filter_options[key] = filter_values[key]
+                continue
+            if key == "salary_to":
+                filter_options[key] = filter_values[key]
+                continue
+            if filter_values[key] == None:
+                continue
+            
+            filter_options[key] = filter_values[key] if len(filter_values[key]) > 0 else filter_options[key]
+        
+    # print(selected_filters)
+
+    if filter_values is None:
+        filter_values = {}
+        
+    if selected_filters is None:
+        selected_filters = filter_options.keys()
+
+    generated_urls = []
+    
+    # print(filter_values)
+
+    for status in filter_options["job_search_status"] if "job_search_status" in selected_filters else [None]:
+        for gender in filter_options["gender"] if "gender" in selected_filters else [None]:
+            for age in filter_options["age"] if "age" in selected_filters else [None]:
+                for experience in filter_options["experience"] if "experience" in selected_filters else [None]:
+                    for education in filter_options["education_level"] if "education_level" in selected_filters else [None]:
+                        url = f"https://hh.ru/search/resume?search_period=0&order_by=relevance&filter_exp_period=all_time&relocation=living_or_relocation&job_search_status_changed_by_user=true&logic=normal&pos=full_text&exp_period=all_time&text={encoded_text}"
+                                                
+                        if status:
+                            url += f"&job_search_status={status}"
+                        if gender:
+                            url += f"&gender={gender}"
+                        if age:
+                            url += f"&label=only_with_age" 
+                            url += f"&age_from={age}&age_to={age}"
+                        if experience:
+                            url += f"&experience={experience}"
+                        if education:
+                            url += f"&education_level={education}"
+                            
+                        if "salary" in selected_filters:
+                            url +=f"&label=only_with_salary"
+                            url += f"&salary_from={filter_options['salary_from']}&salary_to={filter_options['salary_to']}"
+
+                        if "area" in query_params:
+                            url += f"&area={query_params['area'][0]}"
+                        if "university" in query_params:
+                            url += f"&university={query_params['university'][0]}"
+
+                        generated_urls.append(url)
+
+    if use_incredible_values:
+        for status in filter_options["job_search_status"] if "job_search_status" in selected_filters else [None]:
+            for gender in filter_options["gender"] if "gender" in selected_filters else [None]:
+                for experience in filter_options["experience"] if "experience" in selected_filters else [None]:
+                    for education in filter_options["education_level"] if "education_level" in selected_filters else [None]:
+                        url = f"https://hh.ru/search/resume?search_period=0&order_by=relevance&salary_from=10000000&salary_to=1000000000000&filter_exp_period=all_time&relocation=living_or_relocation&job_search_status_changed_by_user=true&logic=normal&pos=full_text&exp_period=all_time&age_from=100000000&age_to=1000000&text={encoded_text}"
+
+                        if status:
+                            url += f"&job_search_status={status}"
+                        if gender:
+                            url += f"&gender={gender}"
+                        if experience:
+                            url += f"&experience={experience}"
+                        if education:
+                            url += f"&education_level={education}"
+
+                        if "area" in query_params:
+                            url += f"&area={query_params['area'][0]}"
+                        if "university" in query_params:
+                            url += f"&university={query_params['university'][0]}"
+
+                        generated_urls.append(url)
+
+    for key, value in query_params.items():
+        if key not in ["text", "job_search_status", "experience", "gender", "age_from", "age_to", "education_level", "area", "university"]:
+            for url in generated_urls:
+                url += f"&{key}={value[0]}"
+
+    return generated_urls
+
+
+def generate_specific_urls(base_url):
+    parsed_url = urllib.parse.urlparse(base_url)
+    query_params = urllib.parse.parse_qs(parsed_url.query)
+
+    experience_options = ["noExperience", "between1And3", "between3And6", "moreThan6"]
+    employment_options = ["full", "part", "project", "volunteer", "probation"]
+
+    specific_urls = []
+
+    for emp in employment_options:
+            query_params["employment"] = [emp]
+
+            # Add area and university parameters if present
+            if "area" in query_params:
+                query_params["area"] = query_params["area"]
+            if "university" in query_params:
+                query_params["university"] = query_params["university"]
+
+            new_query_string = urllib.parse.urlencode(query_params, doseq=True)
+            new_url = parsed_url._replace(query=new_query_string).geturl()
+            specific_urls.append(new_url)
+
+    return specific_urls
+
+
+# urls = generate_resume_urls("https://hh.ru/resumes/generalnyj-direktor-stroitelnoj-kompanii",use_incredible_values=True)
+# print(len(urls))

requirements.txt

@@ -0,0 +1,43 @@
+attrs==23.2.0
+Automat==22.10.0
+beautifulsoup4==4.12.3
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+colorama==0.4.6
+constantly==23.10.4
+crochet==2.1.1
+cryptography==42.0.5
+cssselect==1.2.0
+filelock==3.13.3
+hyperlink==21.0.0
+idna==3.6
+incremental==22.10.0
+itemadapter==0.8.0
+itemloaders==1.1.0
+jmespath==1.0.1
+lxml==5.1.0
+packaging==24.0
+parsel==1.9.0
+Protego==0.3.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pycparser==2.21
+PyDispatcher==2.0.7
+pyOpenSSL==24.1.0
+queuelib==1.6.2
+requests==2.31.0
+requests-file==2.0.0
+Scrapy==2.11.1
+service-identity==24.1.0
+six==1.16.0
+soupsieve==2.5
+tldextract==5.1.2
+tqdm==4.66.2
+Twisted==24.3.0
+twisted-iocpsupport==1.0.4
+typing_extensions==4.10.0
+urllib3==2.2.1
+w3lib==2.1.2
+wrapt==1.16.0
+zope.interface==6.2