Repository: zeshuaro/LinkedRW

Issue:
Error when using
Hello I am trying to use this but i get the error using both chrome driver and firefox driver

>>>
Enter your LinkedIn login email: fastah@gmail.com
Enter your LinkedIn login password: 
[2019-06-02 16:16:46] NOTICE: Scraping LinkedIn profile
[2019-06-02 16:16:46] NOTICE: Please keep the browser window on top
Traceback (most recent call last):
  File "/home/arun/.local/lib/python3.7/site-packages/linkedrw/scraper/background.py", line 163, in get_education
    '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-no-more.ember-view')
  File "/home/arun/.local/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 430, in find_element_by_css_selector
    return self.find_element(by=By.CSS_SELECTOR, value=css_selector)
  File "/home/arun/.local/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 659, in find_element
    {"using": by, "value": value})['value']
  File "/home/arun/.local/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 633, in _execute
    return self._parent.execute(command, params)
  File "/home/arun/.local/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
    self.error_handler.check_response(response)
  File "/home/arun/.local/lib/python3.7/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-no-more.ember-view


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/bin/linkedrw", line 11, in <module>
    load_entry_point('linkedrw==1.1.1', 'console_scripts', 'linkedrw')()
  File "/home/arun/.local/lib/python3.7/site-packages/linkedrw/main.py", line 36, in main
    args.method(**vars(args))
  File "/home/arun/.local/lib/python3.7/site-packages/linkedrw/main.py", line 70, in run
    profile = scrape(driver.lower(), email, password, output_dir, timeout)
  File "/home/arun/.local/lib/python3.7/site-packages/linkedrw/scraper/scrape.py", line 63, in scrape
    EDUCATION: get_background_details(driver, By.ID, 'education-section', EDUCATION),
  File "/home/arun/.local/lib/python3.7/site-packages/linkedrw/scraper/background.py", line 46, in get_background_details
    return get_education(section)
  File "/home/arun/.local/lib/python3.7/site-packages/linkedrw/scraper/background.py", line 166, in get_education
    '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-no-more')
  File "/home/arun/.local/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 430, in find_element_by_css_selector
    return self.find_element(by=By.CSS_SELECTOR, value=css_selector)
  File "/home/arun/.local/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 659, in find_element
    {"using": by, "value": value})['value']
  File "/home/arun/.local/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py", line 633, in _execute
    return self._parent.execute(command, params)
  File "/home/arun/.local/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
    self.error_handler.check_response(response)
  File "/home/arun/.local/lib/python3.7/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-no-more

>>>

can u help?

Discussion:

Patch:
diff --git a/linkedrw/__init__.py b/linkedrw/__init__.py
index e9b95b7..8ef52e7 100644
--- a/linkedrw/__init__.py
+++ b/linkedrw/__init__.py
@@ -1,4 +1,4 @@
 from .main import main
 
-__version__ = "1.1.5"
+__version__ = "1.1.6"
 VERSION = __version__
diff --git a/linkedrw/constants.py b/linkedrw/constants.py
index f4e90b8..38bff18 100644
--- a/linkedrw/constants.py
+++ b/linkedrw/constants.py
@@ -50,8 +50,8 @@ ISSUER = 'issuer'
 RESUME_TEMPLATE = 'templates/resume_template.tex'
 PERSONAL_INFO = [NAME, POSITION, ADDRESS, MOBILE, EMAIL, HOMEPAGE, GITHUB, LINKEDIN, GITLAB, STACKOVERFLOW, TWITTER,
                  SKYPE, REDDIT, MEDIUM, GOOGLE_SCHOLAR]
-RESUME_CONTENT = [EDUCATION, EXPERIENCE, PUBLICATIONS, HONORS, PROJECTS, VOLUNTEERING, SKILLS]
-RESUME_SECTIONS = [EDUCATION, EXPERIENCE, HONORS, PROJECTS, VOLUNTEERING]
+RESUME_CONTENT = [EXPERIENCE, EDUCATION, PUBLICATIONS, HONORS, PROJECTS, VOLUNTEERING, SKILLS]
+RESUME_SECTIONS = [EXPERIENCE, EDUCATION, HONORS, PROJECTS, VOLUNTEERING]
 SECTION_ITEMS = {
     EDUCATION: [DEGREE, NAME, LOCATION, DATES, DESCRIPTION],
     EXPERIENCE: [TITLE, NAME, LOCATION, DATES, DESCRIPTION],
@@ -60,6 +60,24 @@ SECTION_ITEMS = {
     VOLUNTEERING: [TITLE, NAME, LOCATION, DATES, DESCRIPTION]
 }
 
+LATEX_CHARS = {
+    '&': r'\&',
+    '%': r'\%',
+    '$': r'\$',
+    '#': r'\#',
+    '_': r'\_',
+    '{': r'\{',
+    '}': r'\}',
+    '~': r'\textasciitilde{}',
+    '^': r'\^{}',
+    '\\': r'\textbackslash{}',
+    '\n': '\\newline%\n',
+    '-': r'{-}',
+    '\xA0': '~',  # Non-breaking space
+    '[': r'{[}',
+    ']': r'{]}',
+}
+
 PORTFOLIO_TEMPLATE = 'templates/index_template.html'
 PORTFOLIO_SECTIONS = [SUMMARY, EXPERIENCE, EDUCATION, PROJECTS, SKILLS, CONTACT]
 CONTACTS = [GITHUB, LINKEDIN, GITLAB, STACKOVERFLOW, TWITTER, REDDIT, MEDIUM]
diff --git a/linkedrw/linkedr/publication.py b/linkedrw/linkedr/publication.py
index 72b37fb..79234c4 100644
--- a/linkedrw/linkedr/publication.py
+++ b/linkedrw/linkedr/publication.py
@@ -7,6 +7,7 @@ from requests.exceptions import HTTPError
 from urllib.parse import urlparse
 
 from linkedrw.constants import *
+from linkedrw.utils import escape_latex
 
 
 def make_publication_section(publications, output_dir):
@@ -24,7 +25,7 @@ def make_publication_section(publications, output_dir):
         lines = [f'\\cvsection{{{PUBLICATIONS.title()}}}\n', '\\begin{refsection}']
 
         for reference in references:
-            lines.append(f'{LATEX_INDENT}\\nocite{{{reference}}}')
+            lines.append(f'{LATEX_INDENT}\\nocite{{{escape_latex(reference)}}}')
 
         lines.append(f'{LATEX_INDENT}\\printbibliography[heading=none]')
         lines.append('\\end{refsection}')
diff --git a/linkedrw/linkedr/section.py b/linkedrw/linkedr/section.py
index da534c6..e004aab 100644
--- a/linkedrw/linkedr/section.py
+++ b/linkedrw/linkedr/section.py
@@ -1,6 +1,7 @@
 import os
 
 from linkedrw.constants import *
+from linkedrw.utils import escape_latex
 
 
 def make_resume_section(profile, section, output_dir):
@@ -46,11 +47,11 @@ def make_grouped_section(profile, section):
             lines.append(f'{LATEX_INDENT}\\cventry')
             for key in SECTION_ITEMS[section]:
                 if key == NAME and i == 0:
-                    lines.append(f'{LATEX_INDENT * 2}{{{name}}} % {NAME}')
+                    lines.append(f'{LATEX_INDENT * 2}{{{escape_latex(name)}}} % {NAME}')
                 elif key == DESCRIPTION:
                     lines += get_descriptions(item)
                 elif key and key != NAME:
-                    lines.append(f'{LATEX_INDENT * 2}{{{item[key]}}} % {key}')
+                    lines.append(f'{LATEX_INDENT * 2}{{{escape_latex(item[key])}}} % {key}')
                 else:
                     lines.append(f'{LATEX_INDENT * 2}{{}}')
 
@@ -78,7 +79,7 @@ def make_ungrouped_section(profile, section):
             if key == DESCRIPTION:
                 lines += get_descriptions(entry)
             elif key:
-                lines.append(f'{LATEX_INDENT * 2}{{{entry[key]}}} % {key}')
+                lines.append(f'{LATEX_INDENT * 2}{{{escape_latex(entry[key])}}} % {key}')
             else:
                 lines.append(f'{LATEX_INDENT * 2}{{}}')
 
@@ -101,7 +102,9 @@ def get_descriptions(item):
 
         for description in item[DESCRIPTION].split('\n'):
             description = description.strip('-').strip()
-            lines.append(f'{LATEX_INDENT * 4}\\item{{{description}}}')
+            if description:
+                description = escape_latex(description)
+                lines.append(f'{LATEX_INDENT * 4}\\item{{{description}}}')
 
         lines.append(f'{LATEX_INDENT * 3}\\end{{cvitems}}')
         lines.append(f'{LATEX_INDENT * 2}}}\n')
diff --git a/linkedrw/linkedr/skill.py b/linkedrw/linkedr/skill.py
index 163e300..54973b4 100644
--- a/linkedrw/linkedr/skill.py
+++ b/linkedrw/linkedr/skill.py
@@ -2,6 +2,7 @@ import os
 import pkg_resources
 
 from linkedrw.constants import SKILLS, LATEX_INDENT, PACKAGE_NAME
+from linkedrw.utils import escape_latex
 
 
 def make_skill_section(skills, languages, output_dir):
@@ -59,6 +60,6 @@ def make_skill_subsection(skills, skills_type):
     if skills:
         lines.append(f'{LATEX_INDENT}\\cvskill')
         lines.append(f'{LATEX_INDENT * 2}{{{skills_type}}}')
-        lines.append(f'{LATEX_INDENT * 2}{{{", ".join(skills)}}}')
+        lines.append(f'{LATEX_INDENT * 2}{{{escape_latex(", ".join(skills))}}}')
 
     return lines
diff --git a/linkedrw/linkedw/website.py b/linkedrw/linkedw/website.py
index e92428a..124f235 100644
--- a/linkedrw/linkedw/website.py
+++ b/linkedrw/linkedw/website.py
@@ -3,13 +3,15 @@ import os
 import pkg_resources
 import re
 
+from collections import defaultdict
 from datetime import datetime
 from logbook import Logger
-from queue import PriorityQueue
 
 from linkedrw.constants import *
 from linkedrw.utils import make_dir, copy_files
 
+SECTION_ENDS = ['End #about', 'End #experience', 'End #education', 'End #projects', 'End #skills']
+
 
 def make_website_files(profile, output_dir):
     """
@@ -119,7 +121,7 @@ def make_website_files(profile, output_dir):
                 lines += make_contact_section(profile[CONTACT], indent)
 
             # Comment out sections
-            elif comment_line and any(x in line for x in ['End #about', 'End #experience', 'End #education']):
+            elif comment_line and any(x in line for x in SECTION_ENDS):
                 comment_line = False
             elif comment_line:
                 lines += make_comment_line(line)
@@ -180,8 +182,7 @@ def make_experience_section(exps, indent):
     sorted_exps = sort_entries(exps)
     lines = []
 
-    while sorted_exps:
-        exp = sorted_exps.pop()
+    for exp in sorted_exps:
         lines += [
             f'{indent}<div data-date="{exp[DATES]}">',
             f'{indent}{HTML_INDENT}<h3>{exp[NAME]}</h3>',
@@ -206,8 +207,7 @@ def make_education_section(edus, indent):
     sorted_edus = sort_entries(edus, date_format='YYYY')
     lines = []
 
-    while sorted_edus:
-        edu = sorted_edus.pop()
+    for edu in sorted_edus:
         lines += [
             f'{indent}<div class="education-block">',
             f'{indent}{HTML_INDENT}<h3>{edu[NAME]}</h3>',
@@ -298,7 +298,7 @@ def sort_entries(entries, date_format='MMM YYYY'):
     Returns:
         A list of sorted entries
     """
-    sorted_entries = PriorityQueue()
+    all_entries = defaultdict(list)
     for exp in entries:
         name = exp[NAME]
         for entry in exp[ENTRIES]:
@@ -306,19 +306,19 @@ def sort_entries(entries, date_format='MMM YYYY'):
             date = entry[DATES].split(' - ')[-1]
 
             if date.lower() == 'present':
-                arrow_date = arrow.utcnow()
+                arrow_date = arrow.utcnow().date()
             elif date:
-                arrow_date = arrow.get(date, date_format)
+                arrow_date = arrow.get(date, date_format).date()
             else:
-                arrow_date = arrow.get(datetime.min)
+                arrow_date = arrow.get(datetime.min).date()
 
-            sorted_entries.put((arrow_date, entry))
+            all_entries[arrow_date].append(entry)
 
-    all_entries = []
-    while not sorted_entries.empty():
-        all_entries.append(sorted_entries.get()[1])
+    sorted_entries = []
+    for key in sorted(all_entries, reverse=True):
+        sorted_entries += all_entries[key]
 
-    return all_entries
+    return sorted_entries
 
 
 def get_description(descs, indent):
@@ -336,7 +336,8 @@ def get_description(descs, indent):
         lines.append(f'{indent}{HTML_INDENT}<ul>')
         for desc in descs.split('\n'):
             desc = desc.strip('-').strip()
-            lines.append(f'{indent}{HTML_INDENT * 2}<li>{desc}')
+            if desc:
+                lines.append(f'{indent}{HTML_INDENT * 2}<li>{desc}')
 
         lines.append(f'{indent}{HTML_INDENT}</ul>')
 
diff --git a/linkedrw/scraper/accomplishment.py b/linkedrw/scraper/accomplishment.py
index d2f7c0b..1cdaefe 100644
--- a/linkedrw/scraper/accomplishment.py
+++ b/linkedrw/scraper/accomplishment.py
@@ -15,9 +15,12 @@ def get_accomplishment_details(driver, section_type):
         A list of details of all the items under the given section
     """
     # Check if the section exists
-    section = driver.find_element_by_css_selector(
-        f'.accordion-panel.pv-profile-section.pv-accomplishments-block.{section_type}.ember-view')
-    if 'display: none' in section.get_attribute('style'):
+    try:
+        section = driver.find_element_by_css_selector(
+            f'.accordion-panel.pv-profile-section.pv-accomplishments-block.{section_type}.ember-view')
+        if 'display: none' in section.get_attribute('style'):
+            return []
+    except NoSuchElementException:
         return []
 
     if section_type in (PROJECTS, PUBLICATIONS, HONORS):
diff --git a/linkedrw/scraper/background.py b/linkedrw/scraper/background.py
index 17b9fd7..77d4828 100644
--- a/linkedrw/scraper/background.py
+++ b/linkedrw/scraper/background.py
@@ -59,20 +59,24 @@ def get_experience(section):
     Returns:
         A list of details of all experiences
     """
-    divs = section.find_elements_by_css_selector(
+    ul = get_section(section)
+    entries = ul.find_elements_by_css_selector(
+        '.pv-profile-section__sortable-item.pv-profile-section__section-info-item.relative.'
+        'pv-profile-section__list-item.sortable-item.ember-view')
+    entries += ul.find_elements_by_css_selector(
         '.pv-entity__position-group-pager.pv-profile-section__list-item.ember-view')
     exps = []
 
-    for div in divs:
+    for entry in entries:
         # Check if it is a single role in a company or multiple roles in a company
         try:
-            summary = div.find_element_by_css_selector(
+            summary = entry.find_element_by_css_selector(
                 '.pv-entity__summary-info.pv-entity__summary-info--background-section')
-            exps.append(get_single_role(div, summary))
+            exps.append(get_single_role(entry, summary))
         except NoSuchElementException:
-            summary = div.find_element_by_css_selector(
+            summary = entry.find_element_by_css_selector(
                 '.pv-profile-section__card-item-v2.pv-profile-section.pv-position-entity.ember-view')
-            exps.append(get_multiple_roles(div, summary))
+            exps.append(get_multiple_roles(entry, summary))
 
     return exps
 
@@ -156,7 +160,7 @@ def get_education(section):
     Returns:
         A list of details of all educations
     """
-    ul = get_edu_sec(section)
+    ul = get_section(section)
     edu_dict = defaultdict(list)
 
     for li in ul.find_elements_by_tag_name('li'):
@@ -187,42 +191,6 @@ def get_education(section):
     return edu_list
 
 
-def get_edu_sec(section):
-    """
-    Get the education items section
-    Args:
-        section: the education section
-
-    Returns:
-        The education items section
-    """
-    # Check if the section is expandable, if so expand the section and get the ul element
-    try:
-        section.find_element_by_css_selector(
-            '.pv-profile-section__see-more-inline.pv-profile-section__text-truncate-toggle.link').click()
-        time.sleep(1)
-
-        try:
-            elem = section.find_element_by_css_selector(
-                '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-more.ember-view')
-        except NoSuchElementException:
-            elem = section.find_element_by_css_selector(
-                '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-more')
-
-    # The section is not expandable, simply get the ul element
-    except NoSuchElementException:
-        # The ul element can appear in two different classes
-        try:
-            elem = section.find_element_by_css_selector(
-                '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-no-more.'
-                'ember-view')
-        except NoSuchElementException:
-            elem = section.find_element_by_css_selector(
-                '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-no-more')
-
-    return elem
-
-
 def get_degree(li, degree_name):
     """
     Get the full degree description
@@ -255,8 +223,7 @@ def get_volunteering(section):
     Returns:
         A list of details of all volunteering
     """
-    ul = section.find_element_by_css_selector(
-        '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-no-more.ember-view')
+    ul = get_section(section)
     vol_dict = defaultdict(list)
 
     for li in ul.find_elements_by_tag_name('li'):
@@ -306,7 +273,7 @@ def get_skills(section):
             '.pv-skill-category-entity__top-skill.pv-skill-category-entity.pb3.pt4.pv-skill-endorsedSkill-entity.'
             'relative.ember-view'):
         skill = top_skill.find_element_by_css_selector('.pv-skill-category-entity__name-text.t-16.t-black.t-bold').text
-        skills.append(skill)
+        skills.append(skill.split('\n')[0])
 
     # Locate Tools & Technologies section
     target_div = None
@@ -320,6 +287,42 @@ def get_skills(section):
     # Extract the rest of the skills
     if target_div is not None:
         for li in target_div.find_elements_by_tag_name('li'):
-            skills.append(li.text)
+            skills.append(li.text.split('\n')[0])
 
     return skills
+
+
+def get_section(section):
+    """
+    Get the items section
+    Args:
+        section: the section
+
+    Returns:
+        The items section
+    """
+    # Check if the section is expandable, if so expand the section and get the ul element
+    try:
+        section.find_element_by_css_selector(
+            '.pv-profile-section__see-more-inline.pv-profile-section__text-truncate-toggle.link').click()
+        time.sleep(1)
+
+        try:
+            elem = section.find_element_by_css_selector(
+                '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-more.ember-view')
+        except NoSuchElementException:
+            elem = section.find_element_by_css_selector(
+                '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-more')
+
+    # The section is not expandable, simply get the ul element
+    except NoSuchElementException:
+        # The ul element can appear in two different classes
+        try:
+            elem = section.find_element_by_css_selector(
+                '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-no-more.'
+                'ember-view')
+        except NoSuchElementException:
+            elem = section.find_element_by_css_selector(
+                '.pv-profile-section__section-info.section-info.pv-profile-section__section-info--has-no-more')
+
+    return elem
diff --git a/linkedrw/utils/__init__.py b/linkedrw/utils/__init__.py
index 42785c1..6c8542d 100644
--- a/linkedrw/utils/__init__.py
+++ b/linkedrw/utils/__init__.py
@@ -4,6 +4,8 @@ import shutil
 
 from selenium.common.exceptions import NoSuchElementException
 
+from linkedrw.constants import LATEX_CHARS
+
 
 def make_dir(dir_name):
     """
@@ -122,6 +124,8 @@ def get_description(element, name):
     except NoSuchElementException:
         description = ''
 
+    description = description.replace('•', '-')
+
     return description
 
 
@@ -138,3 +142,15 @@ def get_accomplishment_link(element):
         return element.find_element_by_class_name('pv-accomplishment-entity__external-source').get_attribute('href')
     except NoSuchElementException:
         return ''
+
+
+def escape_latex(s):
+    """
+    Escape LaTeX special characters
+    Args:
+        s: the string
+
+    Returns:
+        a string with escaped LaTeX special characters
+    """
+    return ''.join(LATEX_CHARS.get(c, c) for c in s)


Test patch:
diff --git a/tests/test_publication.py b/tests/test_publication.py
index d7b318b..0298eec 100644
--- a/tests/test_publication.py
+++ b/tests/test_publication.py
@@ -45,7 +45,7 @@ def test_make_publication_section():
                 if '\\nocite' in line:
                     cites.add(line.strip().lstrip('\\nocite').strip('{}'))
 
-        assert cites == set(CITES)
+        assert cites == set([x.replace('_', '\\_') for x in CITES])
 
 
 def test_make_references_no_doi():
diff --git a/tests/test_section.py b/tests/test_section.py
index 60f6522..82cc2b0 100644
--- a/tests/test_section.py
+++ b/tests/test_section.py
@@ -36,7 +36,7 @@ def test_make_resume_section_grouped():
                     assert title == entry[TITLE]
                     assert name == profile[EXPERIENCE][0][NAME]
                     assert location == entry[LOCATION]
-                    assert dates == entry[DATES]
+                    assert dates == entry[DATES].replace('-', '{-}')
 
                     break
 
@@ -63,7 +63,7 @@ def test_make_resume_section_ungrouped():
                     dates = get_text(next(f))
 
                     assert name == entry[NAME]
-                    assert dates == entry[DATES]
+                    assert dates == entry[DATES].replace('-', '{-}')
 
                     break
 
@@ -98,4 +98,4 @@ def test_make_resume_section_honors():
 
 
 def get_text(line):
-    return re.sub(r'[}%].*', '', line.strip().strip('{}')).strip()
+    return re.sub(r'(.*)[}%][^.]*', '\\1', line.strip().strip('{}')).strip().strip('}')