Repository: benbusby/whoogle-search
File: app/filter.py (Python)

Code before:
        if self.dark:
            page = page.replace('fff', '000').replace('202124', 'ddd').replace('1967D2', '3b85ea')

        return page

    def clean(self, soup):
        self.remove_ads(soup)
        self.update_image_paths(soup)
        self.update_styling(soup)
        self.update_links(soup)

        input_form = soup.find('form')
        if input_form is not None:
            input_form['method'] = 'POST'

        # Ensure no extra scripts passed through
        for script in soup('script'):
            script.decompose()

        footer = soup.find('div', id='sfooter')
        if footer is not None:
            footer.decompose()

        return soup

    def get_first_url(self, soup):
        # Replace hrefs with only the intended destination (no "utm" type tags)
        for a in soup.find_all('a', href=True):
            href = a['href'].replace('https://www.google.com', '')
            
            result_link = urlparse.urlparse(href)
            query_link = parse_qs(result_link.query)['q'][0] if '?q=' in href else ''

            # Return the first search result URL
            if 'url?q=' in href:
                parsed_link = urlparse.urlparse(query_link)
                link_args = parse_qs(parsed_link.query)
                safe_args = {}

                if len(link_args) == 0 and len(parsed_link) > 0:
                    return query_link
                    

                for arg in link_args.keys():
                    if arg in SKIP_ARGS:
                        continue

                    safe_args[arg] = link_args[arg]

                # Remove original link query and replace with filtered args
                query_link = query_link.replace(parsed_link.query, '')

Review comment (performance):
Since this reuses some of the link parsing functionality from the `update_links` method, I think it'd be better to structure this as follows:

- Change `get_first_url(self, soup)` to be a static method outside of the Filter class, i.e.:
```python
def get_first_url(soup):
    # Replace hrefs with only the intended destination (no "utm" type tags)
    for a in soup.find_all('a', href=True):
        href = a['href'].replace('https://www.google.com', '')

        result_link = urlparse.urlparse(href)
        query_link = parse_qs(result_link.query)['q'][0] if '?q=' in href else ''

        # Return the first search result URL
        if 'url?q=' in href:
            return filter_link_args(query_link)


class Filter:
...
```

- Add in the `filter_link_args` method seen in the previous suggested change:
```python
def filter_link_args(query_link):
    parsed_link = urlparse.urlparse(query_link)
    link_args = parse_qs(parsed_link.query)
    safe_args = {}

    if len(link_args) == 0 and len(parsed_link) > 0:
        return query_link

    for arg in link_args.keys():
        if arg in SKIP_ARGS:
            continue

        safe_args[arg] = link_args[arg]

    # Remove original link query and replace with filtered args
    query_link = query_link.replace(parsed_link.query, '')
    if len(safe_args) > 0:
        query_link = query_link + urlparse.urlencode(safe_args, doseq=True)
    else:
        query_link = query_link.replace('?', '')

    return query_link
```

- Then we can also update the `update_links` Filter class method to make use of this new helper function, i.e.:
```python
def update_links(self, soup):
...
            elif 'url?q=' in href:
                # Strip unneeded arguments
                query_link = filter_link_args(query_link)
                a['href'] = query_link

                # Add no-js option
                if self.nojs:
                    gen_nojs(soup, query_link, a)
...
```

___
With these changes, `routes.py` would also need another small update to import `get_first_url` separately from `filter`, and then would just be called as `get_first_url` instead of `content_filter.get_first_url`.

Let me know if you have any thoughts regarding these changes or need anything clarified. Not trying to nitpick or anything, just trying to make the most efficient use of the code and avoid any duplication.

Code after:
    for arg in link_args.keys():
        if arg in SKIP_ARGS:
            continue

        safe_args[arg] = link_args[arg]

    # Remove original link query and replace with filtered args
    query_link = query_link.replace(parsed_link.query, '')
    if len(safe_args) > 0:
        query_link = query_link + urlparse.urlencode(safe_args, doseq=True)
    else:
        query_link = query_link.replace('?', '')

    return query_link

class Filter:
    def __init__(self, mobile=False, config=None, secret_key=''):
        if config is None:
            config = {}

        self.near = config['near'] if 'near' in config else ''
        self.dark = config['dark'] if 'dark' in config else False
        self.nojs = config['nojs'] if 'nojs' in config else False
        self.mobile = mobile
        self.secret_key = secret_key

    def __getitem__(self, name):
        return getattr(self, name)

    def reskin(self, page):
        # Aesthetic only re-skinning
        page = page.replace('>G<', '>Wh<')
        pattern = re.compile('4285f4|ea4335|fbcc05|34a853|fbbc05', re.IGNORECASE)
        page = pattern.sub('685e79', page)
        if self.dark:
            page = page.replace('fff', '000').replace('202124', 'ddd').replace('1967D2', '3b85ea')

        return page

    def clean(self, soup):
        self.remove_ads(soup)
        self.update_image_paths(soup)
        self.update_styling(soup)
        self.update_links(soup)

        input_form = soup.find('form')
        if input_form is not None:
            input_form['method'] = 'POST'

        # Ensure no extra scripts passed through
        for script in soup('script'):