Repository: wevote/WeVoteServer

Issue:
Knock the score of any Google search user at this URL https://uk.linkedin.com down to zero
Any person we find at https://uk.linkedin.com won't be an American politician, so we can knock these entries down to a score of zero. Actually any URL that starts with any of these URLs could be zero:
https://www.linkedin.com/pub/dir
https://www.facebook.com/events
https://twitter.com/laufer4council/status/867543532504043520 <-- if it includes "/status/"
https://nz.linkedin.com



Discussion:
@DaleMcGrew Could I work on this issue? 
Hi @utsab, yes that would be great! Thank you. I think this would be a good issue to discuss via screenshare though before you get started. Could we connect on Slack to find a time to review this?
I am working this
What would be the best way to store and reference the list of URL patterns?
For the custom Google search engine, what should the configuration be?

Resolving patch:
diff --git a/google_custom_search/controllers.py b/google_custom_search/controllers.py
--- a/google_custom_search/controllers.py
+++ b/google_custom_search/controllers.py
@@ -5,7 +5,7 @@
 # See also WeVoteServer/import_export_twitter/controllers.py for routines that manage incoming twitter data
 from image.functions import analyze_remote_url
 from .models import GoogleSearchUserManager, GOOGLE_SEARCH_API_NAME, GOOGLE_SEARCH_API_VERSION, GOOGLE_SEARCH_API_KEY, \
-    GOOGLE_SEARCH_ENGINE_ID, BALLOTPEDIA_LOGO_URL, MAXIMUM_CHARACTERS_LENGTH, MAXIMUM_GOOGLE_SEARCH_USERS
+    GOOGLE_SEARCH_ENGINE_ID, BALLOTPEDIA_LOGO_URL, MAXIMUM_CHARACTERS_LENGTH, MAXIMUM_GOOGLE_SEARCH_USERS, URL_PATTERNS_TO_IGNORE
 from googleapiclient.discovery import build
 from image.controllers import IMAGE_SOURCE_BALLOTPEDIA, LINKEDIN, FACEBOOK, TWITTER, WIKIPEDIA
 from import_export_facebook.models import FacebookManager
@@ -230,6 +230,12 @@ def retrieve_possible_google_search_users(candidate, voter_device_id):
 
     return results
 
+def should_ignore_google_json(google_json):
+    '''
+    Tests if the google_json object should have a likelihood score of 0 and therefore be ignored
+    based on the list of url patterns
+    '''
+    return any((pattern.match(google_json['item_link']) for pattern in URL_PATTERNS_TO_IGNORE))
 
 def analyze_google_search_results(search_results, search_term, candidate_name,
                                   candidate, voter_device_id):
@@ -254,6 +260,9 @@ def analyze_google_search_results(search_results, search_term, candidate_name,
             from_wikipedia = False
             google_json = parse_google_search_results(search_term, one_result)
 
+            if should_ignore_google_json(google_json):
+                continue
+
             if FACEBOOK in google_json['item_link']:
                 current_candidate_facebook_search_info = analyze_facebook_search_results(
                     google_json, search_term, candidate_name, candidate, voter_device_id)
diff --git a/google_custom_search/models.py b/google_custom_search/models.py
--- a/google_custom_search/models.py
+++ b/google_custom_search/models.py
@@ -8,6 +8,7 @@
 
 from config.base import get_environment_variable
 from wevote_functions.functions import positive_value_exists
+from re import compile
 
 logger = wevote_functions.admin.get_logger(__name__)
 GOOGLE_SEARCH_ENGINE_ID = get_environment_variable("GOOGLE_SEARCH_ENGINE_ID")
@@ -18,6 +19,16 @@
 MAXIMUM_GOOGLE_SEARCH_USERS = 10
 MAXIMUM_CHARACTERS_LENGTH = 1024
 
+URL_PATTERNS_TO_IGNORE = [
+    r"^https?://uk.linkedin.com",
+    r"^https?://nz.linkedin.com",
+    r"^https?://www.linkedin.com/pub/dir",
+    r"^https?://www.facebook.com/events",
+    r"^https?://twitter.com/\w+/status"
+]
+
+URL_PATTERNS_TO_IGNORE = [compile(pattern) for pattern in URL_PATTERNS_TO_IGNORE]
+
 
 class GoogleSearchUser(models.Model):
     """


Test patch: