Repository: searx/searx

Issue:
Flickr engine is broken
The html seems to have changed, but it's seems there is [REST API](https://api.flickr.com/services/rest?sort=relevance&parse_tags=1&content_type=7&extras=can_comment%2Ccount_comments%2Ccount_faves%2Cisfavorite%2Clicense%2Cmedia%2Cneeds_interstitial%2Cowner_name%2Cpath_alias%2Crealname%2Crotation%2Curl_c%2Curl_l%2Curl_m%2Curl_n%2Curl_q%2Curl_s%2Curl_sq%2Curl_t%2Curl_z&per_page=25&page=1&lang=en-US&rb=1&text=proxy&viewerNSID=&method=flickr.photos.search&csrf=&api_key=3e5918155f464baad83cce2efcf8b57e&format=json&hermes=1&hermesClient=1&reqId=rgb38n1&nojsoncallback=1)

In all parameters there is an api_key : I don't know how long it is valid, in which condition.
The call to this URL is trigger inside another minified javascript.



Discussion:
Yes, normally, the flickr_engine.py use the REST API, with a key you had to get from Flickr.
But searx.me uses the flicknoapi_engine.py, which uses the HTML part of the site.

So, unless you want to get an API KEY from Flickr, we have to manage all with the changes on the HTML part.

What I mean: the flicknoapi_engine.py "browses" the HTML page intended to the browser. Now this HTML page uses a REST API with api key, and doesn't contains reference to the thumbnail etc... (not sure, I made a quick look).

Display a [this search](https://www.flickr.com/search/?q=proxy) with chrome dev tools / firebug, you will see a request to /rest..... with the results.

Ah yes, I understand what you mean.

But on my side, there is no request to the rest api.  The page still contains the api response directly in it, like it was when I wrote flickrnoapi_engine. Sure the regex may need to be changed (I don't have the time to look deeply into it right now), but globally, not much have changed.

I confirmed it with Chromium, Firefox and Opera 12.

So maybe it come from the user agent, or from the beta search in Flickr. But if you do a wget on the page, you'll get the "blob" with all the stuff that the noapi engine parse. You'll see it around line 2644.

EDIT : Oh and I forgot, try to load the page without JS, it might help.


Resolving patch:
diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py
--- a/searx/engines/flickr_noapi.py
+++ b/searx/engines/flickr_noapi.py
@@ -25,7 +25,7 @@
 url = 'https://www.flickr.com/'
 search_url = url + 'search?{query}&page={page}'
 photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
-regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
+regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
 image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
 
 paging = True
@@ -38,6 +38,7 @@ def build_flickr_url(user_id, photo_id):
 def request(query, params):
     params['url'] = search_url.format(query=urlencode({'text': query}),
                                       page=params['pageno'])
+
     return params
 
 
@@ -75,10 +76,10 @@ def response(resp):
             logger.debug('cannot find valid image size: {0}'.format(repr(photo)))
             continue
 
-        if 'id' not in photo['owner']:
+        if 'ownerNsid' not in photo:
             continue
 
-# For a bigger thumbnail, keep only the url_z, not the url_n
+        # For a bigger thumbnail, keep only the url_z, not the url_n
         if 'n' in photo['sizes']:
             thumbnail_src = photo['sizes']['n']['url']
         elif 'z' in photo['sizes']:
@@ -86,20 +87,14 @@ def response(resp):
         else:
             thumbnail_src = img_src
 
-        url = build_flickr_url(photo['owner']['id'], photo['id'])
+        url = build_flickr_url(photo['ownerNsid'], photo['id'])
 
         title = photo.get('title', '')
 
         content = '<span class="photo-author">' +\
-                  photo['owner']['username'] +\
+                  photo['username'] +\
                   '</span><br />'
 
-        if 'description' in photo:
-            content = content +\
-                '<span class="description">' +\
-                photo['description'] +\
-                '</span>'
-
         # append result
         results.append({'url': url,
                         'title': title,


Test patch:
diff --git a/searx/tests/engines/test_flickr_noapi.py b/searx/tests/engines/test_flickr_noapi.py
--- a/searx/tests/engines/test_flickr_noapi.py
+++ b/searx/tests/engines/test_flickr_noapi.py
@@ -26,19 +26,29 @@ def test_response(self):
         self.assertRaises(AttributeError, flickr_noapi.response, '')
         self.assertRaises(AttributeError, flickr_noapi.response, '[]')
 
-        response = mock.Mock(text='"search-photos-models","photos":{},"totalItems":')
+        response = mock.Mock(text='"search-photos-lite-models","photos":{},"totalItems":')
         self.assertEqual(flickr_noapi.response(response), [])
 
-        response = mock.Mock(text='search-photos-models","photos":{"data": []},"totalItems":')
+        response = mock.Mock(text='search-photos-lite-models","photos":{"data": []},"totalItems":')
         self.assertEqual(flickr_noapi.response(response), [])
 
+        # everthing is ok test
         json = """
-        "search-photos-models","photos":
+        "search-photos-lite-models","photos":
         {
           "_data": [
             {
-              "_flickrModelRegistry": "photo-models",
+              "_flickrModelRegistry": "photo-lite-models",
               "title": "This is the title",
+              "username": "Owner",
+              "pathAlias": "klink692",
+              "realname": "Owner",
+              "license": 0,
+              "ownerNsid": "59729010@N00",
+              "canComment": false,
+              "commentCount": 14,
+              "faveCount": 21,
+              "id": "14001294434",
               "sizes": {
                 "c": {
                   "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg",
@@ -117,40 +127,7 @@ def test_response(self):
                   "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
                   "key": "z"
                 }
-              },
-              "canComment": false,
-              "rotation": 0,
-              "owner": {
-                "_flickrModelRegistry": "person-models",
-                "pathAlias": "klink692",
-                "username": "Owner",
-                "buddyicon": {
-                  "retina": null,
-                  "large": null,
-                  "medium": null,
-                  "small": null,
-                  "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00"
-                },
-                "isPro": true,
-                "id": "59729010@N00"
-              },
-              "engagement": {
-                "_flickrModelRegistry": "photo-engagement-models",
-                "ownerNsid": "59729010@N00",
-                "faveCount": 21,
-                "commentCount": 14,
-                "viewCount": 10160,
-                "id": "14001294434"
-              },
-              "description": "Description",
-              "isHD": false,
-              "secret": "410f653777",
-              "canAddMeta": false,
-              "license": 0,
-              "oWidth": 1803,
-              "oHeight": 2669,
-              "safetyLevel": 0,
-              "id": "14001294434"
+              }
             }
           ],
           "fetchedStart": true,
@@ -168,15 +145,24 @@ def test_response(self):
         self.assertIn('k.jpg', results[0]['img_src'])
         self.assertIn('n.jpg', results[0]['thumbnail_src'])
         self.assertIn('Owner', results[0]['content'])
-        self.assertIn('Description', results[0]['content'])
 
+        # no n size, only the z size
         json = """
-        "search-photos-models","photos":
+        "search-photos-lite-models","photos":
         {
           "_data": [
             {
-              "_flickrModelRegistry": "photo-models",
+              "_flickrModelRegistry": "photo-lite-models",
               "title": "This is the title",
+              "username": "Owner",
+              "pathAlias": "klink692",
+              "realname": "Owner",
+              "license": 0,
+              "ownerNsid": "59729010@N00",
+              "canComment": false,
+              "commentCount": 14,
+              "faveCount": 21,
+              "id": "14001294434",
               "sizes": {
                 "z": {
                   "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
@@ -185,40 +171,7 @@ def test_response(self):
                   "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
                   "key": "z"
                 }
-              },
-              "canComment": false,
-              "rotation": 0,
-              "owner": {
-                "_flickrModelRegistry": "person-models",
-                "pathAlias": "klink692",
-                "username": "Owner",
-                "buddyicon": {
-                  "retina": null,
-                  "large": null,
-                  "medium": null,
-                  "small": null,
-                  "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00"
-                },
-                "isPro": true,
-                "id": "59729010@N00"
-              },
-              "engagement": {
-                "_flickrModelRegistry": "photo-engagement-models",
-                "ownerNsid": "59729010@N00",
-                "faveCount": 21,
-                "commentCount": 14,
-                "viewCount": 10160,
-                "id": "14001294434"
-              },
-              "description": "Description",
-              "isHD": false,
-              "secret": "410f653777",
-              "canAddMeta": false,
-              "license": 0,
-              "oWidth": 1803,
-              "oHeight": 2669,
-              "safetyLevel": 0,
-              "id": "14001294434"
+              }
             }
           ],
           "fetchedStart": true,
@@ -235,15 +188,24 @@ def test_response(self):
         self.assertIn('z.jpg', results[0]['img_src'])
         self.assertIn('z.jpg', results[0]['thumbnail_src'])
         self.assertIn('Owner', results[0]['content'])
-        self.assertIn('Description', results[0]['content'])
 
+        # no z or n size
         json = """
-        "search-photos-models","photos":
+        "search-photos-lite-models","photos":
         {
           "_data": [
             {
-              "_flickrModelRegistry": "photo-models",
+              "_flickrModelRegistry": "photo-lite-models",
               "title": "This is the title",
+              "username": "Owner",
+              "pathAlias": "klink692",
+              "realname": "Owner",
+              "license": 0,
+              "ownerNsid": "59729010@N00",
+              "canComment": false,
+              "commentCount": 14,
+              "faveCount": 21,
+              "id": "14001294434",
               "sizes": {
                 "o": {
                   "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
@@ -252,39 +214,7 @@ def test_response(self):
                   "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
                   "key": "o"
                 }
-              },
-              "canComment": false,
-              "rotation": 0,
-              "owner": {
-                "_flickrModelRegistry": "person-models",
-                "pathAlias": "klink692",
-                "username": "Owner",
-                "buddyicon": {
-                  "retina": null,
-                  "large": null,
-                  "medium": null,
-                  "small": null,
-                  "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00"
-                },
-                "isPro": true,
-                "id": "59729010@N00"
-              },
-              "engagement": {
-                "_flickrModelRegistry": "photo-engagement-models",
-                "ownerNsid": "59729010@N00",
-                "faveCount": 21,
-                "commentCount": 14,
-                "viewCount": 10160,
-                "id": "14001294434"
-              },
-              "isHD": false,
-              "secret": "410f653777",
-              "canAddMeta": false,
-              "license": 0,
-              "oWidth": 1803,
-              "oHeight": 2669,
-              "safetyLevel": 0,
-              "id": "14001294434"
+              }
             }
           ],
           "fetchedStart": true,
@@ -302,48 +232,25 @@ def test_response(self):
         self.assertIn('o.jpg', results[0]['thumbnail_src'])
         self.assertIn('Owner', results[0]['content'])
 
+        # no image test
         json = """
-        "search-photos-models","photos":
+        "search-photos-lite-models","photos":
         {
           "_data": [
             {
-              "_flickrModelRegistry": "photo-models",
+              "_flickrModelRegistry": "photo-lite-models",
               "title": "This is the title",
-              "sizes": {
-              },
-              "canComment": false,
-              "rotation": 0,
-              "owner": {
-                "_flickrModelRegistry": "person-models",
-                "pathAlias": "klink692",
-                "username": "Owner",
-                "buddyicon": {
-                  "retina": null,
-                  "large": null,
-                  "medium": null,
-                  "small": null,
-                  "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00"
-                },
-                "isPro": true,
-                "id": "59729010@N00"
-              },
-              "engagement": {
-                "_flickrModelRegistry": "photo-engagement-models",
-                "ownerNsid": "59729010@N00",
-                "faveCount": 21,
-                "commentCount": 14,
-                "viewCount": 10160,
-                "id": "14001294434"
-              },
-              "description": "Description",
-              "isHD": false,
-              "secret": "410f653777",
-              "canAddMeta": false,
+              "username": "Owner",
+              "pathAlias": "klink692",
+              "realname": "Owner",
               "license": 0,
-              "oWidth": 1803,
-              "oHeight": 2669,
-              "safetyLevel": 0,
-              "id": "14001294434"
+              "ownerNsid": "59729010@N00",
+              "canComment": false,
+              "commentCount": 14,
+              "faveCount": 21,
+              "id": "14001294434",
+              "sizes": {
+              }
             }
           ],
           "fetchedStart": true,
@@ -356,6 +263,7 @@ def test_response(self):
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 0)
 
+        # null test
         json = """
         "search-photos-models","photos":
         {
@@ -370,13 +278,22 @@ def test_response(self):
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 0)
 
+        # no ownerNsid test
         json = """
-        "search-photos-models","photos":
+        "search-photos-lite-models","photos":
         {
           "_data": [
             {
-              "_flickrModelRegistry": "photo-models",
+              "_flickrModelRegistry": "photo-lite-models",
               "title": "This is the title",
+              "username": "Owner",
+              "pathAlias": "klink692",
+              "realname": "Owner",
+              "license": 0,
+              "canComment": false,
+              "commentCount": 14,
+              "faveCount": 21,
+              "id": "14001294434",
               "sizes": {
                 "o": {
                   "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
@@ -385,39 +302,7 @@ def test_response(self):
                   "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
                   "key": "o"
                 }
-              },
-              "canComment": false,
-              "rotation": 0,
-              "owner": {
-                "_flickrModelRegistry": "person-models",
-                "pathAlias": "klink692",
-                "username": "Owner",
-                "buddyicon": {
-                  "retina": null,
-                  "large": null,
-                  "medium": null,
-                  "small": null,
-                  "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00"
-                },
-                "isPro": true
-              },
-              "engagement": {
-                "_flickrModelRegistry": "photo-engagement-models",
-                "ownerNsid": "59729010@N00",
-                "faveCount": 21,
-                "commentCount": 14,
-                "viewCount": 10160,
-                "id": "14001294434"
-              },
-              "description": "Description",
-              "isHD": false,
-              "secret": "410f653777",
-              "canAddMeta": false,
-              "license": 0,
-              "oWidth": 1803,
-              "oHeight": 2669,
-              "safetyLevel": 0,
-              "id": "14001294434"
+              }
             }
           ],
           "fetchedStart": true,
@@ -430,6 +315,7 @@ def test_response(self):
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 0)
 
+        # garbage test
         json = """
         {"toto":[
             {"id":200,"name":"Artist Name",