Repository: AnthonyBloomer/daftlistings

Issue:
Incorrect results for listings grouped in the same development
When we find grouped listings in a search, the price is returned as "from x" where x is the lowest price in the development. Similarly in such cases the number of bedrooms we get back contains all those in the development e.g. "1, 2, 3 & 4 bed", and the number of bathrooms is not obtained at all. For example if we run the following example with a minimum price of 7500:

```
import pandas as pd
from daftlistings import Daft, Location, SearchType, PropertyType, SortType, MapVisualization
 
daft = Daft()
daft.set_location(Location.DUBLIN)
daft.set_search_type(SearchType.RESIDENTIAL_RENT)
daft.set_property_type(PropertyType.APARTMENT)
daft.set_sort_type(SortType.PRICE_DESC)
daft.set_min_price(7500)

listings = daft.search()

# cache the listings in the local file
with open("result.txt", "w") as fp:
    fp.writelines("%s\n" % listing.as_dict_for_mapping() for listing in listings)

# read from the local file
with open("result.txt") as fp:
  lines = fp.readlines()

properties = []
for line in lines:
  properties.append(eval(line))

df = pd.DataFrame(properties)
print(df)
```

The results look like:

![daft1](https://user-images.githubusercontent.com/52505873/113723641-76d2b980-96e9-11eb-837e-f8b67fe3d854.jpg)

i.e. We are mostly finding prices not satisfying our search criteria.

If we go to the daft link for the second result we find:

![daft2](https://user-images.githubusercontent.com/52505873/113724067-e34db880-96e9-11eb-8b15-1eae8aceb98d.jpg)

i.e. The reason is that there are properties satisfying the search requirements in the development, but the entire development is returned as a single listing so we don't distinguish between these properties leading to incorrect results.

I have a fix ready for this that expands such listings so that each property within the development becomes a separate listing and I'll be ready to pull request it soon. 

Discussion:

Patch:
diff --git a/README.md b/README.md
index 9eba33d..5f28518 100644
--- a/README.md
+++ b/README.md
@@ -33,7 +33,7 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
     # ...
 ```
@@ -60,7 +60,7 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
 ```
 
@@ -80,7 +80,7 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
 ```
 
@@ -97,7 +97,7 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
 ```
 
@@ -113,7 +113,7 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
     print()
 ```
@@ -138,7 +138,7 @@ listings.sort(key=lambda x: x.distance_to(dublin_castle_coords))
 for listing in listings:
     print(f'{listing.title}')
     print(f'{listing.daft_link}')
-    print(f'{listing.abbreviated_price}')
+    print(f'{listing.price}')
     print(f'{listing.distance_to(dublin_castle_coords):.3}km')
     print('')
 
@@ -196,7 +196,7 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
     print()
 ```
diff --git a/daftlistings/daft.py b/daftlistings/daft.py
index e2ff99a..6082ce9 100644
--- a/daftlistings/daft.py
+++ b/daftlistings/daft.py
@@ -3,6 +3,7 @@ import requests
 from typing import Union, Optional, List, Dict
 from math import ceil
 from difflib import SequenceMatcher
+from copy import deepcopy
 
 from .enums import *
 from .listing import Listing
@@ -221,7 +222,6 @@ class Daft:
                           json=_payload)
         listings = r.json()["listings"]
         results_count = r.json()["paging"]["totalResults"]
-        print(f"Got {results_count} results.")
 
         total_pages = ceil(results_count / self._PAGE_SZ)
         limit = min(max_pages, total_pages) if max_pages else total_pages
@@ -232,5 +232,37 @@ class Daft:
                               headers=self._HEADER,
                               json=_payload)
             listings = listings + r.json()["listings"]
-        return [Listing(l) for l in listings]
 
+        # expand out grouped listings as individual listings, commercial searches do not give the necessary information to do this
+
+        expanded_listings = []
+        for l in listings:
+            # the information contained in the key 'prs' for most searches is instead contained in 'newHome' for newHome type searches
+            if 'newHome' in l['listing'].keys():
+                if 'subUnits' in l['listing']['newHome'].keys():
+                    l['listing']['prs'] = l['listing'].pop('newHome')
+            try:
+                num_subUnits = len(l['listing']['prs']['subUnits'])
+                for i in range(num_subUnits):
+                    copy = deepcopy(l)            
+                    for key in copy['listing']['prs']['subUnits'][i].keys(): 
+                        copy['listing'][key] = copy['listing']['prs']['subUnits'][i][key]
+
+                    # studios do not have a 'numBedrooms' so set it separately 
+                    if copy['listing']['propertyType'] == 'Studio':
+                        copy['listing']['numBedrooms'] = '1 bed'                         
+                    expanded_listings.append(copy)
+            except:    
+                # above only sets studio 'numBedrooms' for grouped listings, do ungrouped here
+                if 'propertyType' in l['listing'].keys():
+                    if l['listing']['propertyType'] == 'Studio':
+                            l['listing']['numBedrooms'] = '1 bed'            
+                expanded_listings.append(l)         
+
+        listings = expanded_listings
+
+        print(f"Got {len(listings)} results.")
+        
+        return [Listing(l) for l in listings]    
+
+                             
diff --git a/daftlistings/listing.py b/daftlistings/listing.py
index 9e53740..b5cf753 100644
--- a/daftlistings/listing.py
+++ b/daftlistings/listing.py
@@ -59,8 +59,8 @@ class Listing:
             return price_num
 
     @property
-    def abbreviated_price(self):
-        return self._result["abbreviatedPrice"]
+    def price(self):
+         return self._result["price"]
 
     @property
     def bathrooms(self):
diff --git a/daftlistings/map_visualization.py b/daftlistings/map_visualization.py
index 84eea11..8275452 100644
--- a/daftlistings/map_visualization.py
+++ b/daftlistings/map_visualization.py
@@ -1,4 +1,5 @@
 import folium
+from folium.plugins import MarkerCluster
 import branca.colormap as cm
 
 
@@ -56,6 +57,7 @@ class MapVisualization:
         return folium.Icon(color=self.color_dispatcher(price))
 
     def add_markers(self):
+        markers_dict = {}
         for index, row in self.df.iterrows():
             lat, lon, price = row["latitude"], row["longitude"], row["monthly_price"]
             beds, baths = row["bedrooms"], row["bathrooms"]
@@ -76,7 +78,18 @@ class MapVisualization:
             marker = folium.Marker(
                 [lat, lon], popup=popup_name, tooltip=price, icon=icon
             )
-            marker.add_to(self.map)
+            if (lat,lon) in markers_dict.keys():
+                markers_dict[(lat,lon)].append(marker)
+            else:
+                markers_dict[(lat,lon)] = [marker]
+
+        for key, item in markers_dict.items():
+            if len(item) == 1:
+                item[0].add_to(self.map)
+            else:
+                marker_cluster = MarkerCluster().add_to(self.map)
+                for i in range(len(item)):
+                    item[i].add_to(marker_cluster)
 
     def add_colorbar(self):
         """add a colorbar at the top right corner of the map"""
diff --git a/examples/commercial_listings.py b/examples/commercial_listings.py
index 311afe3..f862409 100644
--- a/examples/commercial_listings.py
+++ b/examples/commercial_listings.py
@@ -7,7 +7,7 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
     print()
 
diff --git a/examples/facilities.py b/examples/facilities.py
index e864f83..f2ccba1 100644
--- a/examples/facilities.py
+++ b/examples/facilities.py
@@ -11,6 +11,6 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
     print()
\ No newline at end of file
diff --git a/examples/properties_for_rent.py b/examples/properties_for_rent.py
index ebfd4cb..2129081 100644
--- a/examples/properties_for_rent.py
+++ b/examples/properties_for_rent.py
@@ -9,6 +9,6 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
     print()
diff --git a/examples/properties_for_sale.py b/examples/properties_for_sale.py
index 0d11a88..1320cf8 100644
--- a/examples/properties_for_sale.py
+++ b/examples/properties_for_sale.py
@@ -11,6 +11,6 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
     print()
diff --git a/examples/sort.py b/examples/sort.py
index 6e41081..f3d1516 100644
--- a/examples/sort.py
+++ b/examples/sort.py
@@ -9,6 +9,6 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
     print()
\ No newline at end of file
diff --git a/examples/sort_by_distance.py b/examples/sort_by_distance.py
index 60f3f4d..ed79f8f 100644
--- a/examples/sort_by_distance.py
+++ b/examples/sort_by_distance.py
@@ -16,6 +16,6 @@ listings.sort(key=lambda x: x.distance_to(dublin_castle_coords))
 for listing in listings:
     print(f'{listing.title}')
     print(f'{listing.daft_link}')
-    print(f'{listing.abbreviated_price}')
+    print(f'{listing.price}')
     print(f'{listing.distance_to(dublin_castle_coords):.3}km')
     print('')
diff --git a/examples/student_accomodation.py b/examples/student_accomodation.py
index c8b9828..c39d8d4 100644
--- a/examples/student_accomodation.py
+++ b/examples/student_accomodation.py
@@ -8,6 +8,6 @@ listings = daft.search()
 
 for listing in listings:
     print(listing.title)
-    print(listing.abbreviated_price)
+    print(listing.price)
     print(listing.daft_link)
     print()


Test patch:
diff --git a/tests/test_daft_search.py b/tests/test_daft_search.py
index 12877bb..9fd991e 100644
--- a/tests/test_daft_search.py
+++ b/tests/test_daft_search.py
@@ -146,8 +146,8 @@ class DaftTest(unittest.TestCase):
         self.assertEqual(listing.id, 1443907)
         self.assertEqual(listing.title, "Capital Dock Residence, Grand Canal, Dublin 2")
         self.assertEqual(listing.agent_id, 9601)
+        self.assertEqual(listing.price, "From €2,970 per month")
         self.assertEqual(listing.bedrooms, "2 & 3 bed")
-        self.assertEqual(listing.abbreviated_price, "€2,970+")
         self.assertEqual(listing.has_brochure, False)
         self.assertEqual(
             listing.daft_link,
@@ -206,6 +206,14 @@ class DaftTest(unittest.TestCase):
         daft.set_location(Location.DUBLIN)
         listings = daft.search(max_pages=1)
         self.assertTrue(len(listings) > 0)
+        self.assertTrue(listings[0].bedrooms == '1 bed')
+
+    def test_new_homes(self):
+        daft = Daft()
+        daft.set_search_type(SearchType.NEW_HOMES)
+        daft.set_location(Location.DUBLIN)
+        listings = daft.search(max_pages=1)
+        self.assertTrue(len(listings) > 0)
 
     def test_distance(self):
         daft = Daft()
@@ -214,7 +222,11 @@ class DaftTest(unittest.TestCase):
         daft.set_min_price(1)
         daft.set_max_price(100000)
         listings = daft.search(max_pages=1)
-        first, second = listings[0], listings[1]
+        first = listings[0]
+        for l in listings[1:]:
+            if (l.latitude, l.longitude) != (first.latitude, first.longitude):
+                second = l
+                break
         coord = [53.3429, -6.2674]
         self.assertGreater(first.distance_to(coord), 0)
-        self.assertGreater(first.distance_to(second), 0)
+        self.assertGreater(first.distance_to(second), 0)
\ No newline at end of file