Repository: alltheplaces/alltheplaces

Issue:
Rompetrol - missing stations
There are about 80 Romanian stations and about 30 Bulgarian stations which are missing. It seems that the problem is caused by duplicated `ref`s on their end (the databases seem to be independent so each site has station with `ref=1`, `ref=2` and so on). Is it possible to resolve that issue? Possibly without having 4 almost identical spiders?


Discussion:

Resolving patch:
diff --git a/locations/spiders/rompetrol.py b/locations/spiders/rompetrol.py
--- a/locations/spiders/rompetrol.py
+++ b/locations/spiders/rompetrol.py
@@ -8,16 +8,14 @@ class RompetrolSpider(Spider):
     name = "rompetrol"
     item_attributes = {"brand": "Rompetrol", "brand_wikidata": "Q1788862"}
     allowed_domains = [
-        "www.rompetrol.bg",
         "www.rompetrol.ge",
         "www.rompetrol.md",
         "www.rompetrol.ro",
     ]
     start_urls = [
-        "https://www.rompetrol.bg/routeplanner/stations?language_id=1",
         "https://www.rompetrol.ge/routeplanner/stations?language_id=1",
         "https://www.rompetrol.md/routeplanner/stations?language_id=1",
-        "https://www.rompetrol.ro/routeplanner/stations?language_id=1",
+        "https://www.rompetrol.ro/routeplanner/stations?language_id=1",  # BG and RO locations
     ]
 
     def parse(self, response):
@@ -25,33 +23,23 @@ def parse(self, response):
             location["street_address"] = location.pop("address", None)
             item = DictParser.parse(location)
 
-            if "www.rompetrol.bg" in response.url:
-                item["country"] = "BG"
-            elif "www.rompetrol.ge" in response.url:
-                item["country"] = "GE"
-            elif "www.rompetrol.md" in response.url:
-                item["country"] = "MD"
-            elif "www.rompetrol.ro" in response.url:
-                item["country"] = "RO"
-
-            apply_yes_no(Fuel.OCTANE_98, item, "27" in location["services"], False)
-            apply_yes_no(Fuel.OCTANE_95, item, "28" in location["services"], False)
-            apply_yes_no(Fuel.GTL_DIESEL, item, "30" in location["services"], False)
-            apply_yes_no(Fuel.DIESEL, item, "31" in location["services"], False)
-            apply_yes_no(Fuel.ADBLUE, item, "14" in location["services"], False)
-            apply_yes_no(Fuel.LPG, item, "5" in location["services"], False)
-            apply_yes_no(Extras.VACUUM_CLEANER, item, "21" in location["services"], False)
-            apply_yes_no(Extras.SHOWERS, item, "15" in location["services"], False)
-            apply_yes_no(Extras.WIFI, item, "7" in location["services"], False)
+            item["ref"] = "{}:{}".format(response.url, item["ref"])
+
+            apply_yes_no(Fuel.OCTANE_98, item, "27" in location["services"])
+            apply_yes_no(Fuel.OCTANE_95, item, "28" in location["services"])
+            apply_yes_no(Fuel.GTL_DIESEL, item, "30" in location["services"])
+            apply_yes_no(Fuel.DIESEL, item, "31" in location["services"])
+            apply_yes_no(Fuel.ADBLUE, item, "14" in location["services"])
+            apply_yes_no(Fuel.LPG, item, "5" in location["services"])
+            apply_yes_no(Extras.VACUUM_CLEANER, item, "21" in location["services"])
+            apply_yes_no(Extras.SHOWERS, item, "15" in location["services"])
+            apply_yes_no(Extras.WIFI, item, "7" in location["services"])
 
             apply_category(Categories.FUEL_STATION, item)
-            if "14" in location["services"]:
-                apply_category(Categories.CHARGING_STATION, item)
-            if "33" in location["services"]:
-                apply_category(Categories.SHOP_LAUNDRY, item)
-            if "20" in location["services"]:
-                apply_category(Categories.COMPRESSED_AIR, item)
-            if "17" in location["services"]:
-                apply_category(Categories.RESTAURANT, item)
+
+            apply_yes_no("fuel:electric", item, "34" in location["services"])
+            apply_yes_no("laundry_service", item, "33" in location["services"])
+            apply_yes_no(Extras.COMPRESSED_AIR, item, "20" in location["services"])
+            apply_yes_no("food", item, "17" in location["services"])
 
             yield item


Test patch: