Repository: data-for-change/anyway

Issue:
Separate marker id from provider code
Loading the complete data on a new DB gives:

```
static/data/lms/Accidents Type 3/H20121161: (DataError) integer out of range
```

@eladkarakuli 

<!---
@huboard:{"order":1.0017524436116219,"milestone_order":279,"custom_state":""}
-->

Separate marker id from provider code
Loading the complete data on a new DB gives:

```
static/data/lms/Accidents Type 3/H20121161: (DataError) integer out of range
```

@eladkarakuli 

<!---
@huboard:{"order":1.0017524436116219,"milestone_order":279,"custom_state":""}
-->



Discussion:
Of course, it's because it's constructing the id by joining the provider code ('1') with the id from the CSV, giving a very large number.
We could keep the id from the CSV as a separate column or something, and use simple autoincrementing integers for the real id.

What's the status here?
@galraij wanted to have the provider code pulled separately from the accident id for further use.

Right, so the request is to separate the id from the provider code.

Of course, it's because it's constructing the id by joining the provider code ('1') with the id from the CSV, giving a very large number.
We could keep the id from the CSV as a separate column or something, and use simple autoincrementing integers for the real id.

What's the status here?
@galraij wanted to have the provider code pulled separately from the accident id for further use.

Right, so the request is to separate the id from the provider code.


Resolving patch:
diff --git a/models.py b/models.py
--- a/models.py
+++ b/models.py
@@ -126,17 +126,15 @@ class Marker(MarkerMixin, Base): # TODO rename to AccidentMarker
         'polymorphic_identity': MARKER_TYPE_ACCIDENT
     }
 
+    provider_code = Column(Integer)
     description = Column(Text)
     subtype = Column(Integer)
     severity = Column(Integer)
     address = Column(Text)
     locationAccuracy = Column(Integer)
     roadType = Column(Integer)
-    # accidentType
     roadShape = Column(Integer)
-    # severityText
     dayType = Column(Integer)
-    # igun
     unit = Column(Integer)
     mainStreet = Column(Text)
     secondaryStreet = Column(Text)
@@ -166,6 +164,7 @@ def json_to_description(msg):
     def serialize(self, is_thin=False):
         fields = {
             "id": str(self.id),
+            "provider_code": self.provider_code,
             "latitude": self.latitude,
             "longitude": self.longitude,
             "severity": self.severity,
@@ -180,11 +179,8 @@ def serialize(self, is_thin=False):
                 "type": self.type,
                 "subtype": self.subtype,
                 "roadType": self.roadType,
-                # "accidentType"
                 "roadShape": self.roadShape,
-                # "severityText"
                 "dayType": self.dayType,
-                # "igun"
                 "unit": self.unit,
                 "mainStreet": self.mainStreet,
                 "secondaryStreet": self.secondaryStreet,
@@ -324,6 +320,7 @@ def bounding_box_query(ne_lat, ne_lng, sw_lat, sw_lng, show_discussions):
 class Involved(Base):
     __tablename__ = "involved"
     id = Column(Integer, primary_key=True)
+    provider_code = Column(Integer)
     accident_id = Column(Integer, ForeignKey("markers.id"))
     involved_type = Column(Integer)
     license_acquiring_date = Column(Integer)
@@ -350,6 +347,7 @@ class Involved(Base):
     def serialize(self):
         return {
             "id": self.id,
+            "provider_code": self.provider_code,
             "accident_id": self.accident_id,
             "involved_type": self.involved_type,
             "license_acquiring_date": self.license_acquiring_date,
@@ -391,6 +389,7 @@ def get_id(self):
 class Vehicle(Base):
     __tablename__ = "vehicles"
     id = Column(Integer, primary_key=True)
+    provider_code = Column(Integer)
     accident_id = Column(Integer, ForeignKey("markers.id"))
     engine_volume = Column(Integer)
     manufacturing_year = Column(Integer)
@@ -404,6 +403,7 @@ class Vehicle(Base):
     def serialize(self):
         return {
             "id": self.id,
+            "provider_code": self.provider_code,
             "accident_id": self.accident_id,
             "engine_volume": self.engine_volume,
             "manufacturing_year": self.manufacturing_year,
diff --git a/process.py b/process.py
--- a/process.py
+++ b/process.py
@@ -218,16 +218,17 @@ def import_accidents(provider_code, accidents, streets, roads, involved, vehicle
             acc_years.append(accident[field_names.accident_year])
 
         marker = {
-            "id":int("{0}{1}".format(provider_code, accident[field_names.id])),
-            "title":"Accident",
-            "description":json.dumps(load_extra_data(accident, streets, roads), encoding=models.db_encoding),
-            "address":get_address(accident, streets),
-            "latitude":lat,
-            "longitude":lng,
-            "subtype":int(accident[field_names.accident_type]),
-            "severity":int(accident[field_names.accident_severity]),
-            "created":parse_date(accident),
-            "locationAccuracy":int(accident[field_names.igun]),
+            "id": int(accident[field_names.id]),
+            "provider_code": int(provider_code),
+            "title": "Accident",
+            "description": json.dumps(load_extra_data(accident, streets, roads), encoding=models.db_encoding),
+            "address": get_address(accident, streets),
+            "latitude": lat,
+            "longitude": lng,
+            "subtype": int(accident[field_names.accident_type]),
+            "severity": int(accident[field_names.accident_severity]),
+            "created": parse_date(accident),
+            "locationAccuracy": int(accident[field_names.igun]),
             "roadType": int(accident[field_names.road_type]),
             "roadShape": int(accident[field_names.road_shape]),
             "dayType": int(accident[field_names.day_type]),
@@ -260,7 +261,8 @@ def import_involved(provider_code, accidents, streets, roads, involved, vehicles
     print("reading involved data from file %s" % (involved.name(),))
     for involve in involved:
         involved_item = {
-            "accident_id": int("{0}{1}".format(provider_code, involve[field_names.id])),
+            "accident_id": int(involve[field_names.id]),
+            "provider_code": int(provider_code),
             "involved_type": int(involve[field_names.involved_type]),
             "license_acquiring_date": int(involve[field_names.license_acquiring_date]),
             "age_group": int(involve[field_names.age_group]),
@@ -291,7 +293,8 @@ def import_vehicles(provider_code, accidents, streets, roads, involved, vehicles
     print("reading involved data from file %s" % (vehicles.name(),))
     for vehicle in vehicles:
         vehicle_item = {
-            "accident_id": int("{0}{1}".format(provider_code, vehicle[field_names.id])),
+            "accident_id": int(vehicle[field_names.id]),
+            "provider_code": int(provider_code),
             "engine_volume": int(vehicle[field_names.engine_volume]),
             "manufacturing_year": get_data_value(vehicle[field_names.manufacturing_year]),
             "driving_directions": get_data_value(vehicle[field_names.driving_directions]),


Test patch: