Repository: data-for-change/anyway

Issue:
Keep years.js out of version control
`statis/js/years.js` is generated when running `process.py`. Therefore, it should be ignored by Git and kept in a special location.

<!---
@huboard:{"order":0.000952152069658041,"milestone_order":356,"custom_state":""}
-->



Discussion:
Also, since we can run `process.py` on one computer with a `DATABASE_URL` that corresponds to another computer's database (e.g. the production server), and `years.js` is created on the computer where the script is run and not on the database, there is a problem.

I can have it create a new DB table with the years. But that's just 1 table for 4 single column entries, and every load would have another ajax call..

Any better ideas?

It would be best if you can make main.py generate the files to send when it
starts running.

On Fri, Jul 10, 2015 at 12:19 AM omerxx notifications@github.com wrote:

> I can have it create a new DB table with the years. But that's just 1
> table for 4 single column entries, and every load would have another ajax
> call..
> 
> Any better ideas?
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/hasadna/anyway/issues/356#issuecomment-120147148.

I have it ready, but it makes main.py loop through the DB markers to find distinct years and then creates the file with them..
Do you want that?
- It adds 6-7 sec to main.py but it works great
- Could be little disturbing for development but most of us use the example file so the DB is very small and it shouldn't be felt at all


Resolving patch:
diff --git a/main.py b/main.py
--- a/main.py
+++ b/main.py
@@ -33,7 +33,12 @@
 from flask.ext.principal import Principal, Permission, RoleNeed
 from flask.ext.security import Security, SQLAlchemyUserDatastore, \
      UserMixin, RoleMixin
+from collections import OrderedDict
+from sqlalchemy import distinct, func
 
+
+app = utilities.init_flask(__name__)
+db = SQLAlchemy(app)
 app = utilities.init_flask(__name__)
 app.config.from_object(__name__)
 
@@ -574,9 +579,36 @@ def login2():
         return render_template('login.html')
 
 
+def create_years_list():
+    """
+    Edits 'years.js', a years structure ready to be presented in app.js
+    as user's last-4-years filter choices.
+    """
+    acc_years = []
+    year_col = db.session.query(distinct(func.substr(Marker.created, 1, 4)))
+
+    for year in year_col:
+        y = str(year)[3:7]
+        if y not in acc_years:
+            acc_years.append(y)
+
+    acc_years_dict = OrderedDict()
+
+    for i, year in enumerate(reversed(acc_years)):
+        if i < 4:
+            acc_years_dict["שנת" + " %s" % year] = ["01/01/%s" % year, "31/12/%s" % year]
+    with open('static/js/years.js', 'w') as outfile:
+        outfile.write("var ACCYEARS = ")
+        json.dump(acc_years_dict, outfile, encoding='utf-8')
+        outfile.write(";\n")
+
+create_years_list()
+
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s')
     ReadDictionaries()
     app.run(debug=True)
 
 
+
diff --git a/process.py b/process.py
--- a/process.py
+++ b/process.py
@@ -14,7 +14,6 @@
 import localization
 import re
 from datetime import datetime
-from collections import OrderedDict
 
 directories_not_processes = {}
 
@@ -42,11 +41,9 @@
     VEHICLES: "VehData.csv"
 }
 
-acc_years = []
 coordinates_converter = ItmToWGS84()
 app = init_flask(__name__)
 db = SQLAlchemy(app)
-acc_years = []
 
 
 def get_street(settlement_sign, street_sign, streets):
@@ -197,23 +194,7 @@ def get_data_value(value):
     return int(value) if value else -1
 
 
-def create_years_list():
-    """
-    Edits 'years.js', a years structure ready to be presented in app.js
-    as user's last-4-years filter choices.
-    """
-    acc_years_dict = OrderedDict()
-    for i, year in enumerate(reversed(acc_years)):
-        if i < 4:
-            acc_years_dict["שנת" + " %s" % year] = ["01/01/%s" % year, "31/12/%s" % year]
-    with open('static/js/years.js', 'w') as outfile:
-        outfile.write("var ACCYEARS = ")
-        json.dump(acc_years_dict, outfile, encoding='utf-8')
-        outfile.write(";\n")
-
-
 def import_accidents(provider_code, accidents, streets, roads, **kwargs):
-    global acc_years
     print("reading accidents from file %s" % (accidents.name(),))
     for accident in accidents:
         if field_names.x_coordinate not in accident or field_names.y_coordinate not in accident:
@@ -224,8 +205,6 @@ def import_accidents(provider_code, accidents, streets, roads, **kwargs):
         else:
             lng, lat = None, None   # Must insert everything to avoid foreign key failure
         main_street, secondary_street = get_streets(accident, streets)
-        if accident[field_names.accident_year] not in acc_years:
-            acc_years.append(accident[field_names.accident_year])
 
         marker = {
             "id": int(accident[field_names.id]),
@@ -430,7 +409,6 @@ def main():
     failed = ["{0}: {1}".format(directory, fail_reason) for directory, fail_reason in
               directories_not_processes.iteritems()]
     print("finished processing all directories, except: %s" % "\n".join(failed))
-    create_years_list()
 
 if __name__ == "__main__":
     main()


Test patch: