Repository: ibi-group/datatools-server
Issue: MTC sync fails (if not first sync)

## Observed behavior

MTC sync fails if not attempting sync for the first time. This is because the code attempts to create a new feed source each time sync is initiated and hits a duplicate ID error in the MongoDB database when re-creating a feed that already exists.

## Steps to reproduce the problem

Create project. Click MTC sync. Wait. Click MTC sync again => fail.

## Any special notes on configuration used

MTC extension should be enabled.

## Version of datatools-server and datatools-ui if applicable (exact commit hash or branch name)

Current dev branches.


Changed files:
src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java

diff --git a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java
index 822de30c..5b7eb075 100644
--- a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java
+++ b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java
@@ -22,10 +22,20 @@ import java.lang.reflect.Field;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.Collection;
 
 import static com.conveyal.datatools.manager.models.ExternalFeedSourceProperty.constructId;
 
 /**
+ * This class implements the {@link ExternalFeedResource} interface for the MTC RTD database list of carriers (transit
+ * operators) and allows the Data Tools application to read and sync the list of carriers to a set of feed sources for a
+ * given project.
+ *
+ * This is generally intended as an initialization step to importing feed sources into a project; however, it should
+ * support subsequent sync requests (e.g., if new agencies are expected in the external feed resource, syncing should
+ * import those OR if feed properties are expected to have changed in the external feed resource, they should be updated
+ * accordingly in Data Tools).
+ *
  * Created by demory on 3/30/16.
  */
 public class MtcFeedResource implements ExternalFeedResource {
@@ -48,11 +58,15 @@ public class MtcFeedResource implements ExternalFeedResource {
         return RESOURCE_TYPE;
     }
 
+    /**
+     * Fetch the list of feeds from the MTC endpoint, create any feed sources that do not match on agencyID, and update
+     * the external feed source properties.
+     */
     @Override
     public void importFeedsForProject(Project project, String authHeader) throws IOException, IllegalAccessException {
         URL url;
         ObjectMapper mapper = new ObjectMapper();
-        // single list from MTC
+        // A single list of feeds is returned from the MTC Carrier endpoint.
         try {
             url = new URL(rtdApi + "/Carrier");
         } catch(MalformedURLException ex) {
@@ -61,76 +75,54 @@ public class MtcFeedResource implements ExternalFeedResource {
         }
 
         try {
-            HttpURLConnection con = (HttpURLConnection) url.openConnection();
-
-            // optional default is GET
-            con.setRequestMethod("GET");
-
+            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
             //add request header
-            con.setRequestProperty("User-Agent", "User-Agent");
-
+            conn.setRequestProperty("User-Agent", "User-Agent");
             // add auth header
-            LOG.info("authHeader="+authHeader);
-            con.setRequestProperty("Authorization", authHeader);
-
-            int responseCode = con.getResponseCode();
-            LOG.info("Sending 'GET' request to URL : " + url);
-            LOG.info("Response Code : " + responseCode);
-
-            BufferedReader in = new BufferedReader(
-                    new InputStreamReader(con.getInputStream()));
-            String inputLine;
-            StringBuffer response = new StringBuffer();
+            conn.setRequestProperty("Authorization", authHeader);
 
-            while ((inputLine = in.readLine()) != null) {
-                response.append(inputLine);
-            }
-            in.close();
-
-            String json = response.toString();
-            RtdCarrier[] results = mapper.readValue(json, RtdCarrier[].class);
-            for (int i = 0; i < results.length; i++) {
-                //                    String className = "RtdCarrier";
-                //                    Object car = Class.forName(className).newInstance();
-                RtdCarrier car = results[i];
-                //LOG.info("car id=" + car.AgencyId + " name=" + car.AgencyName);
+            LOG.info("Sending 'GET' request to URL : {}", url);
+            LOG.info("Response Code : {}", conn.getResponseCode());
 
+            RtdCarrier[] carriers = mapper.readValue(conn.getInputStream(), RtdCarrier[].class);
+            Collection<FeedSource> projectFeedSources = project.retrieveProjectFeedSources();
+            // Iterate over carriers found in response and update properties. Also, create a feed source for any carriers
+            // found in the response that do not correspond to an agency ID found in the external feed source properties.
+            for (int i = 0; i < carriers.length; i++) {
+                RtdCarrier carrier = carriers[i];
                 FeedSource source = null;
 
-                // check if a FeedSource with this AgencyId already exists
-                for (FeedSource existingSource : project.retrieveProjectFeedSources()) {
+                // Check if a FeedSource with this AgencyId already exists.
+                for (FeedSource existingSource : projectFeedSources) {
                     ExternalFeedSourceProperty agencyIdProp;
-                    agencyIdProp = Persistence.externalFeedSourceProperties.getById(constructId(existingSource, this.getResourceType(),
-                                                                                                AGENCY_ID_FIELDNAME));
-                    if (agencyIdProp != null && agencyIdProp.value != null && agencyIdProp.value.equals(car.AgencyId)) {
-                        //LOG.info("already exists: " + car.AgencyId);
+                    String propertyId = constructId(existingSource, this.getResourceType(), AGENCY_ID_FIELDNAME);
+                    agencyIdProp = Persistence.externalFeedSourceProperties.getById(propertyId);
+                    if (agencyIdProp != null && agencyIdProp.value != null && agencyIdProp.value.equals(carrier.AgencyId)) {
                         source = existingSource;
                     }
                 }
-
-                String feedName;
-                if (car.AgencyName != null) {
-                    feedName = car.AgencyName;
-                } else if (car.AgencyShortName != null) {
-                    feedName = car.AgencyShortName;
-                } else {
-                    feedName = car.AgencyId;
-                }
-
+                // Feed source does not exist. Create one using carrier properties.
                 if (source == null) {
+                    // Derive the name from carrier properties found in response.
+                    String feedName = carrier.AgencyName != null
+                        ? carrier.AgencyName
+                        : carrier.AgencyShortName != null
+                            ? carrier.AgencyShortName
+                            : carrier.AgencyId;
+                    // Create new feed source to store in application database.
                     source = new FeedSource(feedName);
+                    source.projectId = project.id;
+                    LOG.info("Creating feed source {} from carrier response. (Did not previously exist.)", feedName);
+                    // Store the feed source if it does not already exist.
+                    Persistence.feedSources.create(source);
                 }
-                else source.name = feedName;
-
-                source.projectId = project.id;
-                // Store the feed source.
-                Persistence.feedSources.create(source);
-
-                // create / update the properties
+                // TODO: Does any property on the feed source need to be updated from the carrier (e.g., name).
 
-                for(Field carrierField : car.getClass().getDeclaredFields()) {
+                // Create / update the properties
+                LOG.info("Updating props for {}", source.name);
+                for(Field carrierField : carrier.getClass().getDeclaredFields()) {
                     String fieldName = carrierField.getName();
-                    String fieldValue = carrierField.get(car) != null ? carrierField.get(car).toString() : null;
+                    String fieldValue = carrierField.get(carrier) != null ? carrierField.get(carrier).toString() : null;
                     ExternalFeedSourceProperty prop = new ExternalFeedSourceProperty(source, this.getResourceType(), fieldName, fieldValue);
                     if (Persistence.externalFeedSourceProperties.getById(prop.id) == null) {
                         Persistence.externalFeedSourceProperties.create(prop);