private static List<Tick> buildEmptyTicks(ZonedDateTime beginTime, ZonedDateTime endTime, int duration) {

        List<Tick> emptyTicks = new ArrayList<>();

        Duration tickDuration = Duration.ofSeconds(duration);
        ZonedDateTime tickEndTime = beginTime;
        do {
            tickEndTime = tickEndTime.plus(tickDuration);
            emptyTicks.add(new BaseTick(tickDuration, tickEndTime));
        } while (tickEndTime.isBefore(endTime));

        return emptyTicks;
    }

private static List < Tick > buildEmptyTicks ( ZonedDateTime beginTime , ZonedDateTime endTime , int duration ) { List < Tick > emptyTicks = new ArrayList <> ( ) ; Duration tickDuration = Duration . ofSeconds ( duration ) ; ZonedDateTime tickEndTime = beginTime ; do { tickEndTime = tickEndTime . plus ( tickDuration ) ; emptyTicks . add ( new BaseTick ( tickDuration , tickEndTime ) ) ; } while ( tickEndTime . isBefore ( endTime ) ) ; return emptyTicks ; }

Builds a list of empty ticks.
@param beginTime the begin time of the whole period
@param endTime the end time of the whole period
@param duration the tick duration (in seconds)
@return the list of empty ticks


void destroy() {
    destroyed = true;

    for (Map<String, ManagedObservable<?>> observableMap : groupMap.values()) {
      for (ManagedObservable<?> managedObservable : observableMap.values()) {
        managedObservable.cancel();
      }
      observableMap.clear();
    }
    groupMap.clear();
  }

void destroy ( ) { destroyed = true ; for ( Map < String , ManagedObservable < ? > > observableMap : groupMap . values ( ) ) { for ( ManagedObservable < ? > managedObservable : observableMap . values ( ) ) { managedObservable . cancel ( ) ; } observableMap . clear ( ) ; } groupMap . clear ( ) ; }

Cancels all subscriptions and releases references to Observables and Observers. No more
Observables can be added to this group after it has been destroyed and it becomes unusable.

Cancels all subscriptions and releases references to Observables and Observers . No more Observables can be added to this group after it has been destroyed and it becomes unusable .


public void onSaveInstanceState(Bundle outState) {
    hasSavedState = true;
    outState.putParcelable(KEY_STATE, new State(observableManager.id(), group.id()));
  }

public void onSaveInstanceState ( Bundle outState ) { hasSavedState = true ; outState . putParcelable ( KEY_STATE , new State ( observableManager . id ( ) , group . id ( ) ) ) ; }

Call this method from your Activity or Fragment's onSaveInstanceState method

Call this method from your Activity or Fragment s onSaveInstanceState method


final void signalEvent(WatchEvent.Kind<Path> kind, Path context) {
    post(new Event<>(kind, 1, context));
    signal();
  }

final void signalEvent ( WatchEvent . Kind < Path > kind , Path context ) { post ( new Event <> ( kind , 1 , context ) ) ; signal ( ) ; }


public CompletableFuture<Void> watchAsync(Executor executor) {
    return CompletableFuture.supplyAsync(
        () -> {
          watch();
          return null;
        },
        executor);
  }

public CompletableFuture < Void > watchAsync ( Executor executor ) { return CompletableFuture . supplyAsync ( ( ) -> { watch ( ) ; return null ; } , executor ) ; }

Asynchronously watch the directories.

@param executor the executor to use to watch asynchronously


private void register(Path directory, boolean useFileTreeModifier) throws IOException {
    logger.debug("Registering [{}].", directory);
    Watchable watchable = isMac ? new WatchablePath(directory) : directory;
    WatchEvent.Modifier[] modifiers =
        useFileTreeModifier
            ? new WatchEvent.Modifier[] {ExtendedWatchEventModifier.FILE_TREE}
            : new WatchEvent.Modifier[] {};
    WatchEvent.Kind<?>[] kinds =
        new WatchEvent.Kind<?>[] {ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY};
    WatchKey watchKey = watchable.register(watchService, kinds, modifiers);
    keyRoots.put(watchKey, directory);
  }

private
void
register
(
Path
directory
,
boolean
useFileTreeModifier
)
throws
IOException
{
logger
.
debug
(
"Registering [{}]."
,
directory
)
;
Watchable
watchable
=
isMac
new
WatchablePath
(
directory
)
:
directory
;
WatchEvent
.
Modifier
[
]
modifiers
=
useFileTreeModifier
new
WatchEvent
.
Modifier
[
]
{
ExtendedWatchEventModifier
.
FILE_TREE
}
:
new
WatchEvent
.
Modifier
[
]
{
}
;
WatchEvent
.
Kind
<
>
[
]
kinds
=
new
WatchEvent
.
Kind
<
>
[
]
{
ENTRY_CREATE
,
ENTRY_DELETE
,
ENTRY_MODIFY
}
;
WatchKey
watchKey
=
watchable
.
register
(
watchService
,
kinds
,
modifiers
)
;
keyRoots
.
put
(
watchKey
,
directory
)
;
}

Internal method to be used by registerAll


private ClassLoader getClassLoader()
        throws MojoExecutionException
    {
        List<Path> classpathURLs = new ArrayList<>();
        this.addRelevantPluginDependenciesToClasspath( classpathURLs );
        this.addRelevantProjectDependenciesToClasspath( classpathURLs );
        this.addAdditionalClasspathElements( classpathURLs );

        try
        {
            return LoaderFinder.find( classpathURLs, mainClass );
        }
        catch ( NullPointerException | IOException e )
        {
            throw new MojoExecutionException( e.getMessage(), e );
        }

    }

Set up a classloader for the execution of the main class.

@return the classloader
@throws MojoExecutionException if a problem happens

Set up a classloader for the execution of the main class .


private void addRelevantPluginDependenciesToClasspath( List<Path> path )
        throws MojoExecutionException
    {
        if ( hasCommandlineArgs() )
        {
            arguments = parseCommandlineArgs();
        }

        for ( Artifact classPathElement : this.determineRelevantPluginDependencies() )
        {
            getLog().debug( "Adding plugin dependency artifact: " + classPathElement.getArtifactId()
                + " to classpath" );
            path.add( classPathElement.getFile().toPath() );
        }
    }

Add any relevant project dependencies to the classpath. Indirectly takes includePluginDependencies and
ExecutableDependency into consideration.

@param path classpath of {@link java.net.URL} objects
@throws MojoExecutionException if a problem happens

Add any relevant project dependencies to the classpath . Indirectly takes includePluginDependencies and ExecutableDependency into consideration .


private void addRelevantProjectDependenciesToClasspath( List<Path> path )
        throws MojoExecutionException
    {
        if ( this.includeProjectDependencies )
        {
            getLog().debug( "Project Dependencies will be included." );

            List<Artifact> artifacts = new ArrayList<>();
            List<Path> theClasspathFiles = new ArrayList<>();

            collectProjectArtifactsAndClasspath( artifacts, theClasspathFiles );

            for ( Path classpathFile : theClasspathFiles )
            {
                getLog().debug( "Adding to classpath : " + classpathFile );
                path.add( classpathFile );
            }

            for ( Artifact classPathElement : artifacts )
            {
                getLog().debug( "Adding project dependency artifact: " + classPathElement.getArtifactId()
                    + " to classpath" );
                path.add( classPathElement.getFile().toPath() );
            }
        }
        else
        {
            getLog().debug( "Project Dependencies will be excluded." );
        }

    }

Add any relevant project dependencies to the classpath. Takes includeProjectDependencies into consideration.

@param path classpath of {@link java.net.URL} objects
@throws MojoExecutionException if a problem happens

Add any relevant project dependencies to the classpath . Takes includeProjectDependencies into consideration .


private Set<Artifact> resolveExecutableDependencies( Artifact executablePomArtifact )
        throws MojoExecutionException
    {

        Set<Artifact> executableDependencies = new LinkedHashSet<>();
        try
        {
            ProjectBuildingRequest buildingRequest = getSession().getProjectBuildingRequest();

            MavenProject executableProject =
                this.projectBuilder.build( executablePomArtifact, buildingRequest ).getProject();

            for ( ArtifactResult artifactResult : dependencyResolver.resolveDependencies( buildingRequest, executableProject.getModel(), null ) )
            {
                executableDependencies.add( artifactResult.getArtifact() );
            }
        }
        catch ( Exception ex )
        {
            throw new MojoExecutionException( "Encountered problems resolving dependencies of the executable "
                + "in preparation for its execution.", ex );
        }

        return executableDependencies;
    }

Resolve the executable dependencies for the specified project

@param executablePomArtifact the project's POM
@return a set of Artifacts
@throws MojoExecutionException if a failure happens


protected Artifact findExecutableArtifact()
        throws MojoExecutionException
    {
        // ILimitedArtifactIdentifier execToolAssembly = this.getExecutableToolAssembly();

        Artifact executableTool = null;
        for ( Artifact pluginDep : this.pluginDependencies )
        {
            if ( this.executableDependency.matches( pluginDep ) )
            {
                executableTool = pluginDep;
                break;
            }
        }

        if ( executableTool == null )
        {
            throw new MojoExecutionException( "No dependency of the plugin matches the specified executableDependency."
                + "  Specified executableToolAssembly is: " + executableDependency.toString() );
        }

        return executableTool;
    }

Examine the plugin dependencies to find the executable artifact.

@return an artifact which refers to the actual executable tool (not a POM)
@throws MojoExecutionException if no executable artifact was found

Examine the plugin dependencies to find the executable artifact .


private void handleWorkingDirectory()
        throws MojoExecutionException
    {
        if ( workingDirectory == null )
        {
            workingDirectory = basedir;
        }

        if ( !workingDirectory.exists() )
        {
            getLog().debug( "Making working directory '" + workingDirectory.getAbsolutePath() + "'." );
            if ( !workingDirectory.mkdirs() )
            {
                throw new MojoExecutionException( "Could not make working directory: '"
                    + workingDirectory.getAbsolutePath() + "'" );
            }
        }
    }

private
void
handleWorkingDirectory
(
)
throws
MojoExecutionException
{
if
(
workingDirectory
==
)
{
workingDirectory
=
basedir
;
}
if
(
!
workingDirectory
.
exists
(
)
)
{
getLog
(
)
.
debug
(
"Making working directory '"
+
workingDirectory
.
getAbsolutePath
(
)
+
"'."
)
;
if
(
!
workingDirectory
.
mkdirs
(
)
)
{
throw
new
MojoExecutionException
(
"Could not make working directory: '"
+
workingDirectory
.
getAbsolutePath
(
)
+
"'"
)
;
}
}
}

This is a convenient method to make the execute method a little bit more readable. It will define the
workingDirectory to be the baseDir in case of workingDirectory is null. If the workingDirectory does not exist it
will created.

@throws MojoExecutionException

This is a convenient method to make the execute method a little bit more readable . It will define the workingDirectory to be the baseDir in case of workingDirectory is null . If the workingDirectory does not exist it will created .


public LocalDate getCalendarServiceRangeStart() {

        int startDate = 0;
        for (Service service : feed.services.values()) {
            if (service.calendar == null)
                continue;
//            if (startDate == 0 || service.calendar.start_date < startDate) {
//                startDate = service.calendar.start_date;
//            }
        }
        if (startDate == 0)
            return null;

        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
        return LocalDate.parse(String.valueOf(startDate), formatter);
    }

returns null for schedules without calendar service schedules


public Rectangle2D getBounds () {
        Rectangle2D ret = null;

        for (Stop stop : feed.stops.values()) {

            // skip over stops that don't have any stop times
            if (!feed.stopCountByStopTime.containsKey(stop.stop_id)) {
                continue;
            }
            if (ret == null) {
                ret = new Rectangle2D.Double(stop.stop_lon, stop.stop_lat, 0, 0);
            }
            else {
                ret.add(new Point2D.Double(stop.stop_lon, stop.stop_lat));
            }
        }

        return ret;
    }

Get the bounding box of this GTFS feed.
We use a Rectangle2D rather than a Geotools envelope because GTFS is always in WGS 84.
Note that stops do not have agencies in GTFS.

Get the bounding box of this GTFS feed . We use a Rectangle2D rather than a Geotools envelope because GTFS is always in WGS 84 . Note that stops do not have agencies in GTFS .


private boolean checkDistanceAndTime (double distanceMeters, double travelTimeSeconds, StopTime stopTime) {
        boolean good = true;
        // TODO Use Epsilon for very tiny travel e.g. < 5 meters
        if (distanceMeters == 0) {
            registerError(stopTime, TRAVEL_DISTANCE_ZERO);
            good = false;
        }
        if (travelTimeSeconds < 0) {
            registerError(stopTime, TRAVEL_TIME_NEGATIVE, travelTimeSeconds);
            good = false;
        } else if (travelTimeSeconds == 0) {
            // Only register the travel time zero error if not all travel times are rounded. Otherwise, hold onto the
            // error in the travelTimeZeroErrors collection until the completion of this validator.
            if (!allTravelTimesAreRounded) registerError(stopTime, TRAVEL_TIME_ZERO);
            else travelTimeZeroErrors.add(createUnregisteredError(stopTime, TRAVEL_TIME_ZERO));
            good = false;
        }
        return good;
    }

This just pulls some of the range checking logic out of the main trip checking loop so it's more readable.
@return true if all values are OK

This just pulls some of the range checking logic out of the main trip checking loop so it s more readable .


private TableLoadResult load (Table table) {
        // This object will be returned to the caller to summarize the contents of the table and any errors.
        TableLoadResult tableLoadResult = new TableLoadResult();
        int initialErrorCount = errorStorage.getErrorCount();
        try {
            tableLoadResult.rowCount = loadInternal(table);
            tableLoadResult.fileSize = getTableSize(table);
            LOG.info(String.format("loaded in %d %s records", tableLoadResult.rowCount, table.name));
        } catch (Exception ex) {
            LOG.error("Fatal error loading table", ex);
            tableLoadResult.fatalException = ex.toString();
            // Rollback connection so that fatal exception does not impact loading of other tables.
            try {
                connection.rollback();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        } finally {
            // Explicitly delete the tmp file now that load is finished (either success or failure).
            // Otherwise these multi-GB files clutter the drive.
            if (tempTextFile != null) {
                tempTextFile.delete();
            }
        }
        int finalErrorCount = errorStorage.getErrorCount();
        tableLoadResult.errorCount = finalErrorCount - initialErrorCount;
        return tableLoadResult;
    }

This wraps the main internal table loader method to catch exceptions and figure out how many errors happened.

This wraps the main internal table loader method to catch exceptions and figure out how many errors happened .


private int getTableSize(Table table) {
        ZipEntry zipEntry = zip.getEntry(table.name + ".txt");
        if (zipEntry == null) return 0;
        return (int) zipEntry.getSize();
    }

private int getTableSize ( Table table ) { ZipEntry zipEntry = zip . getEntry ( table . name + ".txt" ) ; if ( zipEntry == null ) return 0 ; return ( int ) zipEntry . getSize ( ) ; }

Get the uncompressed file size in bytes for the specified GTFS table.

Get the uncompressed file size in bytes for the specified GTFS table .


private void setFieldToNull(boolean postgresText, String[] transformedStrings, int fieldIndex, Field field) {
        if (postgresText) transformedStrings[fieldIndex + 1] = POSTGRES_NULL_TEXT;
        // Adjust parameter index by two: indexes are one-based and the first one is the CSV line number.
        else try {
            // LOG.info("setting {} index to null", fieldIndex + 2);
            field.setNull(insertStatement, fieldIndex + 2);
        } catch (SQLException e) {
            e.printStackTrace();
            // FIXME: store error here? It appears that an exception should only be thrown if the type value is invalid,
            // the connection is closed, or the index is out of bounds. So storing an error may be unnecessary.
        }
    }

Sets field to null in statement or string array depending on whether postgres is being used.

Sets field to null in statement or string array depending on whether postgres is being used .


public SnapshotResult copyTables() {
        // This result object will be returned to the caller to summarize the feed and report any critical errors.
        SnapshotResult result = new SnapshotResult();

        try {
            long startTime = System.currentTimeMillis();
            // We get a single connection object and share it across several different methods.
            // This ensures that actions taken in one method are visible to all subsequent SQL statements.
            // If we create a schema or table on one connection, then access it in a separate connection, we have no
            // guarantee that it exists when the accessing statement is executed.
            connection = dataSource.getConnection();
            // Generate a unique prefix that will identify this feed.
            this.tablePrefix = randomIdString();
            result.uniqueIdentifier = tablePrefix;
            // Create entry in snapshots table.
            registerSnapshot();
            // Include the dot separator in the table prefix.
            // This allows everything to work even when there's no prefix.
            this.tablePrefix += ".";
            // Copy each table in turn
            // FIXME: NO non-fatal exception errors are being captured during copy operations.
            result.agency = copy(Table.AGENCY, true);
            result.calendar = copy(Table.CALENDAR, true);
            result.calendarDates = copy(Table.CALENDAR_DATES, true);
            result.fareAttributes = copy(Table.FARE_ATTRIBUTES, true);
            result.fareRules = copy(Table.FARE_RULES, true);
            result.feedInfo = copy(Table.FEED_INFO, true);
            result.frequencies = copy(Table.FREQUENCIES, true);
            result.routes = copy(Table.ROUTES, true);
            // FIXME: Find some place to store errors encountered on copy for patterns and pattern stops.
            copy(Table.PATTERNS, true);
            copy(Table.PATTERN_STOP, true);
            // see method comments fo why different logic is needed for this table
            result.scheduleExceptions = createScheduleExceptionsTable();
            result.shapes = copy(Table.SHAPES, true);
            result.stops = copy(Table.STOPS, true);
            // TODO: Should we defer index creation on stop times?
            // Copying all tables for STIF w/ stop times idx = 156 sec; w/o = 28 sec
            // Other feeds w/ stop times AC Transit = 3 sec; Brooklyn bus =
            result.stopTimes = copy(Table.STOP_TIMES, true);
            result.transfers = copy(Table.TRANSFERS, true);
            result.trips = copy(Table.TRIPS, true);
            result.completionTime = System.currentTimeMillis();
            result.loadTimeMillis = result.completionTime - startTime;
            LOG.info("Copying tables took {} sec", (result.loadTimeMillis) / 1000);
        } catch (Exception ex) {
            // Note: Exceptions that occur during individual table loads are separately caught and stored in
            // TableLoadResult.
            LOG.error("Exception while creating snapshot: {}", ex.toString());
            ex.printStackTrace();
            result.fatalException = ex.toString();
        }
        return result;
    }

Copy primary entity tables as well as Pattern and PatternStops tables.

Copy primary entity tables as well as Pattern and PatternStops tables .


private boolean tableExists(String namespace, String tableName) {
        // Preempt SQL check with null check of either namespace or table name.
        if (namespace == null || tableName == null) return false;
        try {
            // This statement is postgres-specific.
            PreparedStatement tableExistsStatement = connection.prepareStatement(
                "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?)"
            );
            tableExistsStatement.setString(1, namespace);
            tableExistsStatement.setString(2, tableName);
            ResultSet resultSet = tableExistsStatement.executeQuery();
            resultSet.next();
            return resultSet.getBoolean(1);
        } catch (SQLException e) {
            e.printStackTrace();
            return false;
        }
    }

Helper method to determine if a table exists within a namespace.

Helper method to determine if a table exists within a namespace .


private void addEditorSpecificFields(Connection connection, String tablePrefix, Table table) throws SQLException {
        LOG.info("Adding any missing columns for {}", tablePrefix + table.name);
        Statement statement = connection.createStatement();
        for (Field field : table.editorFields()) {
            // The following statement requires PostgreSQL 9.6+.
            String addColumnSql = String.format("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s",
                    tablePrefix + table.name,
                    field.name,
                    field.getSqlTypeName());
            LOG.info(addColumnSql);
            statement.execute(addColumnSql);
        }
    }

Add columns for any required, optional, or editor fields that don't already exist as columns on the table.
This method contains a SQL statement that requires PostgreSQL 9.6+.

Add columns for any required optional or editor fields that don t already exist as columns on the table . This method contains a SQL statement that requires PostgreSQL 9 . 6 + .


private void registerSnapshot () {
        try {
            // We cannot simply insert into the feeds table because if we are creating an empty snapshot (to create/edit
            // a GTFS feed from scratch), the feed registry table will not exist.
            // TODO copy over feed_id and feed_version from source namespace?
            // TODO: Record total snapshot processing time?
            createFeedRegistryIfNotExists(connection);
            createSchema(connection, tablePrefix);
            PreparedStatement insertStatement = connection.prepareStatement(
                    "insert into feeds values (?, null, null, null, null, null, current_timestamp, ?)");
            insertStatement.setString(1, tablePrefix);
            insertStatement.setString(2, feedIdToSnapshot);
            insertStatement.execute();
            connection.commit();
            LOG.info("Created new snapshot namespace: {}", insertStatement);
        } catch (Exception ex) {
            LOG.error("Exception while registering snapshot namespace in feeds table", ex);
            DbUtils.closeQuietly(connection);
        }
    }

Add a line to the list of loaded feeds to record the snapshot and which feed the snapshot replicates.

Add a line to the list of loaded feeds to record the snapshot and which feed the snapshot replicates .


public String getId() {
        StringBuilder sb = new StringBuilder();
         sb.append(trip_id);
         sb.append('_');
         sb.append(convertToGtfsTime(start_time));
         sb.append("_to_");
         sb.append(convertToGtfsTime(end_time));
         sb.append("_every_");
         sb.append(String.format("%dm%02ds", headway_secs / 60, headway_secs % 60));
         if (exact_times == 1) sb.append("_exact");
         return sb.toString();
     }

public String getId ( ) { StringBuilder sb = new StringBuilder ( ) ; sb . append ( trip_id ) ; sb . append ( ' ' ) ; sb . append ( convertToGtfsTime ( start_time ) ) ; sb . append ( "_to_" ) ; sb . append ( convertToGtfsTime ( end_time ) ) ; sb . append ( "_every_" ) ; sb . append ( String . format ( "%dm%02ds" , headway_secs / 60 , headway_secs % 60 ) ) ; if ( exact_times == 1 ) sb . append ( "_exact" ) ; return sb . toString ( ) ; }

Frequency entries have no ID in GTFS so we define one based on the fields in the frequency entry.

It is possible to have two identical frequency entries in the GTFS, which under our understanding of the situation
would mean that two sets of vehicles were randomly running the same trip at the same headway, but uncorrelated
with each other, which is almost certain to be an error.

Frequency entries have no ID in GTFS so we define one based on the fields in the frequency entry .


public int executeRemaining() throws SQLException {
        if (currentBatchSize > 0) {
            totalRecordsProcessed += currentBatchSize;
            preparedStatement.executeBatch();
            currentBatchSize = 0;
        }
        // Avoid reuse, signal that this was cleanly closed.
        preparedStatement = null;
        LOG.info(String.format("Processed %d %s records", totalRecordsProcessed, recordType));
        return totalRecordsProcessed;
    }

Execute any remaining statements and return the total records processed.

Execute any remaining statements and return the total records processed .


private void cleanUpZipFile() {
        long startTime = System.currentTimeMillis();
        // Define ZIP File System Properties in HashMap
        Map<String, String> zip_properties = new HashMap<>();
        // We want to read an existing ZIP File, so we set this to False
        zip_properties.put("create", "false");

        // Specify the path to the ZIP File that you want to read as a File System
        // (File#toURI allows this to work across different operating systems, including Windows)
        URI zip_disk = URI.create("jar:" + new File(outFile).toURI());

        // Create ZIP file System
        try (FileSystem fileSystem = FileSystems.newFileSystem(zip_disk, zip_properties)) {
            // Get the Path inside ZIP File to delete the ZIP Entry
            for (String fileName : emptyTableList) {
                Path filePath = fileSystem.getPath(fileName);
                // Execute Delete
                Files.delete(filePath);
                LOG.info("Empty file {} successfully deleted", fileName);
            }
        } catch (IOException e) {
            LOG.error("Could not remove empty zip files");
            e.printStackTrace();
        }
        LOG.info("Deleted {} empty files in {} ms", emptyTableList.size(), System.currentTimeMillis() - startTime);
    }

Removes any empty zip files from the final zip file.

Removes any empty zip files from the final zip file .


private TableLoadResult export (Table table, String filterSql) {
        long startTime = System.currentTimeMillis();
        TableLoadResult tableLoadResult = new TableLoadResult();
        try {
            if (filterSql == null) {
                throw new IllegalArgumentException("filterSql argument cannot be null");
            } else {
                // Surround filter SQL in parentheses.
                filterSql = String.format("(%s)", filterSql);
            }

            // Create entry for table
            String textFileName = table.name + ".txt";
            ZipEntry zipEntry = new ZipEntry(textFileName);
            zipOutputStream.putNextEntry(zipEntry);

            // don't let CSVWriter close the stream when it is garbage-collected
            OutputStream protectedOut = new FilterOutputStream(zipOutputStream);
            String copySql = String.format("copy %s to STDOUT DELIMITER ',' CSV HEADER", filterSql);
            LOG.info(copySql);
            // Our connection pool wraps the Connection objects, so we need to unwrap the Postgres connection interface.
            CopyManager copyManager = new CopyManager(connection.unwrap(BaseConnection.class));
            tableLoadResult.rowCount = (int) copyManager.copyOut(copySql, protectedOut);
            if (tableLoadResult.rowCount == 0) {
                // If no rows were exported, keep track of table name for later removal.
                emptyTableList.add(textFileName);
            }
            zipOutputStream.closeEntry();
            LOG.info("Copied {} {} in {} ms.", tableLoadResult.rowCount, table.name, System.currentTimeMillis() - startTime);
            connection.commit();
        } catch (SQLException | IOException | IllegalArgumentException e) {
            // Rollback connection so that fatal exception does not impact loading of other tables.
            try {
                connection.rollback();
            } catch (SQLException ex) {
                ex.printStackTrace();
            }
            tableLoadResult.fatalException = e.toString();
            LOG.error("Exception while exporting tables", e);
        }
        return tableLoadResult;
    }

Export a table to the zipOutputStream to be written to the GTFS.

Export a table to the zipOutputStream to be written to the GTFS .


public void registerError(Entity entity, NewGTFSErrorType errorType) {
        errorStorage.storeError(NewGTFSError.forEntity(entity, errorType));
    }

public void registerError ( Entity entity , NewGTFSErrorType errorType ) { errorStorage . storeError ( NewGTFSError . forEntity ( entity , errorType ) ) ; }

Store an error that affects a single line of a single table. Wraps the underlying error factory method.

Store an error that affects a single line of a single table . Wraps the underlying error factory method .


public void registerError(Entity entity, NewGTFSErrorType errorType, Object badValue) {
        errorStorage.storeError(NewGTFSError.forEntity(entity, errorType).setBadValue(badValue.toString()));
    }

public void registerError ( Entity entity , NewGTFSErrorType errorType , Object badValue ) { errorStorage . storeError ( NewGTFSError . forEntity ( entity , errorType ) . setBadValue ( badValue . toString ( ) ) ) ; }

Store an error that affects a single line of a single table.
Add a bad value to it.

Store an error that affects a single line of a single table . Add a bad value to it .


public static FeedLoadResult export (String feedId, String outFile, DataSource dataSource, boolean fromEditor) {
        JdbcGtfsExporter exporter = new JdbcGtfsExporter(feedId, outFile, dataSource, fromEditor);
        FeedLoadResult result = exporter.exportTables();
        return result;
    }

public static FeedLoadResult export ( String feedId , String outFile , DataSource dataSource , boolean fromEditor ) { JdbcGtfsExporter exporter = new JdbcGtfsExporter ( feedId , outFile , dataSource , fromEditor ) ; FeedLoadResult result = exporter . exportTables ( ) ; return result ; }

Export a feed ID from the database to a zipped GTFS file in the specified export directory.

Export a feed ID from the database to a zipped GTFS file in the specified export directory .


public static FeedLoadResult load (String filePath, DataSource dataSource) {
        JdbcGtfsLoader loader = new JdbcGtfsLoader(filePath, dataSource);
        FeedLoadResult result = loader.loadTables();
        return result;
    }

public static FeedLoadResult load ( String filePath , DataSource dataSource ) { JdbcGtfsLoader loader = new JdbcGtfsLoader ( filePath , dataSource ) ; FeedLoadResult result = loader . loadTables ( ) ; return result ; }

Load the GTFS data in the specified file into the given JDBC DataSource.

About loading from an object on S3:
We've got a problem with keeping the identifier on S3 the same as the identifier in the database.
Question: do we really want to load the feed from S3 or do we just want to move the local file to S3 when done?
i.e. user uploads to the datatools back end,
The advantage of having the end users upload to s3, then is that only the files currently being processed need
to be held on our backend server, not the whole queue waiting to be processed.
But I think currently we have some hack where the end user gets a token to upload to S3, which seems bad.
Probably we should have end users upload directly to the backend server, which then loads the feed into RDBMS,
uploads the file to S3 with the unique name supplied by gtfs-lib, and deletes the local file.

@return a unique identifier for the newly loaded feed in the database

Load the GTFS data in the specified file into the given JDBC DataSource .


public static ValidationResult validate (String feedId, DataSource dataSource) {
        Feed feed = new Feed(dataSource, feedId);
        ValidationResult result = feed.validate();
        return result;
    }

public static ValidationResult validate ( String feedId , DataSource dataSource ) { Feed feed = new Feed ( dataSource , feedId ) ; ValidationResult result = feed . validate ( ) ; return result ; }

Once a feed has been loaded into the database, examine its contents looking for various problems and errors.

Once a feed has been loaded into the database examine its contents looking for various problems and errors .


public static void delete (String feedId, DataSource dataSource) throws SQLException, InvalidNamespaceException {
        LOG.info("Deleting all tables (dropping schema) for {} feed namespace.", feedId);
        Connection connection = null;
        try {
            connection = dataSource.getConnection();
            ensureValidNamespace(feedId);
            // Mark entry in feeds table as deleted.
            String deleteFeedEntrySql = "update feeds set deleted = true where namespace = ?";
            PreparedStatement deleteFeedStatement = connection.prepareStatement(deleteFeedEntrySql);
            deleteFeedStatement.setString(1, feedId);
            deleteFeedStatement.executeUpdate();
            // Drop all tables bearing the feedId namespace.
            // Note: It does not appear to be possible to use prepared statements with "drop schema."
            String dropSchemaSql = String.format("DROP SCHEMA %s CASCADE", feedId);
            Statement statement = connection.createStatement();
            statement.executeUpdate(dropSchemaSql);
            // Commit the changes.
            connection.commit();
        } catch (InvalidNamespaceException | SQLException e) {
            LOG.error(String.format("Could not drop feed for namespace %s", feedId), e);
            throw e;
        } finally {
            if (connection != null) DbUtils.closeQuietly(connection);
        }
    }

Deletes all tables for the specified feed. Simply put, this is a "drop schema" SQL statement called on the feed's
namespace.

Deletes all tables for the specified feed . Simply put this is a drop schema SQL statement called on the feed s namespace .


public static DataSource createDataSource (String url, String username, String password) {
        String characterEncoding = Charset.defaultCharset().toString();
        LOG.debug("Default character encoding: {}", characterEncoding);
        if (!Charset.defaultCharset().equals(StandardCharsets.UTF_8)) {
            // Character encoding must be set to UTF-8 in order for the database connection to work without error.
            // To override default encoding at runtime, run application jar with encoding environment variable set to
            // UTF-8 (or update IDE settings). TODO we should also check that JDBC and the database know to use UTF-8.
            throw new RuntimeException("Your system's default encoding (" + characterEncoding + ") is not supported. Please set it to UTF-8. Example: java -Dfile.encoding=UTF-8 application.jar");
        }
        // ConnectionFactory can handle null username and password (for local host-based authentication)
        ConnectionFactory connectionFactory = new DriverManagerConnectionFactory(url, username, password);
        PoolableConnectionFactory poolableConnectionFactory = new PoolableConnectionFactory(connectionFactory, null);
        GenericObjectPool connectionPool = new GenericObjectPool(poolableConnectionFactory);
        // TODO: set other options on connectionPool?
        connectionPool.setMaxTotal(300);
        connectionPool.setMaxIdle(4);
        connectionPool.setMinIdle(2);
        poolableConnectionFactory.setPool(connectionPool);
        // We also want auto-commit switched off for bulk inserts, and also because fetches are super-slow with
        // auto-commit turned on. Apparently it interferes with result cursors.
        poolableConnectionFactory.setDefaultAutoCommit(false);
        return new PoolingDataSource(connectionPool);
        // We might want already-loaded feeds to be treated as read-only.
        // But we need to call this on the connection, not the connectionSource.
        // connection.setReadOnly();
        // Not sure we need to close cursors - closing the pool-wrapped connection when we're done with it should also close cursors.
        // will this help? https://stackoverflow.com/a/18300252
        // connection.setHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT);
    }

Create an automatically managed pool of database connections to the supplied JDBC database URL.

Creating new database connections is usually an expensive operation,
taking up to seconds to prepare for a transaction that only lasts milliseconds.
The JDBC in Java 7 and 8 support connection pooling, which is critical for performance.
We are using the standard JDBC DataSource interface, with the Apache commons-dbcp implementation.
This should be roughly the same connection pooling used in the Tomcat application server.

Here are some sample database URLs to serve as patterns:
H2_FILE_URL = "jdbc:h2:file:~/test-db"; // H2 memory does not seem faster than H2 file
SQLITE_FILE_URL = "jdbc:sqlite:/Users/abyrd/test-db";
POSTGRES_LOCAL_URL = "jdbc:postgresql://localhost/catalogue";

For local Postgres connections, you can supply a null username and password to use host-based authentication.

Create an automatically managed pool of database connections to the supplied JDBC database URL .


public NewGTFSError addInfo (String key, String value) {
        errorInfo.put(key, value);
        return this;
    }

public NewGTFSError addInfo ( String key , String value ) { errorInfo . put ( key , value ) ; return this ; }

Add a single key-value pair of supplemental info to this error.
@return the modified object (builder pattern that allows adding info to newly created error object without assigning to a variable).

Add a single key - value pair of supplemental info to this error .


public static NewGTFSError forLine (Table table, int lineNumber, NewGTFSErrorType errorType, String badValue) {
        NewGTFSError error = new NewGTFSError(table.getEntityClass(), errorType);
        error.lineNumber = lineNumber;
        error.badValue = badValue;
        return error;
    }

public static NewGTFSError forLine ( Table table , int lineNumber , NewGTFSErrorType errorType , String badValue ) { NewGTFSError error = new NewGTFSError ( table . getEntityClass ( ) , errorType ) ; error . lineNumber = lineNumber ; error . badValue = badValue ; return error ; }

Factory Builder for cases where an entity has not yet been constructed, but we know the line number.

Factory Builder for cases where an entity has not yet been constructed but we know the line number .


public static NewGTFSError forEntity(Entity entity, NewGTFSErrorType errorType) {
        NewGTFSError error = new NewGTFSError(entity.getClass(), errorType);
        error.lineNumber = entity.id;
        error.entityId = entity.getId();
        error.entitySequenceNumber = entity.getSequenceNumber();
        return error;
    }

public static NewGTFSError forEntity ( Entity entity , NewGTFSErrorType errorType ) { NewGTFSError error = new NewGTFSError ( entity . getClass ( ) , errorType ) ; error . lineNumber = entity . id ; error . entityId = entity . getId ( ) ; error . entitySequenceNumber = entity . getSequenceNumber ( ) ; return error ; }

Factory Builder for cases where the entity has already been decoded and an error is discovered during validation


public static NewGTFSError forFeed (NewGTFSErrorType errorType, String badValue) {
        return new NewGTFSError(null, errorType).setBadValue(badValue);
    }

public static NewGTFSError forFeed ( NewGTFSErrorType errorType , String badValue ) { return new NewGTFSError ( null , errorType ) . setBadValue ( badValue ) ; }


public double getPatternSpeed (String pattern_id, LocalDate date, LocalTime from, LocalTime to) {

        List<Trip> trips = getTripsForDate(pattern_id, date);

        return getAverageSpeedForTrips(trips, from, to);
    }

public double getPatternSpeed ( String pattern_id , LocalDate date , LocalTime from , LocalTime to ) { List < Trip > trips = getTripsForDate ( pattern_id , date ) ; return getAverageSpeedForTrips ( trips , from , to ) ; }

Gets the pattern speed for a given pattern for a specified date and time window.
@param pattern_id pattern ID
@param date service date
@param from beginning of time window
@param to end of time window
@return

Gets the pattern speed for a given pattern for a specified date and time window .


public double getAverageSpeedForTrips (Collection<Trip> trips, LocalTime from, LocalTime to) {
        TDoubleList speeds = new TDoubleArrayList();

        for (Trip trip : trips) {
            StopTime firstStopTime = feed.stop_times.ceilingEntry(Fun.t2(trip.trip_id, null)).getValue();
            LocalTime tripBeginTime = LocalTime.ofSecondOfDay(firstStopTime.departure_time % 86399); // convert 24hr+ seconds to 0 - 86399

            // skip trip if begin time is before or after specified time period
            if (tripBeginTime.isAfter(to) || tripBeginTime.isBefore(from)) {
                continue;
            }
            // TODO: swap straight lines for actual geometry?
            double speed = feed.getTripSpeed(trip.trip_id, true);

            if (!Double.isNaN(speed)) {
                speeds.add(speed);
            }
        }

        if (speeds.isEmpty()) return -1;

        return speeds.sum() / speeds.size();
    }

Get average speed for set of trips that begin within the time window in meters per second.
@param trips
@param from
@param to
@return avg. speed (meters per second)

Get average speed for set of trips that begin within the time window in meters per second .


public LocalTime getStartTimeForTrips (Collection<Trip> trips) {
        int earliestDeparture = Integer.MAX_VALUE;

        for (Trip trip : trips) {
            StopTime st = feed.getOrderedStopTimesForTrip(trip.trip_id).iterator().next();
            int dep = st.departure_time;

            // these trips begin on the next day, so we need to cast them to 0 - 86399
            if (dep > 86399) {
                dep = dep % 86399;
            }

            if (dep <= earliestDeparture) {
                earliestDeparture = dep;
            }
        }
        return LocalTime.ofSecondOfDay(earliestDeparture);
    }

Get earliest departure time for a set of trips.
@param trips
@return earliest departure time

Get earliest departure time for a set of trips .


public LocalTime getEndTimeForTrips (Collection<Trip> trips) {
        int latestArrival = Integer.MIN_VALUE;

        for (Trip trip : trips) {
            StopTime st = feed.getOrderedStopTimesForTrip(trip.trip_id).iterator().next();

            if (st.arrival_time >= latestArrival) {
                latestArrival = st.arrival_time;
            }
        }

        // return end time as 2:00 am if last arrival occurs after midnight
        return LocalTime.ofSecondOfDay(latestArrival % 86399);
    }

Get last arrival time for a set of trips.
@param trips
@return last arrival time (if arrival occurs after midnight, time is expressed in terms of following day, e.g., 2:00 AM)

Get last arrival time for a set of trips .


public double getPatternDistance (String pattern_id) {
        Pattern pattern = feed.patterns.get(pattern_id);

        return feed.getTripDistance(pattern.associatedTrips.iterator().next(), false);
    }

public double getPatternDistance ( String pattern_id ) { Pattern pattern = feed . patterns . get ( pattern_id ) ; return feed . getTripDistance ( pattern . associatedTrips . iterator ( ) . next ( ) , false ) ; }

Get distance for a pattern. Uses the first trip associated with the pattern.
@param pattern_id
@return distance (in meters)

Get distance for a pattern . Uses the first trip associated with the pattern .


public double getAverageStopSpacing (String pattern_id) {
        Pattern pattern = feed.patterns.get(pattern_id);
        return getPatternDistance(pattern_id) / pattern.orderedStops.size();
    }

public double getAverageStopSpacing ( String pattern_id ) { Pattern pattern = feed . patterns . get ( pattern_id ) ; return getPatternDistance ( pattern_id ) / pattern . orderedStops . size ( ) ; }

Get average stop spacing for a pattern.
@param pattern_id
@return avg. stop spacing (in meters)

Get average stop spacing for a pattern .


private <E extends Entity> void copyEntityToSql(Iterable<E> entities, Table table) throws SQLException {
        table.createSqlTable(connection, namespace, true);
        String entityInsertSql = table.generateInsertSql(namespace, true);
        PreparedStatement insertStatement = connection.prepareStatement(entityInsertSql);
        // Iterate over agencies and add to prepared statement
        int count = 0, batchSize = 0;
        for (E entity : entities) {
            entity.setStatementParameters(insertStatement, true);
            insertStatement.addBatch();
            count++;
            batchSize++;
            // FIXME: Add batching execute on n
            if (batchSize > JdbcGtfsLoader.INSERT_BATCH_SIZE) {
                insertStatement.executeBatch();
                batchSize = 0;
            }
        }
        // Handle remaining
        insertStatement.executeBatch();
        LOG.info("Inserted {} {}", count, table.name);

        // FIXME: Should some tables not have indexes?
        table.createIndexes(connection, namespace);
    }

Creates table for the specified Table, inserts all entities for the iterable in batches, and, finally, creates
indexes on the table.

Creates table for the specified Table inserts all entities for the iterable in batches and finally creates indexes on the table .


public static ValidateFieldResult<String> from(ValidateFieldResult result) {
        ValidateFieldResult<String> stringResult = new ValidateFieldResult<>();
        stringResult.clean = String.valueOf(result.clean);
        stringResult.errors.addAll(result.errors);
        return stringResult;
    }

public static ValidateFieldResult < String > from ( ValidateFieldResult result ) { ValidateFieldResult < String > stringResult = new ValidateFieldResult <> ( ) ; stringResult . clean = String . valueOf ( result . clean ) ; stringResult . errors . addAll ( result . errors ) ; return stringResult ; }

Builder method that constructs a ValidateFieldResult with type String from the input result.

Builder method that constructs a ValidateFieldResult with type String from the input result .


public int getErrorCount () {
        try {
            // Ensure any outstanding inserts are committed so that count is accurate.
            this.commit();
            Statement statement = connection.createStatement();
            statement.execute(String.format("select count(*) from %serrors", tablePrefix));
            ResultSet resultSet = statement.getResultSet();
            resultSet.next();
            int count = resultSet.getInt(1);
            return count;
        } catch (SQLException ex) {
            throw new StorageException(ex);
        }
    }

Commits any outstanding error inserts and returns the error count via a SQL query.

Commits any outstanding error inserts and returns the error count via a SQL query .


private void commit() {
        try {
            // Execute any remaining batch inserts and commit the transaction.
            insertError.executeBatch();
            insertInfo.executeBatch();
            connection.commit();
        } catch (SQLException ex) {
            throw new StorageException(ex);
        }
    }

This executes any remaining inserts and commits the transaction.

This executes any remaining inserts and commits the transaction .


@Override
    public ValidateFieldResult<String> validateAndConvert(String hhmmss) {
        return ValidateFieldResult.from(getSeconds(hhmmss));
    }

@ Override public ValidateFieldResult < String > validateAndConvert ( String hhmmss ) { return ValidateFieldResult . from ( getSeconds ( hhmmss ) ) ; }

Actually this is converting the string. Can we use some JDBC existing functions for this?

Actually this is converting the string . Can we use some JDBC existing functions for this?


@Override
    public int compareTo (GTFSError o) {
        int compare = super.compareTo(o);
        if (compare != 0) return compare;
        return this.badReference.compareTo((((ReferentialIntegrityError) o).badReference));
    }

@ Override public int compareTo ( GTFSError o ) { int compare = super . compareTo ( o ) ; if ( compare != 0 ) return compare ; return this . badReference . compareTo ( ( ( ( ReferentialIntegrityError ) o ) . badReference ) ) ; }


public static double getDistance(LineString tripGeometry) {
      double distance = 0;
      for (int i = 0; i < tripGeometry.getNumPoints() - 1; i++) {
        try {
          distance += JTS.orthodromicDistance(tripGeometry.getCoordinateN(i), tripGeometry.getCoordinateN(i + 1), DefaultGeographicCRS.WGS84);
        } catch (TransformException e) {
          throw new RuntimeException(e);
        }
      }

      return distance;
    }

public static double getDistance ( LineString tripGeometry ) { double distance = 0 ; for ( int i = 0 ; i < tripGeometry . getNumPoints ( ) - 1 ; i ++ ) { try { distance += JTS . orthodromicDistance ( tripGeometry . getCoordinateN ( i ) , tripGeometry . getCoordinateN ( i + 1 ) , DefaultGeographicCRS . WGS84 ) ; } catch ( TransformException e ) { throw new RuntimeException ( e ) ; } } return distance ; }

Get the length of a linestring in meters


public static GraphQLFieldDefinition field (String fieldName, String tableName) {
        return newFieldDefinition()
                .name(fieldName)
                .type(GraphQLInt)
                .dataFetcher(new RowCountFetcher(tableName))
                .build();
    }

public static GraphQLFieldDefinition field ( String fieldName , String tableName ) { return newFieldDefinition ( ) . name ( fieldName ) . type ( GraphQLInt ) . dataFetcher ( new RowCountFetcher ( tableName ) ) . build ( ) ; }

Convenience method to create a field in a GraphQL schema that fetches the number of rows in a table.
Must be on a type that has a "namespace" field for context.

Convenience method to create a field in a GraphQL schema that fetches the number of rows in a table . Must be on a type that has a namespace field for context .


public static GraphQLFieldDefinition groupedField(String tableName, String groupByColumn) {
        return newFieldDefinition()
                .name(groupByColumn)
                .type(groupCountType)
                .argument(stringArg("pattern_id"))
                .type(new GraphQLList(groupCountType))
                .dataFetcher(new RowCountFetcher(tableName, null, groupByColumn))
                .build();
    }

public static GraphQLFieldDefinition groupedField ( String tableName , String groupByColumn ) { return newFieldDefinition ( ) . name ( groupByColumn ) . type ( groupCountType ) . argument ( stringArg ( "pattern_id" ) ) . type ( new GraphQLList ( groupCountType ) ) . dataFetcher ( new RowCountFetcher ( tableName , null , groupByColumn ) ) . build ( ) ; }

A GraphQL field used to deliver lists of group counts. Currently the primary use of this field is for delivering
trip counts grouped by either service_id, pattern_id, or route_id. These quantities can also be filtered by
providing a string value for pattern_id

A GraphQL field used to deliver lists of group counts . Currently the primary use of this field is for delivering trip counts grouped by either service_id pattern_id or route_id . These quantities can also be filtered by providing a string value for pattern_id


public int[] deduplicateIntArray(int[] original) {
        if (original == null) return null;
        IntArray intArray = new IntArray(original);
        IntArray canonical = canonicalIntArrays.get(intArray);
        if (canonical == null) {
            canonical = intArray;
            canonicalIntArrays.put(canonical, canonical);
        }
        return canonical.array;
    }

public int [ ] deduplicateIntArray ( int [ ] original ) { if ( original == null ) return null ; IntArray intArray = new IntArray ( original ) ; IntArray canonical = canonicalIntArrays . get ( intArray ) ; if ( canonical == null ) { canonical = intArray ; canonicalIntArrays . put ( canonical , canonical ) ; } return canonical . array ; }

Used to deduplicate time and stop sequence arrays. The same times may occur in many trips.

Used to deduplicate time and stop sequence arrays . The same times may occur in many trips .


public List<Trip> getTripsForDate (String stop_id, LocalDate date) {
        List<String> tripIds = stats.getTripsForDate(date).stream()
                .map(trip -> trip.trip_id)
                .collect(Collectors.toList());

        return feed.getDistinctTripsForStop(stop_id).stream()
                .filter(t -> tripIds.contains(t.trip_id)) // filter by trip_id list for date
                .collect(Collectors.toList());
    }

public
List
<
Trip
>
getTripsForDate
(
String
stop_id
,
LocalDate
date
)
{
List
<
String
>
tripIds
=
stats
.
getTripsForDate
(
date
)
.
stream
(
)
.
map
(
trip
->
trip
.
trip_id
)
.
collect
(
Collectors
.
toList
(
)
)
;
return
feed
.
getDistinctTripsForStop
(
stop_id
)
.
stream
(
)
.
filter
(
t
->
tripIds
.
contains
(
t
.
trip_id
)
)
// filter by trip_id list for date
.
collect
(
Collectors
.
toList
(
)
)
;
}

Get list of trips that visit a stop for a specified date of service.
@param stop_id
@param date
@return list of trips for date

Get list of trips that visit a stop for a specified date of service .


public int getAverageHeadwayForStop (String stop_id, LocalDate date, LocalTime from, LocalTime to) {
        List<Trip> tripsForStop = getTripsForDate(stop_id, date);

        return getStopHeadwayForTrips(stop_id, tripsForStop, from, to);
    }

public int getAverageHeadwayForStop ( String stop_id , LocalDate date , LocalTime from , LocalTime to ) { List < Trip > tripsForStop = getTripsForDate ( stop_id , date ) ; return getStopHeadwayForTrips ( stop_id , tripsForStop , from , to ) ; }

Get the average headway, in seconds, for all trips at a stop over a time window.
@param stop_id
@param date
@param from
@param to
@return avg. headway (in seconds)

Get the average headway in seconds for all trips at a stop over a time window .


public Map<String, Integer> getRouteHeadwaysForStop (String stop_id, LocalDate date, LocalTime from, LocalTime to) {
        Map<String, Integer> routeHeadwayMap = new HashMap<>();
        List<Route> routes = feed.patterns.values().stream()
                .filter(p -> p.orderedStops.contains(stop_id))
                .map(p -> feed.routes.get(p.route_id))
                .collect(Collectors.toList());

        for (Route route : routes) {
            routeHeadwayMap.put(route.route_id, getHeadwayForStopByRoute(stop_id, route.route_id, date, from, to));
        }
        return routeHeadwayMap;
    }

public Map < String , Integer > getRouteHeadwaysForStop ( String stop_id , LocalDate date , LocalTime from , LocalTime to ) { Map < String , Integer > routeHeadwayMap = new HashMap <> ( ) ; List < Route > routes = feed . patterns . values ( ) . stream ( ) . filter ( p -> p . orderedStops . contains ( stop_id ) ) . map ( p -> feed . routes . get ( p . route_id ) ) . collect ( Collectors . toList ( ) ) ; for ( Route route : routes ) { routeHeadwayMap . put ( route . route_id , getHeadwayForStopByRoute ( stop_id , route . route_id , date , from , to ) ) ; } return routeHeadwayMap ; }

Get the route headway for a given service date at a stop over a time window, in seconds

Get the route headway for a given service date at a stop over a time window in seconds


public int getHeadwayForStopByRoute (String stop_id, String route_id, LocalDate date, LocalTime from, LocalTime to) {

        List<Trip> tripsForStop = feed.getDistinctTripsForStop(stop_id).stream()
                .filter(trip -> feed.trips.get(trip.trip_id).route_id.equals(route_id))
                .filter(trip -> feed.services.get(trip.service_id).activeOn(date))
                .collect(Collectors.toList());

        return getStopHeadwayForTrips(stop_id, tripsForStop, from, to);
    }

public int getHeadwayForStopByRoute ( String stop_id , String route_id , LocalDate date , LocalTime from , LocalTime to ) { List < Trip > tripsForStop = feed . getDistinctTripsForStop ( stop_id ) . stream ( ) . filter ( trip -> feed . trips . get ( trip . trip_id ) . route_id . equals ( route_id ) ) . filter ( trip -> feed . services . get ( trip . service_id ) . activeOn ( date ) ) . collect ( Collectors . toList ( ) ) ; return getStopHeadwayForTrips ( stop_id , tripsForStop , from , to ) ; }

Get the average headway, in seconds, for a route at a stop over a time window.
@param stop_id
@param route_id
@param date
@param from
@param to
@return avg. headway (in seconds)

Get the average headway in seconds for a route at a stop over a time window .


private static JsonNode getJsonNode (String json) throws IOException {
        try {
            return mapper.readTree(json);
        } catch (IOException e) {
            LOG.error("Bad JSON syntax", e);
            throw e;
        }
    }

private
static
JsonNode
getJsonNode
(
String
json
)
throws
IOException
{
try
{
return
mapper
.
readTree
(
json
)
;
}
catch
(
IOException
e
)
{
LOG
.
error
(
"Bad JSON syntax"
,
e
)
;
throw
e
;
}
}

Wrapper method to call Jackson to deserialize a JSON string into JsonNode.

Wrapper method to call Jackson to deserialize a JSON string into JsonNode .


@Override
    public String create(String json, boolean autoCommit) throws SQLException, IOException {
        return update(null, json, autoCommit);
    }

@ Override public String create ( String json , boolean autoCommit ) throws SQLException , IOException { return update ( null , json , autoCommit ) ; }

Create a new entity in the database from the provided JSON string. Note, any call to create or update must provide
a JSON string with the full set of fields matching the definition of the GTFS table in the Table class.

Create a new entity in the database from the provided JSON string . Note any call to create or update must provide a JSON string with the full set of fields matching the definition of the GTFS table in the Table class .


private int updateStopTimesForPatternStop(ObjectNode patternStop, int previousTravelTime) throws SQLException {
        String sql = String.format(
            "update %s.stop_times st set arrival_time = ?, departure_time = ? from %s.trips t " +
                "where st.trip_id = t.trip_id AND t.pattern_id = ? AND st.stop_sequence = ?",
            tablePrefix,
            tablePrefix
        );
        // Prepare the statement and set statement parameters
        PreparedStatement statement = connection.prepareStatement(sql);
        int oneBasedIndex = 1;
        int travelTime = patternStop.get("default_travel_time").asInt();
        int arrivalTime = previousTravelTime + travelTime;
        statement.setInt(oneBasedIndex++, arrivalTime);
        int dwellTime = patternStop.get("default_dwell_time").asInt();
        statement.setInt(oneBasedIndex++, arrivalTime + dwellTime);
        // Set "where clause" with value for pattern_id and stop_sequence
        statement.setString(oneBasedIndex++, patternStop.get("pattern_id").asText());
        // In the editor, we can depend on stop_times#stop_sequence matching pattern_stops#stop_sequence because we
        // normalize stop sequence values for stop times during snapshotting for the editor.
        statement.setInt(oneBasedIndex++, patternStop.get("stop_sequence").asInt());
        // Log query, execute statement, and log result.
        LOG.debug(statement.toString());
        int entitiesUpdated = statement.executeUpdate();
        LOG.debug("{} stop_time arrivals/departures updated", entitiesUpdated);
        return travelTime + dwellTime;
    }

Updates the stop times that reference the specified pattern stop.
@param patternStop the pattern stop for which to update stop times
@param previousTravelTime the travel time accumulated up to the previous stop_time's departure time (or the
previous pattern stop's dwell time)
@return the travel and dwell time added by this pattern stop
@throws SQLException

Updates the stop times that reference the specified pattern stop .


private void verifyReferencesExist(String referringTableName, Multimap<Table, String> referencesPerTable) throws SQLException {
        for (Table referencedTable: referencesPerTable.keySet()) {
            LOG.info("Checking {} references to {}", referringTableName, referencedTable.name);
            Collection<String> referenceStrings = referencesPerTable.get(referencedTable);
            String referenceFieldName = referencedTable.getKeyFieldName();
            String questionMarks = String.join(", ", Collections.nCopies(referenceStrings.size(), "?"));
            String checkCountSql = String.format(
                    "select %s from %s.%s where %s in (%s)",
                    referenceFieldName,
                    tablePrefix,
                    referencedTable.name,
                    referenceFieldName,
                    questionMarks);
            PreparedStatement preparedStatement = connection.prepareStatement(checkCountSql);
            int oneBasedIndex = 1;
            for (String ref : referenceStrings) {
                preparedStatement.setString(oneBasedIndex++, ref);
            }
            LOG.info(preparedStatement.toString());
            ResultSet resultSet = preparedStatement.executeQuery();
            Set<String> foundReferences = new HashSet<>();
            while (resultSet.next()) {
                String referenceValue = resultSet.getString(1);
                foundReferences.add(referenceValue);
            }
            // Determine if any references were not found.
            referenceStrings.removeAll(foundReferences);
            if (referenceStrings.size() > 0) {
                throw new SQLException(
                        String.format(
                                "%s entities must contain valid %s references. (Invalid references: %s)",
                                referringTableName,
                                referenceFieldName,
                                String.join(", ", referenceStrings)));
            } else {
                LOG.info("All {} {} {} references are valid.", foundReferences.size(), referencedTable.name, referenceFieldName);
            }
        }
    }

Checks that a set of string references to a set of reference tables are all valid. For each set of references
mapped to a reference table, the method queries for all of the references. If there are any references that were
not returned in the query, one of the original references was invalid and an exception is thrown.
@param referringTableName    name of the table which contains references for logging/exception message only
@param referencesPerTable    string references mapped to the tables to which they refer
@throws SQLException

Checks that a set of string references to a set of reference tables are all valid . For each set of references mapped to a reference table the method queries for all of the references . If there are any references that were not returned in the query one of the original references was invalid and an exception is thrown .


private static void verifyInteriorStopsAreUnchanged(
        List<String> originalStopIds,
        List<PatternStop> newStops,
        int firstDifferentIndex,
        int lastDifferentIndex,
        boolean movedRight
    ) {
        //Stops mapped to list of stop IDs simply for easier viewing/comparison with original IDs while debugging with
        // breakpoints.
        List<String> newStopIds = newStops.stream().map(s -> s.stop_id).collect(Collectors.toList());
        // Determine the bounds of the region that should be identical between the two lists.
        int beginRegion = movedRight ? firstDifferentIndex : firstDifferentIndex + 1;
        int endRegion = movedRight ? lastDifferentIndex - 1 : lastDifferentIndex;
        for (int i = beginRegion; i <= endRegion; i++) {
            // Shift index when selecting stop from original list to account for displaced stop.
            int shiftedIndex = movedRight ? i + 1 : i - 1;
            String newStopId = newStopIds.get(i);
            String originalStopId = originalStopIds.get(shiftedIndex);
            if (!newStopId.equals(originalStopId)) {
                // If stop ID for new stop at the given index does not match the original stop ID, the order of at least
                // one stop within the changed region has been changed, which is illegal according to the rule enforcing
                // only a single addition, deletion, or transposition per update.
                throw new IllegalStateException(RECONCILE_STOPS_ERROR_MSG);
            }
        }
    }

Check the stops in the changed region to ensure they remain in the same order. If not, throw an exception to
cancel the transaction.

Check the stops in the changed region to ensure they remain in the same order . If not throw an exception to cancel the transaction .


private void insertBlankStopTimes(
        List<String> tripIds,
        List<PatternStop> newStops,
        int startingStopSequence,
        int stopTimesToAdd,
        Connection connection
    ) throws SQLException {
        if (tripIds.isEmpty()) {
            // There is no need to insert blank stop times if there are no trips for the pattern.
            return;
        }
        String insertSql = Table.STOP_TIMES.generateInsertSql(tablePrefix, true);
        PreparedStatement insertStatement = connection.prepareStatement(insertSql);
        int count = 0;
        int totalRowsUpdated = 0;
        // Create a new stop time for each sequence value (times each trip ID) that needs to be inserted.
        for (int i = startingStopSequence; i < stopTimesToAdd + startingStopSequence; i++) {
            PatternStop patternStop = newStops.get(i);
            StopTime stopTime = new StopTime();
            stopTime.stop_id = patternStop.stop_id;
            stopTime.drop_off_type = patternStop.drop_off_type;
            stopTime.pickup_type = patternStop.pickup_type;
            stopTime.timepoint = patternStop.timepoint;
            stopTime.shape_dist_traveled = patternStop.shape_dist_traveled;
            stopTime.stop_sequence = i;
            // Update stop time with each trip ID and add to batch.
            for (String tripId : tripIds) {
                stopTime.trip_id = tripId;
                stopTime.setStatementParameters(insertStatement, true);
                insertStatement.addBatch();
                if (count % INSERT_BATCH_SIZE == 0) {
                    int[] rowsUpdated = insertStatement.executeBatch();
                    totalRowsUpdated += rowsUpdated.length;
                }
            }
        }
        int[] rowsUpdated = insertStatement.executeBatch();
        totalRowsUpdated += rowsUpdated.length;
        LOG.info("{} blank stop times inserted", totalRowsUpdated);
    }

You must call this method after updating sequences for any stop times following the starting stop sequence to
avoid overwriting these other stop times.

You must call this method after updating sequences for any stop times following the starting stop sequence to avoid overwriting these other stop times .


@Override
    public int delete(Integer id, boolean autoCommit) throws SQLException {
        try {
            // Handle "cascading" delete or constraints on deleting entities that other entities depend on
            // (e.g., keep a calendar from being deleted if trips reference it).
            // FIXME: actually add "cascading"? Currently, it just deletes one level down.
            deleteFromReferencingTables(tablePrefix, specTable, id);
            // Next, delete the actual record specified by id.
            PreparedStatement statement = connection.prepareStatement(specTable.generateDeleteSql(tablePrefix));
            statement.setInt(1, id);
            LOG.info(statement.toString());
            // Execute query
            int result = statement.executeUpdate();
            if (result == 0) {
                LOG.error("Could not delete {} entity with id: {}", specTable.name, id);
                throw new SQLException("Could not delete entity");
            }
            if (autoCommit) connection.commit();
            // FIXME: change return message based on result value
            return result;
        } catch (Exception e) {
            LOG.error("Could not delete {} entity with id: {}", specTable.name, id);
            e.printStackTrace();
            // Rollback changes if errors encountered.
            connection.rollback();
            throw e;
        } finally {
            // Always close connection if auto-committing. Otherwise, leave open (for potential further updates).
            if (autoCommit) connection.close();
        }
    }

Deletes an entity for the specified ID .


private static long handleStatementExecution(PreparedStatement statement, boolean isCreating) throws SQLException {
        // Log the SQL for the prepared statement
        LOG.info(statement.toString());
        int affectedRows = statement.executeUpdate();
        // Determine operation-specific action for any error messages
        String messageAction = isCreating ? "Creating" : "Updating";
        if (affectedRows == 0) {
            // No update occurred.
            // TODO: add some clarity on cause (e.g., where clause found no entity with provided ID)?
            throw new SQLException(messageAction + " entity failed, no rows affected.");
        }
        try (ResultSet generatedKeys = statement.getGeneratedKeys()) {
            if (generatedKeys.next()) {
                // Get the auto-generated ID from the update execution
                long newId = generatedKeys.getLong(1);
                return newId;
            } else {
                throw new SQLException(messageAction + " entity failed, no ID obtained.");
            }
        } catch (SQLException e) {
            e.printStackTrace();
            throw e;
        }
    }

Handle executing a prepared statement and return the ID for the newly-generated or updated entity.

Handle executing a prepared statement and return the ID for the newly - generated or updated entity .


private static int getRowCount(String tableName, Connection connection) throws SQLException {
        String rowCountSql = String.format("SELECT COUNT(*) FROM %s", tableName);
        LOG.info(rowCountSql);
        // Create statement for counting rows selected
        Statement statement = connection.createStatement();
        ResultSet resultSet = statement.executeQuery(rowCountSql);
        if (resultSet.next()) return resultSet.getInt(1);
        else return 0;
    }

Get number of rows for a table. This is currently just used to check the number of entities for the agency table.

Get number of rows for a table . This is currently just used to check the number of entities for the agency table .


private static Set<Table> getReferencingTables(Table table) {
        Set<Table> referencingTables = new HashSet<>();
        for (Table gtfsTable : Table.tablesInOrder) {
            // IMPORTANT: Skip the table for the entity we're modifying or if loop table does not have field.
            if (table.name.equals(gtfsTable.name)) continue;
            for (Field field : gtfsTable.fields) {
                if (field.isForeignReference() && field.referenceTable.name.equals(table.name)) {
                    // If any of the table's fields are foreign references to the specified table, add to the return set.
                    referencingTables.add(gtfsTable);
                }
            }
        }
        return referencingTables;
    }

Finds the set of tables that reference the parent entity being updated.

Finds the set of tables that reference the parent entity being updated .


private static String getValueForId(int id, String fieldName, String namespace, Table table, Connection connection) throws SQLException {
        String tableName = String.join(".", namespace, table.name);
        String selectIdSql = String.format("select %s from %s where id = %d", fieldName, tableName, id);
        LOG.info(selectIdSql);
        Statement selectIdStatement = connection.createStatement();
        ResultSet selectResults = selectIdStatement.executeQuery(selectIdSql);
        String value = null;
        while (selectResults.next()) {
            value = selectResults.getString(1);
        }
        return value;
    }

private
static
String
getValueForId
(
int
id
,
String
fieldName
,
String
namespace
,
Table
table
,
Connection
connection
)
throws
SQLException
{
String
tableName
=
String
.
join
(
"."
,
namespace
,
table
.
name
)
;
String
selectIdSql
=
String
.
format
(
"select %s from %s where id = %d"
,
fieldName
,
tableName
,
id
)
;
LOG
.
info
(
selectIdSql
)
;
Statement
selectIdStatement
=
connection
.
createStatement
(
)
;
ResultSet
selectResults
=
selectIdStatement
.
executeQuery
(
selectIdSql
)
;
String
value
=
;
while
(
selectResults
.
next
(
)
)
{
value
=
selectResults
.
getString
(
1
)
;
}
return
value
;
}

For a given integer ID, return the value for the specified field name for that entity.

For a given integer ID return the value for the specified field name for that entity .


public T put (String id, File feedFile) throws Exception {
        return put(id, feedFile, null);
    }

public T put ( String id , File feedFile ) throws Exception { return put ( id , feedFile , null ) ; }

Add a GTFS feed to this cache with the given ID. NB this is not the feed ID, because feed IDs are not
unique when you load multiple versions of the same feed.

Add a GTFS feed to this cache with the given ID . NB this is not the feed ID because feed IDs are not unique when you load multiple versions of the same feed .


public T put (Function<GTFSFeed, String> idGenerator, File feedFile) throws Exception {
        return put(null, feedFile, idGenerator);
    }

public T put ( Function < GTFSFeed , String > idGenerator , File feedFile ) throws Exception { return put ( null , feedFile , idGenerator ) ; }

Add a GTFS feed to this cache where the ID is calculated from the feed itself


public boolean activeOn (LocalDate date) {
        // first check for exceptions
        CalendarDate exception = calendar_dates.get(date);

        if (exception != null)
            return exception.exception_type == 1;

        else if (calendar == null)
            return false;

        else {
            boolean outsideValidityRange = date.isAfter(calendar.end_date) || date.isBefore(calendar.start_date);
            if (outsideValidityRange) return false;
            switch (date.getDayOfWeek()) {
                case MONDAY:
                    return calendar.monday == 1;
                case TUESDAY:
                    return calendar.tuesday == 1;
                case WEDNESDAY:
                    return calendar.wednesday == 1;
                case THURSDAY:
                    return calendar.thursday == 1;
                case FRIDAY:
                    return calendar.friday == 1;
                case SATURDAY:
                    return calendar.saturday == 1;
                case SUNDAY:
                    return calendar.sunday == 1;
                default:
                    throw new IllegalArgumentException("unknown day of week constant!");
            }
        }
    }

public
boolean
activeOn
(
LocalDate
date
)
{
// first check for exceptions
CalendarDate
exception
=
calendar_dates
.
get
(
date
)
;
if
(
exception
!=
)
return
exception
.
exception_type
==
1
;
else
if
(
calendar
==
)
return
false
;
else
{
boolean
outsideValidityRange
=
date
.
isAfter
(
calendar
.
end_date
)
||
date
.
isBefore
(
calendar
.
start_date
)
;
if
(
outsideValidityRange
)
return
false
;
switch
(
date
.
getDayOfWeek
(
)
)
{
case
MONDAY
:
return
calendar
.
monday
==
1
;
case
TUESDAY
:
return
calendar
.
tuesday
==
1
;
case
WEDNESDAY
:
return
calendar
.
wednesday
==
1
;
case
THURSDAY
:
return
calendar
.
thursday
==
1
;
case
FRIDAY
:
return
calendar
.
friday
==
1
;
case
SATURDAY
:
return
calendar
.
saturday
==
1
;
case
SUNDAY
:
return
calendar
.
sunday
==
1
;
default
:
throw
new
IllegalArgumentException
(
"unknown day of week constant!"
)
;
}
}
}

Is this service active on the specified date?


public static void initialize (DataSource dataSource) {
        GTFSGraphQL.dataSource = dataSource;
        GRAPHQL = GraphQL.newGraphQL(GraphQLGtfsSchema.feedBasedSchema)
            .build();
    }

public static void initialize ( DataSource dataSource ) { GTFSGraphQL . dataSource = dataSource ; GRAPHQL = GraphQL . newGraphQL ( GraphQLGtfsSchema . feedBasedSchema ) . build ( ) ; }

Username and password can be null if connecting to a local instance with host-based authentication.

Username and password can be null if connecting to a local instance with host - based authentication .


public List<Field> editorFields() {
        List<Field> editorFields = new ArrayList<>();
        for (Field f : fields) if (f.requirement == REQUIRED || f.requirement == OPTIONAL || f.requirement == EDITOR) {
            editorFields.add(f);
        }
        return editorFields;
    }

public List < Field > editorFields ( ) { List < Field > editorFields = new ArrayList <> ( ) ; for ( Field f : fields ) if ( f . requirement == REQUIRED || f . requirement == OPTIONAL || f . requirement == EDITOR ) { editorFields . add ( f ) ; } return editorFields ; }

Get only those fields included in the official GTFS specification for this table or used by the editor.

Get only those fields included in the official GTFS specification for this table or used by the editor .


public List<Field> requiredFields () {
        // Filter out fields not used in editor (i.e., extension fields).
        List<Field> requiredFields = new ArrayList<>();
        for (Field f : fields) if (f.requirement == REQUIRED) requiredFields.add(f);
        return requiredFields;
    }

Get only those fields marked as required in the official GTFS specification for this table.

Get only those fields marked as required in the official GTFS specification for this table .


public List<Field> specFields () {
        List<Field> specFields = new ArrayList<>();
        for (Field f : fields) if (f.requirement == REQUIRED || f.requirement == OPTIONAL) specFields.add(f);
        return specFields;
    }

public List < Field > specFields ( ) { List < Field > specFields = new ArrayList <> ( ) ; for ( Field f : fields ) if ( f . requirement == REQUIRED || f . requirement == OPTIONAL ) specFields . add ( f ) ; return specFields ; }

Get only those fields included in the official GTFS specification for this table, i.e., filter out fields used
in the editor or extensions.

Get only those fields included in the official GTFS specification for this table i . e . filter out fields used in the editor or extensions .


public boolean createSqlTable (Connection connection, String namespace, boolean makeIdSerial, String[] primaryKeyFields) {
        // Optionally join namespace and name to create full table name if namespace is not null (i.e., table object is
        // a spec table).
        String tableName = namespace != null ? String.join(".", namespace, name) : name;
        String fieldDeclarations = Arrays.stream(fields)
                .map(Field::getSqlDeclaration)
                .collect(Collectors.joining(", "));
        if (primaryKeyFields != null) {
            fieldDeclarations += String.format(", primary key (%s)", String.join(", ", primaryKeyFields));
        }
        String dropSql = String.format("drop table if exists %s", tableName);
        // Adding the unlogged keyword gives about 12 percent speedup on loading, but is non-standard.
        String idFieldType = makeIdSerial ? "serial" : "bigint";
        String createSql = String.format("create table %s (id %s not null, %s)", tableName, idFieldType, fieldDeclarations);
        try {
            Statement statement = connection.createStatement();
            LOG.info(dropSql);
            statement.execute(dropSql);
            LOG.info(createSql);
            return statement.execute(createSql);
        } catch (Exception ex) {
            throw new StorageException(ex);
        }
    }

Create an SQL table with all the fields specified by this table object,
plus an integer CSV line number field in the first position.

Create an SQL table with all the fields specified by this table object plus an integer CSV line number field in the first position .


public String generateInsertSql (String namespace, boolean setDefaultId) {
        String tableName = namespace == null
                ? name
                : String.join(".", namespace, name);
        String joinedFieldNames = commaSeparatedNames(editorFields());
        String idValue = setDefaultId ? "DEFAULT" : "?";
        return String.format(
            "insert into %s (id, %s) values (%s, %s)",
            tableName,
            joinedFieldNames,
            idValue,
            String.join(", ", Collections.nCopies(editorFields().size(), "?"))
        );
    }

Create SQL string for use in insert statement. Note, this filters table's fields to only those used in editor.

Create SQL string for use in insert statement . Note this filters table s fields to only those used in editor .


public static String commaSeparatedNames(List<Field> fieldsToJoin, String prefix, boolean csvOutput) {
        return fieldsToJoin.stream()
                // NOTE: This previously only prefixed fields that were foreign refs or key fields. However, this
                // caused an issue where shared fields were ambiguously referenced in a select query (specifically,
                // wheelchair_accessible in routes and trips). So this filter has been removed.
                .map(f -> f.getColumnExpression(prefix, csvOutput))
                .collect(Collectors.joining(", "));
    }

Prepend a prefix string to each field and join them with a comma + space separator.
Also, if an export to GTFS is being performed, certain fields need a translation from the database format to the
GTFS format.  Otherwise, the fields are assumed to be asked in order to do a database-to-database export and so
the verbatim values of the fields are needed.

Prepend a prefix string to each field and join them with a comma + space separator . Also if an export to GTFS is being performed certain fields need a translation from the database format to the GTFS format . Otherwise the fields are assumed to be asked in order to do a database - to - database export and so the verbatim values of the fields are needed .


public String generateUpdateSql (String namespace, int id) {
        // Collect field names for string joining from JsonObject.
        String joinedFieldNames = editorFields().stream()
                // If updating, add suffix for use in set clause
                .map(field -> field.name + " = ?")
                .collect(Collectors.joining(", "));

        String tableName = namespace == null ? name : String.join(".", namespace, name);
        return String.format("update %s set %s where id = %d", tableName, joinedFieldNames, id);
    }

Create SQL string for use in update statement. Note, this filters table's fields to only those used in editor.

Create SQL string for use in update statement . Note this filters table s fields to only those used in editor .


public String generateSelectSql (String namespace, Requirement minimumRequirement) {
        String fieldsString;
        String tableName = String.join(".", namespace, name);
        String fieldPrefix = tableName + ".";
        if (minimumRequirement.equals(EDITOR)) {
            fieldsString = commaSeparatedNames(editorFields(), fieldPrefix, true);
        } else if (minimumRequirement.equals(OPTIONAL)) {
            fieldsString = commaSeparatedNames(specFields(), fieldPrefix, true);
        } else if (minimumRequirement.equals(REQUIRED)) {
            fieldsString = commaSeparatedNames(requiredFields(), fieldPrefix, true);
        } else fieldsString = "*";
        return String.format("select %s from %s", fieldsString, tableName);
    }

public
String
generateSelectSql
(
String
namespace
,
Requirement
minimumRequirement
)
{
String
fieldsString
;
String
tableName
=
String
.
join
(
"."
,
namespace
,
name
)
;
String
fieldPrefix
=
tableName
+
"."
;
if
(
minimumRequirement
.
equals
(
EDITOR
)
)
{
fieldsString
=
commaSeparatedNames
(
editorFields
(
)
,
fieldPrefix
,
true
)
;
}
else
if
(
minimumRequirement
.
equals
(
OPTIONAL
)
)
{
fieldsString
=
commaSeparatedNames
(
specFields
(
)
,
fieldPrefix
,
true
)
;
}
else
if
(
minimumRequirement
.
equals
(
REQUIRED
)
)
{
fieldsString
=
commaSeparatedNames
(
requiredFields
(
)
,
fieldPrefix
,
true
)
;
}
else
fieldsString
=
"*"
;
return
String
.
format
(
"select %s from %s"
,
fieldsString
,
tableName
)
;
}

Generate select all SQL string. The minimum requirement parameter is used to determine which fields ought to be
included in the select statement. For example, if "OPTIONAL" is passed in, both optional and required fields
are included in the select. If "EDITOR" is the minimum requirement, editor, optional, and required fields will
all be included.

Generate select all SQL string . The minimum requirement parameter is used to determine which fields ought to be included in the select statement . For example if OPTIONAL is passed in both optional and required fields are included in the select . If EDITOR is the minimum requirement editor optional and required fields will all be included .


public String generateSelectAllExistingFieldsSql(Connection connection, String namespace) throws SQLException {
        // select all columns from table
        // FIXME This is postgres-specific and needs to be made generic for non-postgres databases.
        PreparedStatement statement = connection.prepareStatement(
            "SELECT column_name FROM information_schema.columns WHERE table_schema = ? AND table_name = ?"
        );
        statement.setString(1, namespace);
        statement.setString(2, name);
        ResultSet result = statement.executeQuery();

        // get result and add fields that are defined in this table
        List<Field> existingFields = new ArrayList<>();
        while (result.next()) {
            String columnName = result.getString(1);
            existingFields.add(getFieldForName(columnName));
        }

        String tableName = String.join(".", namespace, name);
        String fieldPrefix = tableName + ".";
        return String.format(
            "select %s from %s", commaSeparatedNames(existingFields, fieldPrefix, true), tableName
        );
    }

Generate a select statement from the columns that actually exist in the database table.  This method is intended
to be used when exporting to a GTFS and eventually generates the select all with each individual field and
applicable transformations listed out.

Generate a select statement from the columns that actually exist in the database table . This method is intended to be used when exporting to a GTFS and eventually generates the select all with each individual field and applicable transformations listed out .


public String generateDeleteSql (String namespace, String fieldName) {
        String whereField = fieldName == null ? "id" : fieldName;
        return String.format("delete from %s where %s = ?", String.join(".", namespace, name), whereField);
    }

public
String
generateDeleteSql
(
String
namespace
,
String
fieldName
)
{
String
whereField
=
fieldName
==
"id"
:
fieldName
;
return
String
.
format
(
"delete from %s where %s = ?"
,
String
.
join
(
"."
,
namespace
,
name
)
,
whereField
)
;
}


public boolean createSqlTableFrom(Connection connection, String tableToClone) {
        long startTime = System.currentTimeMillis();
        try {
            Statement statement = connection.createStatement();
            // Drop target table to avoid a conflict.
            String dropSql = String.format("drop table if exists %s", name);
            LOG.info(dropSql);
            statement.execute(dropSql);
            if (tableToClone.endsWith("stop_times")) {
                normalizeAndCloneStopTimes(statement, name, tableToClone);
            } else {
                // Adding the unlogged keyword gives about 12 percent speedup on loading, but is non-standard.
                // FIXME: Which create table operation is more efficient?
                String createTableAsSql = String.format("create table %s as table %s", name, tableToClone);
                // Create table in the image of the table we're copying (indexes are not included).
                LOG.info(createTableAsSql);
                statement.execute(createTableAsSql);
            }
            applyAutoIncrementingSequence(statement);
            // FIXME: Is there a need to add primary key constraint here?
            if (usePrimaryKey) {
                // Add primary key to ID column for any tables that require it.
                String addPrimaryKeySql = String.format("ALTER TABLE %s ADD PRIMARY KEY (id)", name);
                LOG.info(addPrimaryKeySql);
                statement.execute(addPrimaryKeySql);
            }
            return true;
        } catch (SQLException ex) {
            LOG.error("Error cloning table {}: {}", name, ex.getSQLState());
            LOG.error("details: ", ex);
            try {
                connection.rollback();
                // It is likely that if cloning the table fails, the reason was that the table did not already exist.
                // Try to create the table here from scratch.
                // FIXME: Maybe we should check that the reason the clone failed was that the table already exists.
                createSqlTable(connection, true);
                return true;
            } catch (SQLException e) {
                e.printStackTrace();
                return false;
            }
        } finally {
            LOG.info("Cloned table {} as {} in {} ms", tableToClone, name, System.currentTimeMillis() - startTime);
        }
    }

Creates a SQL table from the table to clone. This uses the SQL syntax "create table x as y" not only copies the
table structure, but also the data from the original table. Creating table indexes is not handled by this method.

Creates a SQL table from the table to clone . This uses the SQL syntax create table x as y not only copies the table structure but also the data from the original table . Creating table indexes is not handled by this method .


private void normalizeAndCloneStopTimes(Statement statement, String name, String tableToClone) throws SQLException {
        // Create table with matching columns first and then insert all rows with a special select query that
        // normalizes the stop sequences before inserting.
        // "Create table like" can optionally include indexes, but we want to avoid creating the indexes beforehand
        // because this will slow down our massive insert for stop times.
        String createTableLikeSql = String.format("create table %s (like %s)", name, tableToClone);
        LOG.info(createTableLikeSql);
        statement.execute(createTableLikeSql);
        long normalizeStartTime = System.currentTimeMillis();
        LOG.info("Normalizing stop sequences");
        // First get the column names (to account for any non-standard fields that may be present)
        List<String> columns = new ArrayList<>();
        ResultSet resultSet = statement.executeQuery(String.format("select * from %s limit 1", tableToClone));
        ResultSetMetaData metadata = resultSet.getMetaData();
        int nColumns = metadata.getColumnCount();
        for (int i = 1; i <= nColumns; i++) {
            columns.add(metadata.getColumnName(i));
        }
        // Replace stop sequence column with the normalized sequence values.
        columns.set(columns.indexOf("stop_sequence"), "-1 + row_number() over (partition by trip_id order by stop_sequence) as stop_sequence");
        String insertAllSql = String.format("insert into %s (select %s from %s)", name, String.join(", ", columns), tableToClone);
        LOG.info(insertAllSql);
        statement.execute(insertAllSql);
        LOG.info("Normalized stop times sequences in {} ms", System.currentTimeMillis() - normalizeStartTime);
    }

Normalize stop sequences for stop times table so that sequences are all zero-based and increment
by one. This ensures that sequence values for stop_times and pattern_stops are not initially out
of sync for feeds imported into the editor.

NOTE: This happens here instead of as part of post-processing because it's much faster overall to perform
this as an INSERT vs. an UPDATE. It also needs to be done before creating table indexes. There may be some
messiness here as far as using the column metadata to perform the SELECT query with the correct column names, but
it is about an order of magnitude faster than the UPDATE approach.

For example, with the Bronx bus feed, the UPDATE approach took 53 seconds on an un-normalized table (i.e., snapshotting
a table from a direct GTFS load), but it only takes about 8 seconds with the INSERT approach. Additionally, this
INSERT approach seems to dramatically cut down on the time needed for indexing large tables.

Normalize stop sequences for stop times table so that sequences are all zero - based and increment by one . This ensures that sequence values for stop_times and pattern_stops are not initially out of sync for feeds imported into the editor .


public int getKeyFieldIndex(Field[] fields) {
        String keyField = getKeyFieldName();
        return Field.getFieldIndex(fields, keyField);
    }

public int getKeyFieldIndex ( Field [ ] fields ) { String keyField = getKeyFieldName ( ) ; return Field . getFieldIndex ( fields , keyField ) ; }

Returns the index of the key field within the array of fields provided for a given table.
@param fields array of fields (intended to be derived from the headers of a csv text file)

Returns the index of the key field within the array of fields provided for a given table .


public static void ensureValidNamespace(String namespace) throws InvalidNamespaceException {
        Pattern pattern = Pattern.compile("[^a-zA-Z0-9_\\.]");
        Matcher matcher = pattern.matcher(namespace);
        if (matcher.find()) {
            throw new InvalidNamespaceException();
        }
    }

public static void ensureValidNamespace ( String namespace ) throws InvalidNamespaceException { Pattern pattern = Pattern . compile ( "[^a-zA-Z0-9_\\.]" ) ; Matcher matcher = pattern . matcher ( namespace ) ; if ( matcher . find ( ) ) { throw new InvalidNamespaceException ( ) ; } }

Ensures namespace is valid by ensuring it only has alphanumeric characters or the underscore
for all characters in the string
@param namespace


public static int getFieldIndex (Field[] fields, String name) {
        // Linear search, assuming a small number of fields per table.
        for (int i = 0; i < fields.length; i++) if (fields[i].name.equals(name)) return i;
        return -1;
    }

Finds the index of the field given a string name.
@return the index of the field or -1 if no match is found

Finds the index of the field given a string name .


@Override
    public Set<Integer> keySet() {
        // use a linkedhashset so values come out in order
        Set<Integer> ret = new LinkedHashSet<>();

        for (Tuple2<String, Integer> t : wrapped.keySet()) {
            ret.add(t.b);
        }

        // Don't let the user modify the set as it won't do what they expect (change the map)
        return Collections.unmodifiableSet(ret);
    }

We currently just expose them as immutable sets in RAM, since all of the modification operations are optional.

We currently just expose them as immutable sets in RAM since all of the modification operations are optional .


public static GTFSFeed fromFile(String file, String feedId) {
        GTFSFeed feed = new GTFSFeed();
        ZipFile zip;
        try {
            zip = new ZipFile(file);
            if (feedId == null) {
                feed.loadFromFile(zip);
            }
            else {
                feed.loadFromFile(zip, feedId);
            }
            zip.close();
            return feed;
        } catch (Exception e) {
            LOG.error("Error loading GTFS: {}", e.getMessage());
            throw new RuntimeException(e);
        }
    }

public
static
GTFSFeed
fromFile
(
String
file
,
String
feedId
)
{
GTFSFeed
feed
=
new
GTFSFeed
(
)
;
ZipFile
zip
;
try
{
zip
=
new
ZipFile
(
file
)
;
if
(
feedId
==
)
{
feed
.
loadFromFile
(
zip
)
;
}
else
{
feed
.
loadFromFile
(
zip
,
feedId
)
;
}
zip
.
close
(
)
;
return
feed
;
}
catch
(
Exception
e
)
{
LOG
.
error
(
"Error loading GTFS: {}"
,
e
.
getMessage
(
)
)
;
throw
new
RuntimeException
(
e
)
;
}
}

Static factory method returning a new instance of GTFSFeed containing the contents of
the GTFS file at the supplied filesystem path. Forces the feedId to the
supplied value if the parameter is non-null.

Static factory method returning a new instance of GTFSFeed containing the contents of the GTFS file at the supplied filesystem path . Forces the feedId to the supplied value if the parameter is non - null .