# Buggy code
def populate_collection_elements(
    self,
    collection,
    root_collection_builder,
    filenames,
    name=None,
    metadata_source_name=None,
    final_job_state="ok",
):
    # TODO: allow configurable sorting.
    #    <sort by="lexical" /> <!-- default -->
    #    <sort by="reverse_lexical" />
    #    <sort regex="example.(\d+).fastq" by="1:numerical" />
    #    <sort regex="part_(\d+)_sample_([^_]+).fastq" by="2:lexical,1:numerical" />
    if name is None:
        name = "unnamed output"

    element_datasets = {
        "element_identifiers": [],
        "datasets": [],
        "tag_lists": [],
        "paths": [],
        "extra_files": [],
    }
    for filename, discovered_file in filenames.items():
        create_dataset_timer = ExecutionTimer()
        fields_match = discovered_file.match
        if not fields_match:
            raise Exception("Problem parsing metadata fields for file %s" % filename)
        element_identifiers = fields_match.element_identifiers
        designation = fields_match.designation
        visible = fields_match.visible
        ext = fields_match.ext
        dbkey = fields_match.dbkey
        extra_files = fields_match.extra_files
        # galaxy.tools.parser.output_collection_def.INPUT_DBKEY_TOKEN
        if dbkey == "__input__":
            dbkey = self.input_dbkey

        # Create new primary dataset
        dataset_name = fields_match.name or designation

        link_data = discovered_file.match.link_data

        sources = discovered_file.match.sources
        hashes = discovered_file.match.hashes
        created_from_basename = discovered_file.match.created_from_basename

        dataset = self.create_dataset(
            ext=ext,
            designation=designation,
            visible=visible,
            dbkey=dbkey,
            name=dataset_name,
            metadata_source_name=metadata_source_name,
            link_data=link_data,
            sources=sources,
            hashes=hashes,
            created_from_basename=created_from_basename,
            final_job_state=final_job_state,
        )
        log.debug(
            "(%s) Created dynamic collection dataset for path [%s] with element identifier [%s] for output [%s] %s",
            self.job_id(),
            filename,
            designation,
            name,
            create_dataset_timer,
        )
        element_datasets["element_identifiers"].append(element_identifiers)
        element_datasets["extra_files"].append(extra_files)
        element_datasets["datasets"].append(dataset)
        element_datasets["tag_lists"].append(discovered_file.match.tag_list)
        element_datasets["paths"].append(filename)

    self.add_tags_to_datasets(
        datasets=element_datasets["datasets"], tag_lists=element_datasets["tag_lists"]
    )
    for element_identifiers, dataset in zip(
        element_datasets["element_identifiers"], element_datasets["datasets"]
    ):
        current_builder = root_collection_builder
        for element_identifier in element_identifiers[:-1]:
            current_builder = current_builder.get_level(element_identifier)
        current_builder.add_dataset(element_identifiers[-1], dataset)

        # Associate new dataset with job
        element_identifier_str = ":".join(element_identifiers)
        association_name = "__new_primary_file_{}|{}__".format(
            name, element_identifier_str
        )
        self.add_output_dataset_association(association_name, dataset)

    self.flush()
    self.update_object_store_with_datasets(
        datasets=element_datasets["datasets"],
        paths=element_datasets["paths"],
        extra_files=element_datasets["extra_files"],
    )
    add_datasets_timer = ExecutionTimer()
    self.add_datasets_to_history(element_datasets["datasets"])
    log.debug(
        "(%s) Add dynamic collection datasets to history for output [%s] %s",
        self.job_id(),
        name,
        add_datasets_timer,
    )
    self.set_datasets_metadata(datasets=element_datasets["datasets"])


# Traceback (sqlalchemy.exc.IntegrityError)
Nov 01 14:58:58 sn04.bi.uni-freiburg.de python[1827028]: galaxy.jobs DEBUG 2020-11-01 14:58:58,991 (12059080) Working directory for job is: /data/dnb03/galaxy_db/job_working_directory/012/059/12059080
Nov 01 14:58:58 sn04.bi.uni-freiburg.de python[1847375]: galaxy.jobs DEBUG 2020-11-01 14:58:58,998 (12059086) Working directory for job is: /data/dnb03/galaxy_db/job_working_directory/012/059/12059086
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1847375]: galaxy.objectstore DEBUG 2020-11-01 14:58:59,011 Using preferred backend 'files10' for creation of Dataset 27240484
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: galaxy.job_execution.output_collect ERROR 2020-11-01 14:58:59,036 Problem gathering output collection.
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: Traceback (most recent call last):
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1284, in _execute_context
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: cursor, statement, parameters, context
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 590, in do_execute
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: cursor.execute(statement, parameters)
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: psycopg2.errors.NotNullViolation: null value in column "dataset_collection_id" violates not-null constraint
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: DETAIL:  Failing row contains (13330967, null, 30637828, null, null, 0, ERR4597396__single).
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: The above exception was the direct cause of the following exception:
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: Traceback (most recent call last):
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/server/lib/galaxy/job_execution/output_collect.py", line 156, in collect_dynamic_outputs
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: final_job_state=job_context.final_job_state,
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/server/lib/galaxy/model/store/discover.py", line 286, in populate_collection_elements
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: self.flush()
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/server/lib/galaxy/job_execution/output_collect.py", line 214, in flush
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: self.sa_session.flush()
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/orm/scoping.py", line 163, in do
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: return getattr(self.registry(), name)(*args, **kwargs)
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/orm/session.py", line 2523, in flush
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: self._flush(objects)
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/orm/session.py", line 2664, in _flush
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: transaction.rollback(_capture_exception=True)
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 69, in __exit__
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: exc_value, with_traceback=exc_tb,
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: raise exception
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/orm/session.py", line 2624, in _flush
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: flush_context.execute()
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: rec.execute(self)
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: uow,
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 236, in save_obj
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: update,
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 995, in _emit_update_statements
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: statement, multiparams
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1020, in execute
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: return meth(self, multiparams, params)
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: return connection._execute_clauseelement(self, multiparams, params)
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1139, in _execute_clauseelement
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: distilled_params,
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1324, in _execute_context
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: e, statement, parameters, cursor, context
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1518, in _handle_dbapi_exception
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: sqlalchemy_exception, with_traceback=exc_info[2], from_=e
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: raise exception
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1284, in _execute_context
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: cursor, statement, parameters, context
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: File "/opt/galaxy/venv/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 590, in do_execute
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: cursor.execute(statement, parameters)
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: sqlalchemy.exc.IntegrityError: (psycopg2.errors.NotNullViolation) null value in column "dataset_collection_id" violates not-null constraint
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: DETAIL:  Failing row contains (13330967, null, 30637828, null, null, 0, ERR4597396__single).
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: [SQL: UPDATE dataset_collection_element SET dataset_collection_id=%(dataset_collection_id)s WHERE dataset_collection_element.id = %(dataset_collection_element_id)s]
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: [parameters: {'dataset_collection_id': None, 'dataset_collection_element_id': 13330967}]
Nov 01 14:58:59 sn04.bi.uni-freiburg.de python[1819184]: (Background on this error at: http://sqlalche.me/e/gkpj)

# Fixed code
def populate_collection_elements(
    self,
    collection,
    root_collection_builder,
    filenames,
    name=None,
    metadata_source_name=None,
    final_job_state="ok",
):
    # TODO: allow configurable sorting.
    #    <sort by="lexical" /> <!-- default -->
    #    <sort by="reverse_lexical" />
    #    <sort regex="example.(\d+).fastq" by="1:numerical" />
    #    <sort regex="part_(\d+)_sample_([^_]+).fastq" by="2:lexical,1:numerical" />
    if name is None:
        name = "unnamed output"

    element_datasets = {
        "element_identifiers": [],
        "datasets": [],
        "tag_lists": [],
        "paths": [],
        "extra_files": [],
    }
    for filename, discovered_file in filenames.items():
        create_dataset_timer = ExecutionTimer()
        fields_match = discovered_file.match
        if not fields_match:
            raise Exception("Problem parsing metadata fields for file %s" % filename)
        element_identifiers = fields_match.element_identifiers
        designation = fields_match.designation
        visible = fields_match.visible
        ext = fields_match.ext
        dbkey = fields_match.dbkey
        extra_files = fields_match.extra_files
        # galaxy.tools.parser.output_collection_def.INPUT_DBKEY_TOKEN
        if dbkey == "__input__":
            dbkey = self.input_dbkey

        # Create new primary dataset
        dataset_name = fields_match.name or designation

        link_data = discovered_file.match.link_data

        sources = discovered_file.match.sources
        hashes = discovered_file.match.hashes
        created_from_basename = discovered_file.match.created_from_basename

        dataset = self.create_dataset(
            ext=ext,
            designation=designation,
            visible=visible,
            dbkey=dbkey,
            name=dataset_name,
            metadata_source_name=metadata_source_name,
            link_data=link_data,
            sources=sources,
            hashes=hashes,
            created_from_basename=created_from_basename,
            final_job_state=final_job_state,
        )
        log.debug(
            "(%s) Created dynamic collection dataset for path [%s] with element identifier [%s] for output [%s] %s",
            self.job_id(),
            filename,
            designation,
            name,
            create_dataset_timer,
        )
        element_datasets["element_identifiers"].append(element_identifiers)
        element_datasets["extra_files"].append(extra_files)
        element_datasets["datasets"].append(dataset)
        element_datasets["tag_lists"].append(discovered_file.match.tag_list)
        element_datasets["paths"].append(filename)

    self.add_tags_to_datasets(
        datasets=element_datasets["datasets"], tag_lists=element_datasets["tag_lists"]
    )
    for element_identifiers, dataset in zip(
        element_datasets["element_identifiers"], element_datasets["datasets"]
    ):
        current_builder = root_collection_builder
        for element_identifier in element_identifiers[:-1]:
            current_builder = current_builder.get_level(element_identifier)
        current_builder.add_dataset(element_identifiers[-1], dataset)

        # Associate new dataset with job
        element_identifier_str = ":".join(element_identifiers)
        association_name = "__new_primary_file_{}|{}__".format(
            name, element_identifier_str
        )
        self.add_output_dataset_association(association_name, dataset)

    root_collection_builder.populate()
    self.flush()
    self.update_object_store_with_datasets(
        datasets=element_datasets["datasets"],
        paths=element_datasets["paths"],
        extra_files=element_datasets["extra_files"],
    )
    add_datasets_timer = ExecutionTimer()
    self.add_datasets_to_history(element_datasets["datasets"])
    log.debug(
        "(%s) Add dynamic collection datasets to history for output [%s] %s",
        self.job_id(),
        name,
        add_datasets_timer,
    )
    self.set_datasets_metadata(datasets=element_datasets["datasets"])