Repository: fair-research/bdbag (language: python)

Issue:
Support external fetch transports via plug-in architecture.
The current set of fetch transports is basically hard-wired into the code, whereby the only way to add to (or override) the default set of handlers is to modify the `bdbag.fetch.transports` module source code directly. This has proven to be too limiting in practice.

Refactoring the `bdbag.fetch.transports` module so that it supports run-time loading of externally provided fetch transport providers (similar to how the `bdbag.fetch.resolvers` module works) should be sufficient address this.
Support external fetch transports via plug-in architecture.
The current set of fetch transports is basically hard-wired into the code, whereby the only way to add to (or override) the default set of handlers is to modify the `bdbag.fetch.transports` module source code directly. This has proven to be too limiting in practice.

Refactoring the `bdbag.fetch.transports` module so that it supports run-time loading of externally provided fetch transport providers (similar to how the `bdbag.fetch.resolvers` module works) should be sufficient address this.


Discussion / hints:

Fix patch:
diff --git a/.travis.yml b/.travis.yml
index 03a5103..b191672 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,10 +1,10 @@
 language: python
 python:
   - "2.7"
-  - "3.4"
   - "3.5"
   - "3.6"
   - "3.7"
+  - "3.8"
 os:
   - linux
 #  - osx
diff --git a/bdbag/__init__.py b/bdbag/__init__.py
index 7256af5..8d64e1e 100644
--- a/bdbag/__init__.py
+++ b/bdbag/__init__.py
@@ -19,11 +19,13 @@
 import json
 import logging
 import mimetypes
+import shutil
+from datetime import datetime
 from requests.utils import requote_uri
 from distutils.util import strtobool
-from pkg_resources import get_distribution, DistributionNotFound
+from pkg_resources import parse_version, get_distribution, DistributionNotFound
 
-__version__ = "1.5.6"
+__version__ = "1.6.0-dev0"
 __bagit_version__ = "1.7.0"
 
 if sys.version_info > (3,):  # pragma: no cover
@@ -209,3 +211,43 @@ def inspect_path(path):
         is_dir = os.path.isdir(abs_path)
 
     return is_file, is_dir, is_uri
+
+
+def safe_move(old_path, new_path=None):
+    path_qualifier = '-' + datetime.strftime(datetime.now(), "%Y-%m-%d_%H.%M.%S")
+    if old_path and os.path.exists(old_path):
+        if not new_path:
+            new_path = ''.join([old_path, path_qualifier])
+            logging.info("Destination %s already exists, moving to %s" % (old_path, new_path))
+        elif new_path and os.path.exists(new_path):
+            override_path = ''.join([new_path, path_qualifier])
+            logging.info("Source %s and destination %s already exist, moving destination to %s" %
+                         (old_path, new_path, override_path))
+            new_path = override_path
+        shutil.move(old_path, new_path)
+    return new_path
+
+
+def bag_parent_dir_from_archive(file_list):
+    root_paths = set()
+    parent_paths = set()
+    child_paths = set()
+    if not isinstance(file_list, list):
+        return None
+    for path in file_list:
+        root = path.partition("/")
+        root_paths.add(root[0])
+        if root[1]:
+            parent_paths.add(root[0])
+        if "/" in root[2]:
+            child_paths.add(root[2].partition("/")[0])
+
+    if len(parent_paths - child_paths) > 1:
+        logging.warning("Unable to determine bag parent directory from archive file list. "
+                        "Expecting single bag parent dir but got: %s" % parent_paths)
+        return None
+    if len(root_paths - parent_paths) > 0:
+        logging.warning("Unable to determine bag parent directory from archive file list. "
+                        "Expecting single bag parent dir in archive but found files in the archive root")
+        return None
+    return parent_paths.pop()
diff --git a/bdbag/bdbag_api.py b/bdbag/bdbag_api.py
index 78e3c1c..3b1c9d7 100644
--- a/bdbag/bdbag_api.py
+++ b/bdbag/bdbag_api.py
@@ -59,10 +59,7 @@ def read_metadata(metadata_file):
 def cleanup_bag(bag_path, save=False):
     logger.info("Cleaning up bag dir: %s" % bag_path)
     if save:
-        saved_bag_path = ''.join([bag_path, '_', datetime.strftime(datetime.now(), "%Y-%m-%d_%H.%M.%S")])
-        logger.info("Moving bag %s to %s" % (bag_path, saved_bag_path))
-        shutil.move(bag_path, saved_bag_path)
-        return saved_bag_path
+        return safe_move(bag_path)
     else:
         shutil.rmtree(bag_path)
         return None
@@ -81,7 +78,7 @@ def ensure_bag_path_exists(bag_path, save=True):
 
 def revert_bag(bag_path):
     if not is_bag(bag_path):
-        logger.warning("Cannot revert the bag %s because it is not a bag directory!")
+        logger.warning("Cannot revert the bag %s because it is not a bag directory!" % bag_path)
         return
 
     for path in os.listdir(bag_path):
@@ -90,12 +87,15 @@ def revert_bag(bag_path):
                 os.remove(os.path.join(bag_path, path))
 
     data_path = os.path.join(bag_path, 'data')
-    for path in os.listdir(data_path):
-        old_path = os.path.join(data_path, path)
-        new_path = os.path.join(bag_path, path)
-        logger.debug("Bag revert: moving payload file %s to %s", old_path, new_path)
-        os.rename(old_path, new_path)
-    os.rmdir(data_path)
+    if os.path.isdir(data_path):
+        for path in os.listdir(data_path):
+            old_path = os.path.join(data_path, path)
+            new_path = os.path.join(bag_path, path)
+            logger.debug("Bag revert: moving payload file %s to %s", old_path, new_path)
+            os.rename(old_path, new_path)
+        os.rmdir(data_path)
+    else:
+        logger.warning("Bag directory %s does not contain a \"data\" directory to revert." % bag_path)
     logging.info("Bag directory %s has been reverted back to a normal directory." % bag_path)
 
 
@@ -227,7 +227,7 @@ def make_bag(bag_path,
              metadata=None,
              metadata_file=None,
              remote_file_manifest=None,
-             config_file=DEFAULT_CONFIG_FILE,
+             config_file=None,
              ro_metadata=None,
              ro_metadata_file=None):
     bag = None
@@ -358,40 +358,50 @@ def extract_bag(bag_path, output_path=None, temp=False):
     if not os.path.exists(bag_path):
         raise RuntimeError("Specified bag path not found: %s" % bag_path)
 
+    # determine output path for extraction
+    extracted_path = None
     bag_dir = os.path.splitext(os.path.basename(bag_path))[0]
     if os.path.isfile(bag_path):
         if temp:
             output_path = tempfile.mkdtemp(prefix='bag_')
+        elif output_path:
+            safe_move(output_path, output_path)
         elif not output_path:
             output_path = os.path.splitext(bag_path)[0]
-            if os.path.exists(output_path):
-                newpath = ''.join([output_path, '-', datetime.strftime(datetime.now(), "%Y-%m-%d_%H.%M.%S")])
-                logger.info("Specified output path %s already exists, moving existing directory to %s" %
-                            (output_path, newpath))
-                shutil.move(output_path, newpath)
+            safe_move(output_path)
             output_path = os.path.dirname(bag_path)
+
+        # perform the extraction
         if zipfile.is_zipfile(bag_path):
             logger.info("Extracting ZIP archived file: %s" % bag_path)
             with open(bag_path, 'rb') as bag_file:
                 zipped = zipfile.ZipFile(bag_file)
+                files = zipped.namelist()
+                extracted_path = bag_parent_dir_from_archive(files)
                 zipped.extractall(output_path)
                 zipped.close()
         elif tarfile.is_tarfile(bag_path):
             logger.info("Extracting TAR/GZ/BZ2 archived file: %s" % bag_path)
             tarred = tarfile.open(bag_path)
+            files = tarred.getnames()
+            extracted_path = bag_parent_dir_from_archive(files)
             tarred.extractall(output_path)
             tarred.close()
         else:
             raise RuntimeError("Archive format not supported for file: %s"
                                "\nSupported archive formats are ZIP or TAR/GZ/BZ2" % bag_path)
 
-    extracted_path = os.path.join(output_path, bag_dir)
+    if not extracted_path:
+        extracted_path = os.path.join(output_path, bag_dir)
+    else:
+        extracted_path = os.path.join(output_path, extracted_path)
+
     logger.info("File %s was successfully extracted to directory %s" % (bag_path, extracted_path))
 
     return extracted_path
 
 
-def validate_bag(bag_path, fast=False, callback=None, config_file=DEFAULT_CONFIG_FILE):
+def validate_bag(bag_path, fast=False, callback=None, config_file=None):
     config = read_config(config_file)
     bag_config = config['bag_config']
     bag_processes = bag_config.get('bag_processes', 1)
@@ -409,7 +419,7 @@ def validate_bag(bag_path, fast=False, callback=None, config_file=DEFAULT_CONFIG
     except bdbagit.BaggingInterruptedError as e:
         logger.warning(get_typed_exception(e))
         raise e
-    except Exception as e:
+    except Exception as e:  # pragma: no cover
         raise RuntimeError("Unhandled exception while validating bag: %s" % e)
 
 
@@ -504,7 +514,7 @@ def generate_remote_files_from_manifest(remote_file_manifest, algs, strict=False
     return remote_files
 
 
-def generate_ro_manifest(bag_path, overwrite=False, config_file=DEFAULT_CONFIG_FILE):
+def generate_ro_manifest(bag_path, overwrite=False, config_file=None):
     bag = bdbagit.BDBag(bag_path)
     bag_ro_metadata_path = os.path.abspath(os.path.join(bag_path, "metadata", "manifest.json"))
     exists = os.path.isfile(bag_ro_metadata_path)
@@ -551,7 +561,7 @@ def resolve_fetch(bag_path,
                   force=False,
                   callback=None,
                   keychain_file=DEFAULT_KEYCHAIN_FILE,
-                  config_file=DEFAULT_CONFIG_FILE,
+                  config_file=None,
                   filter_expr=None,
                   **kwargs):
     bag = bdbagit.BDBag(bag_path)
@@ -576,7 +586,7 @@ def materialize(input_path,
                 fetch_callback=None,
                 validation_callback=None,
                 keychain_file=DEFAULT_KEYCHAIN_FILE,
-                config_file=DEFAULT_CONFIG_FILE,
+                config_file=None,
                 filter_expr=None,
                 force=False,
                 **kwargs):
@@ -598,7 +608,7 @@ def materialize(input_path,
             raise RuntimeError("Unable to retrieve bag from: %s" % input_path)
 
     if bag_file:
-        bag_path = extract_bag(bag_file)
+        bag_path = extract_bag(bag_file, output_path)
 
     if bag_path:
         if not is_bag(bag_path):
diff --git a/bdbag/bdbag_cli.py b/bdbag/bdbag_cli.py
index 1880773..17bbc2d 100644
--- a/bdbag/bdbag_cli.py
+++ b/bdbag/bdbag_cli.py
@@ -20,7 +20,7 @@
 import traceback
 from bdbag.bdbagit import STANDARD_BAG_INFO_HEADERS
 from bdbag import bdbag_api as bdb, inspect_path, get_typed_exception, FILTER_DOCSTRING, VERSION, BAGIT_VERSION
-from bdbag.bdbag_config import bootstrap_config, DEFAULT_CONFIG_FILE
+from bdbag.bdbag_config import bootstrap_config, DEFAULT_CONFIG_FILE, DEFAULT_CONFIG_FILE_ENVAR
 from bdbag.fetch import fetcher
 from bdbag.fetch.auth.keychain import DEFAULT_KEYCHAIN_FILE
 
@@ -158,9 +158,10 @@ def parse_cli():
 
     config_file_arg = "--config-file"
     standard_args.add_argument(
-        config_file_arg, default=DEFAULT_CONFIG_FILE, metavar='<file>',
+        config_file_arg, metavar='<file>',
         help="Optional path to a configuration file. If this argument is not specified, the configuration file "
-             "defaults to: %s " % DEFAULT_CONFIG_FILE)
+             "will be set to the value of the environment variable %s (if present) or otherwise default to: %s"
+             % (DEFAULT_CONFIG_FILE_ENVAR, DEFAULT_CONFIG_FILE))
 
     keychain_file_arg = "--keychain-file"
     standard_args.add_argument(
@@ -195,7 +196,10 @@ def parse_cli():
         '--debug', action="store_true", help="Enable debug logging output.")
 
     standard_args.add_argument(
-        'path', metavar="<path>", help="Path to a bag directory or bag archive file.")
+        '--output-path', metavar="<dir>", help="User-defined output path for use with supported operations.")
+
+    standard_args.add_argument(
+        'path', metavar="<path>", default=None, help="Path to a bag directory or bag archive file.")
 
     metadata_args = parser.add_argument_group('Bag metadata arguments')
     headers = list(STANDARD_BAG_INFO_HEADERS)
@@ -315,7 +319,7 @@ def main():
     try:
         if args.materialize:
             bdb.materialize(path,
-                            output_path=None,
+                            output_path=args.output_path,
                             fetch_callback=None,
                             validation_callback=None,
                             keychain_file=args.keychain_file,
@@ -352,7 +356,7 @@ def main():
 
         # otherwise just extract the bag if it is an archive and no other conflicting options specified
         elif not (args.validate or args.validate_profile or args.resolve_fetch):
-            bdb.extract_bag(path)
+            bdb.extract_bag(path, output_path=args.output_path)
             if not args.quiet:
                 sys.stdout.write('\n')
             return result
@@ -372,7 +376,7 @@ def main():
 
         if args.validate:
             if is_file:
-                temp_path = bdb.extract_bag(path, temp=True)
+                temp_path = bdb.extract_bag(path, args.output_path, temp=True if not args.output_path else False)
             if args.validate == 'structure':
                 bdb.validate_bag_structure(temp_path if temp_path else path)
             elif args.validate == 'completeness':
@@ -391,7 +395,7 @@ def main():
         if args.validate_profile:
             if is_file:
                 if not temp_path:
-                    temp_path = bdb.extract_bag(path, temp=True)
+                    temp_path = bdb.extract_bag(path, args.output_path, temp=True if not args.output_path else False)
             profile = bdb.validate_bag_profile(temp_path if temp_path else path)
             bdb.validate_bag_serialization(archive if archive else path, profile)
 
diff --git a/bdbag/bdbag_config.py b/bdbag/bdbag_config.py
index 36487f3..87f6c7e 100644
--- a/bdbag/bdbag_config.py
+++ b/bdbag/bdbag_config.py
@@ -18,7 +18,8 @@
 import logging
 import json
 from collections import OrderedDict
-from bdbag import get_typed_exception, DEFAULT_CONFIG_PATH, BAG_PROFILE_TAG, BDBAG_PROFILE_ID, VERSION
+from bdbag import parse_version, get_typed_exception, safe_move, \
+    DEFAULT_CONFIG_PATH, BAG_PROFILE_TAG, BDBAG_PROFILE_ID, VERSION
 from bdbag.fetch import Megabyte
 from bdbag.fetch.auth.keychain import DEFAULT_KEYCHAIN_FILE, write_keychain
 
@@ -31,6 +32,7 @@
 BAG_METADATA_TAG = "bag_metadata"
 CONFIG_VERSION_TAG = "bdbag_config_version"
 DEFAULT_BAG_SPEC_VERSION = "0.97"
+DEFAULT_CONFIG_FILE_ENVAR = "BDBAG_CONFIG_FILE"
 DEFAULT_CONFIG_FILE = os.path.join(DEFAULT_CONFIG_PATH, 'bdbag.json')
 DEFAULT_BAG_ALGORITHMS = ['md5', 'sha256']
 
@@ -55,8 +57,20 @@
 DEFAULT_FETCH_CONFIG = {
     "http": {
         "session_config": {
-            "retry_connect": 5,
-            "retry_read": 5,
+            "retry_connect": 2,
+            "retry_read": 4,
+            "retry_backoff_factor": 1.0,
+            "retry_status_forcelist": [500, 502, 503, 504]
+        },
+        "allow_redirects": True,
+        "redirect_status_codes": DEFAULT_FETCH_HTTP_REDIRECT_STATUS_CODES,
+        COOKIE_JAR_TAG: DEFAULT_COOKIE_JAR_SEARCH_CONFIG
+
+    },
+    "https": {
+        "session_config": {
+            "retry_connect": 2,
+            "retry_read": 4,
             "retry_backoff_factor": 1.0,
             "retry_status_forcelist": [500, 502, 503, 504]
         },
@@ -67,18 +81,17 @@
     },
     "s3": {
         "read_chunk_size": 10 * Megabyte,
-        "read_timeout_seconds": 120,
-        "max_read_retries": 5
+        "read_timeout_seconds": 60,
+        "max_read_retries": 3
     }
 }
 
 ID_RESOLVER_TAG = "identifier_resolvers"
-DEFAULT_ID_RESOLVERS = ['n2t.net', 'identifiers.org']
+DEFAULT_ID_RESOLVERS = ['identifiers.org', 'n2t.net']
 RESOLVER_CONFIG_TAG = "resolver_config"
 DEFAULT_RESOLVER_CONFIG = {
     "ark": [
         {
-            "prefix": None,
             ID_RESOLVER_TAG: DEFAULT_ID_RESOLVERS
         },
         {
@@ -135,6 +148,20 @@
 }
 
 
+def get_updated_config_keys(config):
+    updated_config_keys = list()
+    if parse_version(config.get(CONFIG_VERSION_TAG, "0")) < parse_version("1.6.0"):
+        updated_config_keys.append([FETCH_CONFIG_TAG, ID_RESOLVER_TAG, RESOLVER_CONFIG_TAG])
+
+    return updated_config_keys
+
+
+def get_deprecated_config_keys(config):
+    deprecated_config_keys = list()
+
+    return deprecated_config_keys
+
+
 def write_config(config=DEFAULT_CONFIG, config_file=DEFAULT_CONFIG_FILE):
     try:
         config_path = os.path.dirname(config_file)
@@ -147,23 +174,33 @@ def write_config(config=DEFAULT_CONFIG, config_file=DEFAULT_CONFIG_FILE):
         with open(config_file, 'w') as cf:
             cf.write(json.dumps(config if config is not None else DEFAULT_CONFIG, indent=4, sort_keys=True))
             cf.close()
+            logging.info("Wrote configuration file: %s" % config_file)
     except Exception as e:
-        logger.debug("Unable to create configuration file %s. %s" %
-                     (config_file, get_typed_exception(e)))
+        logger.warning("Unable to create configuration file %s. %s" % (config_file, get_typed_exception(e)))
+
 
+def read_config(config_file=None, create_default=True, auto_upgrade=False):
+    if not config_file:
+        config_file = os.getenv(DEFAULT_CONFIG_FILE_ENVAR, DEFAULT_CONFIG_FILE)
+        if not os.path.isfile(config_file) and (config_file != DEFAULT_CONFIG_FILE) and not create_default:
+            logging.warning("Invalid configuration file path specified using environment variable %s: [%s]. "
+                            "Falling back to default configuration file path: [%s]" %
+                            (DEFAULT_CONFIG_FILE_ENVAR, config_file, DEFAULT_CONFIG_FILE))
+            config_file = DEFAULT_CONFIG_FILE
 
-def read_config(config_file, create_default=True, auto_upgrade=False):
-    if config_file == DEFAULT_CONFIG_FILE and not os.path.isfile(config_file) and create_default:
-        write_config()
-    elif auto_upgrade:
+    if not os.path.isfile(config_file) and create_default:
+        write_config(config_file=config_file)
+
+    if os.path.isfile(config_file) and auto_upgrade:
         upgrade_config(config_file)
 
     if os.path.isfile(config_file):
+        logging.debug("Loading configuration file from: %s" % config_file)
         with open(config_file) as cf:
             config = cf.read()
     else:
         config = json.dumps(DEFAULT_CONFIG)
-        logger.warning("Unable to read configuration file: [%s]. Using internal defaults." % DEFAULT_CONFIG_FILE)
+        logger.warning("Unable to read configuration file: [%s]. Using internal defaults." % config_file)
 
     return json.loads(config, object_pairs_hook=OrderedDict)
 
@@ -177,21 +214,35 @@ def upgrade_config(config_file):
         config = json.loads(cf.read(), object_pairs_hook=OrderedDict)
 
     new_config = None
-    config_version = config.get(CONFIG_VERSION_TAG)
-    if VERSION != config_version:
-        if not config_version:
-            new_config = DEFAULT_CONFIG.copy()
-            for k, v in config.get(BAG_CONFIG_TAG, {}).items():
-                new_config[BAG_CONFIG_TAG][k] = v
-            if config.get(ID_RESOLVER_TAG):
-                new_config[ID_RESOLVER_TAG] = config[ID_RESOLVER_TAG]
-            updated = True
+    if parse_version(VERSION) > parse_version(config.get(CONFIG_VERSION_TAG, "0")):
+        new_config = DEFAULT_CONFIG.copy()
+        config_items = [BAG_CONFIG_TAG, FETCH_CONFIG_TAG, RESOLVER_CONFIG_TAG, ID_RESOLVER_TAG]
+        copy_config_items(config, new_config, config_items)
+        updated = True
 
     if updated and new_config:
+        safe_move(config_file)
         write_config(new_config, config_file)
         print("Updated configuration file [%s] to current version format: %s" % (config_file, str(VERSION)))
 
 
+def copy_config_items(old_config, new_config, key_names):
+    for key_name in key_names:
+        if key_name in get_deprecated_config_keys(old_config):
+            continue
+        item = old_config.get(key_name)
+        if (item is None) or (key_name in get_updated_config_keys(old_config)):
+            new_config[key_name] = DEFAULT_CONFIG[key_name]
+        elif isinstance(item, dict):
+            for k, v in item.items():
+                if k not in get_deprecated_config_keys(old_config):
+                    new_config[key_name][k] = v
+        else:
+            new_config[key_name] = item
+
+    return new_config
+
+
 def bootstrap_config(config_file=DEFAULT_CONFIG_FILE, keychain_file=DEFAULT_KEYCHAIN_FILE, base_dir=None):
     if not base_dir:
         base_dir = os.path.expanduser('~')
diff --git a/bdbag/bdbagit.py b/bdbag/bdbagit.py
index 5fbf40f..7b4c7d3 100644
--- a/bdbag/bdbagit.py
+++ b/bdbag/bdbagit.py
@@ -23,7 +23,7 @@
 # bagit-python fixes the underlying issue
 try:
     algorithms_guaranteed = hashlib.algorithms_guaranteed
-except AttributeError:
+except AttributeError:  # pragma: no cover
     # Python 2.7.0-2.7.8
     hashlib.algorithms_guaranteed = set(hashlib.algorithms)
 
@@ -41,7 +41,7 @@
 def parse_version(version):
     try:
         return tuple(int(i) for i in version.split(".", 1))
-    except ValueError:
+    except ValueError:  # pragma: no cover
         raise BagError(
             _("Bag version numbers must be MAJOR.MINOR numbers, not %s") % version
         )
@@ -587,7 +587,7 @@ def _validate_entries(self, processes, callback=None):
         except BaggingInterruptedError:
             raise
         # Any unhandled exceptions are probably fatal
-        except:
+        except:  # pragma: no cover
             LOGGER.error(_("Unable to calculate file hashes for %s"), self)
             raise
 
diff --git a/bdbag/fetch/__init__.py b/bdbag/fetch/__init__.py
index fc201b9..b246fbc 100644
--- a/bdbag/fetch/__init__.py
+++ b/bdbag/fetch/__init__.py
@@ -14,11 +14,21 @@
 # limitations under the License.
 #
 import os
+import logging
 from bdbag import urlsplit, urlunquote
 
 Kilobyte = 1024
 Megabyte = Kilobyte ** 2
 
+SCHEME_HTTP = "http"
+SCHEME_HTTPS = "https"
+SCHEME_S3 = "s3"
+SCHEME_GS = "gs"
+SCHEME_GLOBUS = "globus"
+SCHEME_FTP = "ftp"
+SCHEME_SFTP = "sftp"
+SCHEME_TAG = "tag"
+
 
 def get_transfer_summary(total_bytes, elapsed_time):
     total_secs = elapsed_time.total_seconds()
@@ -31,6 +41,15 @@ def get_transfer_summary(total_bytes, elapsed_time):
     return summary
 
 
+def check_transfer_size_mismatch(path, expected, total):
+    if isinstance(expected, int) and isinstance(total, int):
+        if expected != total:
+            logging.warning("File [%s] transfer size mismatch. Expected %s bytes but received %s bytes." %
+                            (path, expected, total))
+            return True
+    return False
+
+
 def ensure_valid_output_path(url, output_path=None):
     if not output_path:
         upr = urlsplit(url, allow_fragments=False)
diff --git a/bdbag/fetch/auth/cookies.py b/bdbag/fetch/auth/cookies.py
index 450a657..387d29e 100644
--- a/bdbag/fetch/auth/cookies.py
+++ b/bdbag/fetch/auth/cookies.py
@@ -68,3 +68,14 @@ def load_and_merge_cookie_jars(cookie_jar_paths):
             cookie.expires = None
 
     return cookie_jar
+
+
+def get_request_cookies(config):
+    cookie_jars = set()
+    fetch_config = config.get(FETCH_CONFIG_TAG, DEFAULT_FETCH_CONFIG)
+    for config in fetch_config.values():
+        cookie_jar_config = config.get(COOKIE_JAR_TAG, DEFAULT_COOKIE_JAR_SEARCH_CONFIG)
+        if cookie_jar_config.get(COOKIE_JAR_SEARCH_TAG, True):
+            cookie_jars.update(set(find_cookie_jars(cookie_jar_config)))
+    cookies = load_and_merge_cookie_jars(cookie_jars)
+    return cookies
diff --git a/bdbag/fetch/auth/keychain.py b/bdbag/fetch/auth/keychain.py
index 03b42f6..260c6e2 100644
--- a/bdbag/fetch/auth/keychain.py
+++ b/bdbag/fetch/auth/keychain.py
@@ -95,7 +95,7 @@ def read_keychain(keychain_file=DEFAULT_KEYCHAIN_FILE, create_default=True):
         logger.debug("No keychain file specified and no default keychain file found, attempting to create one.")
         try:
             write_keychain(keychain_file=keychain_file)
-        except Exception as e:
+        except Exception as e:  # pragma: no cover
             logger.warning(
                 "Unable to create default keychain file. A keychain file is required for authentication when "
                 "retrieving files from protected remote resources. Either ensure that the default keychain "
diff --git a/bdbag/fetch/fetcher.py b/bdbag/fetch/fetcher.py
index 0eb67e3..5f27228 100644
--- a/bdbag/fetch/fetcher.py
+++ b/bdbag/fetch/fetcher.py
@@ -13,47 +13,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import os
 import datetime
+import logging
 from collections import namedtuple
 from bdbag import urlsplit, urlunquote, filter_dict
-from bdbag.bdbag_config import *
-from bdbag.fetch.transports import *
-from bdbag.fetch.auth.keychain import *
-from bdbag.fetch.auth.cookies import *
+from bdbag.bdbag_config import read_config, DEFAULT_CONFIG, DEFAULT_CONFIG_FILE, DEFAULT_KEYCHAIN_FILE, \
+    FETCH_CONFIG_TAG, DEFAULT_FETCH_CONFIG, RESOLVER_CONFIG_TAG, DEFAULT_RESOLVER_CONFIG
+from bdbag.fetch.auth.keychain import read_keychain, DEFAULT_KEYCHAIN_FILE
+from bdbag.fetch.auth.cookies import get_request_cookies
 from bdbag.fetch.resolvers import resolve
+from bdbag.fetch.transports import find_fetcher
+from bdbag.fetch.transports.base_transport import BaseFetchTransport
 
 logger = logging.getLogger(__name__)
 
-UNIMPLEMENTED = "Transfer protocol \"%s\" is not supported by this implementation"
-
-SCHEME_HTTP = 'http'
-SCHEME_HTTPS = 'https'
-SCHEME_S3 = 's3'
-SCHEME_GS = 'gs'
-SCHEME_GLOBUS = 'globus'
-SCHEME_FTP = 'ftp'
-SCHEME_SFTP = 'sftp'
-SCHEME_TAG = 'tag'
+UNIMPLEMENTED = "Transfer protocol \"%s\" is not supported."
 
 FetchEntry = namedtuple("FetchEntry", ["url", "length", "filename"])
 
 
 def fetch_bag_files(bag,
                     keychain_file=DEFAULT_KEYCHAIN_FILE,
-                    config_file=DEFAULT_CONFIG_FILE,
+                    config_file=None,
                     force=False,
                     callback=None,
                     filter_expr=None,
                     **kwargs):
 
-    auth = read_keychain(keychain_file)
+    keychain = read_keychain(keychain_file)
     config = read_config(config_file)
-    cookies = get_request_cookies(config) if kwargs.get("cookie_scan", True) else None
-
+    fetchers = kwargs.get("fetchers") or dict()
     success = True
     current = 0
     total = 0 if not callback else len(set(bag.files_to_be_fetched()))
     start = datetime.datetime.now()
+
     for entry in map(FetchEntry._make, bag.fetch_entries()):
         filename = urlunquote(entry.filename)
         if filter_expr:
@@ -71,17 +66,9 @@ def fetch_bag_files(bag,
                 missing = False
 
         if not force and not missing:
-            if logger.isEnabledFor(logging.DEBUG):
-                logger.debug("Not fetching already present file: %s" % output_path)
-            pass
+            logger.debug("Not fetching already present file: %s" % output_path)
         else:
-            result_path = fetch_file(entry.url,
-                                     output_path,
-                                     auth,
-                                     size=entry.length,
-                                     config=config,
-                                     cookies=cookies,
-                                     **kwargs)
+            result_path = fetch_file(entry.url, output_path, config, keychain, fetchers, size=remote_size, **kwargs)
             if not result_path:
                 success = False
 
@@ -93,68 +80,53 @@ def fetch_bag_files(bag,
                 break
     elapsed = datetime.datetime.now() - start
     logger.info("Fetch complete. Elapsed time: %s" % elapsed)
-    cleanup_transports()
+    cleanup_fetchers(fetchers)
     return success
 
 
-def fetch_file(url, path, auth, **kwargs):
+def fetch_single_file(url,
+                      output_path=None,
+                      config_file=None,
+                      keychain_file=DEFAULT_KEYCHAIN_FILE,
+                      **kwargs):
+
+    keychain = read_keychain(keychain_file)
+    config = read_config(config_file)
+    fetchers = kwargs.get("fetchers") or dict()
+    result_path = fetch_file(url, output_path, config, keychain, fetchers, **kwargs)
+    cleanup_fetchers(fetchers)
+
+    return result_path
+
 
+def fetch_file(url, output_path, config, keychain, fetchers, **kwargs):
     scheme = urlsplit(url).scheme.lower()
-    if SCHEME_HTTP == scheme or SCHEME_HTTPS == scheme:
-        return fetch_http.get_file(url, path, auth, **kwargs)
-    if SCHEME_FTP == scheme:
-        return fetch_ftp.get_file(url, path, auth, **kwargs)
-    if SCHEME_S3 == scheme or SCHEME_GS == scheme:
-        return fetch_boto3.get_file(url, path, auth, **kwargs)
-    if SCHEME_GLOBUS == scheme:
-        return fetch_globus.get_file(url, path, auth, **kwargs)
-    if SCHEME_TAG == scheme:  # pragma: no cover
-        logger.info("The fetch entry for file %s specifies the tag URI %s. Tag URIs may represent objects that "
-                    "cannot be directly resolved as network resources and therefore cannot be automatically fetched. "
-                    "Such files must be acquired outside of the context of this software." % (path, url))
-        return path
-
-    # if we get here, assume the url is an identifier and try to resolve it
-    config = kwargs.get("config")
+    fetch_config = config.get(FETCH_CONFIG_TAG) or DEFAULT_FETCH_CONFIG
+    fetcher = fetchers.get(scheme)
+    if not fetcher:
+        fetcher = find_fetcher(scheme, fetch_config, keychain, **kwargs)
+        if fetcher:
+            fetchers[scheme] = fetcher
+    if fetcher:
+        return fetcher.fetch(url, output_path, **kwargs)
+
+    # if we get here, assume the url contains an identifier scheme and try to resolve it as such
     resolver_config = config.get(RESOLVER_CONFIG_TAG, DEFAULT_RESOLVER_CONFIG) if config else DEFAULT_RESOLVER_CONFIG
     supported_resolvers = resolver_config.keys()
     if scheme in supported_resolvers:
         for entry in resolve(url, resolver_config):
             url = entry.get("url")
             if url:
-                output_path = fetch_file(url, path, auth, **kwargs)
-                if output_path:
-                    return output_path
+                result_path = fetch_file(url, output_path, config, keychain, fetchers, **kwargs)
+                if result_path:
+                    return result_path
         return None
 
     logger.warning(UNIMPLEMENTED % scheme)
     return None
 
 
-def fetch_single_file(url,
-                      output_path=None,
-                      config_file=DEFAULT_CONFIG_FILE,
-                      keychain_file=DEFAULT_KEYCHAIN_FILE,
-                      **kwargs):
-
-    auth = read_keychain(keychain_file)
-    config = read_config(config_file)
-    cookies = get_request_cookies(config) if kwargs.get("cookie_scan", True) else None
-    result_path = fetch_file(url, output_path, auth, config=config, cookies=cookies, **kwargs)
-    cleanup_transports()
-
-    return result_path
-
-
-def get_request_cookies(config):
-    fetch_config = config.get(FETCH_CONFIG_TAG, DEFAULT_FETCH_CONFIG)
-    http_fetch_config = fetch_config.get("http", dict())
-    cookie_jar_config = http_fetch_config.get(COOKIE_JAR_TAG, DEFAULT_COOKIE_JAR_SEARCH_CONFIG)
-    cookies = load_and_merge_cookie_jars(find_cookie_jars(cookie_jar_config)) if \
-        cookie_jar_config.get(COOKIE_JAR_SEARCH_TAG, True) else None
-    return cookies
-
-
-def cleanup_transports():
-    fetch_http.cleanup()
-    fetch_ftp.cleanup()
+def cleanup_fetchers(fetchers):
+    for fetcher in fetchers.values():
+        if isinstance(fetcher, BaseFetchTransport):
+            fetcher.cleanup()
diff --git a/bdbag/fetch/resolvers/__init__.py b/bdbag/fetch/resolvers/__init__.py
index 7dc1c74..39f3423 100644
--- a/bdbag/fetch/resolvers/__init__.py
+++ b/bdbag/fetch/resolvers/__init__.py
@@ -30,9 +30,13 @@ def find_resolver(identifier, resolver_config):
 
     resolver = None
     resolvers = resolver_config.get(scheme, [])
-    for resolver in resolvers:
-        prefix = resolver.get("prefix")
+    for entry in resolvers:
+        prefix = entry.get("prefix")
+        if not prefix:
+            resolver = entry
+            continue
         if prefix and prefix in path.lstrip("/"):
+            resolver = entry
             break
 
     if not resolver:
@@ -52,10 +56,10 @@ def find_resolver(identifier, resolver_config):
         except KeyError:
             module = import_module(module_name)
         clazz = getattr(module, class_name) if module else None
-    except (ImportError, AttributeError):
+    except (ImportError, AttributeError, ValueError):
         pass
     if not clazz:
-        raise RuntimeError("Unable to import specified resolver class %s" % resolver_class)
+        raise RuntimeError("Unable to import specified resolver class: [%s]" % resolver_class)
 
     return clazz(resolver.get(ID_RESOLVER_TAG, DEFAULT_ID_RESOLVERS), resolver_args)
 
diff --git a/bdbag/fetch/resolvers/ark_resolver.py b/bdbag/fetch/resolvers/ark_resolver.py
index 939f224..6eaef0b 100644
--- a/bdbag/fetch/resolvers/ark_resolver.py
+++ b/bdbag/fetch/resolvers/ark_resolver.py
@@ -36,7 +36,7 @@ def handle_response(self, response):
         entries = list()
         try:
             content = response.json()
-        except Exception as e:
+        except Exception as e:  # pragma: no cover
             logger.warning(
                 "Unable to parse identifier resolution result: a valid JSON structure was not found. Exception: %s. "
                 "Server response: %s" % (get_typed_exception(e), response.content))
diff --git a/bdbag/fetch/resolvers/base_resolver.py b/bdbag/fetch/resolvers/base_resolver.py
index 78cdf58..0b9d3af 100644
--- a/bdbag/fetch/resolvers/base_resolver.py
+++ b/bdbag/fetch/resolvers/base_resolver.py
@@ -61,8 +61,8 @@ def resolve(self, identifier, headers=None):
                         break
             r = session.get(resolver_url, headers=headers)
             if r.status_code != 200:
-                logger.error('HTTP GET Failed for %s with code: %s' % (r.url, r.status_code))
-                logger.error("Host %s responded:\n\n%s" % (urlsplit(r.url).netloc, r.text))
+                logger.error('HTTP GET Failed for %s with code: %s' % (resolver_url, r.status_code))
+                logger.error("Host %s responded:\n\n%s" % (urlsplit(resolver_url).netloc, r.text))
                 continue
             else:
                 urls = self.handle_response(r)
@@ -73,7 +73,7 @@ def resolve(self, identifier, headers=None):
                     (identifier, ', '.join([url["url"] for url in urls])))
                 break
             else:
-                logger.warning("No file locations were found for identifier %s" % identifier)
+                logger.warning("No file locations were found for identifier: [%s]" % identifier)
 
         session.close()
         return urls
diff --git a/bdbag/fetch/resolvers/dataguid_resolver.py b/bdbag/fetch/resolvers/dataguid_resolver.py
index 211e157..a605a53 100644
--- a/bdbag/fetch/resolvers/dataguid_resolver.py
+++ b/bdbag/fetch/resolvers/dataguid_resolver.py
@@ -37,7 +37,7 @@ def handle_response(self, response):
         entries = list()
         try:
             content = response.json()
-        except Exception as e:
+        except Exception as e:  # pragma: no cover
             logger.warning(
                 "Unable to parse identifier resolution result: a valid JSON structure was not found. Exception: %s. "
                 "Server response: %s" % (get_typed_exception(e), response.content))
diff --git a/bdbag/fetch/resolvers/doi_resolver.py b/bdbag/fetch/resolvers/doi_resolver.py
index a191925..b0a724f 100644
--- a/bdbag/fetch/resolvers/doi_resolver.py
+++ b/bdbag/fetch/resolvers/doi_resolver.py
@@ -35,14 +35,14 @@ def handle_response(self, response):
         entries = list()
         try:
             content = response.json()
-        except Exception as e:
+        except Exception as e:  # pragma: no cover
             logger.warning(
                 "Unable to parse identifier resolution result: a valid JSON structure was not found. Exception: %s. "
                 "Server response: %s" % (get_typed_exception(e), response.content))
             return entries
 
         base_entry = dict()
-        locations = content.get('contentUrl')
+        locations = content.get('contentUrl', content.get('url'))
         if locations:
             length = content.get("contentSize")
             if length:
@@ -64,6 +64,5 @@ def handle_response(self, response):
                     entry = dict(base_entry)
                     entry["url"] = location
                     entries.append(entry)
-
         return entries
 
diff --git a/bdbag/fetch/transports/__init__.py b/bdbag/fetch/transports/__init__.py
index 7bc6139..6f78ba2 100644
--- a/bdbag/fetch/transports/__init__.py
+++ b/bdbag/fetch/transports/__init__.py
@@ -13,7 +13,60 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import bdbag.fetch.transports.fetch_http
-import bdbag.fetch.transports.fetch_ftp
-import bdbag.fetch.transports.fetch_globus
-import bdbag.fetch.transports.fetch_boto3
+
+import sys
+import logging
+from importlib import import_module
+
+from bdbag import stob
+from bdbag.fetch import *
+from bdbag.fetch.transports.fetch_http import HTTPFetchTransport
+from bdbag.fetch.transports.fetch_ftp import FTPFetchTransport
+from bdbag.fetch.transports.fetch_globus import GlobusTransferFetchTransport
+from bdbag.fetch.transports.fetch_boto3 import BOTO3FetchTransport
+from bdbag.fetch.transports.fetch_tag import TAGFetchTransport
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_FETCH_TRANSPORTS = {
+    SCHEME_HTTP: HTTPFetchTransport,
+    SCHEME_HTTPS: HTTPFetchTransport,
+    SCHEME_FTP: FTPFetchTransport,
+    SCHEME_S3: BOTO3FetchTransport,
+    SCHEME_GS: BOTO3FetchTransport,
+    SCHEME_GLOBUS: GlobusTransferFetchTransport,
+    SCHEME_TAG: TAGFetchTransport
+}
+
+DEFAULT_SUPPORTED_SCHEMES = [DEFAULT_FETCH_TRANSPORTS.keys()]
+
+
+def find_fetcher(scheme, fetch_config, keychain, **kwargs):
+    clazz = None
+    config = fetch_config.get(scheme, fetch_config.get(scheme.lower(), fetch_config.get(scheme.upper()))) or {}
+    handler = config.get("handler")
+    if not handler:
+        clazz = DEFAULT_FETCH_TRANSPORTS.get(scheme.lower())
+        if not clazz:
+            return None
+
+    if not clazz:
+        try:
+            module_name, class_name = handler.rsplit(".", 1)
+            try:
+                module = sys.modules[module_name]
+            except KeyError:
+                module = import_module(module_name)
+            clazz = getattr(module, class_name) if module else None
+        except (ImportError, AttributeError, ValueError):
+            pass
+        if not clazz:
+            raise RuntimeError("Unable to import specified fetch handler class: [%s]" % handler)
+
+        if not stob(config.get("allow_keychain", False)):
+            keychain = None
+            logging.debug("Keychain will not be passed to fetch handler class [%s]. Set \"allow_keychain\":\"True\" in "
+                          "fetch config to enable keychain propagation." % handler)
+
+    return clazz(config, keychain, **kwargs)
+
diff --git a/bdbag/fetch/transports/base_transport.py b/bdbag/fetch/transports/base_transport.py
new file mode 100644
index 0000000..8c9853a
--- /dev/null
+++ b/bdbag/fetch/transports/base_transport.py
@@ -0,0 +1,30 @@
+#
+# Copyright 2016 University of Southern California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+class BaseFetchTransport(object):
+    def __init__(self, config, keychain, **kwargs):
+        self.config = config or dict()
+        self.keychain = keychain or list()
+        self.kwargs = kwargs
+
+    @classmethod
+    def fetch(cls, url, output_path, **kwargs):
+        raise NotImplementedError("Method must be implemented by subclass")
+
+    @classmethod
+    def cleanup(cls):
+        raise NotImplementedError("Method must be implemented by subclass")
diff --git a/bdbag/fetch/transports/fetch_boto3.py b/bdbag/fetch/transports/fetch_boto3.py
index 10ba7cc..4ad7437 100644
--- a/bdbag/fetch/transports/fetch_boto3.py
+++ b/bdbag/fetch/transports/fetch_boto3.py
@@ -19,8 +19,9 @@
 from importlib import import_module
 from bdbag import urlsplit, urlunsplit, stob, get_typed_exception
 from bdbag.bdbag_config import DEFAULT_CONFIG, DEFAULT_FETCH_CONFIG, FETCH_CONFIG_TAG
-from bdbag.fetch import Megabyte, get_transfer_summary, ensure_valid_output_path
-from bdbag.fetch.auth import keychain
+from bdbag.fetch import *
+from bdbag.fetch.transports.base_transport import BaseFetchTransport
+from bdbag.fetch.auth import keychain as kc
 
 logger = logging.getLogger(__name__)
 
@@ -29,136 +30,139 @@
 CHUNK_SIZE = 10 * Megabyte
 
 
-def import_boto3():
-    # locate library
-    global BOTO3, BOTOCORE
-    if BOTO3 is None and BOTOCORE is None:
-        try:
-            BOTO3 = import_module("boto3")
-            BOTOCORE = import_module("botocore")
-        except ImportError as e:
-            raise RuntimeError(
-                "Unable to find required module. Ensure that the Python package \"boto3\" is installed.", e)
-    if logging.getLogger().isEnabledFor(logging.DEBUG):
-        BOTO3.set_stream_logger('')
-
-
-def validate_auth_config(auth):
-    if not keychain.has_auth_attr(auth, 'auth_type'):
-        return False
-
-    return True
-
-
-def get_credentials(url, auth_config):
-
-    credentials = None
-    for auth in keychain.get_auth_entries(url, auth_config):
+class BOTO3FetchTransport(BaseFetchTransport):
 
-        if not validate_auth_config(auth):
-            continue
+    def __init__(self, config, keychain, **kwargs):
+        super(BOTO3FetchTransport, self).__init__(config, keychain, **kwargs)
+        self.config = config or DEFAULT_FETCH_CONFIG[SCHEME_S3]
 
-        auth_type = auth.get("auth_type")
-        auth_params = auth.get("auth_params")
-        if auth_type == 'aws-credentials':
-            credentials = auth_params
-            break
-
-    return credentials
-
-
-def get_file(url, output_path, auth_config, **kwargs):
-    success = False
-    output_path = ensure_valid_output_path(url, output_path)
-
-    try:
-        import_boto3()
-
-        bdbag_config = kwargs.get("config", DEFAULT_CONFIG)
-        fetch_config = bdbag_config.get(FETCH_CONFIG_TAG, DEFAULT_FETCH_CONFIG)
-        config = fetch_config.get("s3", DEFAULT_FETCH_CONFIG["s3"])
-        credentials = get_credentials(url, auth_config) or {}
-        key = credentials.get("key")
-        secret = credentials.get("secret")
-        token = credentials.get("token")
-        role_arn = credentials.get("role_arn")
-        profile_name = credentials.get("profile")
+    @staticmethod
+    def import_boto3():
+        # locate library
+        global BOTO3, BOTOCORE
+        if BOTO3 is None and BOTOCORE is None:
+            try:
+                BOTO3 = import_module("boto3")
+                BOTOCORE = import_module("botocore")
+            except ImportError as e:
+                raise RuntimeError(
+                    "Unable to find required module. Ensure that the Python package \"boto3\" is installed.", e)
+        if logging.getLogger().isEnabledFor(logging.DEBUG):
+            BOTO3.set_stream_logger("")
+
+    @staticmethod
+    def validate_auth_config(auth):
+        if not kc.has_auth_attr(auth, "auth_type"):
+            return False
+
+        return True
+
+    def get_credentials(self, url):
+        credentials = None
+        for auth in kc.get_auth_entries(url, self.keychain):
+            if not self.validate_auth_config(auth):
+                continue
+            auth_type = auth.get("auth_type")
+            auth_params = auth.get("auth_params")
+            if auth_type == "aws-credentials":
+                credentials = auth_params
+                break
+
+        return credentials
+
+    def fetch(self, url, output_path, **kwargs):
+        success = False
+        output_path = ensure_valid_output_path(url, output_path)
 
         try:
-            session = BOTO3.session.Session(profile_name=profile_name)
-        except Exception as e:
-            raise RuntimeError("Unable to create Boto3 session: %s" % get_typed_exception(e))
+            self.import_boto3()
+
+            credentials = self.get_credentials(url) or {}
+            key = credentials.get("key")
+            secret = credentials.get("secret")
+            token = credentials.get("token")
+            role_arn = credentials.get("role_arn")
+            profile_name = credentials.get("profile")
 
-        if role_arn:
             try:
-                sts = session.client('sts')
-                response = sts.assume_role(RoleArn=role_arn, RoleSessionName='BDBag-Fetch', DurationSeconds=3600)
-                temp_credentials = response['Credentials']
-                key = temp_credentials['AccessKeyId']
-                secret = temp_credentials['SecretAccessKey']
-                token = temp_credentials['SessionToken']
+                session = BOTO3.session.Session(profile_name=profile_name)
             except Exception as e:
-                raise RuntimeError(
-                    "Unable to get temporary credentials using arn [%s]. %s" % (role_arn, get_typed_exception(e)))
-
-        upr = urlsplit(url, allow_fragments=False)
-        try:
-            if upr.scheme == "gs":
-                endpoint_url = "https://storage.googleapis.com"
-                session_config = BOTO3.session.Config(signature_version="s3v4")
-                kwargs = {"aws_access_key_id": key,
-                          "aws_secret_access_key": secret,
-                          "endpoint_url": endpoint_url,
-                          "config": session_config}
-            else:
-                kwargs = {"aws_access_key_id": key, "aws_secret_access_key": secret}
-                if token:
-                    kwargs.update({"aws_session_token": token})
-            s3_client = session.client("s3", **kwargs)
-        except Exception as e:
-            raise RuntimeError("Unable to create Boto3 storage client: %s" % get_typed_exception(e))
-
-        logger.info("Attempting GET from URL: %s" % url)
-        response = s3_client.get_object(Bucket=upr.netloc, Key=upr.path.lstrip("/"))
-        chunk_size = config.get("read_chunk_size", CHUNK_SIZE)
-        max_retries = config.get("max_read_retries", 5)
-        retry_count = 0
-        total = 0
-
-        logger.debug("Transferring file %s to %s" % (url, output_path))
-        start = datetime.datetime.now()
-        with open(output_path, 'wb') as data_file:
-            stream = response["Body"]
-            stream.set_socket_timeout(config.get("read_timeout_seconds", 120))
-            chunk = None
-            while True:
-                while retry_count < max_retries:
-                    try:
-                        chunk = stream.read(chunk_size)
+                raise RuntimeError("Unable to create Boto3 session: %s" % get_typed_exception(e))
+
+            if role_arn:
+                try:
+                    sts = session.client("sts")
+                    response = sts.assume_role(RoleArn=role_arn, RoleSessionName="BDBag-Fetch", DurationSeconds=3600)
+                    temp_credentials = response["Credentials"]
+                    key = temp_credentials["AccessKeyId"]
+                    secret = temp_credentials["SecretAccessKey"]
+                    token = temp_credentials["SessionToken"]
+                except Exception as e:
+                    raise RuntimeError(
+                        "Unable to get temporary credentials using arn [%s]. %s" % (role_arn, get_typed_exception(e)))
+
+            upr = urlsplit(url, allow_fragments=False)
+            try:
+                if upr.scheme == SCHEME_GS:
+                    endpoint_url = "https://storage.googleapis.com"
+                    session_config = BOTO3.session.Config(signature_version="s3v4")
+                    kwargs = {"aws_access_key_id": key,
+                              "aws_secret_access_key": secret,
+                              "endpoint_url": endpoint_url,
+                              "config": session_config}
+                else:
+                    kwargs = {"aws_access_key_id": key, "aws_secret_access_key": secret}
+                    if token:
+                        kwargs.update({"aws_session_token": token})
+                s3_client = session.client("s3", **kwargs)
+            except Exception as e:
+                raise RuntimeError("Unable to create Boto3 storage client: %s" % get_typed_exception(e))
+
+            logger.info("Attempting GET from URL: %s" % url)
+            response = s3_client.get_object(Bucket=upr.netloc, Key=upr.path.lstrip("/"))
+            chunk_size = self.config.get("read_chunk_size", CHUNK_SIZE)
+            max_retries = self.config.get("max_read_retries", 5)
+            retry_count = 0
+            total = 0
+
+            logger.debug("Transferring file %s to %s" % (url, output_path))
+            start = datetime.datetime.now()
+            with open(output_path, "wb") as data_file:
+                stream = response["Body"]
+                stream.set_socket_timeout(self.config.get("read_timeout_seconds", 120))
+                chunk = None
+                while True:
+                    while retry_count < max_retries:
+                        try:
+                            chunk = stream.read(chunk_size)
+                            break
+                        except BOTOCORE.exceptions.ReadTimeoutError as rt:
+                            retry_count += 1
+                            logging.warning("Boto3 read timeout. Retrying attempt %s of %s" %
+                                            (retry_count, max_retries))
+                            if retry_count == max_retries:
+                                raise rt
+                    if chunk == b"" or chunk is None:
                         break
-                    except BOTOCORE.exceptions.ReadTimeoutError as rt:
-                        retry_count += 1
-                        logging.warning("Boto3 read timeout. Retrying attempt %s of %s" % (retry_count, max_retries))
-                        if retry_count == max_retries:
-                            raise rt
-                if chunk == b"" or chunk is None:
-                    break
-                data_file.write(chunk)
-                total += len(chunk)
-            stream.close()
-        elapsed_time = datetime.datetime.now() - start
-        summary = get_transfer_summary(total, elapsed_time)
-        logger.info('File [%s] transfer successful. %s' % (output_path, summary))
-        success = True
-    except BOTOCORE.exceptions.ClientError as e:
-        logger.error('Boto3 Client Error: %s' % get_typed_exception(e))
-    except BOTOCORE.exceptions.BotoCoreError as e:
-        logger.error('Boto3 Error: %s' % get_typed_exception(e))
-    except Exception as e:
-        logger.error(get_typed_exception(e))
-    finally:
-        if not success:
-            logger.error('Boto3 GET Failed for URL: %s' % url)
-            logger.warning('File transfer failed: [%s]' % output_path)
-
-    return output_path if success else None
+                    data_file.write(chunk)
+                    total += len(chunk)
+                stream.close()
+            elapsed_time = datetime.datetime.now() - start
+            check_transfer_size_mismatch(output_path, kwargs.get("size"), total)
+            logger.info("File [%s] transfer successful. %s" % (output_path, get_transfer_summary(total, elapsed_time)))
+            success = True
+        except BOTOCORE.exceptions.ClientError as e:
+            logger.error("Boto3 Client Error: %s" % get_typed_exception(e))
+        except BOTOCORE.exceptions.BotoCoreError as e:
+            logger.error("Boto3 Error: %s" % get_typed_exception(e))
+        except Exception as e:
+            logger.error(get_typed_exception(e))
+        finally:
+            if not success:
+                logger.error("Boto3 GET Failed for URL: %s" % url)
+                logger.warning("File transfer failed: [%s]" % output_path)
+
+        return output_path if success else None
+
+    def cleanup(self):
+        pass
diff --git a/bdbag/fetch/transports/fetch_ftp.py b/bdbag/fetch/transports/fetch_ftp.py
index 8edc609..caf9706 100644
--- a/bdbag/fetch/transports/fetch_ftp.py
+++ b/bdbag/fetch/transports/fetch_ftp.py
@@ -17,69 +17,70 @@
 import datetime
 import logging
 from bdbag import urlsplit, urlunsplit, urlretrieve, urlcleanup, get_typed_exception
-from bdbag.fetch import get_transfer_summary, ensure_valid_output_path
-import bdbag.fetch.auth.keychain as keychain
+from bdbag.fetch import *
+from bdbag.fetch.transports.base_transport import BaseFetchTransport
+import bdbag.fetch.auth.keychain as kc
 
 logger = logging.getLogger(__name__)
 
 
-def validate_auth_config(auth):
-    if not keychain.has_auth_attr(auth, 'uri'):
-        return False
-    if not keychain.has_auth_attr(auth, 'auth_type'):
-        return False
-    if not keychain.has_auth_attr(auth, 'auth_params'):
-        return False
-
-    return True
-
-
-def get_credentials(url, auth_config):
-
-    credentials = (None, None)
-    for auth in keychain.get_auth_entries(url, auth_config):
-
-        if not validate_auth_config(auth):
-            continue
-
-        auth_type = auth.get("auth_type")
-        auth_params = auth.get("auth_params", {})
-        username = auth_params.get("username")
-        password = auth_params.get("password")
-        if auth_type == 'ftp-basic':
-            credentials = (username, password)
-            break
-
-    return credentials
-
-
-def get_file(url, output_path, auth_config, **kwargs):
-
-    try:
-        credentials = kwargs.get("credentials")
-        if not credentials:
-            credentials = get_credentials(url, auth_config)
-        output_path = ensure_valid_output_path(url, output_path)
-        logger.info("Attempting FTP retrieve from URL: %s" % url)
-        creds = "%s:%s@" % (credentials[0] or "anonymous", credentials[1] or "bdbag@users.noreply.github.com")
-        url_parts = urlsplit(url)
-        full_url = urlunsplit(
-            (url_parts.scheme, "%s%s" % (creds, url_parts.netloc), url_parts.path, url_parts.query, url_parts.fragment))
-        start = datetime.datetime.now()
-        logger.debug("Transferring file %s to %s" % (url, output_path))
-        urlretrieve(full_url, output_path)
-        elapsed = datetime.datetime.now() - start
-        total = os.path.getsize(output_path)
-        summary = get_transfer_summary(total, elapsed)
-        logger.info('File [%s] transfer successful. %s' % (output_path, summary))
-        return output_path
-
-    except Exception as e:
-        logger.error('FTP Request Exception: %s' % (get_typed_exception(e)))
-        logger.warning('File transfer failed: [%s]' % output_path)
-
-    return None
-
-
-def cleanup():
-    urlcleanup()
+class FTPFetchTransport(BaseFetchTransport):
+
+    def __init__(self, config, keychain, **kwargs):
+        super(FTPFetchTransport, self).__init__(config, keychain, **kwargs)
+
+    @staticmethod
+    def validate_auth_config(auth):
+        if not kc.has_auth_attr(auth, "uri"):
+            return False
+        if not kc.has_auth_attr(auth, "auth_type"):
+            return False
+        if not kc.has_auth_attr(auth, "auth_params"):
+            return False
+
+        return True
+
+    def get_credentials(self, url):
+        credentials = (None, None)
+        for auth in kc.get_auth_entries(url, self.keychain):
+            if not self.validate_auth_config(auth):
+                continue
+            auth_type = auth.get("auth_type")
+            auth_params = auth.get("auth_params", {})
+            username = auth_params.get("username")
+            password = auth_params.get("password")
+            if auth_type == "ftp-basic":
+                credentials = (username, password)
+                break
+
+        return credentials
+
+    def fetch(self, url, output_path, **kwargs):
+        try:
+            credentials = kwargs.get("credentials")
+            if not credentials:
+                credentials = self.get_credentials(url)
+            output_path = ensure_valid_output_path(url, output_path)
+            logger.info("Attempting FTP retrieve from URL: %s" % url)
+            creds = "%s:%s@" % (credentials[0] or "anonymous", credentials[1] or "bdbag@users.noreply.github.com")
+            url_parts = urlsplit(url)
+            full_url = urlunsplit(
+                (url_parts.scheme, "%s%s" % (creds, url_parts.netloc),
+                 url_parts.path, url_parts.query, url_parts.fragment))
+            start = datetime.datetime.now()
+            logger.debug("Transferring file %s to %s" % (url, output_path))
+            urlretrieve(full_url, output_path)
+            elapsed = datetime.datetime.now() - start
+            total = os.path.getsize(output_path)
+            check_transfer_size_mismatch(output_path, kwargs.get("size"), total)
+            logger.info("File [%s] transfer successful. %s" % (output_path, get_transfer_summary(total, elapsed)))
+            return output_path
+
+        except Exception as e:
+            logger.error("FTP Request Exception: %s" % (get_typed_exception(e)))
+            logger.warning("File transfer failed: [%s]" % output_path)
+
+        return None
+
+    def cleanup(self):
+        urlcleanup()
diff --git a/bdbag/fetch/transports/fetch_globus.py b/bdbag/fetch/transports/fetch_globus.py
index 4d85d53..cda9104 100644
--- a/bdbag/fetch/transports/fetch_globus.py
+++ b/bdbag/fetch/transports/fetch_globus.py
@@ -18,111 +18,116 @@
 import logging
 import platform
 from bdbag import urlsplit, get_typed_exception
-from bdbag.fetch import ensure_valid_output_path
-import bdbag.fetch.auth.keychain as keychain
+from bdbag.fetch import *
+from bdbag.fetch.transports.base_transport import BaseFetchTransport
+import bdbag.fetch.auth.keychain as kc
 
 logger = logging.getLogger(__name__)
 globus_sdk = None
 globus_sdk_name = "globus_sdk"
 
 
-def validate_auth_config(auth):
-    if not keychain.has_auth_attr(auth, 'auth_type'):
-        return False
-    if not keychain.has_auth_attr(auth, 'auth_params'):
-        return False
-    if not keychain.has_auth_attr(auth.auth_params, 'transfer_token'):
-        return False
-    if not keychain.has_auth_attr(auth.auth_params, 'local_endpoint'):
-        return False
+class GlobusTransferFetchTransport(BaseFetchTransport):
+
+    def __init__(self, config, keychain, **kwargs):
+        super(GlobusTransferFetchTransport, self).__init__(config, keychain, **kwargs)
+
+    @staticmethod
+    def validate_auth_config(auth):
+        if not kc.has_auth_attr(auth, "auth_type"):
+            return False
+        if not kc.has_auth_attr(auth, "auth_params"):
+            return False
+        if not kc.has_auth_attr(auth.auth_params, "transfer_token"):
+            return False
+        if not kc.has_auth_attr(auth.auth_params, "local_endpoint"):
+            return False
+
+        return True
+
+    def get_credentials(self, url):
+        credentials = (None, None)
+        for auth in kc.get_auth_entries(url, self.keychain):
+            if not self.validate_auth_config(auth):
+                continue
+            auth_type = auth.get("auth_type")
+            auth_params = auth.get("auth_params", {})
+            if auth_type == "globus_transfer":
+                transfer_token = auth_params.get("transfer_token")
+                local_endpoint = auth_params.get("local_endpoint")
+                credentials = (transfer_token, local_endpoint)
+                break
+
+        return credentials
+
+    def fetch(self, url, output_path, **kwargs):
+
+        # locate library
+        global globus_sdk
+        if globus_sdk is None:
+            try:
+                globus_sdk = importlib.import_module(globus_sdk_name)
+            except ImportError:
+                pass
+        if globus_sdk is None:
+            raise RuntimeError("Cannot fetch file using Globus Transfer: unable to find the Globus SDK. "
+                               "Ensure that the Python module \"%s\" is installed." % globus_sdk_name)
 
-    return True
-
-
-def get_credentials(url, auth_config):
-
-    credentials = (None, None)
-    for auth in keychain.get_auth_entries(url, auth_config):
-
-        if not validate_auth_config(auth):
-            continue
-
-        auth_type = auth.get("auth_type")
-        auth_params = auth.get("auth_params", {})
-        if auth_type == 'globus_transfer':
-            transfer_token = auth_params.get("transfer_token")
-            local_endpoint = auth_params.get("local_endpoint")
-            credentials = (transfer_token, local_endpoint)
-            break
-
-    return credentials
-
-
-def get_file(url, output_path, auth_config, **kwargs):
-
-    # locate library
-    global globus_sdk
-    if globus_sdk is None:
         try:
-            globus_sdk = importlib.import_module(globus_sdk_name)
-        except ImportError:
-            pass
-    if globus_sdk is None:
-        raise RuntimeError("Cannot fetch file using Globus Transfer: unable to find the Globus SDK. "
-                           "Ensure that the Python module \"%s\" is installed." % globus_sdk_name)
-
-    try:
-        src_endpoint = urlsplit(url).hostname
-        src_path = urlsplit(url).path
-        output_path = ensure_valid_output_path(url, output_path)
-        if platform.system() == "Windows":
-            dest_path = ''.join(('/', output_path.replace('\\', '/').replace(':', '')))
-        else:
-            dest_path = os.path.abspath(output_path)
-
-        token, dest_endpoint = get_credentials(url, auth_config)
-        if token is None:
-            logger.warn("A valid Globus Transfer access token is required to create transfers. "
-                        "Check keychain.json for invalid parameters.")
-            return None
-
-        if dest_endpoint is None:
-            logger.warn("A valid Globus Transfer destination endpoint must be specified. "
-                        "Check keychain.json for invalid parameters.")
-            return None
-
-        # initialize transfer client
-        authorizer = globus_sdk.AccessTokenAuthorizer(token)
-        client = globus_sdk.TransferClient(authorizer=authorizer)
-
-        # Activate source endpoint
-        logger.debug("Activating source endpoint: %s" % src_endpoint)
-        data = client.endpoint_autoactivate(src_endpoint, if_expires_in=600)
-
-        # Activate destination endpoint
-        logger.debug("Activating destination endpoint: %s" % dest_endpoint)
-        data = client.endpoint_autoactivate(dest_endpoint, if_expires_in=600)
-
-        filename = src_path.rsplit('/', 1)[-1]
-        label = "".join(("BDBag Fetch -- ", filename.replace('.', '_')))
-
-        # get a unique ID for this transfer
-        tdata = globus_sdk.TransferData(client,
-                                        src_endpoint,
-                                        dest_endpoint,
-                                        label=label)
-
-        tdata.add_item(src_path, dest_path, recursive=False)
-
-        # start the transfer
-        data = client.submit_transfer(tdata)
-        task_id = data["task_id"]
-
-        logger.info("Globus Transfer started with ID %s" % task_id)
-        logger.debug("Transferring file %s to %s" % (url, output_path))
-        return output_path
-
-    except Exception as e:
-        logger.error('Globus Transfer request exception: %s' % get_typed_exception(e))
-
-    return None
+            src_endpoint = urlsplit(url).hostname
+            src_path = urlsplit(url).path
+            output_path = ensure_valid_output_path(url, output_path)
+            if platform.system() == "Windows":
+                dest_path = "".join(("/", output_path.replace("\\", "/").replace(":", "")))
+            else:
+                dest_path = os.path.abspath(output_path)
+
+            token, dest_endpoint = self.get_credentials(url)
+            if token is None:
+                logger.warning("A valid Globus Transfer access token is required to create transfers. "
+                               "Check keychain.json for invalid parameters.")
+                return None
+
+            if dest_endpoint is None:
+                logger.warning("A valid Globus Transfer destination endpoint must be specified. "
+                               "Check keychain.json for invalid parameters.")
+                return None
+
+            # initialize transfer client
+            authorizer = globus_sdk.AccessTokenAuthorizer(token)
+            client = globus_sdk.TransferClient(authorizer=authorizer)
+
+            # Activate source endpoint
+            logger.debug("Activating source endpoint: %s" % src_endpoint)
+            data = client.endpoint_autoactivate(src_endpoint, if_expires_in=600)
+
+            # Activate destination endpoint
+            logger.debug("Activating destination endpoint: %s" % dest_endpoint)
+            data = client.endpoint_autoactivate(dest_endpoint, if_expires_in=600)
+
+            filename = src_path.rsplit("/", 1)[-1]
+            label = "".join(("BDBag Fetch -- ", filename.replace(".", "_")))
+
+            # get a unique ID for this transfer
+            tdata = globus_sdk.TransferData(client,
+                                            src_endpoint,
+                                            dest_endpoint,
+                                            label=label)
+
+            tdata.add_item(src_path, dest_path, recursive=False)
+
+            # start the transfer
+            data = client.submit_transfer(tdata)
+            task_id = data["task_id"]
+
+            logger.info("Globus Transfer started with ID %s" % task_id)
+            logger.debug("Transferring file %s to %s" % (url, output_path))
+            return output_path
+
+        except Exception as e:
+            logger.error("Globus Transfer request exception: %s" % get_typed_exception(e))
+
+        return None
+
+    def cleanup(self):
+        pass
diff --git a/bdbag/fetch/transports/fetch_http.py b/bdbag/fetch/transports/fetch_http.py
index 457a176..69ca295 100644
--- a/bdbag/fetch/transports/fetch_http.py
+++ b/bdbag/fetch/transports/fetch_http.py
@@ -23,215 +23,218 @@
 from bdbag import urlsplit, stob, get_typed_exception
 from bdbag.bdbag_config import DEFAULT_CONFIG, DEFAULT_FETCH_CONFIG, FETCH_CONFIG_TAG, \
     FETCH_HTTP_REDIRECT_STATUS_CODES_TAG, DEFAULT_FETCH_HTTP_REDIRECT_STATUS_CODES
-from bdbag.fetch import Megabyte, get_transfer_summary, ensure_valid_output_path
-import bdbag.fetch.auth.keychain as keychain
+from bdbag.fetch import *
+from bdbag.fetch.transports.base_transport import BaseFetchTransport
+from bdbag.fetch.auth.cookies import get_request_cookies
+import bdbag.fetch.auth.keychain as kc
 
 logger = logging.getLogger(__name__)
 
 CHUNK_SIZE = 10 * Megabyte
-SESSIONS = dict()
-HEADERS = {'Connection': 'keep-alive'}
-
-
-def validate_auth_config(auth):
-    if not keychain.has_auth_attr(auth, 'auth_type'):
-        return False
-    if not keychain.has_auth_attr(auth, 'auth_params'):
-        return False
-
-    return True
-
-
-def get_auth(url, auth_config):
-    for auth in keychain.get_auth_entries(url, auth_config):
-        if validate_auth_config(auth):
-            return auth
-    return None
+HEADERS = {"Connection": "keep-alive"}
+
+
+class HTTPFetchTransport(BaseFetchTransport):
+
+    def __init__(self, config, keychain, **kwargs):
+        super(HTTPFetchTransport, self).__init__(config, keychain, **kwargs)
+        self.config = config or DEFAULT_FETCH_CONFIG[SCHEME_HTTP]
+        self.cookies = get_request_cookies(self.config) if kwargs.get("cookie_scan", True) else None
+        self.sessions = dict()
+
+    @staticmethod
+    def validate_auth_config(auth):
+        if not kc.has_auth_attr(auth, "auth_type"):
+            return False
+        if not kc.has_auth_attr(auth, "auth_params"):
+            return False
+
+        return True
+
+    def get_auth(self, url):
+        for auth in kc.get_auth_entries(url, self.keychain):
+            if self.validate_auth_config(auth):
+                return auth
+        return None
+
+    @staticmethod
+    def init_new_session(session_config):
+        session = requests.session()
+        retries = Retry(connect=session_config["retry_connect"],
+                        read=session_config["retry_read"],
+                        backoff_factor=session_config["retry_backoff_factor"],
+                        status_forcelist=session_config["retry_status_forcelist"])
+        session.mount("http://", HTTPAdapter(max_retries=retries))
+        session.mount("https://", HTTPAdapter(max_retries=retries))
+
+        return session
+
+    def get_session(self, url):
+        session = None
+        response = None
+
+        for auth in kc.get_auth_entries(url, self.keychain):
+            try:
+                if not self.validate_auth_config(auth):
+                    continue
+
+                uri = auth.get("uri")
+                if uri in self.sessions:
+                    session = self.sessions[uri]
+                    break
+                else:
+                    session = self.init_new_session(self.config["session_config"])
+
+                auth_type = auth.get("auth_type")
+                auth_params = auth.get("auth_params", {})
+
+                if auth_type == "cookie":
+                    if auth_params:
+                        cookies = auth_params.get("cookies", [])
+                        if cookies:
+                            for cookie in cookies:
+                                name, value = cookie.split("=", 1)
+                                session.cookies.set(name, value, domain=urlsplit(uri).hostname, path="/")
+                        session.headers.update(auth_params.get("additional_request_headers", {}))
+                        self.sessions[uri] = session
+                        break
+
+                if auth_type == "bearer-token":
+                    token = auth_params.get("token")
+                    if token:
+                        session.headers.update({"Authorization": "Bearer " + token})
+                        session.headers.update(auth_params.get("additional_request_headers", {}))
+                        self.sessions[uri] = session
+                        break
+                    else:
+                        logging.warning("Missing required parameters [token] for auth_type [%s] for keychain entry [%s]"
+                                        % (auth_type, uri))
+
+                # if we get here the assumption is that the auth_type is either http-basic or http-form and that an
+                # actual session "login" request is necessary
+                auth_uri = auth.get("auth_uri", uri)
+                username = auth_params.get("username")
+                password = auth_params.get("password")
+                if not (username and password):
+                    logging.warning(
+                        "Missing required parameters [username, password] for auth_type [%s] for keychain entry [%s]" %
+                        (auth_type, uri))
+                    continue
+
+                session.headers.update(auth_params.get("additional_request_headers", {}))
+
+                auth_method = auth_params.get("auth_method", "post")
+                if auth_type == "http-basic":
+                    session.auth = (username, password)
+                    if auth_method:
+                        auth_method = auth_method.lower()
+                    if auth_method == "post":
+                        response = session.post(auth_uri, auth=session.auth)
+                    elif auth_method == "get":
+                        response = session.get(auth_uri, auth=session.auth)
+                    else:
+                        logging.warning("Unsupported auth_method [%s] for auth_type [%s] for keychain entry [%s]" %
+                                        (auth_method, auth_type, uri))
+                elif auth_type == "http-form":
+                    username_field = auth_params.get("username_field", "username")
+                    password_field = auth_params.get("password_field", "password")
+                    response = session.post(auth_uri, {username_field: username, password_field: password})
+                if response.status_code > 203:
+                    logger.warning(
+                        "Authentication failed with Status Code: %s %s\n" % (response.status_code, response.text))
+                else:
+                    logger.info("Session established: %s", uri)
+                    self.sessions[uri] = session
+                    break
 
+            except Exception as e:  # pragma: no cover
+                logger.warning("Unhandled exception during HTTP(S) authentication: %s" % get_typed_exception(e))
 
-def get_session(url, auth_config, config):
+        if not session:
+            url_parts = urlsplit(url)
+            base_url = str("%s://%s" % (url_parts.scheme, url_parts.netloc))
+            session = self.sessions.get(base_url, None)
+            if not session:
+                session = self.init_new_session(self.config["session_config"])
+                self.sessions[base_url] = session
 
-    session = None
-    response = None
+        return session
 
-    for auth in keychain.get_auth_entries(url, auth_config):
+    def fetch(self, url, output_path, **kwargs):
         try:
-            if not validate_auth_config(auth):
-                continue
-
-            uri = auth.get("uri")
-            if uri in SESSIONS:
-                session = SESSIONS[uri]
-                break
-            else:
-                session = init_new_session(config["session_config"])
-
+            headers = kwargs.get("headers", HEADERS)
+            redirect_status_codes = self.config.get(
+                FETCH_HTTP_REDIRECT_STATUS_CODES_TAG, DEFAULT_FETCH_HTTP_REDIRECT_STATUS_CODES)
+
+            session = self.get_session(url)
+            output_path = ensure_valid_output_path(url, output_path)
+            allow_redirects = stob(self.config.get("allow_redirects", True))
+            allow_redirects_with_token = False
+            auth = self.get_auth(url) or {}
             auth_type = auth.get("auth_type")
-            auth_params = auth.get("auth_params", {})
-
-            if auth_type == 'cookie':
+            auth_params = auth.get("auth_params")
+            if auth_type == "bearer-token":
+                allow_redirects = False
+                # Force setting the "X-Requested-With": "XMLHttpRequest" header is a workaround for some OIDC servers
+                # which on an unauthenticated request redirect to a login flow instead of responding with a 401.
+                headers.update({"X-Requested-With": "XMLHttpRequest"})
                 if auth_params:
-                    cookies = auth_params.get("cookies", [])
-                    if cookies:
-                        for cookie in cookies:
-                            name, value = cookie.split('=', 1)
-                            session.cookies.set(name, value, domain=urlsplit(uri).hostname, path='/')
-                    session.headers.update(auth_params.get("additional_request_headers", {}))
-                    SESSIONS[uri] = session
-                    break
-
-            if auth_type == 'bearer-token':
-                token = auth_params.get("token")
-                if token:
-                    session.headers.update({"Authorization": "Bearer " + token})
-                    session.headers.update(auth_params.get("additional_request_headers", {}))
-                    SESSIONS[uri] = session
-                    break
-                else:
-                    logging.warning("Missing required parameters [token] for auth_type [%s] for keychain entry [%s]" %
-                                    (auth_type, uri))
-
-            # if we get here the assumption is that the auth_type is either http-basic or http-form and that an
-            # actual session "login" request is necessary
-            auth_uri = auth.get("auth_uri", uri)
-            username = auth_params.get("username")
-            password = auth_params.get("password")
-            if not (username and password):
-                logging.warning(
-                    "Missing required parameters [username, password] for auth_type [%s] for keychain entry [%s]" %
-                    (auth_type, uri))
-                continue
-
-            session.headers.update(auth_params.get("additional_request_headers", {}))
-
-            auth_method = auth_params.get("auth_method", "post")
-            if auth_type == 'http-basic':
-                session.auth = (username, password)
-                if auth_method:
-                    auth_method = auth_method.lower()
-                if auth_method == 'post':
-                    response = session.post(auth_uri, auth=session.auth)
-                elif auth_method == 'get':
-                    response = session.get(auth_uri, auth=session.auth)
+                    allow_redirects_with_token = stob(auth_params.get("allow_redirects_with_token", False))
+
+            while True:
+                logger.info("Attempting GET from URL: %s" % url)
+                r = session.get(url,
+                                stream=True,
+                                headers=headers,
+                                allow_redirects=allow_redirects,
+                                verify=certifi.where(),
+                                cookies=self.cookies)
+                if r.status_code in redirect_status_codes:
+                    url = r.headers["Location"]
+                    logger.info("Server responded with redirect to: %s" % url)
+                    if auth_type == "bearer-token":
+                        if allow_redirects_with_token:
+                            authorization = session.headers.get("Authorization")
+                            if authorization:
+                                headers.update({"Authorization": authorization})
+                            else:
+                                logger.warning(
+                                    "Unable to locate Authorization header in requests session headers after redirect")
+                        else:
+                            logger.warning("Authorization bearer token propagation on redirect is disabled for "
+                                           "security reasons. Enable token propagation for this URL in keychain.json")
+                            if session.headers.get("Authorization"):
+                                del session.headers["Authorization"]
+                    elif not allow_redirects:
+                        logger.warning("Redirects for this scheme have been disabled via the configuration file.")
+                        break
                 else:
-                    logging.warning("Unsupported auth_method [%s] for auth_type [%s] for keychain entry [%s]" %
-                                    (auth_method, auth_type, uri))
-            elif auth_type == 'http-form':
-                username_field = auth_params.get("username_field", "username")
-                password_field = auth_params.get("password_field", "password")
-                response = session.post(auth_uri, {username_field: username, password_field: password})
-            if response.status_code > 203:
-                logger.warning(
-                    'Authentication failed with Status Code: %s %s\n' % (response.status_code, response.text))
-            else:
-                logger.info("Session established: %s", uri)
-                SESSIONS[uri] = session
-                break
-
-        except Exception as e:
-            logger.warning("Unhandled exception during HTTP(S) authentication: %s" % get_typed_exception(e))
+                    break
 
-    if not session:
-        url_parts = urlsplit(url)
-        base_url = str("%s://%s" % (url_parts.scheme, url_parts.netloc))
-        session = SESSIONS.get(base_url, None)
-        if not session:
-            session = init_new_session(config["session_config"])
-            SESSIONS[base_url] = session
-
-    return session
-
-
-def init_new_session(session_config):
-    session = requests.session()
-    retries = Retry(connect=session_config['retry_connect'],
-                    read=session_config['retry_read'],
-                    backoff_factor=session_config['retry_backoff_factor'],
-                    status_forcelist=session_config['retry_status_forcelist'])
-    session.mount('http://', HTTPAdapter(max_retries=retries))
-    session.mount('https://', HTTPAdapter(max_retries=retries))
-
-    return session
-
-
-def get_file(url, output_path, auth_config, **kwargs):
-
-    try:
-        headers = kwargs.get("headers", HEADERS)
-        bdbag_config = kwargs.get("config", DEFAULT_CONFIG)
-        fetch_config = bdbag_config.get(FETCH_CONFIG_TAG, DEFAULT_FETCH_CONFIG)
-        config = fetch_config.get("http", DEFAULT_FETCH_CONFIG["http"])
-        redirect_status_codes = config.get(
-            FETCH_HTTP_REDIRECT_STATUS_CODES_TAG, DEFAULT_FETCH_HTTP_REDIRECT_STATUS_CODES)
-
-        session = get_session(url, auth_config, config)
-        output_path = ensure_valid_output_path(url, output_path)
-
-        allow_redirects = config.get("allow_redirects", False)
-        allow_redirects_with_token = False
-        auth = get_auth(url, auth_config) or {}
-        auth_type = auth.get("auth_type")
-        auth_params = auth.get("auth_params")
-        if auth_type == 'bearer-token':
-            allow_redirects = False
-            # Force setting the "X-Requested-With": "XMLHttpRequest" header is a workaround for some OIDC servers that
-            # on an unauthenticated request redirect to a login flow instead of responding with a 401 Unauthorized.
-            headers.update({"X-Requested-With": "XMLHttpRequest"})
-            if auth_params:
-                allow_redirects_with_token = stob(auth_params.get("allow_redirects_with_token", False))
-
-        while True:
-            logger.info("Attempting GET from URL: %s" % url)
-            r = session.get(url,
-                            stream=True,
-                            headers=headers,
-                            allow_redirects=allow_redirects,
-                            verify=certifi.where(),
-                            cookies=kwargs.get("cookies"))
-            if r.status_code in redirect_status_codes:
-                url = r.headers['Location']
-                logger.info("Server responded with redirect to: %s" % url)
-                if auth_type == 'bearer-token':
-                    if allow_redirects_with_token:
-                        authorization = session.headers.get("Authorization")
-                        if authorization:
-                            headers.update({"Authorization": authorization})
-                        else:
-                            logger.warning(
-                                "Unable to locate Authorization header in requests session headers after redirect")
-                    else:
-                        logger.warning("Authorization bearer token propagation on redirect is disabled for security "
-                                       "purposes. Enable token propagation for this URL in keychain.json")
-                        if session.headers.get("Authorization"):
-                            del session.headers["Authorization"]
-                continue
+            if r.status_code != 200:
+                logger.error("HTTP GET Failed for URL: %s" % url)
+                logger.error("Host %s responded:\n\n%s" % (urlsplit(url).netloc,  r.text))
+                logger.warning("File transfer failed: [%s]" % output_path)
             else:
-                break
-
-        if r.status_code != 200:
-            logger.error('HTTP GET Failed for URL: %s' % url)
-            logger.error("Host %s responded:\n\n%s" % (urlsplit(url).netloc,  r.text))
-            logger.warning('File transfer failed: [%s]' % output_path)
-        else:
-            total = 0
-            start = datetime.datetime.now()
-            logger.debug("Transferring file %s to %s" % (url, output_path))
-            with open(output_path, 'wb') as data_file:
-                for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
-                    data_file.write(chunk)
-                    total += len(chunk)
-            elapsed_time = datetime.datetime.now() - start
-            summary = get_transfer_summary(total, elapsed_time)
-            logger.info('File [%s] transfer successful. %s' % (output_path, summary))
-            return output_path
-
-    except requests.exceptions.RequestException as e:
-        logger.error('HTTP Request Exception: %s' % (get_typed_exception(e)))
-
-    return None
-
-
-def cleanup():
-    for session in SESSIONS.values():
-        session.close()
-    SESSIONS.clear()
+                total = 0
+                start = datetime.datetime.now()
+                logger.debug("Transferring file %s to %s" % (url, output_path))
+                with open(output_path, "wb") as data_file:
+                    for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
+                        data_file.write(chunk)
+                        total += len(chunk)
+                elapsed_time = datetime.datetime.now() - start
+                check_transfer_size_mismatch(output_path, kwargs.get("size"), total)
+                logger.info("File [%s] transfer successful. %s" %
+                            (output_path, get_transfer_summary(total, elapsed_time)))
+                return output_path
+
+        except requests.exceptions.RequestException as e:
+            logger.error("HTTP Request Exception: %s" % (get_typed_exception(e)))
+
+        return None
+
+    def cleanup(self):
+        for session in self.sessions.values():
+            session.close()
+        self.sessions.clear()
diff --git a/bdbag/fetch/transports/fetch_tag.py b/bdbag/fetch/transports/fetch_tag.py
new file mode 100644
index 0000000..5de3594
--- /dev/null
+++ b/bdbag/fetch/transports/fetch_tag.py
@@ -0,0 +1,35 @@
+#
+# Copyright 2016 University of Southern California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+from bdbag.fetch.transports.base_transport import BaseFetchTransport
+
+logger = logging.getLogger(__name__)
+
+
+class TAGFetchTransport(BaseFetchTransport):
+
+    def __init__(self, config, keychain, **kwargs):
+        super(TAGFetchTransport, self).__init__(config, keychain, **kwargs)
+
+    @staticmethod
+    def fetch(url, output_path, **kwargs):
+        logger.info("The fetch entry for file %s specifies the tag URI %s. Tag URIs may represent objects that "
+                    "cannot be directly resolved as network resources and therefore cannot be automatically fetched. "
+                    "Such files must be acquired outside of the context of this software." % (output_path, url))
+        return output_path
+
+    def cleanup(self):  # pragma: no cover
+        pass
diff --git a/doc/api.md b/doc/api.md
index df0210c..5b4fd32 100644
--- a/doc/api.md
+++ b/doc/api.md
@@ -183,7 +183,7 @@ make_bag(bag_path,
          metadata=None,
          metadata_file=None,
          remote_file_manifest=None,
-         config_file=bdbag.DEFAULT_CONFIG_FILE,
+         config_file=None,
          ro_metadata=None,
          ro_metadata_file=None)
 ```
@@ -215,7 +215,7 @@ materialize(input_path,
             fetch_callback=None,
             validation_callback=None,
             keychain_file=DEFAULT_KEYCHAIN_FILE,
-            config_file=DEFAULT_CONFIG_FILE,
+            config_file=None,
             filter_expr=None,
             force=False,
             **kwargs)
@@ -277,7 +277,7 @@ resolve_fetch(bag_path,
               force=False,
               callback=None,
               keychain_file=DEFAULT_KEYCHAIN_FILE,
-              config_file=DEFAULT_CONFIG_FILE,
+              config_file=None,
               filter_expr=None,
               **kwargs)
 ```
diff --git a/doc/cli.md b/doc/cli.md
index 3f0e0a0..6e50054 100644
--- a/doc/cli.md
+++ b/doc/cli.md
@@ -37,6 +37,7 @@ usage: bdbag
 [--quiet]
 [--debug]
 [--help]
+[--output-path]
 <path>
 ```
 
@@ -73,6 +74,10 @@ In order for a bag to be created in-place, the calling user must have write perm
 * If the target path is an archive file and no other conflicting arguments are specified, the archive file will be extracted.
 * If the target path is an actionable URL/URI, the target of the URL (or resolved URI, if the URI represents a supported identifier scheme) will be downloaded to the current directory.
 
+#### `--output-path`
+Optional output path parameter. Functions that may produce extracted directory output (such as `materialize`, `extract`, or `validate` when 
+operating on bag _archive_ files) can be configured to write such output to the specified `output-path`.
+
 ----
 #### `--update`
 Update an existing bag dir, recalculating tag-manifest checksums and regenerating manifests and fetch.txt if necessary.
@@ -178,7 +183,7 @@ Validate a bag against the profile specified by the bag's `BagIt-Profile-Identif
 #### `--config-file <file>`
 Optional path to a *bdbag* configuration file. The configuration file format is described
 [here](./config.md#bdbag.json).
-If this argument is not specified, the configuration file defaults to: `~/.bdbag/bdbag.json`
+If this argument is not specified, the configuration file will be set to the value of the environment variable `BDBAG_CONFIG_FILE` (if present) or otherwise default to `~/.bdbag/bdbag.json`.
 
 ----
 #### `--keychain-file <file>`
@@ -226,6 +231,7 @@ This following table enumerates the various arguments and compatibility modes.
 | Argument | Context | Description |
 |---:| :---: | --- |
 |`<path>`|all|Required argument. When no other options are specified, creates a bag from the target path if that path is a directory and not already a bag; otherwise, if the path represents an archive file in a supported format, the file is extracted.
+|`--output-path`|bag archive only|For a certain set of functions that may extract bag archive files (currently only `materialize`, `extract`, or `validate`), this argument dictates the directory where the output results of the extraction should be placed.
 |`--update`|bag dir only|An existing bag archive cannot be updated in-place. The bag must first be extracted and then updated.
 |`--revert`|bag dir only|Only a bag directory may be reverted to a non-bag directory.
 |`--archiver`|bag dir only|A bag archive cannot be created from an existing bag archive.
diff --git a/doc/config.md b/doc/config.md
index 3181520..8140fa0 100644
--- a/doc/config.md
+++ b/doc/config.md
@@ -41,8 +41,78 @@ This object contains all bag-related configuration parameters.
 |`bagit_spec_version`|The version of the `bagit` specification that created bags will conform to. Valid values are "0.97" or "1.0".
 
 ##### Object: `fetch_config`
-The `fetch_config` object contains a set of child objects each keyed by the scheme of the transport protocol that the object contains configuration parameters for.
-Currently, only the `http` and `s3` transports have configuration objects that control their behavior.
+The `fetch_config` object contains a set of child objects each keyed by the scheme of the transport protocol that contains the transport handler configuration parameters.
+
+###### Custom Transports
+There is a default set of transport handlers installed with `bdbag`. In addition, `bdbag` supports 
+externally implemented transport handlers that can be plugged-in (i.e., declared as run-time imports) via the `fetch_config` configuration object in the `bdbag.json` config file. 
+This requires developers to perform some integration tasks.
+
+###### Custom Transports: Implementation
+Developers should create a class deriving from `bdbag.fetch.transports.base_transport.BaseFetchTransport` 
+and implement three required functions:
+
+  * `__init__(self, config, keychain, **kwargs)`:
+     The class constructor.  Derived classes _should_ first call `super(<derived class name>, self).__init__(config, keychain, **kwargs)` which sets the `config`, `keychain`, and `kwargs` variables as class member variables with the same names. 
+
+  * `fetch(self, url, output_path, **kwargs)`: 
+    This method should implement the logic required to transfer a file referenced by `url` to the local path referenced by `output_path`.  The `**kwargs` argument is an extensible argument dictionary that the framework _may_ populate with extra data, for example: an integer argument `size` may present (if it can be found in `fetch.txt` for a given fetch entry), representing the expected size of the remote file in bytes.
+
+  * `cleanup(self)`:
+    This method should implement any transport-specific release of resources. Note this function will be called only once per-transport at the end of a entire bag fetch, and not once per-file.
+
+###### Custom Transports: Configuration
+Configure the usage of the external transport via the `fetch_config` object of the `bdbag.json` configuration file.  The `fetch_config` object is comprised of child configuration objects keyed by a lowercase string value representing the URL protocol scheme that is being configured. When configuring an external handler, the following applies:
+
+  * There is a single _required_ top-level string parameter with the key name `handler` which maps to the fully-qualified class name implementing the required methods of the  `bdbag.fetch.transports.base_transport.BaseFetchTransport` base class. At runtime the __bdbag__ fetch framework code will attempt to load this class via `importlib.import_module` machinery and if successful, it will be instantiated and returned to the __bdbag__ fetch framework code and the class instance cached for the duration of the bag fetch operation. Subsequently, whenever a URL is encountered in `fetch.txt` with a protocol scheme matching that of the installed handler, that handler's `fetch` method will be invoked.
+
+  * There is also an _optional_ string parameter, `allow_keychain`, which must be present and evaluate to `true` in order to toggle the propagation of the __bdbag__ `keychain` into the handler code during the `__init__` call. If the `allow_keychain` parameter is missing or set to any other value that cannot be evaluated as a Python boolean `True`, then the value of the `keychain` variable passed to the `__init__` call will be `None`. In general, if the custom handler code has its own mechanism for managing credentials, then this parameter may be omitted. If the handler intends to make use of the __bdbag__ `keychain` that is currently in context for the current user and fetch operation, then this parameter must be present and evaluate to `True`.
+
+  * The remainder of the protocol scheme handler configuration object can consist of any valid JSON; the entire object value assigned to the _scheme_ key will be passed as the `config` parameter to the `__init__` method of the custom handler. 
+
+For example, given the following `fetch_config` section:
+```
+{
+    "fetch_config": {
+        "s3": {
+            "handler":"my.custom.S3Transport",
+            "max_read_retries": 5,
+            "read_chunk_size": 10485760,
+            "read_timeout_seconds": 120
+        },
+	    "foo": {
+            "handler":"my.custom.FooTransport",
+            "allow_keychain": true,
+            "my_foo_complex_config": {
+                "bar":[
+                    "a","b","c"
+                ],
+                "baz":{
+                    "xyz":123
+                }
+            }
+        }
+    }
+}
+```
+For the scheme `foo`, the following object will be passed as the `config` parameter to the `__init__` method of `my.custom.FooTransport` upon class instantiation:
+```
+{
+    "handler":"my.custom.FooTransport",
+    "allow_keychain": true,
+    "my_foo_complex_config": {
+        "bar":[
+            "a","b","c"
+         ],
+        "baz":{
+            "xyz":123
+        }
+    }
+}
+```
+
+###### Default Transports: Configuration
+Currently, only the default `http` and `s3` transport handlers have configuration objects that control their behavior.
 
 | Parameter | Description
 | --- | --- |
@@ -262,7 +332,7 @@ Below is a sample `keychain.json` file:
         "auth_type": "bearer-token",
         "auth_params": {
             "token": "<token>",
-            "allow_redirects_with_token": "True"
+            "allow_redirects_with_token": true
         }
     },
     {
diff --git a/setup.py b/setup.py
index d7c5cdb..5797e36 100644
--- a/setup.py
+++ b/setup.py
@@ -60,7 +60,7 @@
         'bagit'
     ],
     install_requires=['pytz',
-                      'tzlocal',
+                      'tzlocal==2.1',
                       'certifi',
                       'requests>=2.7.0',
                       'setuptools_scm',  # for bagit<=1.7.0 which does not properly include it in install_requires
@@ -80,10 +80,10 @@
         "Operating System :: Microsoft :: Windows",
         'Programming Language :: Python',
         'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7'
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8'
     ]
 )
 


Test patch:
diff --git a/test/test-data/test-archives/test-bag-multi-parent.tgz b/test/test-data/test-archives/test-bag-multi-parent.tgz
new file mode 100644
index 0000000..472cb74
Binary files /dev/null and b/test/test-data/test-archives/test-bag-multi-parent.tgz differ
diff --git a/test/test-data/test-archives/test-bag-multi-parent.zip b/test/test-data/test-archives/test-bag-multi-parent.zip
new file mode 100644
index 0000000..884e8d3
Binary files /dev/null and b/test/test-data/test-archives/test-bag-multi-parent.zip differ
diff --git a/test/test-data/test-archives/test-bag-no-parent.tgz b/test/test-data/test-archives/test-bag-no-parent.tgz
new file mode 100644
index 0000000..a04ec90
Binary files /dev/null and b/test/test-data/test-archives/test-bag-no-parent.tgz differ
diff --git a/test/test-data/test-archives/test-bag-no-parent.zip b/test/test-data/test-archives/test-bag-no-parent.zip
new file mode 100644
index 0000000..d5b0934
Binary files /dev/null and b/test/test-data/test-archives/test-bag-no-parent.zip differ
diff --git a/test/test-data/test-bag-fetch-ark-bad/bag-info.txt b/test/test-data/test-bag-fetch-ark-bad/bag-info.txt
new file mode 100644
index 0000000..055d112
--- /dev/null
+++ b/test/test-data/test-bag-fetch-ark-bad/bag-info.txt
@@ -0,0 +1,7 @@
+Bag-Software-Agent: BDBag version: 1.2.2 (Bagit version: 1.6.4) <https://github.com/fair-research/bdbag>
+BagIt-Profile-Identifier: https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json
+Bagging-Date: 2018-03-19
+Bagging-Time: 15:49:59 PDT
+Contact-Name: mdarcy
+Contact-Orcid: 0000-0003-2280-917X
+Payload-Oxum: 289.2
diff --git a/test/test-data/test-bag-fetch-ark-bad/bagit.txt b/test/test-data/test-bag-fetch-ark-bad/bagit.txt
new file mode 100644
index 0000000..c4aebb4
--- /dev/null
+++ b/test/test-data/test-bag-fetch-ark-bad/bagit.txt
@@ -0,0 +1,2 @@
+BagIt-Version: 0.97
+Tag-File-Character-Encoding: UTF-8
diff --git a/test/test-data/test-bag-fetch-ark-bad/data/README.txt b/test/test-data/test-bag-fetch-ark-bad/data/README.txt
new file mode 100644
index 0000000..ade2506
--- /dev/null
+++ b/test/test-data/test-bag-fetch-ark-bad/data/README.txt
@@ -0,0 +1,1 @@
+This is a test input directory structure for the bdbag unit tests.
\ No newline at end of file
diff --git a/test/test-data/test-bag-fetch-ark-bad/fetch.txt b/test/test-data/test-bag-fetch-ark-bad/fetch.txt
new file mode 100644
index 0000000..9880dd5
--- /dev/null
+++ b/test/test-data/test-bag-fetch-ark-bad/fetch.txt
@@ -0,0 +1,1 @@
+ark:/0/0	223	data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-fetch-ark-bad/manifest-md5.txt b/test/test-data/test-bag-fetch-ark-bad/manifest-md5.txt
new file mode 100644
index 0000000..a462c23
--- /dev/null
+++ b/test/test-data/test-bag-fetch-ark-bad/manifest-md5.txt
@@ -0,0 +1,1 @@
+64ed533554cc55b83c211b6c3e885702  data/README.txt
diff --git a/test/test-data/test-bag-fetch-ark-bad/manifest-sha256.txt b/test/test-data/test-bag-fetch-ark-bad/manifest-sha256.txt
new file mode 100644
index 0000000..9906ab5
--- /dev/null
+++ b/test/test-data/test-bag-fetch-ark-bad/manifest-sha256.txt
@@ -0,0 +1,2 @@
+31d6b84097497c8853b069da5fd0d725857025fa66a8dcf2f52af003bb8f3bbc  data/README.txt
+59e6e0b91b51d49a5fb0e1068980d2e7d2b2001a6d11c59c64156d32e197a626  data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-fetch-ark-bad/tagmanifest-md5.txt b/test/test-data/test-bag-fetch-ark-bad/tagmanifest-md5.txt
new file mode 100644
index 0000000..60b5e2c
--- /dev/null
+++ b/test/test-data/test-bag-fetch-ark-bad/tagmanifest-md5.txt
@@ -0,0 +1,5 @@
+cddd06ba6dc2efcfdaf4747fded70186 bag-info.txt
+9e5ad981e0d29adc278f6a294b8c2aca bagit.txt
+cb31a0316621be309215f68c5dc878e9 fetch.txt
+8181d9cba7f59bb7525895646210630c manifest-md5.txt
+c51bdbc841ad21079b3897964b435636 manifest-sha256.txt
diff --git a/test/test-data/test-bag-fetch-ark-bad/tagmanifest-sha256.txt b/test/test-data/test-bag-fetch-ark-bad/tagmanifest-sha256.txt
new file mode 100644
index 0000000..782d0e8
--- /dev/null
+++ b/test/test-data/test-bag-fetch-ark-bad/tagmanifest-sha256.txt
@@ -0,0 +1,5 @@
+ec1e92090c8684c88dfe180608223cdd16c5777194efd81b5f92139477400120 bag-info.txt
+e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt
+ad216c04add17b8026689de9ae6c89785d72d363c9470fc0943e6c3fbcbf4970 fetch.txt
+c7ebf78acf7b056ddf699e634d0604be8d648eb54f2f23d59b0dd3690afbd1b8 manifest-md5.txt
+567c4d651edc2f5634410498ba4e15493c21d1c3bf8814900abe275350716941 manifest-sha256.txt
diff --git a/test/test-data/test-bag-fetch-http-no-redirect/bag-info.txt b/test/test-data/test-bag-fetch-http-no-redirect/bag-info.txt
new file mode 100644
index 0000000..b6a49d2
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-no-redirect/bag-info.txt
@@ -0,0 +1,7 @@
+Bag-Software-Agent: BDBag version: 1.2.2 (Bagit version: 1.6.4) <https://github.com/fair-research/bdbag>
+BagIt-Profile-Identifier: https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json
+Bagging-Date: 2018-03-19
+Bagging-Time: 14:00:01 PDT
+Contact-Name: mdarcy
+Contact-Orcid: 0000-0003-2280-917X
+Payload-Oxum: 490.3
diff --git a/test/test-data/test-bag-fetch-http-no-redirect/bagit.txt b/test/test-data/test-bag-fetch-http-no-redirect/bagit.txt
new file mode 100644
index 0000000..c4aebb4
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-no-redirect/bagit.txt
@@ -0,0 +1,2 @@
+BagIt-Version: 0.97
+Tag-File-Character-Encoding: UTF-8
diff --git a/test/test-data/test-bag-fetch-http-no-redirect/data/README.txt b/test/test-data/test-bag-fetch-http-no-redirect/data/README.txt
new file mode 100644
index 0000000..ade2506
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-no-redirect/data/README.txt
@@ -0,0 +1,1 @@
+This is a test input directory structure for the bdbag unit tests.
\ No newline at end of file
diff --git a/test/test-data/test-bag-fetch-http-no-redirect/fetch.txt b/test/test-data/test-bag-fetch-http-no-redirect/fetch.txt
new file mode 100644
index 0000000..ab23687
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-no-redirect/fetch.txt
@@ -0,0 +1,2 @@
+https://github.com/fair-research/bdbag/raw/master/test/test-data/test-http/test-fetch-http.txt	201	data/test-fetch-http.txt
+https://raw.githubusercontent.com/fair-research/bdbag/master/test/test-data/test-http/test-fetch-identifier.txt	-	data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-fetch-http-no-redirect/manifest-md5.txt b/test/test-data/test-bag-fetch-http-no-redirect/manifest-md5.txt
new file mode 100644
index 0000000..57dc74d
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-no-redirect/manifest-md5.txt
@@ -0,0 +1,3 @@
+64ed533554cc55b83c211b6c3e885702  data/README.txt
+f3ad851f4213d41ce9690542010bffa0  data/test-fetch-http.txt
+abdaf93ac71675a2df628aa220abf487  data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-fetch-http-no-redirect/manifest-sha256.txt b/test/test-data/test-bag-fetch-http-no-redirect/manifest-sha256.txt
new file mode 100644
index 0000000..137e8ac
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-no-redirect/manifest-sha256.txt
@@ -0,0 +1,3 @@
+31d6b84097497c8853b069da5fd0d725857025fa66a8dcf2f52af003bb8f3bbc  data/README.txt
+861236468065b9b0ae369ae99bbc7df08b4db919438e288d923efc0e77775bbf  data/test-fetch-http.txt
+59e6e0b91b51d49a5fb0e1068980d2e7d2b2001a6d11c59c64156d32e197a626  data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-fetch-http-no-redirect/tagmanifest-md5.txt b/test/test-data/test-bag-fetch-http-no-redirect/tagmanifest-md5.txt
new file mode 100644
index 0000000..d8d7357
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-no-redirect/tagmanifest-md5.txt
@@ -0,0 +1,5 @@
+4becd28d25c53d98148dc26fd777c6e7 bag-info.txt
+9e5ad981e0d29adc278f6a294b8c2aca bagit.txt
+14f7a2505f4641b3cd8b78f47bad9791 fetch.txt
+b123ddb2316f672c223897b6dae1c314 manifest-md5.txt
+116643d6d1efbbab020740e3acc67729 manifest-sha256.txt
diff --git a/test/test-data/test-bag-fetch-http-no-redirect/tagmanifest-sha256.txt b/test/test-data/test-bag-fetch-http-no-redirect/tagmanifest-sha256.txt
new file mode 100644
index 0000000..e153f1f
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-no-redirect/tagmanifest-sha256.txt
@@ -0,0 +1,5 @@
+2ad2821c0afa3a30e45ed2ed543ab817afaafd0518e4cf1be28438b926e7e48b bag-info.txt
+e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt
+246fb36d4bed89d36060cdf868eec2f4492736dd9a2e654c6122afb2b88fda92 fetch.txt
+7b1b71f2a822746abc799e2bc5f64e5a87caf0eef63b221e6001948ae9142db9 manifest-md5.txt
+e8a67e24a84b58c0c25cd26c431b4f7dc281b9583b9a9166d96e9f9bd9bfa06d manifest-sha256.txt
diff --git a/test/test-data/test-bag-fetch-http-unexpected-filesize/bag-info.txt b/test/test-data/test-bag-fetch-http-unexpected-filesize/bag-info.txt
new file mode 100644
index 0000000..e840de0
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-unexpected-filesize/bag-info.txt
@@ -0,0 +1,7 @@
+Bag-Software-Agent: BDBag version: 1.2.2 (Bagit version: 1.6.4) <https://github.com/fair-research/bdbag>
+BagIt-Profile-Identifier: https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json
+Bagging-Date: 2018-03-19
+Bagging-Time: 11:10:44 PDT
+Contact-Name: mdarcy
+Contact-Orcid: 0000-0003-2280-917X
+Payload-Oxum: 490.3
diff --git a/test/test-data/test-bag-fetch-http-unexpected-filesize/bagit.txt b/test/test-data/test-bag-fetch-http-unexpected-filesize/bagit.txt
new file mode 100644
index 0000000..c4aebb4
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-unexpected-filesize/bagit.txt
@@ -0,0 +1,2 @@
+BagIt-Version: 0.97
+Tag-File-Character-Encoding: UTF-8
diff --git a/test/test-data/test-bag-fetch-http-unexpected-filesize/data/README.txt b/test/test-data/test-bag-fetch-http-unexpected-filesize/data/README.txt
new file mode 100644
index 0000000..ade2506
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-unexpected-filesize/data/README.txt
@@ -0,0 +1,1 @@
+This is a test input directory structure for the bdbag unit tests.
\ No newline at end of file
diff --git a/test/test-data/test-bag-fetch-http-unexpected-filesize/fetch.txt b/test/test-data/test-bag-fetch-http-unexpected-filesize/fetch.txt
new file mode 100644
index 0000000..e5afa10
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-unexpected-filesize/fetch.txt
@@ -0,0 +1,2 @@
+https://raw.githubusercontent.com/fair-research/bdbag/master/test/test-data/test-http/test-fetch-http.txt	200	data/test-fetch-http.txt
+https://raw.githubusercontent.com/fair-research/bdbag/master/test/test-data/test-http/test-fetch-identifier.txt	-	data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-fetch-http-unexpected-filesize/manifest-md5.txt b/test/test-data/test-bag-fetch-http-unexpected-filesize/manifest-md5.txt
new file mode 100644
index 0000000..57dc74d
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-unexpected-filesize/manifest-md5.txt
@@ -0,0 +1,3 @@
+64ed533554cc55b83c211b6c3e885702  data/README.txt
+f3ad851f4213d41ce9690542010bffa0  data/test-fetch-http.txt
+abdaf93ac71675a2df628aa220abf487  data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-fetch-http-unexpected-filesize/manifest-sha256.txt b/test/test-data/test-bag-fetch-http-unexpected-filesize/manifest-sha256.txt
new file mode 100644
index 0000000..137e8ac
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-unexpected-filesize/manifest-sha256.txt
@@ -0,0 +1,3 @@
+31d6b84097497c8853b069da5fd0d725857025fa66a8dcf2f52af003bb8f3bbc  data/README.txt
+861236468065b9b0ae369ae99bbc7df08b4db919438e288d923efc0e77775bbf  data/test-fetch-http.txt
+59e6e0b91b51d49a5fb0e1068980d2e7d2b2001a6d11c59c64156d32e197a626  data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-fetch-http-unexpected-filesize/tagmanifest-md5.txt b/test/test-data/test-bag-fetch-http-unexpected-filesize/tagmanifest-md5.txt
new file mode 100644
index 0000000..dc8fd39
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-unexpected-filesize/tagmanifest-md5.txt
@@ -0,0 +1,5 @@
+644b79dca54c5f0fe3984a6b0cb9413e bag-info.txt
+9e5ad981e0d29adc278f6a294b8c2aca bagit.txt
+48a285882f8c4c8f6ea18e47d9a3e418 fetch.txt
+b123ddb2316f672c223897b6dae1c314 manifest-md5.txt
+116643d6d1efbbab020740e3acc67729 manifest-sha256.txt
diff --git a/test/test-data/test-bag-fetch-http-unexpected-filesize/tagmanifest-sha256.txt b/test/test-data/test-bag-fetch-http-unexpected-filesize/tagmanifest-sha256.txt
new file mode 100644
index 0000000..d962218
--- /dev/null
+++ b/test/test-data/test-bag-fetch-http-unexpected-filesize/tagmanifest-sha256.txt
@@ -0,0 +1,5 @@
+66673e3847fabe11c60cb76ffc6a8cae055eed7c93cbd162c341bd4b5646ba2c bag-info.txt
+e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt
+476263fff2fe697fa731056d4e7cc9b891a53213012645c6142e7ac21e40e1f0 fetch.txt
+7b1b71f2a822746abc799e2bc5f64e5a87caf0eef63b221e6001948ae9142db9 manifest-md5.txt
+e8a67e24a84b58c0c25cd26c431b4f7dc281b9583b9a9166d96e9f9bd9bfa06d manifest-sha256.txt
diff --git a/test/test-data/test-bag-fetch-tag/bag-info.txt b/test/test-data/test-bag-fetch-tag/bag-info.txt
new file mode 100644
index 0000000..de8f447
--- /dev/null
+++ b/test/test-data/test-bag-fetch-tag/bag-info.txt
@@ -0,0 +1,7 @@
+Bag-Software-Agent: BDBag version: 1.2.2 (Bagit version: 1.6.4) <https://github.com/fair-research/bdbag>
+BagIt-Profile-Identifier: https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json
+Bagging-Date: 2018-03-19
+Bagging-Time: 13:52:08 PDT
+Contact-Name: mdarcy
+Contact-Orcid: 0000-0003-2280-917X
+Payload-Oxum: 267.2
diff --git a/test/test-data/test-bag-fetch-tag/bagit.txt b/test/test-data/test-bag-fetch-tag/bagit.txt
new file mode 100644
index 0000000..c4aebb4
--- /dev/null
+++ b/test/test-data/test-bag-fetch-tag/bagit.txt
@@ -0,0 +1,2 @@
+BagIt-Version: 0.97
+Tag-File-Character-Encoding: UTF-8
diff --git a/test/test-data/test-bag-fetch-tag/data/README.txt b/test/test-data/test-bag-fetch-tag/data/README.txt
new file mode 100644
index 0000000..ade2506
--- /dev/null
+++ b/test/test-data/test-bag-fetch-tag/data/README.txt
@@ -0,0 +1,1 @@
+This is a test input directory structure for the bdbag unit tests.
\ No newline at end of file
diff --git a/test/test-data/test-bag-fetch-tag/fetch.txt b/test/test-data/test-bag-fetch-tag/fetch.txt
new file mode 100644
index 0000000..a126bed
--- /dev/null
+++ b/test/test-data/test-bag-fetch-tag/fetch.txt
@@ -0,0 +1,1 @@
+tag:fair-research.org,2019-10-15:bdbag/test/test-data/test-http/test-fetch-http.txt	201	data/test-fetch-http.txt
diff --git a/test/test-data/test-bag-fetch-tag/manifest-md5.txt b/test/test-data/test-bag-fetch-tag/manifest-md5.txt
new file mode 100644
index 0000000..a2a0d50
--- /dev/null
+++ b/test/test-data/test-bag-fetch-tag/manifest-md5.txt
@@ -0,0 +1,2 @@
+64ed533554cc55b83c211b6c3e885702  data/README.txt
+f3ad851f4213d41ce9690542010bffa0  data/test-fetch-http.txt
diff --git a/test/test-data/test-bag-fetch-tag/manifest-sha256.txt b/test/test-data/test-bag-fetch-tag/manifest-sha256.txt
new file mode 100644
index 0000000..48794f5
--- /dev/null
+++ b/test/test-data/test-bag-fetch-tag/manifest-sha256.txt
@@ -0,0 +1,2 @@
+31d6b84097497c8853b069da5fd0d725857025fa66a8dcf2f52af003bb8f3bbc  data/README.txt
+861236468065b9b0ae369ae99bbc7df08b4db919438e288d923efc0e77775bbf  data/test-fetch-http.txt
diff --git a/test/test-data/test-bag-fetch-tag/tagmanifest-md5.txt b/test/test-data/test-bag-fetch-tag/tagmanifest-md5.txt
new file mode 100644
index 0000000..0ca402f
--- /dev/null
+++ b/test/test-data/test-bag-fetch-tag/tagmanifest-md5.txt
@@ -0,0 +1,5 @@
+132e473bb36350afea9af7ef9c39f3fa bag-info.txt
+9e5ad981e0d29adc278f6a294b8c2aca bagit.txt
+569f20e7f492532271e3949d19ea6ece fetch.txt
+89a6688120dc33971cb143ad7951bf0b manifest-md5.txt
+ae6d2bd9ba3f770ac0212a2f325ac4f5 manifest-sha256.txt
diff --git a/test/test-data/test-bag-fetch-tag/tagmanifest-sha256.txt b/test/test-data/test-bag-fetch-tag/tagmanifest-sha256.txt
new file mode 100644
index 0000000..b532549
--- /dev/null
+++ b/test/test-data/test-bag-fetch-tag/tagmanifest-sha256.txt
@@ -0,0 +1,5 @@
+69da51555ceb5ce14f2b6d7e90b80dfd6c9ae51c0103bf31f77884c1d4a7be65 bag-info.txt
+e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt
+3dd5a711cc9293773cc9e99c4600415a68636e5ffa4fb5fbc43c0b06a1609640 fetch.txt
+105a4654179426f944f5b69a0b915de2d88f1bc79c6baffae2316b1ef276d062 manifest-md5.txt
+22fbf3b38b56cd8dbba67f1e454a4c82e1666970bde2b3bc8664666ae63df96d manifest-sha256.txt
diff --git a/test/test-data/test-bag-incomplete-fetch/bag-info.txt b/test/test-data/test-bag-incomplete-fetch/bag-info.txt
index de8f447..dbfc790 100644
--- a/test/test-data/test-bag-incomplete-fetch/bag-info.txt
+++ b/test/test-data/test-bag-incomplete-fetch/bag-info.txt
@@ -4,4 +4,4 @@ Bagging-Date: 2018-03-19
 Bagging-Time: 13:52:08 PDT
 Contact-Name: mdarcy
 Contact-Orcid: 0000-0003-2280-917X
-Payload-Oxum: 267.2
+Payload-Oxum: 490.3
diff --git a/test/test-data/test-bag-incomplete-fetch/data/test-fetch-http.txt b/test/test-data/test-bag-incomplete-fetch/data/test-fetch-http.txt
index c596131..6f83b89 100644
--- a/test/test-data/test-bag-incomplete-fetch/data/test-fetch-http.txt
+++ b/test/test-data/test-bag-incomplete-fetch/data/test-fetch-http.txt
@@ -1,2 +1,2 @@
 This is a simple file to be used in HTTP-based fetch tests. It should NOT be edited or changed, as that will cause the
-checksum to be changed and thus tests which are expecting the checksum will fail
+checksum to be changed and thus tests which are expecting the checksum will fail.
diff --git a/test/test-data/test-bag-incomplete-fetch/fetch.txt b/test/test-data/test-bag-incomplete-fetch/fetch.txt
index d781976..59b7c11 100644
--- a/test/test-data/test-bag-incomplete-fetch/fetch.txt
+++ b/test/test-data/test-bag-incomplete-fetch/fetch.txt
@@ -1,1 +1,2 @@
 https://raw.githubusercontent.com/fair-research/bdbag/master/test/test-data/test-http/test-fetch-http.txt	201	data/test-fetch-http.txt
+https://raw.githubusercontent.com/fair-research/bdbag/master/test/test-data/test-http/test-fetch-identifier.txt	223	data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-incomplete-fetch/manifest-sha256.txt b/test/test-data/test-bag-incomplete-fetch/manifest-sha256.txt
index 48794f5..137e8ac 100644
--- a/test/test-data/test-bag-incomplete-fetch/manifest-sha256.txt
+++ b/test/test-data/test-bag-incomplete-fetch/manifest-sha256.txt
@@ -1,2 +1,3 @@
 31d6b84097497c8853b069da5fd0d725857025fa66a8dcf2f52af003bb8f3bbc  data/README.txt
 861236468065b9b0ae369ae99bbc7df08b4db919438e288d923efc0e77775bbf  data/test-fetch-http.txt
+59e6e0b91b51d49a5fb0e1068980d2e7d2b2001a6d11c59c64156d32e197a626  data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-incomplete-fetch/tagmanifest-md5.txt b/test/test-data/test-bag-incomplete-fetch/tagmanifest-md5.txt
index 4ab1348..d6e63bf 100644
--- a/test/test-data/test-bag-incomplete-fetch/tagmanifest-md5.txt
+++ b/test/test-data/test-bag-incomplete-fetch/tagmanifest-md5.txt
@@ -1,5 +1,5 @@
-132e473bb36350afea9af7ef9c39f3fa bag-info.txt
+c43f57a314c1aaf42036f6798acdb985 bag-info.txt
 9e5ad981e0d29adc278f6a294b8c2aca bagit.txt
-9c8daed4d8fb0e7f3e1cf822dfef0cce fetch.txt
+11a2643e83b82225dd9bd8c3e228b2f0 fetch.txt
 89a6688120dc33971cb143ad7951bf0b manifest-md5.txt
-ae6d2bd9ba3f770ac0212a2f325ac4f5 manifest-sha256.txt
+116643d6d1efbbab020740e3acc67729 manifest-sha256.txt
diff --git a/test/test-data/test-bag-incomplete-fetch/tagmanifest-sha256.txt b/test/test-data/test-bag-incomplete-fetch/tagmanifest-sha256.txt
index e615ba4..eeca6a9 100644
--- a/test/test-data/test-bag-incomplete-fetch/tagmanifest-sha256.txt
+++ b/test/test-data/test-bag-incomplete-fetch/tagmanifest-sha256.txt
@@ -1,5 +1,5 @@
-69da51555ceb5ce14f2b6d7e90b80dfd6c9ae51c0103bf31f77884c1d4a7be65 bag-info.txt
+b20ceca333f523fa103d8094ec944d562dc4abf747ed3233c0018e62a657adfb bag-info.txt
 e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt
-f5dfe0c5853dfba46ecbd22ae4f45f0b30a39227d34ebd0f2383181e484befd4 fetch.txt
+407f8ef583bbb27f8d8a6691b5be92ba52c66313ff0123763843c38ae90de2f2 fetch.txt
 105a4654179426f944f5b69a0b915de2d88f1bc79c6baffae2316b1ef276d062 manifest-md5.txt
-22fbf3b38b56cd8dbba67f1e454a4c82e1666970bde2b3bc8664666ae63df96d manifest-sha256.txt
+e8a67e24a84b58c0c25cd26c431b4f7dc281b9583b9a9166d96e9f9bd9bfa06d manifest-sha256.txt
diff --git a/test/test-data/test-bag-invalid-state-fetch-filesize/bag-info.txt b/test/test-data/test-bag-invalid-state-fetch-filesize/bag-info.txt
index 1ad6d21..2e5b933 100644
--- a/test/test-data/test-bag-invalid-state-fetch-filesize/bag-info.txt
+++ b/test/test-data/test-bag-invalid-state-fetch-filesize/bag-info.txt
@@ -3,4 +3,4 @@ BagIt-Profile-Identifier: https://raw.githubusercontent.com/fair-research/bdbag/
 Bagging-Date: 2018-03-19
 Contact-Name: mdarcy
 Contact-Orcid: 0000-0003-2280-917X
-Payload-Oxum: 985.3
+Payload-Oxum: 490.3
diff --git a/test/test-data/test-bag-invalid-state-fetch-filesize/data/test-fetch-http.txt b/test/test-data/test-bag-invalid-state-fetch-filesize/data/test-fetch-http.txt
new file mode 100644
index 0000000..ca64832
--- /dev/null
+++ b/test/test-data/test-bag-invalid-state-fetch-filesize/data/test-fetch-http.txt
@@ -0,0 +1,2 @@
+This is a simple file to be used in HTTP-based fetch tests. It should NOT be edited or changed, as that will cause the
+checksum to be changed and thus tests which are expecting the checksum will fail
\ No newline at end of file
diff --git a/test/test-data/test-bag-invalid-state-fetch-filesize/data/test-fetch-identifier.txt b/test/test-data/test-bag-invalid-state-fetch-filesize/data/test-fetch-identifier.txt
new file mode 100644
index 0000000..5d43b5d
--- /dev/null
+++ b/test/test-data/test-bag-invalid-state-fetch-filesize/data/test-fetch-identifier.txt
@@ -0,0 +1,2 @@
+This is a simple file to be used in HTTP-based identifier resolution fetch tests. It should NOT be edited or changed,
+as that will cause the checksum to be changed and thus tests which are expecting the checksum will fail.
diff --git a/test/test-data/test-bag-invalid-state-fetch-filesize/fetch.txt b/test/test-data/test-bag-invalid-state-fetch-filesize/fetch.txt
index d781976..1297d6f 100644
--- a/test/test-data/test-bag-invalid-state-fetch-filesize/fetch.txt
+++ b/test/test-data/test-bag-invalid-state-fetch-filesize/fetch.txt
@@ -1,1 +1,2 @@
 https://raw.githubusercontent.com/fair-research/bdbag/master/test/test-data/test-http/test-fetch-http.txt	201	data/test-fetch-http.txt
+https://raw.githubusercontent.com/fair-research/bdbag/master/test/test-data/test-http/test-fetch-identifier.txt	-	data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-invalid-state-fetch-filesize/manifest-md5.txt b/test/test-data/test-bag-invalid-state-fetch-filesize/manifest-md5.txt
index 3f4ff47..57dc74d 100644
--- a/test/test-data/test-bag-invalid-state-fetch-filesize/manifest-md5.txt
+++ b/test/test-data/test-bag-invalid-state-fetch-filesize/manifest-md5.txt
@@ -1,3 +1,3 @@
 64ed533554cc55b83c211b6c3e885702  data/README.txt
-0439473ea96d5300a38d69fdc644d1cf  data/bdbag-profile.json
 f3ad851f4213d41ce9690542010bffa0  data/test-fetch-http.txt
+abdaf93ac71675a2df628aa220abf487  data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-invalid-state-fetch-filesize/manifest-sha256.txt b/test/test-data/test-bag-invalid-state-fetch-filesize/manifest-sha256.txt
index 26c8dec..137e8ac 100644
--- a/test/test-data/test-bag-invalid-state-fetch-filesize/manifest-sha256.txt
+++ b/test/test-data/test-bag-invalid-state-fetch-filesize/manifest-sha256.txt
@@ -1,3 +1,3 @@
 31d6b84097497c8853b069da5fd0d725857025fa66a8dcf2f52af003bb8f3bbc  data/README.txt
-d7abf8e50b6f58e58fcb18a2564e0a64bc0a2dbd0d4323e1cbb0442287bfc148  data/bdbag-profile.json
 861236468065b9b0ae369ae99bbc7df08b4db919438e288d923efc0e77775bbf  data/test-fetch-http.txt
+59e6e0b91b51d49a5fb0e1068980d2e7d2b2001a6d11c59c64156d32e197a626  data/test-fetch-identifier.txt
diff --git a/test/test-data/test-bag-invalid-state-fetch-filesize/tagmanifest-md5.txt b/test/test-data/test-bag-invalid-state-fetch-filesize/tagmanifest-md5.txt
index 7b2db63..ce541ac 100644
--- a/test/test-data/test-bag-invalid-state-fetch-filesize/tagmanifest-md5.txt
+++ b/test/test-data/test-bag-invalid-state-fetch-filesize/tagmanifest-md5.txt
@@ -1,5 +1,5 @@
-7137aafb30b44854a56419a420edf3ba bag-info.txt
+2fc9b8b45421301826e3dc97aa628084 bag-info.txt
 9e5ad981e0d29adc278f6a294b8c2aca bagit.txt
-9c8daed4d8fb0e7f3e1cf822dfef0cce fetch.txt
-625c6594046dbf7135565eb355cd0eaa manifest-md5.txt
-89b72785a58a3e0a3dbe9845f216bf1d manifest-sha256.txt
+e5669cfff8b758195c4bcaba5e67f9ea fetch.txt
+b123ddb2316f672c223897b6dae1c314 manifest-md5.txt
+116643d6d1efbbab020740e3acc67729 manifest-sha256.txt
diff --git a/test/test-data/test-bag-invalid-state-fetch-filesize/tagmanifest-sha256.txt b/test/test-data/test-bag-invalid-state-fetch-filesize/tagmanifest-sha256.txt
index 9abf45a..2a9f457 100644
--- a/test/test-data/test-bag-invalid-state-fetch-filesize/tagmanifest-sha256.txt
+++ b/test/test-data/test-bag-invalid-state-fetch-filesize/tagmanifest-sha256.txt
@@ -1,5 +1,5 @@
-2eb33836646b1dc799fa03424286d54c67012ad17e93868d4b1db3f3b2003b0a bag-info.txt
+505a4e6d27d97c018459f73b28ca7d3af5fc329dc6be919173f48a2b1a2d1baf bag-info.txt
 e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt
-f5dfe0c5853dfba46ecbd22ae4f45f0b30a39227d34ebd0f2383181e484befd4 fetch.txt
-445a86c89b7358667a81e16e6584630bf5955ad3f03e5508201a91f0a04e1fdd manifest-md5.txt
-20f66f3cc1c6722db6458fe7621bedc080c8cc080095a2be0c52e8295b0c9d48 manifest-sha256.txt
+81ae1f788299d4dc360092dad59427dcd998d34eb5fa4fce39b078c4dce7de60 fetch.txt
+7b1b71f2a822746abc799e2bc5f64e5a87caf0eef63b221e6001948ae9142db9 manifest-md5.txt
+e8a67e24a84b58c0c25cd26c431b4f7dc281b9583b9a9166d96e9f9bd9bfa06d manifest-sha256.txt
diff --git a/test/test-data/test-bag-no-data/bag-info.txt b/test/test-data/test-bag-no-data/bag-info.txt
new file mode 100644
index 0000000..fe877d5
--- /dev/null
+++ b/test/test-data/test-bag-no-data/bag-info.txt
@@ -0,0 +1,6 @@
+Bag-Software-Agent: BDBag version: 1.2.2 (Bagit version: 1.6.4) <https://github.com/fair-research/bdbag>
+BagIt-Profile-Identifier: https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json
+Bagging-Date: 2018-03-19
+Contact-Name: mdarcy
+Contact-Orcid: 0000-0003-2280-917X
+Payload-Oxum: 167.3
diff --git a/test/test-data/test-bag-no-data/bagit.txt b/test/test-data/test-bag-no-data/bagit.txt
new file mode 100644
index 0000000..c4aebb4
--- /dev/null
+++ b/test/test-data/test-bag-no-data/bagit.txt
@@ -0,0 +1,2 @@
+BagIt-Version: 0.97
+Tag-File-Character-Encoding: UTF-8
diff --git a/test/test-data/test-bag-no-data/manifest-md5.txt b/test/test-data/test-bag-no-data/manifest-md5.txt
new file mode 100644
index 0000000..39d58ad
--- /dev/null
+++ b/test/test-data/test-bag-no-data/manifest-md5.txt
@@ -0,0 +1,3 @@
+64ed533554cc55b83c211b6c3e885702  data/README.txt
+dad6891eb4148dfed2d162370983d06f  data/test1/test1.txt
+11069c18ee13a990265cf944359dbca5  data/test2/test2.txt
diff --git a/test/test-data/test-bag-no-data/manifest-sha1.txt b/test/test-data/test-bag-no-data/manifest-sha1.txt
new file mode 100644
index 0000000..2a3f30b
--- /dev/null
+++ b/test/test-data/test-bag-no-data/manifest-sha1.txt
@@ -0,0 +1,3 @@
+8484402bf07484cc4e26cacea911a8cbcb20b8ac  data/README.txt
+bda0288b428216225977a4441deb6d47b8ea7b21  data/test1/test1.txt
+7dba91b360751a565acc1e2775055d9f8f8267c3  data/test2/test2.txt
diff --git a/test/test-data/test-bag-no-data/manifest-sha256.txt b/test/test-data/test-bag-no-data/manifest-sha256.txt
new file mode 100644
index 0000000..787836a
--- /dev/null
+++ b/test/test-data/test-bag-no-data/manifest-sha256.txt
@@ -0,0 +1,3 @@
+31d6b84097497c8853b069da5fd0d725857025fa66a8dcf2f52af003bb8f3bbc  data/README.txt
+359185267ff38b191b451db1be2cd419cd423568b1e79f36a764e43c9fd604e2  data/test1/test1.txt
+23e9ba3d71a709f4f3885825c614d0c2a4dd42bf03fdb713ff2485aac411b5e3  data/test2/test2.txt
diff --git a/test/test-data/test-bag-no-data/manifest-sha512.txt b/test/test-data/test-bag-no-data/manifest-sha512.txt
new file mode 100644
index 0000000..aaec489
--- /dev/null
+++ b/test/test-data/test-bag-no-data/manifest-sha512.txt
@@ -0,0 +1,3 @@
+01a48356d519886ed980a00b51d930d963097b491482af34afd20cdc256dd02fced58c87dfd25f6ddcb28499fa2ff6a46c59aaff9bf33370aba9a0098dc4016f  data/README.txt
+6f6ecabefdbb7fe7e682078d49e8d934bebaebe76fd21a2e44574b6db52ac3cce05d6eca25dd9a1a9daab0b6e5a30b72bcca1b95c8b0185f790575a9a8b1032e  data/test1/test1.txt
+1d6bed1f987550c6ba8af59d543eff1f44115128424dc97e038d509e9627b258bab9551c3d927d1e33065abaa78fb197e4dbb0e82729803974a142c766bc4ca7  data/test2/test2.txt
diff --git a/test/test-data/test-bag-no-data/tagmanifest-md5.txt b/test/test-data/test-bag-no-data/tagmanifest-md5.txt
new file mode 100644
index 0000000..8954d35
--- /dev/null
+++ b/test/test-data/test-bag-no-data/tagmanifest-md5.txt
@@ -0,0 +1,6 @@
+a054edbf4600d8fbd677b805403adc41 bag-info.txt
+9e5ad981e0d29adc278f6a294b8c2aca bagit.txt
+d0c7f2742808e0f0007f197339110959 manifest-md5.txt
+beb3e836fcab267224d2fc8d87e667db manifest-sha1.txt
+8f98a0b3cc9a383c5a2fb77b034300e9 manifest-sha256.txt
+ef1ab1a06c8466b4a0bb1af49c0c716d manifest-sha512.txt
diff --git a/test/test-data/test-bag-no-data/tagmanifest-sha1.txt b/test/test-data/test-bag-no-data/tagmanifest-sha1.txt
new file mode 100644
index 0000000..41120c3
--- /dev/null
+++ b/test/test-data/test-bag-no-data/tagmanifest-sha1.txt
@@ -0,0 +1,6 @@
+7fe1e63a953b81ef6e9506d22b0b976a56fb0377 bag-info.txt
+e2924b081506bac23f5fffe650ad1848a1c8ac1d bagit.txt
+00f00590ea8c78c27fd5866f337fd900eeeb9d17 manifest-md5.txt
+c392c06018b4dcb5a20db90cb0e33164bd3e2bd6 manifest-sha1.txt
+a7ef096226ef78e49be795835fe4d817f653cc00 manifest-sha256.txt
+5690cc83030704f99c6fc118e57b3c8e7de67938 manifest-sha512.txt
diff --git a/test/test-data/test-bag-no-data/tagmanifest-sha256.txt b/test/test-data/test-bag-no-data/tagmanifest-sha256.txt
new file mode 100644
index 0000000..be512eb
--- /dev/null
+++ b/test/test-data/test-bag-no-data/tagmanifest-sha256.txt
@@ -0,0 +1,6 @@
+879009975e2ea81060e0c54d5ee00453b06f67336f1985247c238fac84634d86 bag-info.txt
+e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt
+3bc03d737c6e0c7432ca7a5ac06c3f411292ff8ddd884c52f6b1df1abba50763 manifest-md5.txt
+70615718807d67bb652bfd1a3bdc35a7f99f8eda7458b073f5808db3f5c3b365 manifest-sha1.txt
+9603969a98d63e51b77e29b14d730a0041734eb953d3e626712f481103944b94 manifest-sha256.txt
+4f70c03f907a390e1b2c70e264c7997f49f9da3f550c664eb37b2afeac729b43 manifest-sha512.txt
diff --git a/test/test-data/test-bag-no-data/tagmanifest-sha512.txt b/test/test-data/test-bag-no-data/tagmanifest-sha512.txt
new file mode 100644
index 0000000..7fea2b2
--- /dev/null
+++ b/test/test-data/test-bag-no-data/tagmanifest-sha512.txt
@@ -0,0 +1,6 @@
+1dea44d89556c490839ac5240818f4c3478c82827a81a233245c470d3d710fdc46c201d7f8302d5364a3931da8eb8093a67f750fcd0ef73f6215ae8c4766af6e bag-info.txt
+418dcfbe17d5f4b454b18630be795462cf7da4ceb6313afa49451aa2568e41f7ca3d34cf0280c7d056dc5681a70c37586aa1755620520b9198eede905ba2d0f6 bagit.txt
+0eecbffb32b44584197afe7e983ebb8c3c321167e916a64f23c3ec8170c4940395da9820bb2074defadd77d9aa59c6aa890c1d652888c8ca12e46f403c540c98 manifest-md5.txt
+7444259042079d10f4ea436d598a7f3485d24bf7e790ff20c775213f22558cb08d92cae1bc7aa15dd83e8bb00873fae2963d8e6204adf172a8b1c147204f89b5 manifest-sha1.txt
+ad20e473a5f535021ca0bcaf7a19aa5ee045a162681cdf8b04ca5945cca6d4257204d694a78c00347c79f851e978fdad45d0dadb77fe881a11bb7b45b1f600e5 manifest-sha256.txt
+3655d3335949ceaec3b64b8dcdf3874951e3d7efc34db23b79a7e85273e887e600264e0a1b849b277c8aec3bf93c29d7f7c53eb743384de05ab7e92a6ae125cf manifest-sha512.txt
diff --git a/test/test-data/test-config/base-config.json b/test/test-data/test-config/base-config.json
index a4112c3..de8e10f 100644
--- a/test/test-data/test-config/base-config.json
+++ b/test/test-data/test-config/base-config.json
@@ -10,7 +10,7 @@
         "bag_processes": 1
     },
     "identifier_resolvers": [
-        "n2t.net",
-        "identifiers.org"
+        "identifiers.org",
+        "n2t.net"
     ]
 }
\ No newline at end of file
diff --git a/test/test-data/test-config/test-config-10.json b/test/test-data/test-config/test-config-10.json
new file mode 100644
index 0000000..9e46722
--- /dev/null
+++ b/test/test-data/test-config/test-config-10.json
@@ -0,0 +1,30 @@
+{
+    "bag_config": {
+        "bag_algorithms": [
+            "md5",
+            "sha256"
+        ],
+        "bag_metadata": {
+            "BagIt-Profile-Identifier": "https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json"
+        },
+        "bag_processes": 1,
+        "bagit_spec_version": "0.97"
+    },
+    "bdbag_config_version": "1.5.0",
+    "fetch_config": {
+        "https": {
+            "handler": "foo.bar.InvalidHandler",
+            "session_config": {
+                "retry_backoff_factor": 1.0,
+                "retry_connect": 1,
+                "retry_read": 1,
+                "retry_status_forcelist": [
+                    500,
+                    502,
+                    503,
+                    504
+                ]
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/test-data/test-config/test-config-3.json b/test/test-data/test-config/test-config-3.json
index b390ba2..20520ab 100644
--- a/test/test-data/test-config/test-config-3.json
+++ b/test/test-data/test-config/test-config-3.json
@@ -12,7 +12,7 @@
     },
     "bdbag_config_version": "1.5.0",
     "fetch_config": {
-        "http": {
+        "https": {
             "session_config": {
                 "retry_backoff_factor": 1.0,
                 "retry_connect": 1,
@@ -42,7 +42,8 @@
                     "n2t.net",
                     "identifiers.org"
                 ],
-                "prefix": null
+                "handler": "bdbag.fetch.resolvers.ark_resolver.MinidResolverHandler",
+                "args": {"allow_automatic_redirects": "True"}
             },
             {
                 "handler": "bdbag.fetch.resolvers.ark_resolver.MinidResolverHandler",
diff --git a/test/test-data/test-config/test-config-4.json b/test/test-data/test-config/test-config-4.json
new file mode 100644
index 0000000..e5b7285
--- /dev/null
+++ b/test/test-data/test-config/test-config-4.json
@@ -0,0 +1,49 @@
+{
+    "bag_config": {
+        "bag_algorithms": [
+            "md5",
+            "sha256"
+        ],
+        "bag_metadata": {
+            "BagIt-Profile-Identifier": "https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json"
+        },
+        "bag_processes": 1,
+        "bagit_spec_version": "0.97"
+    },
+    "bdbag_config_version": "1.5.0",
+    "fetch_config": {
+        "http": {
+            "session_config": {
+                "retry_backoff_factor": 1.0,
+                "retry_connect": 1,
+                "retry_read": 1,
+                "retry_status_forcelist": [
+                    500,
+                    502,
+                    503,
+                    504
+                ]
+            }
+        },
+        "s3": {
+            "max_read_retries": 5,
+            "read_chunk_size": 10485760,
+            "read_timeout_seconds": 120
+        }
+    },
+    "identifier_resolvers": [
+        "n2t.net",
+        "identifiers.org"
+    ],
+    "resolver_config": {
+        "minid": [
+            {
+                "handler": "Invalid Entry",
+                "identifier_resolvers": [
+                    "n2t.net",
+                    "identifiers.org"
+                ]
+            }
+        ]
+    }
+}
\ No newline at end of file
diff --git a/test/test-data/test-config/test-config-5.json b/test/test-data/test-config/test-config-5.json
new file mode 100644
index 0000000..01e5962
--- /dev/null
+++ b/test/test-data/test-config/test-config-5.json
@@ -0,0 +1,48 @@
+{
+    "bag_config": {
+        "bag_algorithms": [
+            "md5",
+            "sha256"
+        ],
+        "bag_metadata": {
+            "BagIt-Profile-Identifier": "https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json"
+        },
+        "bag_processes": 1,
+        "bagit_spec_version": "0.97"
+    },
+    "bdbag_config_version": "1.5.0",
+    "fetch_config": {
+        "http": {
+            "session_config": {
+                "retry_backoff_factor": 1.0,
+                "retry_connect": 1,
+                "retry_read": 1,
+                "retry_status_forcelist": [
+                    500,
+                    502,
+                    503,
+                    504
+                ]
+            }
+        },
+        "s3": {
+            "max_read_retries": 5,
+            "read_chunk_size": 10485760,
+            "read_timeout_seconds": 120
+        }
+    },
+    "identifier_resolvers": [
+        "n2t.net",
+        "identifiers.org"
+    ],
+    "resolver_config": {
+        "minid": [
+            {
+                "identifier_resolvers": [
+                    "n2t.net",
+                    "identifiers.org"
+                ]
+            }
+        ]
+    }
+}
\ No newline at end of file
diff --git a/test/test-data/test-config/test-config-6.json b/test/test-data/test-config/test-config-6.json
new file mode 100644
index 0000000..7e18dda
--- /dev/null
+++ b/test/test-data/test-config/test-config-6.json
@@ -0,0 +1,41 @@
+{
+    "bag_config": {
+        "bag_algorithms": [
+            "md5",
+            "sha256"
+        ],
+        "bag_metadata": {
+            "BagIt-Profile-Identifier": "https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json"
+        },
+        "bag_processes": 1,
+        "bagit_spec_version": "0.97"
+    },
+    "bdbag_config_version": "1.5.0",
+    "fetch_config": {
+        "https": {
+            "allow_redirects": false,
+            "session_config": {
+                "retry_backoff_factor": 1.0,
+                "retry_connect": 1,
+                "retry_read": 1,
+                "retry_status_forcelist": [
+                    500,
+                    502,
+                    503,
+                    504
+                ]
+            }
+        },
+        "s3": {
+            "max_read_retries": 5,
+            "read_chunk_size": 10485760,
+            "read_timeout_seconds": 120
+        }
+    },
+    "identifier_resolvers": [
+        "n2t.net",
+        "identifiers.org"
+    ],
+    "resolver_config": {
+    }
+}
\ No newline at end of file
diff --git a/test/test-data/test-config/test-config-7.json b/test/test-data/test-config/test-config-7.json
new file mode 100644
index 0000000..05de5b1
--- /dev/null
+++ b/test/test-data/test-config/test-config-7.json
@@ -0,0 +1,30 @@
+{
+    "bag_config": {
+        "bag_algorithms": [
+            "md5",
+            "sha256"
+        ],
+        "bag_metadata": {
+            "BagIt-Profile-Identifier": "https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json"
+        },
+        "bag_processes": 1,
+        "bagit_spec_version": "0.97"
+    },
+    "bdbag_config_version": "1.5.0",
+    "fetch_config": {
+        "https": {
+            "handler": "test.test_remote.CustomTestFetchTransport",
+            "session_config": {
+                "retry_backoff_factor": 1.0,
+                "retry_connect": 1,
+                "retry_read": 1,
+                "retry_status_forcelist": [
+                    500,
+                    502,
+                    503,
+                    504
+                ]
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/test-data/test-config/test-config-8.json b/test/test-data/test-config/test-config-8.json
new file mode 100644
index 0000000..e7a89ba
--- /dev/null
+++ b/test/test-data/test-config/test-config-8.json
@@ -0,0 +1,31 @@
+{
+    "bag_config": {
+        "bag_algorithms": [
+            "md5",
+            "sha256"
+        ],
+        "bag_metadata": {
+            "BagIt-Profile-Identifier": "https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json"
+        },
+        "bag_processes": 1,
+        "bagit_spec_version": "0.97"
+    },
+    "bdbag_config_version": "1.5.0",
+    "fetch_config": {
+        "https": {
+            "handler": "test.test_remote.CustomTestFetchTransport",
+            "allow_keychain": true,
+            "session_config": {
+                "retry_backoff_factor": 1.0,
+                "retry_connect": 1,
+                "retry_read": 1,
+                "retry_status_forcelist": [
+                    500,
+                    502,
+                    503,
+                    504
+                ]
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/test-data/test-config/test-config-9.json b/test/test-data/test-config/test-config-9.json
new file mode 100644
index 0000000..0b442ee
--- /dev/null
+++ b/test/test-data/test-config/test-config-9.json
@@ -0,0 +1,30 @@
+{
+    "bag_config": {
+        "bag_algorithms": [
+            "md5",
+            "sha256"
+        ],
+        "bag_metadata": {
+            "BagIt-Profile-Identifier": "https://raw.githubusercontent.com/fair-research/bdbag/master/profiles/bdbag-profile.json"
+        },
+        "bag_processes": 1,
+        "bagit_spec_version": "0.97"
+    },
+    "bdbag_config_version": "1.5.0",
+    "fetch_config": {
+        "https": {
+            "handler": "test.test_remote.BadCustomTestFetchTransport",
+            "session_config": {
+                "retry_backoff_factor": 1.0,
+                "retry_connect": 1,
+                "retry_read": 1,
+                "retry_status_forcelist": [
+                    500,
+                    502,
+                    503,
+                    504
+                ]
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/test-data/test-config/test-keychain-9.json b/test/test-data/test-config/test-keychain-9.json
new file mode 100644
index 0000000..a993e3c
--- /dev/null
+++ b/test/test-data/test-config/test-keychain-9.json
@@ -0,0 +1,32 @@
+[
+  {
+    "uri": "https://foo.com/",
+    "tag": "unit test",
+    "auth_type": "http-basic",
+    "auth_params": {
+      "auth_method": "get",
+      "username": "fake",
+      "password": "fake!"
+    }
+  },
+  {
+    "uri": "https://example.com/",
+    "tag": "unit test",
+    "auth_type": "bearer-token",
+    "auth_params": {
+      "token": "bar",
+      "allow_redirects_with_token": "False",
+      "additional_request_headers": {
+        "X-Requested-With": "XMLHttpRequest"
+      }
+    }
+  },
+  {
+    "uri": "ftp://ftp.nist.gov/",
+    "auth_type": "ftp-basic",
+    "auth_params": {
+      "username": "anonymous",
+      "password": "bdbag@users.noreply.github.com"
+    }
+  }
+]
\ No newline at end of file
diff --git a/test/test_api.py b/test/test_api.py
index f6e39d4..6968608 100644
--- a/test/test_api.py
+++ b/test/test_api.py
@@ -24,7 +24,7 @@
 from os.path import exists as ospe
 from os.path import isfile as ospif
 from bdbag import bdbag_api as bdb, bdbag_config as bdbcfg, bdbag_ro as bdbro, bdbagit as bdbagit, filter_dict, \
-    get_typed_exception
+    get_typed_exception, DEFAULT_CONFIG_PATH
 from bdbag.fetch.auth import keychain
 from test.test_common import BaseTest
 
@@ -73,6 +73,86 @@ def test_read_with_update_base_config(self):
         except Exception as e:
             self.fail(get_typed_exception(e))
 
+    def test_read_with_default_config(self):
+        logger.info(self.getTestHeader('read config with explicit path'))
+        config_envar = os.getenv(bdbcfg.DEFAULT_CONFIG_FILE_ENVAR)
+        try:
+            if config_envar:
+                logging.info("Ignoring already set envar %s: %s" % (bdbcfg.DEFAULT_CONFIG_FILE_ENVAR, config_envar))
+                del os.environ[bdbcfg.DEFAULT_CONFIG_FILE_ENVAR]
+            self.assertIsNone(os.getenv(bdbcfg.DEFAULT_CONFIG_FILE_ENVAR))
+            bdbcfg.read_config()
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["Loading configuration file from: %s" % bdbcfg.DEFAULT_CONFIG_FILE], output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
+    def test_read_with_explicit_config(self):
+        logger.info(self.getTestHeader('read config with explicit path'))
+        try:
+            config_file = ospj(self.test_config_dir, 'test-config.json')
+            bdbcfg.read_config(config_file=config_file, create_default=False)
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["Loading configuration file from: %s" % config_file], output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
+    def test_read_with_invalid_config(self):
+        logger.info(self.getTestHeader('read config with invalid path'))
+        try:
+            config_file = ospj(self.test_config_dir, 'bdbag-invalid.json')
+            bdbcfg.read_config(config_file=config_file, create_default=False)
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["Unable to read configuration file: [%s]. Using internal defaults." %
+                                         config_file], output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
+    def test_read_config_from_env(self):
+        logger.info(self.getTestHeader('read config from env'))
+        config_envar = os.getenv(bdbcfg.DEFAULT_CONFIG_FILE_ENVAR)
+        try:
+            if config_envar:
+                logging.info("Ignoring already set envar %s: %s" % (bdbcfg.DEFAULT_CONFIG_FILE_ENVAR, config_envar))
+                del os.environ[bdbcfg.DEFAULT_CONFIG_FILE_ENVAR]
+            self.assertIsNone(os.getenv(bdbcfg.DEFAULT_CONFIG_FILE_ENVAR))
+            config_file = ospj(self.test_config_dir, 'base-config.json')
+            os.environ[bdbcfg.DEFAULT_CONFIG_FILE_ENVAR] = config_file
+            config = bdbcfg.read_config()
+            self.assertEqual("bdbag test", config["bag_config"]["bag_metadata"]["Contact-Name"],
+                             "Unexpected config value")
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+        finally:
+            if os.getenv(bdbcfg.DEFAULT_CONFIG_FILE_ENVAR):
+                del os.environ[bdbcfg.DEFAULT_CONFIG_FILE_ENVAR]
+
+    def test_read_config_from_env_invalid(self):
+        logger.info(self.getTestHeader('read config from env invalid file path'))
+        config_envar = os.getenv(bdbcfg.DEFAULT_CONFIG_FILE_ENVAR)
+        try:
+            if config_envar:
+                logging.info("Ignoring already set envar %s: %s" % (bdbcfg.DEFAULT_CONFIG_FILE_ENVAR, config_envar))
+                del os.environ[bdbcfg.DEFAULT_CONFIG_FILE_ENVAR]
+            self.assertIsNone(os.getenv(bdbcfg.DEFAULT_CONFIG_FILE_ENVAR))
+            config_file = ospj(self.test_config_dir, 'bse-config.json')
+            os.environ[bdbcfg.DEFAULT_CONFIG_FILE_ENVAR] = config_file
+            bdbcfg.DEFAULT_CONFIG_FILE = ospj(self.test_config_dir, 'base-config.json')
+            config = bdbcfg.read_config(create_default=False)
+            self.assertEqual("bdbag test", config["bag_config"]["bag_metadata"]["Contact-Name"],
+                             "Unexpected config value")
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["Invalid configuration file path specified using environment variable %s: "
+                                         "[%s]. Falling back to default configuration file path: [%s]" %
+                                         (bdbcfg.DEFAULT_CONFIG_FILE_ENVAR, config_file, bdbcfg.DEFAULT_CONFIG_FILE)],
+                                        output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+        finally:
+            bdbcfg.DEFAULT_CONFIG_FILE = ospj(DEFAULT_CONFIG_PATH, 'bdbag.json')
+            if os.getenv(bdbcfg.DEFAULT_CONFIG_FILE_ENVAR):
+                del os.environ[bdbcfg.DEFAULT_CONFIG_FILE_ENVAR]
+
     def test_create_keychain(self):
         logger.info(self.getTestHeader('create keychain'))
         try:
@@ -197,11 +277,20 @@ def test_revert_bag(self):
             self.fail(get_typed_exception(e))
 
     def test_revert_non_bag(self):
-        logger.info(self.getTestHeader('revert bag'))
+        logger.info(self.getTestHeader('revert non-bag'))
         try:
             bdb.revert_bag(self.test_data_dir)
             output = self.stream.getvalue()
-            self.assertExpectedMessages(["Cannot revert the bag %s because it is not a bag directory!"], output)
+            self.assertExpectedMessages(["Cannot revert the bag", "because it is not a bag directory!"], output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
+    def test_revert_no_payload(self):
+        logger.info(self.getTestHeader('revert no data directory'))
+        try:
+            bdb.revert_bag(self.test_bag_no_data_dir)
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["Bag directory", "does not contain a \"data\" directory to revert."], output)
         except Exception as e:
             self.fail(get_typed_exception(e))
 
@@ -498,6 +587,39 @@ def test_extract_bag_archive_zip(self):
         except Exception as e:
             self.fail(get_typed_exception(e))
 
+    def test_extract_bag_archive_zip_to_target(self):
+        logger.info(self.getTestHeader('extract bag zip format to target'))
+        try:
+            bag_path = bdb.extract_bag(ospj(self.test_archive_dir, 'test-bag.zip'),
+                                       output_path=ospj(self.tmpdir, "test-bag-extract-output-dir"),
+                                       temp=False)
+            self.assertTrue(ospe(bag_path))
+            self.assertTrue(bdb.is_bag(bag_path))
+            bdb.cleanup_bag(os.path.dirname(bag_path))
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
+    def test_extract_bag_archive_zip_no_parent_warning(self):
+        logger.info(self.getTestHeader('extract bag zip format with no parent dir archive root'))
+        try:
+            bag_path = bdb.extract_bag(ospj(self.test_archive_dir, 'test-bag-no-parent.zip'), temp=True)
+            bdb.cleanup_bag(os.path.dirname(bag_path))
+            output = self.stream.getvalue()
+            self.assertExpectedMessages([
+                "Expecting single bag parent dir in archive but found files in the archive root"], output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
+    def test_extract_bag_archive_zip_multi_parent_warning(self):
+        logger.info(self.getTestHeader('extract bag zip format with multi parent archive root'))
+        try:
+            bag_path = bdb.extract_bag(ospj(self.test_archive_dir, 'test-bag-multi-parent.zip'), temp=True)
+            bdb.cleanup_bag(os.path.dirname(bag_path))
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["Expecting single bag parent dir but got:"], output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
     def test_extract_bag_archive_zip_with_relocate_existing(self):
         logger.info(self.getTestHeader('extract bag zip format, relocate existing'))
         try:
@@ -509,7 +631,25 @@ def test_extract_bag_archive_zip_with_relocate_existing(self):
             self.assertTrue(bdb.is_bag(bag_path))
             bdb.cleanup_bag(os.path.dirname(bag_path))
             output = self.stream.getvalue()
-            self.assertExpectedMessages(["moving existing directory"], output)
+            self.assertExpectedMessages(["Destination", "already exists, moving to"], output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
+    def test_extract_bag_archive_zip_with_relocate_existing_to_target(self):
+        logger.info(self.getTestHeader('extract bag zip format, relocate existing to target'))
+        try:
+            bag_path = bdb.extract_bag(ospj(self.test_archive_dir, 'test-bag.zip'),
+                                       output_path=ospj(self.tmpdir, "test-bag-extract-output-dir"),
+                                       temp=False)
+            self.assertTrue(ospe(bag_path))
+            self.assertTrue(bdb.is_bag(bag_path))
+            bag_path = bdb.extract_bag(ospj(self.test_archive_dir, 'test-bag.zip'),
+                                       output_path=ospj(self.tmpdir, "test-bag-extract-output-dir"), temp=False)
+            self.assertTrue(ospe(bag_path))
+            self.assertTrue(bdb.is_bag(bag_path))
+            bdb.cleanup_bag(os.path.dirname(bag_path))
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["Source", "and destination", "already exist, moving destination to"], output)
         except Exception as e:
             self.fail(get_typed_exception(e))
 
@@ -523,6 +663,27 @@ def test_extract_bag_archive_tgz(self):
         except Exception as e:
             self.fail(get_typed_exception(e))
 
+    def test_extract_bag_archive_tgz_no_parent_warning(self):
+        logger.info(self.getTestHeader('extract bag tgz format with no parent dir archive root'))
+        try:
+            bag_path = bdb.extract_bag(ospj(self.test_archive_dir, 'test-bag-no-parent.tgz'), temp=True)
+            bdb.cleanup_bag(os.path.dirname(bag_path))
+            output = self.stream.getvalue()
+            self.assertExpectedMessages([
+                "Expecting single bag parent dir in archive but found files in the archive root"], output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
+    def test_extract_bag_archive_tgz_multi_parent_warning(self):
+        logger.info(self.getTestHeader('extract bag tgz format with multi parent archive root'))
+        try:
+            bag_path = bdb.extract_bag(ospj(self.test_archive_dir, 'test-bag-multi-parent.tgz'), temp=True)
+            bdb.cleanup_bag(os.path.dirname(bag_path))
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["Expecting single bag parent dir but got:"], output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
     def test_extract_bag_archive_tar(self):
         logger.info(self.getTestHeader('extract bag tar format'))
         try:
@@ -589,6 +750,15 @@ def test_validate_incomplete_bag_full(self):
         except Exception as e:
             self.fail(get_typed_exception(e))
 
+    def test_validate_unexpected_bag_fetch(self):
+        logger.info(self.getTestHeader('test bag validation with unexpected entries bag in fetch.txt'))
+        try:
+            bdb.validate_bag(self.test_bag_invalid_structure_fetch_dir)
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["exists in fetch.txt but is not in manifest"], output)
+        except Exception as e:
+            self.fail(get_typed_exception(e))
+
     def test_validate_incomplete_bag_fast(self):
         logger.info(self.getTestHeader('test fast validation incomplete bag'))
         try:
@@ -636,7 +806,8 @@ def test_validate_invalid_bag_structure_fetch(self):
         try:
             self.assertRaises(bdbagit.BagValidationError,
                               bdb.validate_bag_structure,
-                              self.test_bag_invalid_structure_fetch_dir)
+                              self.test_bag_invalid_structure_fetch_dir,
+                              skip_remote=False)
         except Exception as e:
             self.fail(get_typed_exception(e))
 
@@ -657,6 +828,10 @@ def test_validate_invalid_bag_state_fetch_filesize(self):
                               bdb.validate_bag_structure,
                               self.test_bag_invalid_state_fetch_filesize_dir,
                               skip_remote=False)
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["The size of the local file",
+                                         "does not match the size of the file",
+                                         "specified in fetch.txt"], output)
         except Exception as e:
             self.fail(get_typed_exception(e))
 
@@ -719,7 +894,7 @@ def test_filter_dict(self):
                 "folder": "../data/",
             },
             "mediatype": "text/plain",
-            "uri": "http://n2t.net/ark:/57799/b9dd5t"
+            "uri": "http://identifiers.org/ark:/57799/b9dd5t"
         },
         {
             "mediatype": "text/plain",
@@ -740,7 +915,8 @@ def test_generate_ro_manifest_update(self):
         try:
             bdb.make_bag(self.test_data_dir, algs=['md5', 'sha1', 'sha256', 'sha512'],
                          remote_file_manifest=ospj(self.test_config_dir, 'test-fetch-manifest.json'))
-            bdb.generate_ro_manifest(self.test_data_dir, overwrite=True)
+            bdb.generate_ro_manifest(self.test_data_dir, config_file=ospj(self.test_config_dir, 'base-config.json'),
+                                     overwrite=True)
             ro = bdbro.read_bag_ro_metadata(self.test_data_dir)
             old_agg_dict = dict()
             for entry in ro.get("aggregates", []):
@@ -762,7 +938,8 @@ def test_generate_ro_manifest_overwrite(self):
         try:
             bdb.make_bag(self.test_data_dir, algs=['md5', 'sha1', 'sha256', 'sha512'],
                          remote_file_manifest=ospj(self.test_config_dir, 'test-fetch-manifest.json'))
-            bdb.generate_ro_manifest(self.test_data_dir, overwrite=True)
+            bdb.generate_ro_manifest(self.test_data_dir, config_file=ospj(self.test_config_dir, 'base-config.json'),
+                                     overwrite=True)
             ro = bdbro.read_bag_ro_metadata(self.test_data_dir)
             agg_dict = dict()
             for entry in ro.get("aggregates", []):
diff --git a/test/test_bdbagit.py b/test/test_bdbagit.py
index 9479600..8076597 100644
--- a/test/test_bdbagit.py
+++ b/test/test_bdbagit.py
@@ -16,8 +16,7 @@
 # limitations under the License.
 #
 
-from __future__ import (absolute_import, division, print_function,
-                        unicode_literals)
+from __future__ import (absolute_import, division, print_function, unicode_literals)
 import platform
 import codecs
 import datetime
@@ -36,6 +35,10 @@
 from bdbag import bdbagit as bagit
 import mock
 
+if sys.version_info > (3,):
+    from io import StringIO
+else:
+    from StringIO import StringIO
 
 logger = logging.getLogger()
 
@@ -54,6 +57,9 @@ class SelfCleaningTestCase(unittest.TestCase):
 
     def setUp(self):
         super(SelfCleaningTestCase, self).setUp()
+        self.stream = StringIO()
+        self.handler = logging.StreamHandler(self.stream)
+        logger.addHandler(self.handler)
 
         self.starting_directory = os.getcwd()   # FIXME: remove this after we stop changing directories in bagit.py
         self.tmpdir = tempfile.mkdtemp()
@@ -799,6 +805,21 @@ def test_save_bag_to_unwritable_directory(self):
         self.assertEqual('Cannot save bag to non-existent or inaccessible directory %s' % self.tmpdir,
                          str(error_catcher.exception))
 
+    @unittest.skipIf(platform.system() == "Windows", 'Unit test compatibility issue on Windows')
+    def test_save_bag_with_unwritable_source(self):
+        logger.info(self.getTestHeader(sys._getframe().f_code.co_name))
+        bag = bagit.make_bag(self.tmpdir, checksums=['sha256'])
+        path_suffixes = ('', 'loc')
+
+        for path_suffix in reversed(path_suffixes):
+            os.chmod(j(self.tmpdir, 'data', path_suffix), 0o500)
+
+        with self.assertRaises(bagit.BagError) as error_catcher:
+            bag.save()
+
+        self.assertEqual('Missing permissions to move all files and directories',
+                         str(error_catcher.exception))
+
     @unittest.skipIf(platform.system() == "Windows", 'Unit test compatibility issue on Windows')
     def test_save_bag_with_unwritable_file(self):
         logger.info(self.getTestHeader(sys._getframe().f_code.co_name))
diff --git a/test/test_cli.py b/test/test_cli.py
index 01d7d34..8bdcffe 100644
--- a/test/test_cli.py
+++ b/test/test_cli.py
@@ -13,13 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import sys
 import atexit
 import unittest
 import subprocess
 from os.path import join as ospj
 from test.test_common import BaseTest
 
-ARGS = ['python', 'bdbag/bdbag_cli.py']
+ARGS = [sys.executable, 'bdbag/bdbag_cli.py', '--debug']
 logfile = open('test_cli.log', mode='w')
 atexit.register(logfile.close)
 
diff --git a/test/test_common.py b/test/test_common.py
index 773913f..ac4d6e2 100644
--- a/test/test_common.py
+++ b/test/test_common.py
@@ -16,6 +16,7 @@
 import os
 import gc
 import sys
+import json
 import shutil
 import tempfile
 import unittest
@@ -47,6 +48,8 @@ def setUp(self):
 
         self.test_bag_dir = os.path.join(self.tmpdir, 'test-data', 'test-bag')
         self.assertTrue(os.path.isdir(self.test_bag_dir))
+        self.test_bag_no_data_dir = os.path.join(self.tmpdir, 'test-data', 'test-bag-no-data')
+        self.assertTrue(os.path.isdir(self.test_bag_no_data_dir))
         self.test_bag_incomplete_dir = os.path.join(self.tmpdir, 'test-data', 'test-bag-incomplete')
         self.assertTrue(os.path.isdir(self.test_bag_incomplete_dir))
         self.test_bag_incomplete_fetch_dir = os.path.join(self.tmpdir, 'test-data', 'test-bag-incomplete-fetch')
@@ -59,6 +62,8 @@ def setUp(self):
         self.assertTrue(os.path.isdir(self.test_bag_fetch_ark_dir))
         self.test_bag_fetch_ark2_dir = os.path.join(self.tmpdir, 'test-data', 'test-bag-fetch-ark2')
         self.assertTrue(os.path.isdir(self.test_bag_fetch_ark2_dir))
+        self.test_bag_fetch_ark_bad_dir = os.path.join(self.tmpdir, 'test-data', 'test-bag-fetch-ark-bad')
+        self.assertTrue(os.path.isdir(self.test_bag_fetch_ark_bad_dir))
         self.test_bag_fetch_doi_dir = os.path.join(self.tmpdir, 'test-data', 'test-bag-fetch-doi')
         self.assertTrue(os.path.isdir(self.test_bag_fetch_doi_dir))
         self.test_bag_fetch_dataguid_dir = os.path.join(self.tmpdir, 'test-data', 'test-bag-fetch-dataguid')
@@ -69,6 +74,8 @@ def setUp(self):
         self.assertTrue(os.path.isdir(self.test_bag_fetch_ftp_dir))
         self.test_bag_fetch_auth_dir = os.path.join(self.tmpdir, 'test-data', 'test-bag-fetch-ftp-auth')
         self.assertTrue(os.path.isdir(self.test_bag_fetch_auth_dir))
+        self.test_bag_fetch_tag_dir = os.path.join(self.tmpdir, 'test-data', 'test-bag-fetch-tag')
+        self.assertTrue(os.path.isdir(self.test_bag_fetch_tag_dir))
         self.test_bag_invalid_structure_manifest_dir = os.path.join(
             self.tmpdir, 'test-data', 'test-bag-invalid-structure-manifest')
         self.assertTrue(os.path.isdir(self.test_bag_invalid_structure_manifest_dir))
@@ -93,6 +100,12 @@ def setUp(self):
         self.test_bag_fetch_http_encoded_filename_dir = os.path.join(
             self.tmpdir, 'test-data', 'test-bag-fetch-http-encoded-filename')
         self.assertTrue(os.path.isdir(self.test_bag_fetch_http_encoded_filename_dir))
+        self.test_bag_fetch_http_unexpected_filesize_dir = os.path.join(
+            self.tmpdir, 'test-data', 'test-bag-fetch-http-unexpected-filesize')
+        self.assertTrue(os.path.isdir(self.test_bag_fetch_http_unexpected_filesize_dir))
+        self.test_bag_fetch_http_no_redirect_dir = os.path.join(
+            self.tmpdir, 'test-data', 'test-bag-fetch-http-no-redirect')
+        self.assertTrue(os.path.isdir(self.test_bag_fetch_http_no_redirect_dir))
 
     def tearDown(self):
         if os.path.isdir(self.tmpdir):
@@ -121,5 +134,9 @@ def __init__(self, json_data, status_code, headers={}):
             self.status_code = status_code
             self.headers = headers
 
+        @property
+        def text(self):
+            return json.dumps(self.json())
+
         def json(self):
             return self.json_data
diff --git a/test/test_remote.py b/test/test_remote.py
index 4fe2380..6181c5d 100644
--- a/test/test_remote.py
+++ b/test/test_remote.py
@@ -29,6 +29,7 @@
 from os.path import isfile as ospif
 from bdbag import bdbag_api as bdb, bdbag_config as bdbcfg
 from bdbag.fetch import fetcher
+from bdbag.fetch.transports.fetch_http import BaseFetchTransport, HTTPFetchTransport
 from bdbag.fetch.auth import cookies
 from bdbag.fetch.auth.keychain import read_keychain, update_keychain, get_auth_entries
 from test.test_common import BaseTest
@@ -41,6 +42,26 @@
 logger = logging.getLogger()
 
 
+class CustomTestFetchTransport(HTTPFetchTransport):
+
+    def __init__(self, config, keychain, **kwargs):
+        super(CustomTestFetchTransport, self).__init__(config, keychain, **kwargs)
+        if self.keychain:
+            logging.debug("Got propagated keychain: %s" % json.dumps(self.keychain))
+
+    def fetch(self, url, output_path, **kwargs):
+        return super(CustomTestFetchTransport, self).fetch(url, output_path, **kwargs)
+
+    def cleanup(self):
+        super(CustomTestFetchTransport, self).cleanup()
+
+
+class BadCustomTestFetchTransport(BaseFetchTransport):
+
+    def __init__(self, config, keychain, **kwargs):
+        super(BadCustomTestFetchTransport, self).__init__(config, keychain, **kwargs)
+
+
 class TestRemoteAPI(BaseTest):
 
     def setUp(self):
@@ -126,7 +147,7 @@ def test_validate_remote_bag_from_rfm(self):
         logger.info(self.getTestHeader('create, resolve, and validate bag from remote file manifest'))
         try:
             self._test_bag_with_remote_file_manifest()
-            bdb.resolve_fetch(self.test_data_dir)
+            self.assertTrue(bdb.resolve_fetch(self.test_data_dir), "Fetch incomplete")
             bdb.validate_bag(self.test_data_dir, fast=True)
             bdb.validate_bag(self.test_data_dir, fast=False)
         except Exception as e:
@@ -135,16 +156,36 @@ def test_validate_remote_bag_from_rfm(self):
     def test_resolve_fetch_http(self):
         logger.info(self.getTestHeader('test resolve fetch http'))
         try:
-            bdb.resolve_fetch(self.test_bag_fetch_http_dir, cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir, cookie_scan=False), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=False)
         except Exception as e:
             self.fail(bdbag.get_typed_exception(e))
 
+    def test_resolve_fetch_http_no_redirects(self):
+        logger.info(self.getTestHeader('test resolve fetch http no redirects'))
+        try:
+            self.assertFalse(bdb.resolve_fetch(self.test_bag_fetch_http_no_redirect_dir,
+                                               config_file=ospj(self.test_config_dir, 'test-config-6.json'),
+                                               cookie_scan=False), "Fetch complete")
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
+    def test_resolve_fetch_http_unexpected_filesize(self):
+        logger.info(self.getTestHeader('test resolve fetch http unexpected filesize warning'))
+        try:
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_unexpected_filesize_dir, cookie_scan=False),
+                            "Fetch incomplete")
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["transfer size mismatch. Expected 200 bytes but received 201 bytes"], output)
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
     def test_resolve_fetch_http_encoded_filename(self):
         logger.info(self.getTestHeader('test resolve fetch http with encoded filename'))
         try:
-            bdb.resolve_fetch(self.test_bag_fetch_http_encoded_filename_dir, cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_encoded_filename_dir, cookie_scan=False),
+                            "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_http_encoded_filename_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_http_encoded_filename_dir, fast=False)
         except Exception as e:
@@ -173,7 +214,7 @@ def test_resolve_fetch_http_bad_request(self):
             self.assertExpectedMessages(["HTTP GET Failed for URL",
                                          "HTTP Request Exception",
                                          "Transfer protocol",
-                                         "is not supported by this implementation"],
+                                         "is not supported"],
                                         output)
         except Exception as e:
             self.fail(bdbag.get_typed_exception(e))
@@ -181,7 +222,9 @@ def test_resolve_fetch_http_bad_request(self):
     def test_resolve_fetch_incomplete(self):
         logger.info(self.getTestHeader('test resolve fetch incomplete'))
         try:
-            bdb.resolve_fetch(self.test_bag_incomplete_fetch_dir, force=False, cookie_scan=False, quiet=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_incomplete_fetch_dir,
+                                              force=False, cookie_scan=False, quiet=False),
+                            "Fetch incomplete")
             bdb.validate_bag(self.test_bag_incomplete_fetch_dir, fast=True)
             bdb.validate_bag(self.test_bag_incomplete_fetch_dir, fast=False)
         except Exception as e:
@@ -190,7 +233,10 @@ def test_resolve_fetch_incomplete(self):
     def _test_resolve_fetch_http_with_filter(self, expr, files):
         logger.info(self.getTestHeader('test resolve fetch http with filter expression "%s"' % expr))
         try:
-            bdb.resolve_fetch(self.test_bag_fetch_http_dir, filter_expr=expr, cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir,
+                                              filter_expr=expr,
+                                              cookie_scan=False),
+                            "Fetch incomplete")
             for test_file in files:
                 self.assertTrue(ospif(ospj(self.test_bag_fetch_http_dir, test_file)))
         except Exception as e:
@@ -222,8 +268,9 @@ def mocked_request_auth_get_success(*args, **kwargs):
                                                        create=True)
 
             patched_requests_get.start()
-            bdb.resolve_fetch(self.test_bag_fetch_http_dir,
-                              keychain_file=ospj(self.test_config_dir, 'test-keychain-1.json'), cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir,
+                                              keychain_file=ospj(self.test_config_dir, 'test-keychain-1.json'),
+                                              cookie_scan=False), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=False)
         except Exception as e:
@@ -232,8 +279,9 @@ def mocked_request_auth_get_success(*args, **kwargs):
     def test_resolve_fetch_http_basic_auth_get_bad_key(self):
         logger.info(self.getTestHeader('test resolve fetch http basic auth GET with bad key'))
         try:
-            bdb.resolve_fetch(self.test_bag_fetch_http_dir,
-                              keychain_file=ospj(self.test_config_dir, 'test-keychain-bad-1.json'), cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir,
+                                              keychain_file=ospj(self.test_config_dir, 'test-keychain-bad-1.json'),
+                                              cookie_scan=False))
             output = self.stream.getvalue()
             self.assertExpectedMessages(["Missing required parameters [username, password]"], output)
         except Exception as e:
@@ -251,8 +299,10 @@ def mocked_request_auth_post_success(*args, **kwargs):
                                                         auth=None,
                                                         create=True)
             patched_requests_post.start()
-            bdb.resolve_fetch(self.test_bag_fetch_http_dir,
-                              keychain_file=ospj(self.test_config_dir, keychain_file), cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir,
+                                              keychain_file=ospj(self.test_config_dir, keychain_file),
+                                              cookie_scan=False),
+                            "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=False)
         except Exception as e:
@@ -269,8 +319,9 @@ def test_resolve_fetch_http_form_auth_post(self):
     def test_resolve_fetch_http_cookie_auth(self):
         logger.info(self.getTestHeader('test resolve fetch http cookie auth'))
         try:
-            bdb.resolve_fetch(self.test_bag_fetch_http_dir,
-                              keychain_file=ospj(self.test_config_dir, 'test-keychain-4.json'), cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir,
+                                              keychain_file=ospj(self.test_config_dir, 'test-keychain-4.json'),
+                                              cookie_scan=False), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=False)
         except Exception as e:
@@ -293,8 +344,9 @@ def mocked_request_auth_token_get_success(*args, **kwargs):
                                                             create=True)
 
             patched_requests_get_auth.start()
-            bdb.resolve_fetch(self.test_bag_fetch_http_dir,
-                              keychain_file=ospj(self.test_config_dir, 'test-keychain-6.json'), cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir,
+                            keychain_file=ospj(self.test_config_dir, 'test-keychain-6.json'),
+                            cookie_scan=False), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=False)
         except Exception as e:
@@ -319,15 +371,16 @@ def mocked_request_auth_token_get_success(*args, **kwargs):
                                                             create=True)
 
             patched_requests_get_auth.start()
-            bdb.resolve_fetch(self.test_bag_fetch_http_dir,
-                              keychain_file=ospj(self.test_config_dir, 'test-keychain-6.json'), cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir,
+                            keychain_file=ospj(self.test_config_dir, 'test-keychain-6.json'),
+                                              cookie_scan=False), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=False)
         except Exception as e:
             self.fail(bdbag.get_typed_exception(e))
 
     def test_resolve_fetch_http_auth_token_get_with_disallowed_redirects(self):
-        logger.info(self.getTestHeader('test resolve fetch http token auth with allowed redirect'))
+        logger.info(self.getTestHeader('test resolve fetch http token auth with disallowed redirect'))
         try:
             patched_requests_get_auth = None
 
@@ -345,8 +398,10 @@ def mocked_request_auth_token_get_success(*args, **kwargs):
                                                             create=True)
 
             patched_requests_get_auth.start()
-            bdb.resolve_fetch(self.test_bag_fetch_http_dir,
-                              keychain_file=ospj(self.test_config_dir, 'test-keychain-7.json'), cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir,
+                                              keychain_file=ospj(self.test_config_dir, 'test-keychain-7.json'),
+                                              cookie_scan=False),
+                            "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_http_dir, fast=False)
             output = self.stream.getvalue()
@@ -357,7 +412,7 @@ def mocked_request_auth_token_get_success(*args, **kwargs):
     def test_resolve_fetch_ark(self):
         logger.info(self.getTestHeader('test resolve fetch ark'))
         try:
-            bdb.resolve_fetch(self.test_bag_fetch_ark_dir, cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_ark_dir, cookie_scan=False), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_ark_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_ark_dir, fast=False)
         except Exception as e:
@@ -407,12 +462,32 @@ def mocked_request_resolver_ark_get_success(*args, **kwargs):
                                                           create=True)
             patched_resolve_ark_get.start()
 
-            bdb.resolve_fetch(self.test_bag_fetch_ark2_dir, cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_ark2_dir, cookie_scan=False), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_ark2_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_ark2_dir, fast=False)
         except Exception as e:
             self.fail(bdbag.get_typed_exception(e))
 
+    def test_resolve_fetch_bad_ark(self):
+        logger.info(self.getTestHeader('test resolve fetch bad ark'))
+        try:
+            self.assertFalse(bdb.resolve_fetch(self.test_bag_fetch_ark_bad_dir,
+                                               config_file=ospj(self.test_config_dir, 'test-config-3.json'),
+                                               cookie_scan=False))
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["No file locations were found for identifier"], output)
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
+    def test_resolve_fetch_ark_bad_response(self):
+        logger.info(self.getTestHeader('test resolve fetch ark bad response'))
+        try:
+            self.assertFalse(bdb.resolve_fetch(self.test_bag_fetch_ark2_dir,
+                                               config_file=ospj(self.test_config_dir, 'test-config-3.json'),
+                                               cookie_scan=False), "Fetch complete")
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
     def test_resolve_fetch_doi(self):
         logger.info(self.getTestHeader('test resolve fetch doi'))
         try:
@@ -472,7 +547,7 @@ def mocked_request_resolver_doi_get_success(*args, **kwargs):
                                                           create=True)
             patched_resolve_doi_get.start()
 
-            bdb.resolve_fetch(self.test_bag_fetch_doi_dir, cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_doi_dir, cookie_scan=False), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_doi_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_doi_dir, fast=False)
         except Exception as e:
@@ -520,7 +595,7 @@ def mocked_request_resolver_dataguid_get_success(*args, **kwargs):
                                                                create=True)
             patched_resolve_dataguid_get.start()
 
-            bdb.resolve_fetch(self.test_bag_fetch_dataguid_dir, cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_dataguid_dir, cookie_scan=False), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_dataguid_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_dataguid_dir, fast=False)
         except Exception as e:
@@ -529,12 +604,39 @@ def mocked_request_resolver_dataguid_get_success(*args, **kwargs):
     def test_resolve_fetch_minid(self):
         logger.info(self.getTestHeader('test resolve fetch minid'))
         try:
-            bdb.resolve_fetch(self.test_bag_fetch_minid_dir, cookie_scan=False)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_minid_dir, cookie_scan=False), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_minid_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_minid_dir, fast=False)
         except Exception as e:
             self.fail(bdbag.get_typed_exception(e))
 
+    def test_resolve_fetch_resolver_handler_not_found(self):
+        logger.info(self.getTestHeader('test resolve fetch with resolver handler not found'))
+        try:
+            self.assertFalse(bdb.resolve_fetch(self.test_bag_fetch_minid_dir,
+                                               config_file=ospj(self.test_config_dir, 'test-config-4.json'),
+                                               cookie_scan=False), "Fetch complete")
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
+    def test_resolve_fetch_resolver_handler_unspecified(self):
+        logger.info(self.getTestHeader('test resolve fetch with resolver handler unspecified'))
+        try:
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_minid_dir,
+                                              config_file=ospj(self.test_config_dir, 'test-config-5.json'),
+                                              cookie_scan=False), "Fetch incomplete")
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
+    def test_resolve_fetch_resolver_scheme_not_supported(self):
+        logger.info(self.getTestHeader('test resolve fetch with resolver scheme not supported'))
+        try:
+            self.assertFalse(bdb.resolve_fetch(self.test_bag_fetch_minid_dir,
+                                               config_file=ospj(self.test_config_dir, 'test-config-6.json'),
+                                               cookie_scan=False), "Fetch complete")
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
     def test_resolve_fetch_ftp_no_auth(self):
         logger.info(self.getTestHeader('test resolve fetch ftp'))
         try:
@@ -548,7 +650,7 @@ def mocked_urlretrieve_success(*args, **kwargs):
                                                       urlretrieve=mocked_urlretrieve_success)
             patched_urlretrieve.start()
 
-            bdb.resolve_fetch(self.test_bag_fetch_ftp_dir, force=True)
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_ftp_dir, force=True), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_ftp_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_ftp_dir, fast=False)
         except Exception as e:
@@ -567,8 +669,8 @@ def mocked_urlretrieve_success(*args, **kwargs):
                                                       urlretrieve=mocked_urlretrieve_success)
             patched_urlretrieve.start()
 
-            bdb.resolve_fetch(self.test_bag_fetch_auth_dir, force=True,
-                              keychain_file=ospj(self.test_config_dir, 'test-keychain-5.json'))
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_auth_dir, force=True,
+                            keychain_file=ospj(self.test_config_dir, 'test-keychain-5.json')), "Fetch incomplete")
             bdb.validate_bag(self.test_bag_fetch_auth_dir, fast=True)
             bdb.validate_bag(self.test_bag_fetch_auth_dir, fast=False)
         except Exception as e:
@@ -587,9 +689,16 @@ def mocked_urlretrieve_success(*args, **kwargs):
                                                       urlretrieve=mocked_urlretrieve_success)
             patched_urlretrieve.start()
 
-            bdb.resolve_fetch(self.test_bag_fetch_ftp_dir, force=True)
+            self.assertFalse(bdb.resolve_fetch(self.test_bag_fetch_ftp_dir, force=True), "Fetch complete")
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
+    def test_resolve_fetch_tag(self):
+        logger.info(self.getTestHeader('test resolve fetch tag'))
+        try:
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_tag_dir, cookie_scan=False), "Fetch incomplete")
             output = self.stream.getvalue()
-            self.assertExpectedMessages(["FTP Request Exception"], output)
+            self.assertExpectedMessages(["The fetch entry for file", "specifies the tag URI"], output)
         except Exception as e:
             self.fail(bdbag.get_typed_exception(e))
 
@@ -679,7 +788,7 @@ def test_materialize_from_identifier(self):
         curdir = os.getcwd()
         os.chdir(self.tmpdir)
         try:
-            bdb.materialize("ark:/57799/b91H6JHBS1u2FTG")
+            bdb.materialize("minid:b91H6JHBS1u2FTG")
         except Exception as e:
             self.fail(bdbag.get_typed_exception(e))
         finally:
@@ -893,6 +1002,59 @@ def test_update_keychain_invalid_params(self):
         except Exception as e:
             self.fail(bdbag.get_typed_exception(e))
 
+    def test_resolve_fetch_custom_handler_no_keychain(self):
+        logger.info(self.getTestHeader('test resolve fetch custom handler no keychain'))
+        try:
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir,
+                                              config_file=ospj(self.test_config_dir, 'test-config-7.json'),
+                                              cookie_scan=False),
+                            "Fetch incomplete")
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["Keychain will not be passed to fetch handler class"], output)
+            bdb.validate_bag(self.test_bag_fetch_http_dir, fast=True)
+            bdb.validate_bag(self.test_bag_fetch_http_dir, fast=False)
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
+    def test_resolve_fetch_custom_handler_with_keychain(self):
+        logger.info(self.getTestHeader('test resolve fetch custom handler with keychain'))
+        try:
+            self.assertTrue(bdb.resolve_fetch(self.test_bag_fetch_http_dir,
+                                              config_file=ospj(self.test_config_dir, 'test-config-8.json'),
+                                              keychain_file=ospj(self.test_config_dir, 'test-keychain-9.json'),
+                                              cookie_scan=False),
+                            "Fetch incomplete")
+            output = self.stream.getvalue()
+            self.assertExpectedMessages(["Got propagated keychain: ", "ftp://ftp.nist.gov/"], output)
+            bdb.validate_bag(self.test_bag_fetch_http_dir, fast=True)
+            bdb.validate_bag(self.test_bag_fetch_http_dir, fast=False)
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
+    def test_resolve_fetch_bad_custom_handler(self):
+        logger.info(self.getTestHeader('test resolve fetch bad custom handler'))
+        try:
+            self.assertRaisesRegex(NotImplementedError,
+                                   "Method must be implemented by subclass",
+                                   bdb.resolve_fetch,
+                                   self.test_bag_fetch_http_dir,
+                                   config_file=ospj(self.test_config_dir, 'test-config-9.json'),
+                                   cookie_scan=False)
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
+    def test_resolve_fetch_invalid_custom_handler(self):
+        logger.info(self.getTestHeader('test resolve fetch invalid custom handler'))
+        try:
+            self.assertRaisesRegex(RuntimeError,
+                                   "Unable to import specified fetch handler class: \\[foo\\.bar\\.InvalidHandler\\]",
+                                   bdb.resolve_fetch,
+                                   self.test_bag_fetch_http_dir,
+                                   config_file=ospj(self.test_config_dir, 'test-config-10.json'),
+                                   cookie_scan=False)
+        except Exception as e:
+            self.fail(bdbag.get_typed_exception(e))
+
     @unittest.skip("Not implemented")
     def test_resolve_fetch_globus(self):
         # TODO