Repository: mapillary/mapillary_tools

Issue:
Checks for authentication too late
<!--
   Thanks for contributing! Request a feature or use the
   template below for reporting bugs.

   To help us diagnose problems, please:

  - Check the console for relevant errors and warnings.
  
  - Provide a link to where the problem occurs (if applicable)

  - Attach screenshots and logs if possible/applicable
  
  - Remove template lines which do not apply to your issue/feature request
-->

### Steps to reproduce behavior

 1. Attempt to upload ~500,000 images
 2. Forget to authenticate first
 3. Collect underpants

### Expected behavior

Sanitize inputs to ensure everything needed to upload is present before uploading.

### Actual behavior

Spends five hours validating geotags and metadata before failing because authentication is checked last.

Hints from issue comments:

Solution patch:
diff --git a/mapillary_tools/api_v4.py b/mapillary_tools/api_v4.py
index fb66062..6e5088a 100644
--- a/mapillary_tools/api_v4.py
+++ b/mapillary_tools/api_v4.py
@@ -70,7 +70,7 @@ def _truncate(s, limit=512):
         return s
 
 
-def _sanitize(headers: T.Dict):
+def _sanitize(headers: T.Mapping[T.Any, T.Any]) -> T.Mapping[T.Any, T.Any]:
     new_headers = {}
 
     for k, v in headers.items():
@@ -81,6 +81,7 @@ def _sanitize(headers: T.Dict):
             "access-token",
             "access_token",
             "password",
+            "user_upload_token",
         ]:
             new_headers[k] = "[REDACTED]"
         else:
@@ -224,6 +225,44 @@ def request_get(
     return resp
 
 
+def is_auth_error(resp: requests.Response) -> bool:
+    if resp.status_code in [401, 403]:
+        return True
+
+    if resp.status_code in [400]:
+        try:
+            error_body = resp.json()
+        except Exception:
+            error_body = {}
+
+        type = error_body.get("debug_info", {}).get("type")
+        if type in ["NotAuthorizedError"]:
+            return True
+
+    return False
+
+
+def extract_auth_error_message(resp: requests.Response) -> str:
+    assert is_auth_error(resp), "has to be an auth error"
+
+    try:
+        error_body = resp.json()
+    except Exception:
+        error_body = {}
+
+    # from Graph APIs
+    message = error_body.get("error", {}).get("message")
+    if message is not None:
+        return str(message)
+
+    # from upload service
+    message = error_body.get("debug_info", {}).get("message")
+    if message is not None:
+        return str(message)
+
+    return resp.text
+
+
 def get_upload_token(email: str, password: str) -> requests.Response:
     resp = request_post(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/login",
@@ -252,6 +291,30 @@ def fetch_organization(
     return resp
 
 
+def fetch_user_or_me(
+    user_access_token: str,
+    user_id: T.Optional[T.Union[int, str]] = None,
+) -> requests.Response:
+    if user_id is None:
+        url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me"
+    else:
+        url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{user_id}"
+
+    resp = request_get(
+        url,
+        params={
+            "fields": ",".join(["id", "username"]),
+        },
+        headers={
+            "Authorization": f"OAuth {user_access_token}",
+        },
+        timeout=REQUESTS_TIMEOUT,
+    )
+
+    resp.raise_for_status()
+    return resp
+
+
 ActionType = T.Literal[
     "upload_started_upload", "upload_finished_upload", "upload_failed_upload"
 ]
diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py
index 4b80b46..af956f5 100644
--- a/mapillary_tools/authenticate.py
+++ b/mapillary_tools/authenticate.py
@@ -1,102 +1,362 @@
+from __future__ import annotations
+
 import getpass
+import json
 import logging
+import re
+import sys
 import typing as T
 
-import jsonschema
 import requests
+import jsonschema
 
-from . import api_v4, config, types
+from . import api_v4, config, constants, exceptions, types
 
 
 LOG = logging.getLogger(__name__)
 
 
 def authenticate(
-    user_name: T.Optional[str] = None,
-    user_email: T.Optional[str] = None,
-    user_password: T.Optional[str] = None,
-    jwt: T.Optional[str] = None,
+    user_name: str | None = None,
+    user_email: str | None = None,
+    user_password: str | None = None,
+    jwt: str | None = None,
+    delete: bool = False,
 ):
-    if user_name:
-        user_name = user_name.strip()
+    """
+    Prompt for authentication information and save it to the config file
+    """
+
+    # We still have to accept --user_name for the back compatibility
+    profile_name = user_name
+
+    all_user_items = config.list_all_users()
+    if all_user_items:
+        _list_all_profiles(all_user_items)
+    else:
+        _welcome()
 
-    while not user_name:
-        user_name = input(
-            "Enter the Mapillary username you would like to (re)authenticate: "
+    # Make sure profile name either validated or existed
+    if profile_name is not None:
+        profile_name = profile_name.strip()
+    else:
+        if not _prompt_enabled():
+            raise exceptions.MapillaryBadParameterError(
+                "Profile name is required, please specify one with --user_name"
+            )
+        profile_name = _prompt_choose_profile_name(
+            list(all_user_items.keys()), must_exist=delete
         )
-        user_name = user_name.strip()
 
-    if jwt:
-        user_items: types.UserItem = {
-            "user_upload_token": jwt,
-        }
-    elif user_email and user_password:
-        resp = api_v4.get_upload_token(user_email, user_password)
-        data = resp.json()
-        user_items = {
-            "MAPSettingsUserKey": data["user_id"],
-            "user_upload_token": data["access_token"],
-        }
+    assert profile_name is not None, "profile_name should be set"
+
+    if delete:
+        config.remove_config(profile_name)
+        LOG.info('Profile "%s" deleted', profile_name)
+    else:
+        if profile_name in all_user_items:
+            LOG.warning(
+                'The profile "%s" already exists and will be overridden',
+                profile_name,
+            )
+        else:
+            LOG.info('Creating new profile: "%s"', profile_name)
+
+        if jwt:
+            user_items: types.UserItem = {"user_upload_token": jwt}
+            user_items = _verify_user_auth(_validate_profile(user_items))
+        else:
+            user_items = _prompt_login(
+                user_email=user_email, user_password=user_password
+            )
+            _validate_profile(user_items)
+
+        # Update the config with the new user items
+        config.update_config(profile_name, user_items)
+
+        # TODO: print more user information
+        if profile_name in all_user_items:
+            LOG.info(
+                'Profile "%s" updated: %s', profile_name, api_v4._sanitize(user_items)
+            )
+        else:
+            LOG.info(
+                'Profile "%s" created: %s', profile_name, api_v4._sanitize(user_items)
+            )
+
+
+def fetch_user_items(
+    user_name: str | None = None,
+    organization_key: str | None = None,
+) -> types.UserItem:
+    """
+    Read user information from the config file,
+    or prompt the user to authenticate if the specified profile does not exist
+    """
+
+    # we still have to accept --user_name for the back compatibility
+    profile_name = user_name
+
+    all_user_items = config.list_all_users()
+    if not all_user_items:
+        authenticate(user_name=profile_name)
+
+    # Fetch user information only here
+    all_user_items = config.list_all_users()
+    assert len(all_user_items) >= 1, "should have at least 1 profile"
+    if profile_name is None:
+        if len(all_user_items) > 1:
+            if not _prompt_enabled():
+                raise exceptions.MapillaryBadParameterError(
+                    "Multiple user profiles found, please choose one with --user_name"
+                )
+            _list_all_profiles(all_user_items)
+            profile_name = _prompt_choose_profile_name(
+                list(all_user_items.keys()), must_exist=True
+            )
+            user_items = all_user_items[profile_name]
+        else:
+            profile_name, user_items = list(all_user_items.items())[0]
     else:
-        user_items = prompt_user_for_user_items(user_name)
+        if profile_name in all_user_items:
+            user_items = all_user_items[profile_name]
+        else:
+            _list_all_profiles(all_user_items)
+            raise exceptions.MapillaryBadParameterError(
+                f'Profile "{profile_name}" not found'
+            )
+
+    assert profile_name is not None, "profile_name should be set"
 
-    config.update_config(user_name, user_items)
+    user_items = _verify_user_auth(_validate_profile(user_items))
 
+    LOG.info(
+        'Uploading to profile "%s": %s', profile_name, api_v4._sanitize(user_items)
+    )
+
+    if organization_key is not None:
+        resp = api_v4.fetch_organization(
+            user_items["user_upload_token"], organization_key
+        )
+        LOG.info("Uploading to Mapillary organization: %s", json.dumps(resp.json()))
+        user_items["MAPOrganizationKey"] = organization_key
+
+    return user_items
+
+
+def _echo(*args, **kwargs):
+    print(*args, **kwargs, file=sys.stderr)
 
-def prompt_user_for_user_items(user_name: str) -> types.UserItem:
-    print(f"Sign in for user {user_name}")
-    user_email = input("Enter your Mapillary user email: ")
-    user_password = getpass.getpass("Enter Mapillary user password: ")
 
+def _prompt(message: str) -> str:
+    """Display prompt on stderr and get input from stdin"""
+    print(message, end="", file=sys.stderr, flush=True)
+    return input()
+
+
+def _validate_profile(user_items: types.UserItem) -> types.UserItem:
     try:
-        resp = api_v4.get_upload_token(user_email, user_password)
+        jsonschema.validate(user_items, types.UserItemSchema)
+    except jsonschema.ValidationError as ex:
+        raise exceptions.MapillaryBadParameterError(
+            f"Invalid profile format: {ex.message}"
+        )
+    return user_items
+
+
+def _verify_user_auth(user_items: types.UserItem) -> types.UserItem:
+    """
+    Verify that the user access token is valid
+    """
+    if constants._AUTH_VERIFICATION_DISABLED:
+        return user_items
+
+    try:
+        resp = api_v4.fetch_user_or_me(
+            user_access_token=user_items["user_upload_token"]
+        )
     except requests.HTTPError as ex:
-        if (
-            isinstance(ex, requests.HTTPError)
-            and isinstance(ex.response, requests.Response)
-            and 400 <= ex.response.status_code < 500
-        ):
-            r = ex.response.json()
-            subcode = r.get("error", {}).get("error_subcode")
-            if subcode in [1348028, 1348092, 3404005, 1348131]:
-                title = r.get("error", {}).get("error_user_title")
-                message = r.get("error", {}).get("error_user_msg")
-                LOG.error("%s: %s", title, message)
-                return prompt_user_for_user_items(user_name)
-            else:
-                raise ex
+        if api_v4.is_auth_error(ex.response):
+            message = api_v4.extract_auth_error_message(ex.response)
+            raise exceptions.MapillaryUploadUnauthorizedError(message)
         else:
             raise ex
 
-    data = resp.json()
-    upload_token = T.cast(str, data.get("access_token"))
-    user_key = T.cast(str, data.get("user_id"))
-    if not isinstance(upload_token, str) or not isinstance(user_key, (str, int)):
-        raise RuntimeError(
-            f"Error extracting user_key or token from the login response: {data}"
+    user_json = resp.json()
+
+    return {
+        **user_items,
+        "MAPSettingsUsername": user_json.get("username"),
+        "MAPSettingsUserKey": user_json.get("id"),
+    }
+
+
+def _validate_profile_name(profile_name: str):
+    if not (2 <= len(profile_name) <= 32):
+        raise exceptions.MapillaryBadParameterError(
+            "Profile name must be between 2 and 32 characters long"
         )
 
-    if isinstance(user_key, int):
-        user_key = str(user_key)
+    pattern = re.compile(r"^[a-zA-Z]+[a-zA-Z0-9_-]*$")
+    if not bool(pattern.match(profile_name)):
+        raise exceptions.MapillaryBadParameterError(
+            "Invalid profile name. Use only letters, numbers, hyphens and underscores"
+        )
 
-    return {
-        "MAPSettingsUserKey": user_key,
-        "user_upload_token": upload_token,
+
+def _list_all_profiles(profiles: dict[str, types.UserItem]) -> None:
+    _echo("Existing Mapillary profiles:")
+
+    # Header
+    _echo(f"{'':>5}  {'Profile name':<32} {'User ID':>16} {'Username':>32}")
+
+    # List all profiles
+    for idx, name in enumerate(profiles, 1):
+        items = profiles[name]
+        user_id = items.get("MAPSettingsUserKey", "N/A")
+        username = items.get("MAPSettingsUsername", "N/A")
+        _echo(f"{idx:>5}. {name:<32} {user_id:>16} {username:>32}")
+
+
+def _is_interactive():
+    # Check if stdout is connected to a terminal
+    stdout_interactive = sys.stdout.isatty() if hasattr(sys.stdout, "isatty") else False
+
+    # Optionally, also check stdin and stderr
+    stdin_interactive = sys.stdin.isatty() if hasattr(sys.stdin, "isatty") else False
+    stderr_interactive = sys.stderr.isatty() if hasattr(sys.stderr, "isatty") else False
+
+    # Return True if any stream is interactive
+    return stdout_interactive or stdin_interactive or stderr_interactive
+
+
+def _prompt_enabled() -> bool:
+    if constants.PROMPT_DISABLED:
+        return False
+
+    if not _is_interactive():
+        return False
+
+    return True
+
+
+def _is_login_retryable(ex: requests.HTTPError) -> bool:
+    if 400 <= ex.response.status_code < 500:
+        r = ex.response.json()
+        subcode = r.get("error", {}).get("error_subcode")
+        if subcode in [1348028, 1348092, 3404005, 1348131]:
+            title = r.get("error", {}).get("error_user_title")
+            message = r.get("error", {}).get("error_user_msg")
+            LOG.error("%s: %s", title, message)
+            return True
+    return False
+
+
+def _prompt_login(
+    user_email: str | None = None,
+    user_password: str | None = None,
+) -> types.UserItem:
+    _enabled = _prompt_enabled()
+
+    if user_email is None:
+        if not _enabled:
+            raise exceptions.MapillaryBadParameterError("user_email is required")
+        while not user_email:
+            user_email = _prompt("Enter Mapillary user email: ").strip()
+    else:
+        user_email = user_email.strip()
+
+    if user_password is None:
+        if not _enabled:
+            raise exceptions.MapillaryBadParameterError("user_password is required")
+        while True:
+            user_password = getpass.getpass("Enter Mapillary user password: ")
+            if user_password:
+                break
+
+    try:
+        resp = api_v4.get_upload_token(user_email, user_password)
+    except requests.HTTPError as ex:
+        if not _enabled:
+            raise ex
+
+        if _is_login_retryable(ex):
+            return _prompt_login()
+
+        raise ex
+
+    data = resp.json()
+
+    user_items: types.UserItem = {
+        "user_upload_token": str(data["access_token"]),
+        "MAPSettingsUserKey": str(data["user_id"]),
     }
 
+    return user_items
+
+
+def _prompt_choose_profile_name(
+    existing_profile_names: T.Sequence[str], must_exist: bool = False
+) -> str:
+    assert _prompt_enabled(), "should not get here if prompting is disabled"
+
+    existed = set(existing_profile_names)
+
+    while True:
+        if must_exist:
+            prompt = "Enter an existing profile: "
+        else:
+            prompt = "Enter an existing profile or create a new one: "
+
+        profile_name = _prompt(prompt).strip()
 
-def authenticate_user(user_name: str) -> types.UserItem:
-    user_items = config.load_user(user_name)
-    if user_items is not None:
+        if not profile_name:
+            continue
+
+        # Exit if it's found
+        if profile_name in existed:
+            break
+
+        # Try to find by index
         try:
-            jsonschema.validate(user_items, types.UserItemSchema)
-        except jsonschema.ValidationError:
+            profile_name = existing_profile_names[int(profile_name) - 1]
+        except (ValueError, IndexError):
             pass
         else:
-            return user_items
+            # Exit if it's found
+            break
 
-    user_items = prompt_user_for_user_items(user_name)
-    jsonschema.validate(user_items, types.UserItemSchema)
-    config.update_config(user_name, user_items)
+        assert profile_name not in existed, (
+            f"Profile {profile_name} must not exist here"
+        )
 
-    return user_items
+        if must_exist:
+            LOG.error('Profile "%s" not found', profile_name)
+        else:
+            try:
+                _validate_profile_name(profile_name)
+            except exceptions.MapillaryBadParameterError as ex:
+                LOG.error("Error validating profile name: %s", ex)
+                profile_name = ""
+            else:
+                break
+
+    if must_exist:
+        assert profile_name in existed, f"Profile {profile_name} must exist"
+
+    return profile_name
+
+
+def _welcome():
+    _echo(
+        """
+================================================================================
+                             Welcome to Mapillary!
+================================================================================
+  If you haven't registered yet, please visit https://www.mapillary.com/signup
+  to create your account first.
+
+  Once registered, proceed here to sign in.
+================================================================================
+    """
+    )
diff --git a/mapillary_tools/commands/authenticate.py b/mapillary_tools/commands/authenticate.py
index b6c6a92..e30c920 100644
--- a/mapillary_tools/commands/authenticate.py
+++ b/mapillary_tools/commands/authenticate.py
@@ -10,7 +10,7 @@ class Command:
 
     def add_basic_arguments(self, parser: argparse.ArgumentParser):
         parser.add_argument(
-            "--user_name", help="Mapillary user name", default=None, required=False
+            "--user_name", help="Mapillary user profile", default=None, required=False
         )
         parser.add_argument(
             "--user_email",
@@ -27,6 +27,13 @@ class Command:
         parser.add_argument(
             "--jwt", help="Mapillary user access token", default=None, required=False
         )
+        parser.add_argument(
+            "--delete",
+            help="Delete the specified user profile",
+            default=False,
+            required=False,
+            action="store_true",
+        )
 
     def run(self, vars_args: dict):
         authenticate(
diff --git a/mapillary_tools/commands/process_and_upload.py b/mapillary_tools/commands/process_and_upload.py
index c7ffadd..268986b 100644
--- a/mapillary_tools/commands/process_and_upload.py
+++ b/mapillary_tools/commands/process_and_upload.py
@@ -1,3 +1,7 @@
+import inspect
+
+from ..authenticate import fetch_user_items
+
 from .process import Command as ProcessCommand
 from .upload import Command as UploadCommand
 
@@ -10,11 +14,20 @@ class Command:
         ProcessCommand().add_basic_arguments(parser)
         UploadCommand().add_basic_arguments(parser)
 
-    def run(self, args: dict):
-        if args.get("desc_path") is None:
+    def run(self, vars_args: dict):
+        if vars_args.get("desc_path") is None:
             # \x00 is a special path similiar to /dev/null
             # it tells process command do not write anything
-            args["desc_path"] = "\x00"
+            vars_args["desc_path"] = "\x00"
+
+        if "user_items" not in vars_args:
+            vars_args["user_items"] = fetch_user_items(
+                **{
+                    k: v
+                    for k, v in vars_args.items()
+                    if k in inspect.getfullargspec(fetch_user_items).args
+                }
+            )
 
-        ProcessCommand().run(args)
-        UploadCommand().run(args)
+        ProcessCommand().run(vars_args)
+        UploadCommand().run(vars_args)
diff --git a/mapillary_tools/commands/upload.py b/mapillary_tools/commands/upload.py
index f8c6748..418ce04 100644
--- a/mapillary_tools/commands/upload.py
+++ b/mapillary_tools/commands/upload.py
@@ -1,6 +1,7 @@
 import inspect
 
 from .. import constants
+from ..authenticate import fetch_user_items
 from ..upload import upload
 from .process import bold_text
 
@@ -41,9 +42,18 @@ class Command:
         Command.add_common_upload_options(group)
 
     def run(self, vars_args: dict):
-        args = {
-            k: v
-            for k, v in vars_args.items()
-            if k in inspect.getfullargspec(upload).args
-        }
-        upload(**args)
+        if "user_items" not in vars_args:
+            user_items_args = {
+                k: v
+                for k, v in vars_args.items()
+                if k in inspect.getfullargspec(fetch_user_items).args
+            }
+            vars_args["user_items"] = fetch_user_items(**user_items_args)
+
+        upload(
+            **{
+                k: v
+                for k, v in vars_args.items()
+                if k in inspect.getfullargspec(upload).args
+            }
+        )
diff --git a/mapillary_tools/commands/video_process_and_upload.py b/mapillary_tools/commands/video_process_and_upload.py
index 12d7748..d1407a7 100644
--- a/mapillary_tools/commands/video_process_and_upload.py
+++ b/mapillary_tools/commands/video_process_and_upload.py
@@ -1,3 +1,7 @@
+import inspect
+
+from ..authenticate import fetch_user_items
+
 from .upload import Command as UploadCommand
 from .video_process import Command as VideoProcessCommand
 
@@ -10,10 +14,20 @@ class Command:
         VideoProcessCommand().add_basic_arguments(parser)
         UploadCommand().add_basic_arguments(parser)
 
-    def run(self, args: dict):
-        if args.get("desc_path") is None:
+    def run(self, vars_args: dict):
+        if vars_args.get("desc_path") is None:
             # \x00 is a special path similiar to /dev/null
             # it tells process command do not write anything
-            args["desc_path"] = "\x00"
-        VideoProcessCommand().run(args)
-        UploadCommand().run(args)
+            vars_args["desc_path"] = "\x00"
+
+        if "user_items" not in vars_args:
+            vars_args["user_items"] = fetch_user_items(
+                **{
+                    k: v
+                    for k, v in vars_args.items()
+                    if k in inspect.getfullargspec(fetch_user_items).args
+                }
+            )
+
+        VideoProcessCommand().run(vars_args)
+        UploadCommand().run(vars_args)
diff --git a/mapillary_tools/config.py b/mapillary_tools/config.py
index a70b8b1..8f10026 100644
--- a/mapillary_tools/config.py
+++ b/mapillary_tools/config.py
@@ -35,37 +35,53 @@ def _load_config(config_path: str) -> configparser.ConfigParser:
 
 
 def load_user(
-    user_name: str, config_path: T.Optional[str] = None
+    profile_name: str, config_path: T.Optional[str] = None
 ) -> T.Optional[types.UserItem]:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     config = _load_config(config_path)
-    if not config.has_section(user_name):
+    if not config.has_section(profile_name):
         return None
-    user_items = dict(config.items(user_name))
+    user_items = dict(config.items(profile_name))
     return T.cast(types.UserItem, user_items)
 
 
-def list_all_users(config_path: T.Optional[str] = None) -> T.List[types.UserItem]:
+def list_all_users(config_path: T.Optional[str] = None) -> T.Dict[str, types.UserItem]:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     cp = _load_config(config_path)
-    users = [
-        load_user(user_name, config_path=config_path) for user_name in cp.sections()
-    ]
-    return [item for item in users if item is not None]
+    users = {
+        profile_name: load_user(profile_name, config_path=config_path)
+        for profile_name in cp.sections()
+    }
+    return {profile: item for profile, item in users.items() if item is not None}
 
 
 def update_config(
-    user_name: str, user_items: types.UserItem, config_path: T.Optional[str] = None
+    profile_name: str, user_items: types.UserItem, config_path: T.Optional[str] = None
 ) -> None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     config = _load_config(config_path)
-    if not config.has_section(user_name):
-        config.add_section(user_name)
+    if not config.has_section(profile_name):
+        config.add_section(profile_name)
     for key, val in user_items.items():
-        config.set(user_name, key, T.cast(str, val))
+        config.set(profile_name, key, T.cast(str, val))
+    os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True)
+    with open(config_path, "w") as fp:
+        config.write(fp)
+
+
+def remove_config(profile_name: str, config_path: T.Optional[str] = None) -> None:
+    if config_path is None:
+        config_path = MAPILLARY_CONFIG_PATH
+
+    config = _load_config(config_path)
+    if not config.has_section(profile_name):
+        return
+
+    config.remove_section(profile_name)
+
     os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True)
     with open(config_path, "w") as fp:
         config.write(fp)
diff --git a/mapillary_tools/constants.py b/mapillary_tools/constants.py
index 60c1302..a58cce8 100644
--- a/mapillary_tools/constants.py
+++ b/mapillary_tools/constants.py
@@ -5,6 +5,15 @@ import appdirs
 
 _ENV_PREFIX = "MAPILLARY_TOOLS_"
 
+
+def _yes_or_no(val: str) -> bool:
+    return val.strip().upper() in [
+        "1",
+        "TRUE",
+        "YES",
+    ]
+
+
 # In meters
 CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600))
 # In seconds
@@ -55,3 +64,9 @@ MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
 MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
 # Max number of pixels per sequence (sum of image pixels in the sequence)
 MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
+
+PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO"))
+
+_AUTH_VERIFICATION_DISABLED: bool = _yes_or_no(
+    os.getenv(_ENV_PREFIX + "_AUTH_VERIFICATION_DISABLED", "NO")
+)
diff --git a/mapillary_tools/upload.py b/mapillary_tools/upload.py
index a396991..055b64a 100644
--- a/mapillary_tools/upload.py
+++ b/mapillary_tools/upload.py
@@ -12,8 +12,6 @@ from tqdm import tqdm
 
 from . import (
     api_v4,
-    authenticate,
-    config,
     constants,
     exceptions,
     history,
@@ -139,36 +137,6 @@ def zip_images(
     uploader.ZipImageSequence.zip_images(image_metadatas, zip_dir)
 
 
-def fetch_user_items(
-    user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None
-) -> types.UserItem:
-    if user_name is None:
-        all_user_items = config.list_all_users()
-        if not all_user_items:
-            raise exceptions.MapillaryBadParameterError(
-                "No Mapillary account found. Add one with --user_name"
-            )
-        if len(all_user_items) == 1:
-            user_items = all_user_items[0]
-        else:
-            raise exceptions.MapillaryBadParameterError(
-                "Found multiple Mapillary accounts. Please specify one with --user_name"
-            )
-    else:
-        user_items = authenticate.authenticate_user(user_name)
-
-    if organization_key is not None:
-        resp = api_v4.fetch_organization(
-            user_items["user_upload_token"], organization_key
-        )
-        org = resp.json()
-        LOG.info("Uploading to organization: %s", json.dumps(org))
-        user_items = T.cast(
-            types.UserItem, {**user_items, "MAPOrganizationKey": organization_key}
-        )
-    return user_items
-
-
 def _setup_cancel_due_to_duplication(emitter: uploader.EventEmitter) -> None:
     @emitter.on("upload_start")
     def upload_start(payload: uploader.Progress):
@@ -476,12 +444,94 @@ def _find_metadata_with_filename_existed_in(
     return [d for d in metadatas if d.filename.resolve() in resolved_image_paths]
 
 
+def _upload_everything(
+    mly_uploader: uploader.Uploader,
+    import_paths: T.Sequence[Path],
+    metadatas: T.Sequence[types.Metadata],
+    skip_subfolders: bool,
+):
+    # upload images
+    image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders)
+    # find descs that match the image paths from the import paths
+    image_metadatas = [
+        metadata
+        for metadata in (metadatas or [])
+        if isinstance(metadata, types.ImageMetadata)
+    ]
+    specified_image_metadatas = _find_metadata_with_filename_existed_in(
+        image_metadatas, image_paths
+    )
+    if specified_image_metadatas:
+        try:
+            clusters = mly_uploader.upload_images(
+                specified_image_metadatas,
+                event_payload={"file_type": FileType.IMAGE.value},
+            )
+        except Exception as ex:
+            raise UploadError(ex) from ex
+
+        if clusters:
+            LOG.debug("Uploaded to cluster: %s", clusters)
+
+    # upload videos
+    video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders)
+    video_metadatas = [
+        metadata
+        for metadata in (metadatas or [])
+        if isinstance(metadata, types.VideoMetadata)
+    ]
+    specified_video_metadatas = _find_metadata_with_filename_existed_in(
+        video_metadatas, video_paths
+    )
+    for idx, video_metadata in enumerate(specified_video_metadatas):
+        video_metadata.update_md5sum()
+        assert isinstance(video_metadata.md5sum, str), "md5sum should be updated"
+
+        # extract telemetry measurements from GoPro videos
+        telemetry_measurements: T.List[camm_parser.TelemetryMeasurement] = []
+        if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES":
+            if video_metadata.filetype is FileType.GOPRO:
+                with video_metadata.filename.open("rb") as fp:
+                    gopro_info = gpmf_parser.extract_gopro_info(fp, telemetry_only=True)
+                if gopro_info is not None:
+                    telemetry_measurements.extend(gopro_info.accl or [])
+                    telemetry_measurements.extend(gopro_info.gyro or [])
+                    telemetry_measurements.extend(gopro_info.magn or [])
+                telemetry_measurements.sort(key=lambda m: m.time)
+
+        generator = camm_builder.camm_sample_generator2(
+            video_metadata, telemetry_measurements=telemetry_measurements
+        )
+
+        with video_metadata.filename.open("rb") as src_fp:
+            camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator)
+            event_payload: uploader.Progress = {
+                "total_sequence_count": len(specified_video_metadatas),
+                "sequence_idx": idx,
+                "file_type": video_metadata.filetype.value,
+                "import_path": str(video_metadata.filename),
+            }
+            try:
+                cluster_id = mly_uploader.upload_stream(
+                    T.cast(T.BinaryIO, camm_fp),
+                    upload_api_v4.ClusterFileType.CAMM,
+                    video_metadata.md5sum,
+                    event_payload=event_payload,
+                )
+            except Exception as ex:
+                raise UploadError(ex) from ex
+            LOG.debug("Uploaded to cluster: %s", cluster_id)
+
+    # upload zip files
+    zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders)
+    _upload_zipfiles(mly_uploader, zip_paths)
+
+
 def upload(
     import_path: T.Union[Path, T.Sequence[Path]],
+    user_items: types.UserItem,
     desc_path: T.Optional[str] = None,
     _metadatas_from_process: T.Optional[T.Sequence[types.MetadataOrError]] = None,
-    user_name: T.Optional[str] = None,
-    organization_key: T.Optional[str] = None,
     dry_run=False,
     skip_subfolders=False,
 ) -> None:
@@ -505,8 +555,6 @@ def upload(
 
     metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths)
 
-    user_items = fetch_user_items(user_name, organization_key)
-
     # Setup the emitter -- the order matters here
 
     emitter = uploader.EventEmitter()
@@ -547,81 +595,7 @@ def upload(
     )
 
     try:
-        image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders)
-        # find descs that match the image paths from the import paths
-        image_metadatas = [
-            metadata
-            for metadata in (metadatas or [])
-            if isinstance(metadata, types.ImageMetadata)
-        ]
-        specified_image_metadatas = _find_metadata_with_filename_existed_in(
-            image_metadatas, image_paths
-        )
-        if specified_image_metadatas:
-            try:
-                clusters = mly_uploader.upload_images(
-                    specified_image_metadatas,
-                    event_payload={"file_type": FileType.IMAGE.value},
-                )
-            except Exception as ex:
-                raise UploadError(ex) from ex
-
-            if clusters:
-                LOG.debug("Uploaded to cluster: %s", clusters)
-
-        video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders)
-        video_metadatas = [
-            metadata
-            for metadata in (metadatas or [])
-            if isinstance(metadata, types.VideoMetadata)
-        ]
-        specified_video_metadatas = _find_metadata_with_filename_existed_in(
-            video_metadatas, video_paths
-        )
-        for idx, video_metadata in enumerate(specified_video_metadatas):
-            video_metadata.update_md5sum()
-            assert isinstance(video_metadata.md5sum, str), "md5sum should be updated"
-
-            # extract telemetry measurements from GoPro videos
-            telemetry_measurements: T.List[camm_parser.TelemetryMeasurement] = []
-            if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES":
-                if video_metadata.filetype is FileType.GOPRO:
-                    with video_metadata.filename.open("rb") as fp:
-                        gopro_info = gpmf_parser.extract_gopro_info(
-                            fp, telemetry_only=True
-                        )
-                    if gopro_info is not None:
-                        telemetry_measurements.extend(gopro_info.accl or [])
-                        telemetry_measurements.extend(gopro_info.gyro or [])
-                        telemetry_measurements.extend(gopro_info.magn or [])
-                    telemetry_measurements.sort(key=lambda m: m.time)
-
-            generator = camm_builder.camm_sample_generator2(
-                video_metadata, telemetry_measurements=telemetry_measurements
-            )
-
-            with video_metadata.filename.open("rb") as src_fp:
-                camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator)
-                event_payload: uploader.Progress = {
-                    "total_sequence_count": len(specified_video_metadatas),
-                    "sequence_idx": idx,
-                    "file_type": video_metadata.filetype.value,
-                    "import_path": str(video_metadata.filename),
-                }
-                try:
-                    cluster_id = mly_uploader.upload_stream(
-                        T.cast(T.BinaryIO, camm_fp),
-                        upload_api_v4.ClusterFileType.CAMM,
-                        video_metadata.md5sum,
-                        event_payload=event_payload,
-                    )
-                except Exception as ex:
-                    raise UploadError(ex) from ex
-                LOG.debug("Uploaded to cluster: %s", cluster_id)
-
-        zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders)
-        _upload_zipfiles(mly_uploader, zip_paths)
-
+        _upload_everything(mly_uploader, import_paths, metadatas, skip_subfolders)
     except UploadError as ex:
         inner_ex = ex.inner_ex
 
@@ -637,16 +611,10 @@ def upload(
         if isinstance(inner_ex, requests.HTTPError) and isinstance(
             inner_ex.response, requests.Response
         ):
-            if inner_ex.response.status_code in [400, 401]:
-                try:
-                    error_body = inner_ex.response.json()
-                except Exception:
-                    error_body = {}
-                debug_info = error_body.get("debug_info", {})
-                if debug_info.get("type") in ["NotAuthorizedError"]:
-                    raise exceptions.MapillaryUploadUnauthorizedError(
-                        debug_info.get("message")
-                    ) from inner_ex
+            if api_v4.is_auth_error(inner_ex.response):
+                raise exceptions.MapillaryUploadUnauthorizedError(
+                    api_v4.extract_auth_error_message(inner_ex.response)
+                ) from inner_ex
             raise inner_ex
 
         raise inner_ex


Test patch:
diff --git a/tests/cli/upload_api_v4.py b/tests/cli/upload_api_v4.py
index 54718aa..1d1ceaf 100644
--- a/tests/cli/upload_api_v4.py
+++ b/tests/cli/upload_api_v4.py
@@ -6,7 +6,7 @@ import typing as T
 
 import requests
 import tqdm
-from mapillary_tools import upload
+from mapillary_tools import api_v4, authenticate
 
 from mapillary_tools.upload_api_v4 import DEFAULT_CHUNK_SIZE, UploadService
 
@@ -14,16 +14,6 @@ from mapillary_tools.upload_api_v4 import DEFAULT_CHUNK_SIZE, UploadService
 LOG = logging.getLogger("mapillary_tools")
 
 
-def wrap_http_exception(ex: requests.HTTPError):
-    resp = ex.response
-    lines = [
-        f"{ex.request.method} {resp.url}",
-        f"> HTTP Status: {ex.response.status_code}",
-        f"{resp.content!r}",
-    ]
-    return Exception("\n".join(lines))
-
-
 def configure_logger(logger: logging.Logger, stream=None) -> None:
     formatter = logging.Formatter("%(asctime)s - %(levelname)-7s - %(message)s")
     handler = logging.StreamHandler(stream)
@@ -67,7 +57,7 @@ def main():
     with open(parsed.filename, "rb") as fp:
         entity_size = _file_stats(fp)
 
-    user_items = upload.fetch_user_items(parsed.user_name)
+    user_items = authenticate.fetch_user_items(parsed.user_name)
 
     session_key = parsed.session_key
     user_access_token = user_items.get("user_upload_token", "")
@@ -81,11 +71,15 @@ def main():
             else DEFAULT_CHUNK_SIZE
         ),
     )
-    initial_offset = service.fetch_offset()
+
+    try:
+        initial_offset = service.fetch_offset()
+    except requests.HTTPError as ex:
+        raise RuntimeError(api_v4.readable_http_error(ex))
 
     LOG.info("Session key: %s", session_key)
-    LOG.info("Entity size: %d", entity_size)
     LOG.info("Initial offset: %s", initial_offset)
+    LOG.info("Entity size: %d", entity_size)
     LOG.info("Chunk size: %s MB", service.chunk_size / (1024 * 1024))
 
     with open(parsed.filename, "rb") as fp:
@@ -101,9 +95,20 @@ def main():
             try:
                 file_handle = service.upload(fp, initial_offset)
             except requests.HTTPError as ex:
-                raise wrap_http_exception(ex)
+                raise RuntimeError(api_v4.readable_http_error(ex))
+            except KeyboardInterrupt:
+                file_handle = None
+                LOG.warning("Upload interrupted")
+
+    try:
+        final_offset = service.fetch_offset()
+    except requests.HTTPError as ex:
+        raise RuntimeError(api_v4.readable_http_error(ex))
+
+    LOG.info("Final offset: %s", final_offset)
+    LOG.info("Entity size: %d", entity_size)
 
-    LOG.info(file_handle)
+    LOG.info("File handle: %s", file_handle)
 
 
 if __name__ == "__main__":
diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py
index cc30f54..0233530 100644
--- a/tests/integration/fixtures.py
+++ b/tests/integration/fixtures.py
@@ -28,6 +28,8 @@ USERNAME = "test_username_MAKE_SURE_IT_IS_UNIQUE_AND_LONG_AND_BORING"
 def setup_config(tmpdir: py.path.local):
     config_path = tmpdir.mkdir("configs").join("CLIENT_ID")
     os.environ["MAPILLARY_CONFIG_PATH"] = str(config_path)
+    os.environ["MAPILLARY_TOOLS_PROMPT_DISABLED"] = "YES"
+    os.environ["MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED"] = "YES"
     x = subprocess.run(
         f"{EXECUTABLE} authenticate --user_name {USERNAME} --jwt test_user_token",
         shell=True,
@@ -36,7 +38,9 @@ def setup_config(tmpdir: py.path.local):
     yield config_path
     if tmpdir.check():
         tmpdir.remove(ignore_errors=True)
-    del os.environ["MAPILLARY_CONFIG_PATH"]
+    os.environ.pop("MAPILLARY_CONFIG_PATH", None)
+    os.environ.pop("MAPILLARY_TOOLS_PROMPT_DISABLED", None)
+    os.environ.pop("MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED", None)
 
 
 @pytest.fixture
@@ -53,15 +57,19 @@ def setup_data(tmpdir: py.path.local):
 def setup_upload(tmpdir: py.path.local):
     upload_dir = tmpdir.mkdir("mapillary_public_uploads")
     os.environ["MAPILLARY_UPLOAD_PATH"] = str(upload_dir)
+    os.environ["MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED"] = "YES"
+    os.environ["MAPILLARY_TOOLS_PROMPT_DISABLED"] = "YES"
     os.environ["MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"] = "YES"
     history_path = tmpdir.join("history")
     os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"] = str(history_path)
     yield upload_dir
     if tmpdir.check():
         tmpdir.remove(ignore_errors=True)
-    del os.environ["MAPILLARY_UPLOAD_PATH"]
-    del os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"]
-    del os.environ["MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"]
+    os.environ.pop("MAPILLARY_UPLOAD_PATH", None)
+    os.environ.pop("MAPILLARY_UPLOAD_HISTORY_PATH", None)
+    os.environ.pop("MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED", None)
+    os.environ.pop("MAPILLARY_TOOLS_PROMPT_DISABLED", None)
+    os.environ.pop("MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", None)
 
 
 def _ffmpeg_installed():
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 9185b30..67af45c 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -23,7 +23,7 @@ def test_config_list_all_users(tmpdir: py.path.local):
 
     x = config.list_all_users(config_path=str(c))
     assert len(x) == 1
-    assert x[0] == {"ThisIsOption": "1"}
+    assert x["hello"] == {"ThisIsOption": "1"}
 
 
 def test_update_config(tmpdir: py.path.local):
@@ -50,3 +50,15 @@ def test_load_user(tmpdir: py.path.local):
     assert x is None
     x = config.load_user("world", config_path=str(c))
     assert x == {"ThisIsOption": "hello"}
+
+
+def test_remove(tmpdir: py.path.local):
+    c = tmpdir.join("empty_config.ini")
+    config.update_config(
+        "world", T.cast(T.Any, {"ThisIsOption": "hello"}), config_path=str(c)
+    )
+    config.remove_config("world", config_path=str(c))
+    u = config.load_user("world", config_path=str(c))
+    assert u is None
+    x = config.list_all_users(config_path=str(c))
+    assert not x