sqlitebiter/_enum.py
class DupDatabase(Enum):
    OVERWRITE = auto()
    APPEND = auto()
    SKIP = auto()  # TODO
sqlitebiter/__version__.py

sqlitebiter/_const.py
PROGRAM_NAME = "sqlitebiter"
sqlitebiter/converter/_url.py
class UrlConverter(TableConverter):
    def __init__(
        self,
        logger,
        con,
        symbol_replace_value,
        add_pri_key_name,
        convert_configs,
        index_list,
        is_type_inference,
        is_type_hint_header,
        verbosity_level,
        max_workers: int,
        format_name,
        encoding,
        proxy,
    ):
        super().__init__(
            logger,
            con,
            symbol_replace_value=symbol_replace_value,
            add_pri_key_name=add_pri_key_name,
            convert_configs=convert_configs,
            index_list=index_list,
            is_type_inference=is_type_inference,
            is_type_hint_header=is_type_hint_header,
            verbosity_level=verbosity_level,
            max_workers=max_workers,
            format_name=format_name,
            encoding=encoding,
        )

        self.__proxy = proxy

    def convert(self, url: str) -> None:
        logger = self._logger
        result_counter = self._result_counter

        source_info_record_base = self.__parse_source_info_url(url)

        if self._format_name in IPYNB_FORMAT_NAME_LIST or is_ipynb_url(url):
            try:
                nb, nb_size = load_ipynb_url(url, proxies=self.__get_proxies())
            except RuntimeError as e:
                logger.error(e)
                return

            changed_table_name_set = self._convert_nb(nb, source_info=source_info_record_base)

            for table_name in changed_table_name_set:
                record = deepcopy(source_info_record_base)
                record.format_name = "ipynb"  # type: ignore
                record.dst_table = table_name
                record.size = nb_size
                SourceInfo.insert(record)

            return

        loader = self.__create_loader(url)
        source_info_record_base.format_name = loader.format_name
        success_count = result_counter.success_count

        try:
            for table_data in loader.load():
                logger.debug(f"loaded table_data: {str(table_data)}")

                sqlite_tabledata = self.normalize_table(table_data)

                try:
                    self._table_creator.create(
                        sqlite_tabledata, self._index_list, source_info=source_info_record_base
                    )
                except sqlite.OperationalError as e:
                    logger.error(
                        "{:s}: failed to convert: url={}, message={}".format(
                            e.__class__.__name__, url, e.message
                        )
                    )
                    result_counter.inc_fail()
                    continue
                except ValueError as e:
                    logger.debug(f"{e.__class__.__name__:s}: url={url}, message={str(e)}")
                    result_counter.inc_fail()
                    continue

                record = deepcopy(source_info_record_base)
                record.dst_table = sqlite_tabledata.table_name  # type: ignore
                SourceInfo.insert(record)
        except ptr.ValidationError as e:
            if loader.format_name == "json":
                for table_name in self._convert_complex_json(
                    loader.loader, source_info_record_base
                ):
                    record = deepcopy(source_info_record_base)
                    record.dst_table = table_name
                    SourceInfo.insert(record)
            else:
                logger.error(f"{e.__class__.__name__:s}: url={url}, message={str(e)}")
                result_counter.inc_fail()
        except ptr.DataError as e:
            logger.error(f"{e.__class__.__name__:s}: invalid data: url={url}, message={str(e)}")
            result_counter.inc_fail()
        except OverflowError as e:
            logger.error(f"{url}: {e}")
            result_counter.inc_fail()

        if result_counter.success_count == success_count:
            logger.warning(TABLE_NOT_FOUND_MSG_FORMAT.format(url))

    def __get_proxies(self) -> Dict:
        return {"http": self.__proxy, "https": self.__proxy}

    def __create_loader(self, url: str) -> AbstractTableReader:
        logger = self._logger
        type_hint_rules = self.__extract_type_hint_rules(url)
        proxies = self.__get_proxies()

        try:
            return create_url_loader(
                logger, url, self._format_name, self._encoding, type_hint_rules, proxies
            )
        except ptr.LoaderNotFoundError as e:
            logger.debug(e)

        try:
            return create_url_loader(logger, url, "html", self._encoding, type_hint_rules, proxies)
        except ptr.LoaderNotFoundError as e:
            logger.error(msgfy.to_error_message(e))
            sys.exit(ExitCode.FAILED_LOADER_NOT_FOUND)

    def __extract_type_hint_rules(self, url: str) -> TypeHintRules:
        if self._is_type_hint_header:
            return TYPE_HINT_FROM_HEADER_RULES

        type_hint_rules = {}

        for config in self._convert_configs:
            if not isinstance(config, dict):
                self._logger.debug(f"unexpected config value: {config}")
                continue

            if config.get("target_url") not in url:  # type: ignore
                continue

            for pattern, params in config["rules"].items():
                if not params.get("type hint"):
                    continue

                type_hint_rules[re.compile(pattern, re.IGNORECASE)] = normalize_type_hint(
                    params["type hint"]
                )

        return type_hint_rules

    def __parse_source_info_url(self, url: str) -> SourceInfo:
        result = urlparse(url)

        source_info = SourceInfo(
            dir_name=result.netloc + os.path.dirname(result.path),
            base_name=os.path.basename(str(result.path)),
            source_id=self._fetch_next_source_id(),
        )

        return source_info
sqlitebiter/_config.py
class ConfigKey:
    DEFAULT_ENCODING = "default_encoding"
    PROXY_SERVER = "proxy_server"
    GS_CREDENTIALS_FILE_PATH = "gs_credentials_file_path"
sqlitebiter/converter/_gs.py
class GoogleSheetsConverter(TableConverter):
    def convert(self, credentials, title: str) -> None:
        logger = self._logger
        result_counter = self._result_counter
        source_id = self._fetch_next_source_id()

        loader = ptr.GoogleSheetsTableLoader()
        loader.source = credentials
        loader.title = title

        # if typepy.is_null_string(loader.source):
        #     loader.source = app_config_mgr.load().get(
        #         ConfigKey.GS_CREDENTIALS_FILE_PATH)

        try:
            for table_data in loader.load():
                logger.debug(f"loaded table_data: {str(table_data)}")

                sqlite_tabledata = self.normalize_table(table_data)
                source_info = SourceInfo(
                    base_name=title,
                    dst_table=sqlite_tabledata.table_name,
                    format_name="google sheets",
                    source_id=source_id,
                )

                try:
                    self._table_creator.create(
                        sqlite_tabledata, self._index_list, source_info=source_info
                    )
                    SourceInfo.insert(source_info)
                except (ptr.ValidationError, ptr.DataError):
                    result_counter.inc_fail()
        except ptr.OpenError as e:
            logger.error(msgfy.to_error_message(e))
            result_counter.inc_fail()
        except (ptr.ValidationError, ptr.DataError) as e:
            logger.error(f"invalid credentials data: path={credentials}, message={str(e)}")
            result_counter.inc_fail()
        except ptr.APIError as e:
            logger.error(msgfy.to_error_message(e))
            result_counter.inc_fail()
        except ImportError as e:
            logger.error(
                "{}: try to install dependencies with 'pip install sqlitebiter[gs]'".format(
                    msgfy.to_error_message(e)
                )
            )
        except OverflowError as e:
            logger.error(f"{title}: {e}")
            result_counter.inc_fail()
sqlitebiter/_common.py
DEFAULT_DUP_COL_HANDLER = "rename"
sqlitebiter/_const.py
IPYNB_FORMAT_NAME_LIST = ["ipynb"]
sqlitebiter/_enum.py
class Context(Enum):
    DUP_DATABASE = auto()
    CONVERT_CONFIG = auto()
    INDEX_LIST = auto()
    ADD_PRIMARY_KEY_NAME = auto()
    TYPE_INFERENCE = auto()
    TYPE_HINT_HEADER = auto()
    LOG_LEVEL = auto()
    OUTPUT_PATH = auto()
    VERBOSITY_LEVEL = auto()
    MAX_WORKERS = auto()
    SYMBOL_REPLACE_VALUE = auto()
sqlitebiter/_const.py
class ExitCode:
    SUCCESS = 0
    FAILED_LOADER_NOT_FOUND = 1
    FAILED_CONVERT = 2
    FAILED_HTTP = 3
    NO_INPUT = 10
sqlitebiter/converter/_text.py
class TextConverter(TableConverter):
    def __get_source_info_base(self, text: str) -> SourceInfo:
        return SourceInfo(
            base_name="stdin",
            size=len(text),
            source_id=self._fetch_next_source_id(),
        )

    def convert(self, text: str) -> None:
        logger = self._logger
        result_counter = self._result_counter
        source_info_record_base = self.__get_source_info_base(text)

        if self._format_name in IPYNB_FORMAT_NAME_LIST:
            try:
                nb, nb_size = load_ipynb_text(text)
            except RuntimeError as e:
                logger.error(e)
                return

            changed_table_name_set = self._convert_nb(nb, source_info=source_info_record_base)

            for table_name in changed_table_name_set:
                record = deepcopy(source_info_record_base)
                record.format_name = "ipynb"  # type: ignore
                record.dst_table = table_name
                record.size = nb_size
                SourceInfo.insert(record)

            return

        loader = create_text_loader(
            logger,
            text,
            self._format_name,
            self._encoding,
            TYPE_HINT_FROM_HEADER_RULES if self._is_type_hint_header else None,
        )
        source_info_record_base.format_name = loader.format_name
        success_count = result_counter.success_count

        try:
            for table_data in loader.load():
                logger.debug(f"loaded table_data: {str(table_data)}")

                sqlite_tabledata = self.normalize_table(table_data)

                try:
                    self._table_creator.create(
                        sqlite_tabledata, self._index_list, source_info=source_info_record_base
                    )
                except sqlite.OperationalError as e:
                    logger.error(
                        "{:s}: failed to convert: text={}, message={}".format(
                            e.__class__.__name__, text, e.message
                        )
                    )
                    result_counter.inc_fail()
                    continue
                except ValueError as e:
                    logger.debug(f"{e.__class__.__name__:s}: text={text}, message={str(e)}")
                    result_counter.inc_fail()
                    continue

                record = deepcopy(source_info_record_base)
                record.dst_table = sqlite_tabledata.table_name  # type: ignore
                SourceInfo.insert(record)
        except ptr.ValidationError as e:
            if loader.format_name == "json":
                for table_name in self._convert_complex_json(
                    loader.loader, source_info_record_base
                ):
                    record = deepcopy(source_info_record_base)
                    record.dst_table = table_name
                    SourceInfo.insert(record)
            else:
                logger.error(f"{e.__class__.__name__:s}: text={text}, message={str(e)}")
                result_counter.inc_fail()
        except ptr.DataError as e:
            logger.error(f"{e.__class__.__name__:s}: invalid data: text={text}, message={str(e)}")
            result_counter.inc_fail()
        except OverflowError as e:
            logger.error(f"{text}: {e}")
            result_counter.inc_fail()

        if result_counter.success_count == success_count:
            logger.warning(TABLE_NOT_FOUND_MSG_FORMAT.format(text))
sqlitebiter/converter/_file.py
class FileConverter(TableConverter):
    SKIP_MSG_FORMAT = "skip '{source:s}': {message:s}"

    def __init__(
        self,
        logger,
        con,
        symbol_replace_value,
        add_pri_key_name,
        convert_configs,
        index_list,
        is_type_inference,
        is_type_hint_header,
        verbosity_level,
        max_workers: int,
        format_name,
        encoding,
        exclude_pattern,
        follow_symlinks,
    ):
        super().__init__(
            logger,
            con=con,
            symbol_replace_value=symbol_replace_value,
            add_pri_key_name=add_pri_key_name,
            convert_configs=convert_configs,
            index_list=index_list,
            is_type_inference=is_type_inference,
            is_type_hint_header=is_type_hint_header,
            verbosity_level=verbosity_level,
            max_workers=max_workers,
            format_name=format_name,
            encoding=encoding,
        )

        self.__exclude_pattern = exclude_pattern
        self.__follow_symlinks = follow_symlinks

    def convert(self, file_path) -> None:
        file_path = Path(file_path)
        logger = self._logger
        result_counter = self._result_counter

        if not self.__is_file(file_path):
            return

        if self.__exclude_pattern and file_path.fnmatch(self.__exclude_pattern):
            logger.debug(
                self.SKIP_MSG_FORMAT.format(source=file_path, message="matching an exclude pattern")
            )
            self._result_counter.inc_skip()
            return

        logger.debug(f"converting '{file_path}'")
        success_count = result_counter.success_count
        fail_count = result_counter.fail_count
        source_info_record_base = self.__get_source_info_base(file_path.realpath())

        if self._format_name in IPYNB_FORMAT_NAME_LIST or is_ipynb_file_path(file_path):
            import nbformat

            try:
                changed_table_name_set = self._convert_nb(
                    nb=load_ipynb_file(file_path, encoding=self._encoding),
                    source_info=source_info_record_base,
                )
            except (nbformat.reader.NotJSONError, RuntimeError) as e:
                logger.error(f"failed to load {file_path}: {e}")
                return

            for table_name in changed_table_name_set:
                record = deepcopy(source_info_record_base)
                record.format_name = "ipynb"  # type: ignore
                record.dst_table = table_name
                SourceInfo.insert(record)
        else:
            self.__convert(file_path, source_info_record_base)

        if result_counter.fail_count > fail_count:
            return

        if result_counter.success_count == success_count:
            logger.warning(TABLE_NOT_FOUND_MSG_FORMAT.format(file_path))

    def __convert(self, file_path: Path, source_info_record_base: SourceInfo) -> None:
        logger = self._logger
        result_counter = self._result_counter

        try:
            loader = ptr.TableFileLoader(
                file_path,
                format_name=self._format_name,
                encoding=self._encoding,
                type_hint_rules=TYPE_HINT_FROM_HEADER_RULES if self._is_type_hint_header else None,
            )
        except ptr.InvalidFilePathError as e:
            logger.debug(msgfy.to_debug_message(e))
            result_counter.inc_fail()
            return
        except ptr.LoaderNotFoundError:
            logger.warning(f"not supported file format: ext={file_path.ext}, path={file_path}")
            result_counter.inc_fail()
            return

        source_info_record_base.format_name = loader.format_name

        try:
            for table_data in loader.load():
                logger.debug(f"loaded tabledata: {str(table_data)}")

                sqlite_tabledata = self.normalize_table(table_data)

                try:
                    self._table_creator.create(
                        sqlite_tabledata, self._index_list, source_info=source_info_record_base
                    )
                except (ValueError, OSError) as e:
                    logger.debug(f"exception={type(e).__name__:s}, path={file_path}, message={e}")
                    result_counter.inc_fail()
                    return

                record = deepcopy(source_info_record_base)
                record.dst_table = sqlite_tabledata.table_name  # type: ignore
                SourceInfo.insert(record)
        except ptr.OpenError as e:
            logger.error(
                "{:s}: open error: file={}, message='{}'".format(
                    e.__class__.__name__, file_path, str(e)
                )
            )
            result_counter.inc_fail()
        except ptr.ValidationError as e:
            if loader.format_name == "json":
                for table_name in self._convert_complex_json(
                    loader.loader, source_info_record_base
                ):
                    record = deepcopy(source_info_record_base)
                    record.dst_table = table_name  # type: ignore
                    SourceInfo.insert(record)
            else:
                logger.error(
                    "{:s}: invalid {} data format: path={}, message={}".format(
                        e.__class__.__name__,
                        _get_format_type_from_path(file_path),
                        file_path,
                        str(e),
                    )
                )
                result_counter.inc_fail()
        except ptr.DataError as e:
            logger.error(
                "{:s}: invalid {} data: path={}, message={}".format(
                    e.__class__.__name__, _get_format_type_from_path(file_path), file_path, str(e)
                )
            )
            result_counter.inc_fail()
        except OverflowError as e:
            logger.error(f"{file_path}: {e}")
            result_counter.inc_fail()

    def __is_fifo(self, file_path: Path) -> bool:
        try:
            return stat.S_ISFIFO(os.stat(file_path).st_mode)
        except OSError as e:
            if e.errno not in (EBADF, ENAMETOOLONG, ENOENT, ENOTDIR):
                raise

            return False
        except ValueError:
            return False

    def __is_file(self, file_path: Path) -> bool:
        if not file_path.exists():
            self._logger.warning(
                self.SKIP_MSG_FORMAT.format(source=file_path, message="no such file or directory")
            )
            self._result_counter.inc_skip()
            return False

        if file_path.islink() and not self.__follow_symlinks:
            self._logger.info(
                self.SKIP_MSG_FORMAT.format(source=file_path, message="skip a symlink to a file")
            )
            self._result_counter.inc_skip()
            return False

        if not file_path.isfile() and not self.__is_fifo(file_path):
            self._logger.warning(
                self.SKIP_MSG_FORMAT.format(source=file_path, message="not a file")
            )
            self._result_counter.inc_skip()
            return False

        if file_path.realpath() == self._con.database_path:
            self._logger.warning(
                self.SKIP_MSG_FORMAT.format(
                    source=file_path, message="same path as the output file"
                )
            )
            self._result_counter.inc_skip()
            return False

        return True

    def __get_source_info_base(self, source: Path) -> SourceInfo:
        return SourceInfo(
            dir_name=source.dirname(),
            base_name=source.basename(),
            size=source.getsize(),
            mtime=int(source.getmtime()),
            source_id=self._fetch_next_source_id(),
        )


Question: import os
import sys
import appconfigpy
import click
import msgfy
import path
import pytablereader as ptr
import simplesqlite as sqlite
import typepy
    import ujson as json
    import json  # type: ignore
    import envinfopy
from textwrap import dedent
from typing import Dict, Tuple
from loguru import logger
from .__version__ import __version__
from ._common import DEFAULT_DUP_COL_HANDLER
from ._config import ConfigKey, app_config_mgr
from ._const import IPYNB_FORMAT_NAME_LIST, PROGRAM_NAME, ExitCode
from ._enum import Context, DupDatabase
from .converter import FileConverter, GoogleSheetsConverter, TextConverter, UrlConverter
    converter = FileConverter(
        logger=logger,
        con=con,
        symbol_replace_value=ctx.obj[Context.SYMBOL_REPLACE_VALUE],
        add_pri_key_name=ctx.obj[Context.ADD_PRIMARY_KEY_NAME],
        convert_configs=convert_configs,
        index_list=ctx.obj.get(Context.INDEX_LIST),
        is_type_inference=ctx.obj[Context.TYPE_INFERENCE],
        is_type_hint_header=ctx.obj[Context.TYPE_HINT_HEADER],
        verbosity_level=ctx.obj.get(Context.VERBOSITY_LEVEL),
        max_workers=max_workers,
        format_name=format_name,
        encoding=encoding,
        exclude_pattern=exclude,
        follow_symlinks=follow_symlinks,
    )

    for file_path in files:
        dir_path_obj = path.Path(file_path)

        if not follow_symlinks and dir_path_obj.islink() and dir_path_obj.isdir():
            logger.debug(
                "skip symlink to a directory: {} -> {}".format(
                    dir_path_obj, dir_path_obj.readlink()
                )
            )
            continue

        if recursive and dir_path_obj.isdir():
            for file_path_obj in dir_path_obj.walkfiles(pattern):
                converter.convert(file_path_obj)
        else:
            converter.convert(file_path)

    sys.exit(finalize(con, converter, is_create_db))


@cmd.command(epilog=COMMAND_EPILOG)
@click.argument(
    "format_name",
    type=click.Choice(ptr.TableTextLoader.get_format_names() + IPYNB_FORMAT_NAME_LIST),
)
@click.pass_context
def stdin(ctx, format_name):
    """
    Convert tabular data within
    CSV/HTML/JSON/Jupyter Notebook/LDJSON/LTSV/Markdown/Mediawiki/SSV/TSV
    text to a SQLite database file.
    """

    initialize_logger(f"{PROGRAM_NAME:s} stdin", ctx.obj[Context.LOG_LEVEL])

    convert_configs = load_convert_config(
        logger, ctx.obj[Context.CONVERT_CONFIG], subcommand="stdin"
    )

    max_workers = ctx.obj.get(Context.MAX_WORKERS)
    con, is_create_db = create_database(
        ctx.obj[Context.OUTPUT_PATH], ctx.obj[Context.DUP_DATABASE], max_workers=max_workers
    )
    converter = TextConverter(
        logger=logger,
        con=con,
        symbol_replace_value=ctx.obj[Context.SYMBOL_REPLACE_VALUE],
        add_pri_key_name=ctx.obj[Context.ADD_PRIMARY_KEY_NAME],
        convert_configs=convert_configs,
        index_list=ctx.obj.get(Context.INDEX_LIST),
        is_type_inference=ctx.obj[Context.TYPE_INFERENCE],
        is_type_hint_header=ctx.obj[Context.TYPE_HINT_HEADER],
        verbosity_level=ctx.obj.get(Context.VERBOSITY_LEVEL),
        max_workers=max_workers,
        format_name=format_name,
    )

    converter.convert(sys.stdin.read())

    sys.exit(finalize(con, converter, is_create_db))


@cmd.command(epilog=COMMAND_EPILOG)
@click.argument("url", type=str)
@click.option(
    "-f",
    "--format",
    "format_name",
    type=click.Choice(ptr.TableUrlLoader.get_format_names() + IPYNB_FORMAT_NAME_LIST),
    help="Data format to loading (defaults to html).",
)
@click.option(
    "-e",
    "--encoding",
    type=str,
    metavar="ENCODING",
    help=f"HTML page read encoding. Defaults to {Default.ENCODING:s}.",
)
@click.option(
    "-p",
    "--proxy",
    type=str,
    metavar="PROXY",
    help="Specify a proxy in the form [user:passwd@]proxy.server:port.",
)
@click.pass_context
def url(ctx, url, format_name, encoding, proxy):
    """
    Scrape tabular data from a URL and convert data to a SQLite database file.
    """

    if typepy.is_empty_sequence(url):
        sys.exit(ExitCode.NO_INPUT)

    initialize_logger(f"{PROGRAM_NAME:s} url", ctx.obj[Context.LOG_LEVEL])

    try:
        app_configs = app_config_mgr.load()
    except ValueError as e:
        logger.debug(msgfy.to_debug_message(e))
        app_configs = {}

    if typepy.is_empty_sequence(encoding):

Answer:         encoding = app_configs.get(ConfigKey.DEFAULT_ENCODING)