Repository: mindsdb/mindsdb

Issue:
OSX Installer using pyinstaller




Discussion:

Resolving patch:
diff --git a/distributions/osx/build_installers.py b/distributions/osx/build_installers.py
new file mode 100644
--- /dev/null
+++ b/distributions/osx/build_installers.py
@@ -0,0 +1,25 @@
+import os
+import sys
+
+about = {}
+with open("mindsdb/__about__.py") as fp:
+    exec(fp.read(), about)
+
+with open('distributions/osx/install.py', 'r') as f:
+    install_py = f.read()
+
+if sys.argv[1] == 'beta':
+    LATEST_NAME = 'MindsDB_Latest_Beta'
+    FIXED_NAME = 'MindsDB_{}_Beta'.format(about['__version__'])
+elif sys.argv[1] == 'release':
+    LATEST_NAME = 'MindsDB_Latest'
+    FIXED_NAME = 'MindsDB-{}'.format(about['__version__'])
+
+with open('distributions/osx/latest.py', 'w+') as f:
+    f.write(install_py.replace('$name', LATEST_NAME).replace('$version', ''))
+
+with open('distributions/osx/fixed.py', 'w+') as f:
+    f.write(install_py.replace('$name', FIXED_NAME).replace('$version', about['__version__']))
+
+os.system('cd distributions/osx && pyinstaller latest.py -F --onefile -n {}-Setup'.format(LATEST_NAME))
+os.system('cd distributions/osx && pyinstaller fixed.py -F --onefile -n {}-Setup'.format(FIXED_NAME))
diff --git a/distributions/osx/install.py b/distributions/osx/install.py
new file mode 100644
--- /dev/null
+++ b/distributions/osx/install.py
@@ -0,0 +1,151 @@
+import os
+import sys
+import atexit
+import tarfile
+import subprocess
+import shutil
+import argparse
+
+import requests
+
+PIP_VERSION = "20.2.4"
+REQUIRED_UTILS = {"make": "brew install make",
+                  "openssl": "brew install openssl",
+                  "zlib": "brew install zlib",
+                  "sqlite": "brew install sqlite",
+                  "bzip2": "brew install bzip2",
+                  "libiconv": "brew install libiconv",
+                  "libzip": "brew install libzip"}
+
+def at_exit():
+    os.system('echo Press Enter to exit ...;read foo')
+
+def check_system_requirements():
+    res = subprocess.run(['which', 'brew'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if res.returncode:
+        print("brew is not installed. How to install: /usr/local/opt/openssl@1.1/bin/openssl")
+        sys.exit("please install required utils and try again")
+    is_good = True
+
+    for util in REQUIRED_UTILS:
+        res = subprocess.run(['brew', "--prefix", util], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if res.returncode:
+            print("{} is required but not installed. Installation instruction: {}".format(util, REQUIRED_UTILS[util]))
+            is_good = False
+    if not is_good:
+        sys.exit("please install required utils and try again")
+
+
+def make_dir(d):
+    if not os.path.isdir(d):
+        os.makedirs(d)
+
+def download_file(url):
+    filename = url.split('/')[-1]
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(filename, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+    return filename
+
+def get_util_dir(util):
+    res = subprocess.check_output(["brew", "--prefix", util])
+    return res.decode('utf-8').rstrip()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='MindsDB Installer')
+    parser.add_argument("--install_dir", type=str, default=os.getenv("HOME"))
+    parser.add_argument("--python_version", type=str, default="3.8.5")
+    args = parser.parse_args()
+
+    assert os.name == 'posix'
+    check_system_requirements()
+    atexit.register(at_exit)
+
+
+    # must be replaced in another script
+    NAME = '$name'
+    VERSION = '$version'
+
+    assert NAME != '$' + 'name'
+    assert VERSION != '$' + 'version'
+
+
+    PY_EMBED_URL = f'https://www.python.org/ftp/python/{args.python_version}/Python-{args.python_version}.tgz'
+    GET_PIP_URL = 'https://bootstrap.pypa.io/get-pip.py'
+    VC_REDIST_URL = 'https://aka.ms/vs/16/release/vc_redist.x64.exe'
+
+    INSTALL_DIR = os.path.join(os.path.abspath(args.install_dir), NAME)
+
+    PYTHON_SRC_DIR = os.path.join(INSTALL_DIR, f"Python-{args.python_version}")
+    PYTHON_DIR = os.path.join(INSTALL_DIR, 'python')
+
+
+    try:
+        if os.path.exists(INSTALL_DIR):
+            print(f"{INSTALL_DIR} is exist. it will be re-created")
+            shutil.rmtree(INSTALL_DIR)
+        make_dir(INSTALL_DIR)
+    except PermissionError as e:
+        print('Please, run the installer as administrator (use "sudo")')
+        os.system('pause')
+        sys.exit(1)
+
+
+    print('extraction python source files')
+    py_embed_filename = download_file(PY_EMBED_URL)
+    with tarfile.open(py_embed_filename, 'r:gz') as z:
+        z.extractall(INSTALL_DIR)
+    os.remove(py_embed_filename)
+
+    for util in ("openssl", "zlib", "sqlite"):
+        util_path = get_util_dir(util)
+        if util_path not in os.getenv("CPPFLAGS", ""):
+            os.environ["CPPFLAGS"] = "{} -I{}/include".format(os.getenv("CPPFLAGS", ""), util_path)
+        if util_path not in os.getenv("LDFLAGS", ""):
+            os.environ["LDFLAGS"] = "{} -L{}/lib".format(os.getenv("LDFLAGS", ""), util_path)
+        if util in ("zlib", "sqlite"):
+            os.environ["PKG_CONFIG_PATH"] = "{} {}/lib/pkgconfig".format(os.getenv("PKG_CONFIG_PATH", ""), util_path)
+    config_cmd = "cd {};./configure --prefix={} --with-openssl={} --with-ssl-default-suites=openssl --silent".format(PYTHON_SRC_DIR, PYTHON_DIR, get_util_dir("openssl"))
+
+
+    print("configuring python: {}".format(config_cmd))
+    print("this may take a while...")
+    print("CPPFLAGS: {}".format(os.getenv("CPPFLAGS", "")))
+    print("LDFLAGS: {}".format(os.getenv("LDFLAGS", "")))
+    print("PKG_CONFIG_PATH: {}".format(os.getenv("PKG_CONFIG_PATH", "")))
+    status = os.system(config_cmd)
+    if status:
+        sys.exit("python configuring process finished with error(s)")
+    print("installing python to %s" % PYTHON_DIR)
+    print("this may take a while...")
+
+    status = os.system("cd {}; make install --silent".format(PYTHON_SRC_DIR))
+    shutil.rmtree(PYTHON_SRC_DIR)
+    if status:
+        sys.exit("installation process finished with error(s)")
+
+
+    PYTHON_EXEC = os.path.join(PYTHON_DIR, 'bin/python3')
+    print("python location: {}".format(PYTHON_EXEC))
+    os.system("{} -V".format(PYTHON_EXEC))
+
+    # it is a clutch to avoid conflicts since old mindsdb_native versions also install torch
+    # ideally we just need to install mindsdb. And torch with other dependencies would be installed properly as mindsdb dependency
+    if os.system('{} -m pip install "torch >= 1.4.0, <= 1.6.0" "torchvision >= 0.5.0, <= 0.7.0" -f https://download.pytorch.org/whl/torch_stable.html --no-warn-script-location'.format(PYTHON_EXEC)):
+        sys.exit("python packages installation finished with error(s)")
+    os.system('{} -m pip install --upgrade pip=={} --no-cache-dir'.format(PYTHON_EXEC, PIP_VERSION))
+    if VERSION == '':
+        os.system('{} -m pip install mindsdb --no-warn-script-location'.format(PYTHON_EXEC))
+    else:
+        os.system('{} -m pip install mindsdb=={} --no-warn-script-location'.format(PYTHON_EXEC, VERSION))
+
+    print('generating run_server.sh')
+    with open(os.path.join(INSTALL_DIR, 'run_server.sh'), 'w') as f:
+        lines = []
+        if VERSION == '':
+            lines.append('{} -m pip install mindsdb --upgrade --no-warn-script-location'.format(PYTHON_EXEC))
+        lines.append('{} -m mindsdb'.format(PYTHON_EXEC))
+        f.write('\n'.join(lines))
diff --git a/distributions/osx/sync_s3.py b/distributions/osx/sync_s3.py
new file mode 100644
--- /dev/null
+++ b/distributions/osx/sync_s3.py
@@ -0,0 +1,53 @@
+import os
+import boto3
+from pathlib import Path
+from bisect import bisect_left
+
+required_env_variables = [
+    'AWS_ACCESS_KEY_ID',
+    'AWS_SECRET_ACCESS_KEY',
+    'AWS_REGION',
+    'SOURCE_DIR',
+    'DEST_DIR',
+    'AWS_S3_BUCKET'
+]
+
+for var in required_env_variables:
+    if os.getenv(var) is None:
+        exit('environment variable {} is not set'.format(var))
+
+s3 = boto3.client(
+    's3',
+    aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
+    aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'),
+    region_name=os.getenv('AWS_REGION')
+)
+
+
+def list_source_objects(source_folder):
+    path = Path(source_folder)
+    paths = []
+    for file_path in path.rglob('*'):
+        if file_path.is_dir():
+            continue
+        paths.append(str(file_path))
+    return paths
+
+
+def sync(source, dest, bucket):
+    for path in list_source_objects(source_folder=source):
+        src_filename = Path(path).name
+        print('Uploading {} ({})'.format(path, src_filename))
+        s3.upload_file(
+            Filename=path,
+            Bucket=bucket,
+            Key=str(Path(dest).joinpath(src_filename).as_posix())
+        )
+
+
+if __name__ == '__main__':
+    sync(
+        source=os.getenv('SOURCE_DIR'),
+        dest=os.getenv('DEST_DIR'),
+        bucket=os.getenv('AWS_S3_BUCKET')
+    )


Test patch: