Repository: piskvorky/gensim

Issue:
gensim.downloader should accept custom directory for downloading embeddings
#### Problem description

I am trying to use gensim.downloader for download embeddings. However, it is creating a directory in my home folder where there is not enough space. I would like to specify a custom folder so that the embeddings get downloaded there. 

#### Steps/code/corpus to reproduce

Fill up your C drive and use gensim.downloader to download one of the embeddings. 

#### Versions

Please provide the output of:

```python
import gensim.downloader as api
dataset = api.load("text8")
```

gensim.downloader should accept custom directory for downloading embeddings
#### Problem description

I am trying to use gensim.downloader for download embeddings. However, it is creating a directory in my home folder where there is not enough space. I would like to specify a custom folder so that the embeddings get downloaded there. 

#### Steps/code/corpus to reproduce

Fill up your C drive and use gensim.downloader to download one of the embeddings. 

#### Versions

Please provide the output of:

```python
import gensim.downloader as api
dataset = api.load("text8")
```



Discussion:
Can you not symlink the directory to another volume?
That's possible. But it isn't clean to not have the possibility of
providing a directory to download. Moreover, the path to which the download
happens is not returned until the download actually happens. Therefore, in
case this folder changes in the future I would have to re-do the symlink
again.

On Fri, Mar 15, 2019 at 04:09 Michael Penkov <notifications@github.com>
wrote:

> Can you not symlink the directory to another volume?
>
> —
> You are receiving this because you authored the thread.
>
>
> Reply to this email directly, view it on GitHub
> <https://github.com/RaRe-Technologies/gensim/issues/2416#issuecomment-473246667>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAddlWANiQUSEWMaa9Uea5ZD9U2G5GBzks5vW39rgaJpZM4b1-BH>
> .
>

Can you not symlink the directory to another volume?
That's possible. But it isn't clean to not have the possibility of
providing a directory to download. Moreover, the path to which the download
happens is not returned until the download actually happens. Therefore, in
case this folder changes in the future I would have to re-do the symlink
again.

On Fri, Mar 15, 2019 at 04:09 Michael Penkov <notifications@github.com>
wrote:

> Can you not symlink the directory to another volume?
>
> —
> You are receiving this because you authored the thread.
>
>
> Reply to this email directly, view it on GitHub
> <https://github.com/RaRe-Technologies/gensim/issues/2416#issuecomment-473246667>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAddlWANiQUSEWMaa9Uea5ZD9U2G5GBzks5vW39rgaJpZM4b1-BH>
> .
>


Resolving patch:
diff --git a/gensim/downloader.py b/gensim/downloader.py
--- a/gensim/downloader.py
+++ b/gensim/downloader.py
@@ -41,6 +41,14 @@
     python -m gensim.downloader --info name # same as api.info(name_only=True)
     python -m gensim.downloader --download <dataname> # same as api.load(dataname, return_path=True)
 
+You may specify the local subdirectory for saving gensim data using the
+GENSIM_DATA_DIR environment variable.  For example:
+
+    $ export GENSIM_DATA_DIR=/tmp/gensim-data
+    $ python -m gensim.downloader --download <dataname>
+
+By default, this subdirectory is ~/gensim-data.
+
 """
 from __future__ import absolute_import
 import argparse
@@ -62,8 +70,20 @@
     import urllib.request as urllib
     from urllib.request import urlopen
 
-user_dir = os.path.expanduser('~')
-base_dir = os.path.join(user_dir, 'gensim-data')
+
+_DEFAULT_BASE_DIR = os.path.expanduser('~/gensim-data')
+BASE_DIR = os.environ.get('GENSIM_DATA_DIR', _DEFAULT_BASE_DIR)
+"""The default location to store downloaded data.
+
+You may override this with the GENSIM_DATA_DIR environment variable.
+
+"""
+
+_PARENT_DIR = os.path.abspath(os.path.join(BASE_DIR, '..'))
+
+
+base_dir = BASE_DIR  # for backward compatibility with some of our test data
+
 logger = logging.getLogger(__name__)
 
 DATA_LIST_URL = "https://raw.githubusercontent.com/RaRe-Technologies/gensim-data/master/list.json"
@@ -125,21 +145,21 @@ def _create_base_dir():
         already exists in the home directory.
 
     """
-    if not os.path.isdir(base_dir):
+    if not os.path.isdir(BASE_DIR):
         try:
-            logger.info("Creating %s", base_dir)
-            os.makedirs(base_dir)
+            logger.info("Creating %s", BASE_DIR)
+            os.makedirs(BASE_DIR)
         except OSError as e:
             if e.errno == errno.EEXIST:
                 raise Exception(
                     "Not able to create folder gensim-data in {}. File gensim-data "
-                    "exists in the direcory already.".format(user_dir)
+                    "exists in the directory already.".format(_PARENT_DIR)
                 )
             else:
                 raise Exception(
                     "Can't create {}. Make sure you have the read/write permissions "
                     "to the directory or you can try creating the folder manually"
-                    .format(base_dir)
+                    .format(BASE_DIR)
                 )
 
 
@@ -297,7 +317,7 @@ def _download(name):
 
     """
     url_load_file = "{base}/{fname}/__init__.py".format(base=DOWNLOAD_BASE_URL, fname=name)
-    data_folder_dir = os.path.join(base_dir, name)
+    data_folder_dir = os.path.join(BASE_DIR, name)
     data_folder_dir_tmp = data_folder_dir + '_tmp'
     tmp_dir = tempfile.mkdtemp()
     init_path = os.path.join(tmp_dir, "__init__.py")
@@ -429,7 +449,7 @@ def load(name, return_path=False):
     file_name = _get_filename(name)
     if file_name is None:
         raise ValueError("Incorrect model/corpus name")
-    folder_dir = os.path.join(base_dir, name)
+    folder_dir = os.path.join(BASE_DIR, name)
     path = os.path.join(folder_dir, file_name)
     if not os.path.exists(folder_dir):
         _download(name)
@@ -437,7 +457,7 @@ def load(name, return_path=False):
     if return_path:
         return path
     else:
-        sys.path.insert(0, base_dir)
+        sys.path.insert(0, BASE_DIR)
         module = __import__(name)
         return module.load_data()
 


Test patch:
diff --git a/gensim/test/test_api.py b/gensim/test/test_api.py
--- a/gensim/test/test_api.py
+++ b/gensim/test/test_api.py
@@ -2,7 +2,6 @@
 import unittest
 import os
 import gensim.downloader as api
-from gensim.downloader import base_dir
 import shutil
 import numpy as np
 
@@ -13,24 +12,24 @@
 )
 class TestApi(unittest.TestCase):
     def test_base_dir_creation(self):
-        if os.path.isdir(base_dir):
-            shutil.rmtree(base_dir)
+        if os.path.isdir(api.BASE_DIR):
+            shutil.rmtree(api.BASE_DIR)
         api._create_base_dir()
-        self.assertTrue(os.path.isdir(base_dir))
-        os.rmdir(base_dir)
+        self.assertTrue(os.path.isdir(api.BASE_DIR))
+        os.rmdir(api.BASE_DIR)
 
     def test_load_dataset(self):
-        dataset_path = os.path.join(base_dir, "__testing_matrix-synopsis", "__testing_matrix-synopsis.gz")
-        if os.path.isdir(base_dir):
-            shutil.rmtree(base_dir)
+        dataset_path = os.path.join(api.BASE_DIR, "__testing_matrix-synopsis", "__testing_matrix-synopsis.gz")
+        if os.path.isdir(api.BASE_DIR):
+            shutil.rmtree(api.BASE_DIR)
         self.assertEqual(api.load("__testing_matrix-synopsis", return_path=True), dataset_path)
-        shutil.rmtree(base_dir)
+        shutil.rmtree(api.BASE_DIR)
         self.assertEqual(len(list(api.load("__testing_matrix-synopsis"))), 1)
-        shutil.rmtree(base_dir)
+        shutil.rmtree(api.BASE_DIR)
 
     def test_load_model(self):
-        if os.path.isdir(base_dir):
-            shutil.rmtree(base_dir)
+        if os.path.isdir(api.BASE_DIR):
+            shutil.rmtree(api.BASE_DIR)
         vector_dead = np.array([
             0.17403787, -0.10167074, -0.00950371, -0.10367849, -0.14034484,
             -0.08751217, 0.10030612, 0.07677923, -0.32563496, 0.01929072,
@@ -44,23 +43,23 @@ def test_load_model(self):
             0.00414471, -0.02296237, 0.18336892, -0.23840059, 0.17924534
         ])
         dataset_path = os.path.join(
-            base_dir, "__testing_word2vec-matrix-synopsis", "__testing_word2vec-matrix-synopsis.gz"
+            api.BASE_DIR, "__testing_word2vec-matrix-synopsis", "__testing_word2vec-matrix-synopsis.gz"
         )
         model = api.load("__testing_word2vec-matrix-synopsis")
         vector_dead_calc = model.wv["dead"]
         self.assertTrue(np.allclose(vector_dead, vector_dead_calc))
-        shutil.rmtree(base_dir)
+        shutil.rmtree(api.BASE_DIR)
         self.assertEqual(api.load("__testing_word2vec-matrix-synopsis", return_path=True), dataset_path)
-        shutil.rmtree(base_dir)
+        shutil.rmtree(api.BASE_DIR)
 
     def test_multipart_load(self):
         dataset_path = os.path.join(
-            base_dir, '__testing_multipart-matrix-synopsis', '__testing_multipart-matrix-synopsis.gz'
+            api.BASE_DIR, '__testing_multipart-matrix-synopsis', '__testing_multipart-matrix-synopsis.gz'
         )
-        if os.path.isdir(base_dir):
-            shutil.rmtree(base_dir)
+        if os.path.isdir(api.BASE_DIR):
+            shutil.rmtree(api.BASE_DIR)
         self.assertEqual(dataset_path, api.load("__testing_multipart-matrix-synopsis", return_path=True))
-        shutil.rmtree(base_dir)
+        shutil.rmtree(api.BASE_DIR)
         dataset = api.load("__testing_multipart-matrix-synopsis")
         self.assertEqual(len(list(dataset)), 1)