Hopefully finished with the requirements debacle, now using conda but freezing requirements with pip as usual

.dvc/config

@@ -1,4 +1,4 @@
 [core]
     analytics = false
 ['remote "dvc-remote"']
-    url = s3://dagshub-savta-depth
+    url = gs://dagshub-savta-depth

.gitignore

@@ -4,3 +4,4 @@
 .ipynb_checkpoints/
 .workspace/
 aws/
+google-cloud-sdk

README.md

@@ -31,13 +31,20 @@ If you'd like to take part, please follow the guide.
         $ conda activate savta_depth
         $ make requirements
         ```
+
+        **Note**: If you don't have a GPU you will need to install pytorch separately and then run make requirements. You can install pytorch for computers without a gpu with the following command:
+
+        ```bash
+        $ conda install pytorch torchvision cpuonly -c pytorch
+        ```
+        
 * Pull the dvc files to your workspace by typing:
 
     ```bash
     $ dvc checkout #use this to get the data, models etc
     ```
 
-    * Note: you might need to install and setup the tools to pull from a remote. Read more in [this guide](https://dagshub.com/docs/getting-started/set-up-remote-storage-for-data-and-models/).
+    **Note**: You might need to install and setup the tools to pull from a remote. Read more in [this guide](https://dagshub.com/docs/getting-started/set-up-remote-storage-for-data-and-models/) on how to setup Google Storage or AWS S3 access.
 * After you are finished your modification, make sure to do the following:
     * Freeze your virtualenv by typing in the terminal:
 

requirements.txt

@@ -1,171 +1,96 @@
-name: savta_depth
-channels:
-  - pytorch
-  - fastai
-  - defaults
-  - conda-forge
-dependencies:
-  - _libgcc_mutex=0.1=main
-  - attrs=19.3.0=py_0
-  - beautifulsoup4=4.9.1=py37_0
-  - blas=1.0=mkl
-  - bottleneck=1.3.2=py37heb32a55_1
-  - brotlipy=0.7.0=py37h7b6447c_1000
-  - ca-certificates=2020.6.24=0
-  - catalogue=1.0.0=py37_1
-  - certifi=2020.6.20=py37_0
-  - cffi=1.14.0=py37h2e261b9_0
-  - chardet=3.0.4=py37_1003
-  - cryptography=2.9.2=py37h1ba5d50_0
-  - cudatoolkit=10.2.89=hfd86e86_1
-  - cycler=0.10.0=py37_0
-  - cymem=2.0.3=py37he6710b0_0
-  - cython-blis=0.4.1=py37h7b6447c_1
-  - dataclasses=0.7=py37_0
-  - fastai=1.0.61=1
-  - fastcore=1.0.0=pyh39e3cac_0
-  - fastprogress=1.0.0=pyh39e3cac_0
-  - freetype=2.10.2=h5ab3b9f_0
-  - idna=2.10=py_0
-  - importlib-metadata=1.7.0=py37_0
-  - importlib_metadata=1.7.0=0
-  - intel-openmp=2020.1=217
-  - joblib=0.16.0=py_0
-  - jpeg=9b=h024ee3a_2
-  - jsonschema=3.0.2=py37_0
-  - kiwisolver=1.2.0=py37hfd86e86_0
-  - lcms2=2.11=h396b838_0
-  - ld_impl_linux-64=2.33.1=h53a641e_7
-  - libedit=3.1.20191231=h14c3975_1
-  - libffi=3.2.1=hd88cf55_4
-  - libgcc-ng=9.1.0=hdf63c60_0
-  - libgfortran-ng=7.3.0=hdf63c60_0
-  - libpng=1.6.37=hbc83047_0
-  - libstdcxx-ng=9.1.0=hdf63c60_0
-  - libtiff=4.1.0=h2733197_1
-  - lz4-c=1.9.2=he6710b0_1
-  - matplotlib=3.3.1=1
-  - matplotlib-base=3.3.1=py37h817c723_0
-  - mkl=2020.1=217
-  - mkl-service=2.3.0=py37he904b0f_0
-  - mkl_fft=1.1.0=py37h23d657b_0
-  - mkl_random=1.1.1=py37h0573a6f_0
-  - murmurhash=1.0.2=py37he6710b0_0
-  - ncurses=6.2=he6710b0_1
-  - ninja=1.10.0=py37hfd86e86_0
-  - numexpr=2.7.1=py37h423224d_0
-  - numpy=1.19.1=py37hbc911f0_0
-  - numpy-base=1.19.1=py37hfa32c7d_0
-  - nvidia-ml-py3=7.352.0=py_0
-  - olefile=0.46=py37_0
-  - openssl=1.1.1g=h7b6447c_0
-  - packaging=20.4=py_0
-  - pandas=1.1.0=py37he6710b0_0
-  - pillow=7.2.0=py37hb39fc2d_0
-  - pip=20.2.2=py37_0
-  - plac=0.9.6=py37_1
-  - preshed=3.0.2=py37he6710b0_1
-  - pycparser=2.20=py_2
-  - pyopenssl=19.1.0=py_1
-  - pyparsing=2.4.7=py_0
-  - pyrsistent=0.16.0=py37h7b6447c_0
-  - pysocks=1.7.1=py37_1
-  - python=3.7.6=cpython_h8356626_6
-  - python-dateutil=2.8.1=py_0
-  - python_abi=3.7=1_cp37m
-  - pytorch=1.6.0=py3.7_cuda10.2.89_cudnn7.6.5_0
-  - pytz=2020.1=py_0
-  - pyyaml=5.3.1=py37h7b6447c_1
-  - readline=8.0=h7b6447c_0
-  - requests=2.24.0=py_0
-  - scikit-learn=0.23.1=py37h423224d_0
-  - scipy=1.5.2=py37h0b6359f_0
-  - setuptools=49.6.0=py37_0
-  - six=1.15.0=py_0
-  - soupsieve=2.0.1=py_0
-  - spacy=2.3.1=py37hfd86e86_0
-  - sqlite=3.33.0=h62c20be_0
-  - srsly=1.0.2=py37he6710b0_0
-  - thinc=7.4.1=py37hfd86e86_0
-  - threadpoolctl=2.1.0=pyh5ca1d4c_0
-  - tk=8.6.10=hbc83047_0
-  - torchvision=0.7.0=py37_cu102
-  - tornado=6.0.4=py37h7b6447c_1
-  - tqdm=4.48.2=py_0
-  - urllib3=1.25.10=py_0
-  - wasabi=0.7.1=py_0
-  - wheel=0.34.2=py37_0
-  - xz=5.2.5=h7b6447c_0
-  - yaml=0.2.5=h7b6447c_0
-  - zipp=3.1.0=py_0
-  - zlib=1.2.11=h7b6447c_3
-  - zstd=1.4.5=h9ceee32_0
-  - pip:
-    - appdirs==1.4.4
-    - atpublic==2.0
-    - backcall==0.2.0
-    - boto3==1.14.47
-    - botocore==1.17.47
-    - cachetools==4.1.1
-    - colorama==0.4.3
-    - commonmark==0.9.1
-    - configobj==5.0.6
-    - decorator==4.4.2
-    - dictdiffer==0.8.1
-    - distro==1.5.0
-    - docutils==0.15.2
-    - dpath==2.0.1
-    - dvc==1.6.0
-    - flatten-json==0.1.7
-    - flufl-lock==3.2
-    - funcy==1.14
-    - future==0.18.2
-    - gitdb==4.0.5
-    - gitpython==3.1.7
-    - google-api-core==1.22.1
-    - google-auth==1.20.1
-    - google-cloud-core==1.4.1
-    - google-cloud-storage==1.19.0
-    - google-crc32c==0.1.0
-    - google-resumable-media==0.7.1
-    - googleapis-common-protos==1.52.0
-    - grandalf==0.6
-    - h5py==2.10.0
-    - ipython==7.17.0
-    - ipython-genutils==0.2.0
-    - jedi==0.17.2
-    - jmespath==0.10.0
-    - jsonpath-ng==1.5.1
-    - nanotime==0.5.2
-    - networkx==2.4
-    - opencv-python==4.4.0.42
-    - parso==0.7.1
-    - pathspec==0.8.0
-    - pexpect==4.8.0
-    - pickleshare==0.7.5
-    - ply==3.11
-    - prompt-toolkit==3.0.6
-    - protobuf==3.13.0
-    - ptyprocess==0.6.0
-    - pyasn1==0.4.8
-    - pyasn1-modules==0.2.8
-    - pydot==1.4.1
-    - pygments==2.6.1
-    - pygtrie==2.3.2
-    - rich==5.2.1
-    - rsa==4.6
-    - ruamel-yaml==0.16.10
-    - ruamel-yaml-clib==0.2.0
-    - s3transfer==0.3.3
-    - shortuuid==1.0.1
-    - shtab==1.3.1
-    - smmap==3.0.4
-    - tabulate==0.8.7
-    - toml==0.10.1
-    - traitlets==4.3.3
-    - typing-extensions==3.7.4.2
-    - voluptuous==0.11.7
-    - wcwidth==0.2.5
-    - zc-lockfile==2.0
-prefix: /opt/conda/envs/savta_depth
-
+appdirs==1.4.4
+atpublic==2.0
+blis==0.4.1
+cachetools==4.1.1
+catalogue==1.0.0
+certifi==2020.6.20
+cffi==1.14.2
+chardet==3.0.4
+colorama==0.4.3
+commonmark==0.9.1
+configobj==5.0.6
+cycler==0.10.0
+cymem==2.0.3
+dataclasses==0.6
+decorator==4.4.2
+dictdiffer==0.8.1
+distro==1.5.0
+dpath==2.0.1
+dvc==1.6.0
+fastai==2.0.0
+fastcore==1.0.0
+fastprogress==1.0.0
+flatten-json==0.1.7
+flufl.lock==3.2
+funcy==1.14
+future==0.18.2
+gitdb==4.0.5
+GitPython==3.1.7
+google-api-core==1.22.1
+google-auth==1.20.1
+google-cloud-core==1.4.1
+google-cloud-storage==1.19.0
+google-crc32c==0.1.0
+google-resumable-media==0.7.1
+googleapis-common-protos==1.52.0
+grandalf==0.6
+h5py==2.10.0
+idna==2.10
+importlib-metadata==1.7.0
+joblib==0.16.0
+jsonpath-ng==1.5.1
+kiwisolver==1.2.0
+matplotlib==3.3.1
+mkl-fft==1.1.0
+mkl-random==1.1.1
+mkl-service==2.3.0
+murmurhash==1.0.2
+nanotime==0.5.2
+networkx==2.4
+numpy==1.19.1
+olefile==0.46
+opencv-python==4.4.0.42
+packaging==20.4
+pandas==1.1.1
+pathspec==0.8.0
+Pillow==7.2.0
+plac==1.1.3
+ply==3.11
+preshed==3.0.2
+protobuf==3.13.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+pydot==1.4.1
+Pygments==2.6.1
+pygtrie==2.3.2
+pyparsing==2.4.7
+python-dateutil==2.8.1
+pytz==2020.1
+PyYAML==5.3.1
+requests==2.24.0
+rich==5.2.1
+rsa==4.6
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.0
+scikit-learn==0.23.2
+scipy==1.5.2
+shortuuid==1.0.1
+shtab==1.3.1
+six==1.15.0
+smmap==3.0.4
+spacy==2.3.2
+srsly==1.0.2
+tabulate==0.8.7
+thinc==7.4.1
+threadpoolctl==2.1.0
+toml==0.10.1
+torch==1.6.0
+torchvision==0.7.0
+tqdm==4.48.2
+typing-extensions==3.7.4.3
+urllib3==1.25.10
+voluptuous==0.11.7
+wasabi==0.7.1
+zc.lockfile==2.0
+zipp==3.1.0

run_dev_env.sh

@@ -2,6 +2,6 @@ docker run -d \
     -p 8080:8080 \
     --name "ml-workspace" -v "${PWD}:/workspace" \
     --env AUTHENTICATE_VIA_JUPYTER="dagshub_savta" \
-    --shm-size 512m \
+    --shm-size 2G \
     --restart always \
-    mltooling/ml-workspace:latest
+    mltooling/ml-workspace-minimal:latest