Leave the nupic build/install process the more "pythonic" as possible

This PR aims make Nupic "pip installable". For this it tries to leave the build/install process the more "pythonic" as possible using:
a) native support from Distutils/SetupTools to build all C++ extensions (https://github.com/numenta/nupic/issues/1573) 
b) Python scripts to perform jobs that were being performed by C++ executables (https://github.com/numenta/nupic/issues/1576).

The motivation of B is because it's very tricky install all current cpp executables and libraries using A solution. Some of these libraries (like `bindings`) are inevitable to be build as extensions, however c++ extensions like `htmtest` and `compare_nupic_core_version` can be replaced to python scripts. Different than bindings, these 2 extensions are complicating the process as they should be installed into souce repository (`bin` folder, to be exact) and not into `nupic` package install folder like the other modules. 

Once that all extensions now can be build in a natural way, we can remove CMakeLists.txt from NuPIC Pyhton. This makes that CMake becomes essential only to build the NuPIC Core library: as this is an optional job, we could bypass CMake/Git use during PIP process decreasing the number of build/install steps and dependencies.

I still am working on the pythonic version of `HtmTest`. Thus it's normal that some tests fail, because Travis still is referencing its c++ version. 

* * *

## TODO:
- [ ] Figure out the `ARCHFLAGS="-arch x86_64"` requirement on darwin64
- [ ] Update supporting documentation (README, wikis) with any build changes

+ commonCompileFlags.append(stdLib)
+
+ commonLinkFlags = [
+ "-std=c++11",
+ "-m" + self.bitness,
+ "-fPIC",
+ "-L" + pythonPrefix + "/lib"]
+ if self.platform == "darwin":
+ commonLinkFlags.append(stdLib)
+
+ commonLibraries = []
+ if self.platform == "linux":
+ commonLibraries.extend(["pthread", "dl"])
+
+ commonObjects = [
+ nupicCoreReleaseDir + "/lib/" +
+ self.getLibPrefix() + "nupic_core" + self.getStaticLibExtension()]
+
+ pythonSupportSources = [
+ "extensions/py_support/NumpyVector.cpp",
+ "extensions/py_support/PyArray.cpp",
+ "extensions/py_support/PyHelpers.cpp",
+ "extensions/py_support/PythonStream.cpp"]
+
+ extensions = []
+
+ libDynamicCppRegion = setuptools.Extension(
+ "nupic." + self.getLibPrefix() + "cpp_region",
+ extra_compile_args=commonCompileFlags,
+ define_macros=commonDefines,
+ extra_link_args=commonLinkFlags,
+ include_dirs=commonIncludeDirs,
+ libraries =
+ commonLibraries +
+ ["dl",
+ "python" + pythonVersion],
+ sources=pythonSupportSources +
+ ["extensions/cpp_region/PyRegion.cpp",
+ "extensions/cpp_region/unittests/PyHelpersTest.cpp"],
+ extra_objects=commonObjects)
+ extensions.append(libDynamicCppRegion)
+
+ #
+ # SWIG
+ #
+ swigDir = self.repositoryDir + "/external/common/share/swig/3.0.2"
+ swigExecutable = self.repositoryDir + "/external/" + self.platform \
+ + self.bitness + "/bin/swig"
+ buildCommands = ["build", "build_ext", "install", "install_lib", "develop"]
+ for arg in sys.argv:
+ if arg in buildCommands:
+ sys.argv.extend(["build_ext", "--swig", swigExecutable])
+ break
+
+ swigFlags = [
+ "-c++",
+ "-features",
+ "autodoc=0,directors=0",
Matthew Taylor Owner
rhyolight added a note 3 days ago

This this right? This this be broken into two list items?
Austin Marshall Owner
oxtopus added a note 3 days ago

I bielieve so. It's one value for the "-features" option.
+ "-noproxyimport",
+ "-keyword",
+ "-modern",
+ "-modernargs",
+ "-noproxydel",
+ "-fvirtual",
+ "-fastunpack",
+ "-nofastproxy",
+ "-fastquery",
+ "-outputtuple",
+ "-castmode",
+ "-w402",
+ "-w503",
+ "-w511",
+ "-w302",
+ "-w362",
+ "-w312",
+ "-w389",
+ "-DSWIG_PYTHON_LEGACY_BOOL",
+ "-DNTA_PLATFORM_" + self.platform + self.bitness,
+ "-I" + self.repositoryDir + "/extensions",
+ "-I" + nupicCoreReleaseDir + "/include",
+ "-I" + swigDir + "/python",
+ "-I" + swigDir]
+
+ swigLibraries = [
+ "dl",
+ "python" + pythonVersion]
+
+ ## TODO: Remove these prints
+ print "\n### DEBUGGING ###"
+ import pprint
+ print "Args: "
+ pprint.pprint( sys.argv )
+ print "Common defines:"
+ pprint.pprint( commonDefines )
+ print "Common include dirs:"
+ pprint.pprint( commonIncludeDirs )
+ print "Common compile flags:"
+ pprint.pprint( commonCompileFlags )
+ print "Common libraries:"
+ pprint.pprint( commonLibraries )
+ print "Common link flags:"
+ pprint.pprint( commonLinkFlags )
+ print "Common objects:"
+ pprint.pprint( commonObjects )
+ print "Swig flags:"
+ pprint.pprint( swigFlags )
+ print "Swig libraries:"
+ pprint.pprint( swigLibraries )
+ print "### DEBUGGING ###\n"
+
+ libSharedEngineInternal = setuptools.Extension(
+ "nupic.bindings._engine_internal",
+ swig_opts=swigFlags,
+ extra_compile_args=commonCompileFlags,
+ define_macros=commonDefines,
+ extra_link_args=commonLinkFlags,
+ include_dirs=commonIncludeDirs,
+ libraries=swigLibraries,
+ sources=pythonSupportSources +
+ ["nupic/bindings/engine_internal.i"],
+ extra_objects=commonObjects)
+ extensions.append(libSharedEngineInternal)
+
+ libSharedMath = setuptools.Extension(
+ "nupic.bindings._math",
+ swig_opts=swigFlags,
+ extra_compile_args=commonCompileFlags,
+ define_macros=commonDefines,
+ extra_link_args=commonLinkFlags,
+ include_dirs=commonIncludeDirs,
+ libraries=swigLibraries,
+ sources=pythonSupportSources +
+ ["nupic/bindings/math.i",
+ "nupic/bindings/PySparseTensor.cpp"],
+ extra_objects=commonObjects)
+ extensions.append(libSharedMath)
+
+ libSharedAlgorithms = setuptools.Extension(
+ "nupic.bindings._algorithms",
+ swig_opts=swigFlags,
+ extra_compile_args=commonCompileFlags,
+ define_macros=commonDefines,
+ extra_link_args=commonLinkFlags,
+ include_dirs=commonIncludeDirs,
+ libraries=swigLibraries,
+ sources=pythonSupportSources +
+ ["nupic/bindings/algorithms.i"],
+ extra_objects=commonObjects)
+ extensions.append(libSharedAlgorithms)
+
+ return extensions
+
+
+
+ def getLibPrefix(self):
+ """
+ Returns the default system prefix of a compiled library.
+ """
+ if self.platform == "linux" or self.platform == "darwin":
+ return "lib"
+ elif self.platform == "windows":
+ return ""
+
+
+
+ def getStaticLibExtension(self):
+ """
+ Returns the default system extension of a compiled static library.
+ """
+ if self.platform == "linux" or self.platform == "darwin":
+ return ".a"
+ elif self.platform == "windows":
+ return ".lib"
+
+
+
+ def getSharedLibExtension(self):
+ """
+ Returns the default system extension of a compiled shared library.
+ """
+ if self.platform == "linux" or self.platform == "darwin":
+ return ".so"
+ elif self.platform == "windows":
+ return ".dll"
+
+
+
+ def extractNupicCoreTarget(self):
+ # First, get the nupic.core SHA and remote location from local config.
+ nupicConfig = {}
+ if os.path.exists(self.repositoryDir + "/.nupic_config"):
+ execfile(
+ os.path.join(self.repositoryDir, ".nupic_config"), {}, nupicConfig
)
- ],
- include_package_data = True,
- description = "Numenta Platform for Intelligent Computing",
- author="Numenta",
- author_email="help@numenta.org",
- url="https://github.com/numenta/nupic",
- classifiers=[
- "Programming Language :: Python",
- "Programming Language :: Python :: 2",
- "License :: OSI Approved :: GNU General Public License (GPL)",
- "Operating System :: OS Independent",
- "Development Status :: 5 - Production/Stable",
- "Environment :: Console",
- "Intended Audience :: Science/Research",
- "Topic :: Scientific/Engineering :: Artificial Intelligence"
- ],
- long_description = """\
-NuPIC is a library that provides the building blocks for online prediction systems. The library contains the Cortical Learning Algorithm (CLA), but also the Online Prediction Framework (OPF) that allows clients to build prediction systems out of encoders, models, and metrics.
-
-For more information, see numenta.org or the NuPIC wiki (https://github.com/numenta/nupic/wiki).
-"""
- )
+ elif os.path.exists(os.environ["HOME"] + "/.nupic_config"):
+ execfile(
+ os.path.join(os.environ["HOME"], ".nupic_config"), {}, nupicConfig
+ )
+ else:
+ execfile(
+ os.path.join(self.repositoryDir, ".nupic_modules"), {}, nupicConfig
+ )
+ return nupicConfig["NUPIC_CORE_REMOTE"], nupicConfig["NUPIC_CORE_COMMITISH"]
+
+
+
+ # Returns nupic.core release directory and source directory in tuple.
+ def getDefaultNupicCoreDirectories(self):
+ # Default nupic.core location is relative to the NuPIC checkout.
+ return self.repositoryDir + "/extensions/core/build/release", \
+ self.repositoryDir + "/extensions/core"
+
+
+
+ def buildNupicCoreFromGitClone(self, nupicCoreCommitish,
+ nupicCoreLocalPackage, nupicCoreReleaseDir,
+ nupicCoreRemote, nupicCoreSourceDir):
+ print ("Building nupic.core from local checkout "
+ + nupicCoreSourceDir + "...")
+ # Remove the local package file, which didn't get populated due to the
+ # download failure.
+ if os.path.exists(nupicCoreLocalPackage):
+ os.remove(nupicCoreLocalPackage)
+
+ # Get nupic.core dependency through git.
+ if not os.path.exists(nupicCoreSourceDir + "/.git"):
+ # There's not a git repo in nupicCoreSourceDir, so we can blow the
+ # whole directory away and clone nupic.core there.
+ shutil.rmtree(nupicCoreSourceDir, True)
+ os.makedirs(nupicCoreSourceDir)
+ cloneCommand = ["git", "clone", nupicCoreRemote, nupicCoreSourceDir, "--depth=50"]
+ print " ".join(cloneCommand)
+ process = subprocess.Popen(cloneCommand,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ process.communicate()
+ if process.returncode != 0:
+ raise Exception("Fatal Error: Unable to clone %s into %s"
+ % (nupicCoreRemote, nupicCoreSourceDir))
+ else:
+ # Fetch if already cloned.
+ gitFetchCmd = ["git", "fetch", nupicCoreRemote]
+ print " ".join(gitFetchCmd)
+ process = subprocess.Popen(gitFetchCmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=nupicCoreSourceDir)
+ process.communicate()
+ if process.returncode != 0:
+ raise Exception("Fatal Error: Unable to fetch %s"
+ % nupicCoreRemote)
+
+ # Get the exact SHA we need for nupic.core.
+ gitResetCmd = ["git", "reset", "--hard", nupicCoreCommitish]
+ print " ".join(gitResetCmd)
+ process = subprocess.Popen(gitResetCmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=nupicCoreSourceDir)
+ process.communicate()
+ if process.returncode != 0:
+ raise Exception("Fatal Error: Unable to checkout %s in %s"
+ % (nupicCoreCommitish, nupicCoreSourceDir))
+
+ # Execute the Make scripts
+ process = subprocess.Popen(["git", "clean", "-fdx"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=nupicCoreSourceDir)
+ process.communicate()
+ if process.returncode != 0:
+ raise Exception(
+ "Fatal Error: Compiling 'nupic.core' library within %s failed."
+ % self.repositoryDir
+ )
+
+ # Build and set external libraries
+ print "Building 'nupic.core' library..."
+ makeWorkingDir = "%s/build/scripts" % nupicCoreSourceDir
+ # Clean 'build/scripts' subfolder at submodule folder
+ shutil.rmtree(nupicCoreSourceDir + "/build/scripts", True)
+ os.makedirs(nupicCoreSourceDir + "/build/scripts")
+ shutil.rmtree(nupicCoreReleaseDir, True)
+ # Generate the Make scripts
+ cmakeCmd = ["cmake",
+ "%s/src" % nupicCoreSourceDir,
+ "-DCMAKE_INSTALL_PREFIX=%s" % nupicCoreReleaseDir]
+ print " ".join(cmakeCmd)
+ process = subprocess.Popen(cmakeCmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=makeWorkingDir)
+ cmakeStdOut, cmakeStdErr = process.communicate()
+ if process.returncode != 0:
+ print cmakeStdErr
+ raise Exception(
+ "Fatal Error: cmake command failed in %s!"
+ % makeWorkingDir
+ )
+ else:
+ print cmakeStdOut
+ print "CMake complete."
+
+ # Execute the Make scripts
+ if "user-make-command" in self.options:
+ userMakeCommand = [self.options["user-make-command"]]
+ else:
+ userMakeCommand = ["make"]
+ userMakeCommand = userMakeCommand + ["install", "-j4"]
+ print " ".join(userMakeCommand)
+ process = subprocess.Popen(userMakeCommand,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=makeWorkingDir)
+ makeStdOut, makeStdErr = process.communicate()
+ if process.returncode != 0:
+ print makeStdErr
+ raise Exception(
+ "Fatal Error: make command failed in %s!"
+ % makeWorkingDir
+ )
+ else:
+ print makeStdOut
+ print "Make complete."
+ print "Done building nupic.core."
+
+
+
+ def prepareNupicCore(self):
+
+ nupicCoreReleaseDir = self.getCommandLineOption("nupic-core-dir")
+ nupicCoreSourceDir = None
+ fetchNupicCore = True
+
+ if nupicCoreReleaseDir:
+ # User specified that they have their own nupic.core
+ fetchNupicCore = False
+ else:
+ nupicCoreReleaseDir, nupicCoreSourceDir = \
+ self.getDefaultNupicCoreDirectories()
+
+ nupicCoreRemote, nupicCoreCommitish = self.extractNupicCoreTarget()
+
+ if fetchNupicCore:
+ # User has not specified 'nupic.core' location, so we'll download the
+ # binaries.
+
+ nupicCoreRemoteUrl = (NUPIC_CORE_BUCKET_URL + "/nupic_core-"
+ + nupicCoreCommitish + "-" + self.platform
+ + self.bitness + ".tar.gz")
+ nupicCoreLocalPackage = (nupicCoreSourceDir + "/nupic_core-"
+ + nupicCoreCommitish + "-" + self.platform
+ + self.bitness + ".tar.gz")
+ nupicCoreLocalDirToUnpack = ("nupic_core-"
+ + nupicCoreCommitish + "-" + self.platform
+ + self.bitness)
+
+ if os.path.exists(nupicCoreLocalPackage):
+ print ("Target nupic.core package already exists at "
+ + nupicCoreLocalPackage + ".")
+ self.unpackFile(
+ nupicCoreLocalPackage, nupicCoreLocalDirToUnpack, nupicCoreReleaseDir
+ )
+ else:
+ print "Attempting to fetch nupic.core binaries..."
+ downloadSuccess = self.downloadFile(
+ nupicCoreRemoteUrl, nupicCoreLocalPackage
+ )
+
+ # TODO: Give user a way to clean up all the downloaded binaries. It can
+ # be manually done with `rm -rf $NUPIC_CORE/extensions/core` but would
+ # be cleaner with something like `python setup.py clean`.
+
+ if not downloadSuccess:
+ print ("WARNING:\n\tRemote nupic.core download of %s failed!"
+ "\n\tBuilding nupic.core locally from SHA: %s.\n"
+ % (nupicCoreRemoteUrl, nupicCoreCommitish))
+ self.buildNupicCoreFromGitClone(nupicCoreCommitish,
+ nupicCoreLocalPackage,
+ nupicCoreReleaseDir, nupicCoreRemote,
+ nupicCoreSourceDir)
+
+ else:
+ print "Download successful."
+ self.unpackFile(nupicCoreLocalPackage,
+ nupicCoreLocalDirToUnpack,
+ nupicCoreReleaseDir)
+
+ else:
+ print "Using nupic.core binaries at " + nupicCoreReleaseDir
+
+ if "skip-compare-versions" in self.options:
+ skipCompareVersions = True
+ else:
+ skipCompareVersions = not fetchNupicCore
+
+ if not skipCompareVersions:
+ # Compare expected version of nupic.core against installed version
+ file = open(nupicCoreReleaseDir + "/include/nupic/Version.hpp", "r")
+ content = file.read()
+ file.close()
+ nupicCoreVersionFound = re.search(
+ "#define NUPIC_CORE_VERSION \"([a-z0-9]+)\"", content
+ ).group(1)
+
+ if nupicCoreCommitish != nupicCoreVersionFound:
+ raise Exception(
+ "Fatal Error: Unexpected version of nupic.core! "
+ "Expected %s, but detected %s."
+ % (nupicCoreCommitish, nupicCoreVersionFound)
+ )
+
+ return nupicCoreReleaseDir
+
+
+
+ def downloadFile(self, url, destFile, silent=False):
+ """
+ Download a file to the specified location
+ """
+
+ if not silent:
+ print "Downloading from\n\t%s\nto\t%s.\n" % (url, destFile);
+
+ destDir = os.path.dirname(destFile)
+ if not os.path.exists(destDir):
+ os.makedirs(destDir)
+
+ try:
+ response = urllib2.urlopen(url)
+ except urllib2.URLError:
+ return False
+
+ file = open(destFile, "wb")
+
+ totalSize = response.info().getheader('Content-Length').strip()
+ totalSize = int(totalSize)
+ bytesSoFar = 0
+
+ # Download chunks writing them to target file
+ chunkSize = 8192
+ oldPercent = 0
+ while True:
+ chunk = response.read(chunkSize)
+ bytesSoFar += len(chunk)
+
+ if not chunk:
+ break
+
+ file.write(chunk)
+
+ # Show progress
+ if not silent:
+ percent = (float(bytesSoFar) / totalSize) * 100
+ percent = int(percent)
+ if percent != oldPercent and percent % 5 == 0:
+ print ("Downloaded %i of %i bytes (%i%%)."
+ % (bytesSoFar, totalSize, int(percent)))
+ oldPercent = percent
+
+ file.close()
+
+ return True
+
+
+
+ def unpackFile(self, package, dirToUnpack, destDir, silent=False):
+ """
+ Unpack package file to the specified directory
+ """
+
+ if not silent:
+ print "Unpacking %s into %s..." % (package, destDir)
+
+ file = tarfile.open(package, 'r:gz')
+ file.extractall(destDir)
+ file.close()
+
+ # Copy subdirectories to a level up
+ subDirs = os.listdir(destDir + "/" + dirToUnpack)
+ for dir in subDirs:
+ shutil.rmtree(destDir + "/" + dir, True)
+ shutil.move(destDir + "/" + dirToUnpack + "/" + dir, destDir + "/" + dir)
+ shutil.rmtree(destDir + "/" + dirToUnpack, True)
+
+
+
+class CustomBuild(build):
+ def run(self):
+ # Compile extensions before python modules to avoid that SWIG generated
+ # modules get out of the dist
+ self.run_command('build_ext')
+ build.run(self)
+
+
+
+class CustomInstall(install):
+ def run(self):
+ # Compile extensions before python modules to avoid that SWIG generated
+ # modules get out of the dist
+ self.run_command('build_ext')
+ self.do_egg_install()
+
-# Build and setup NuPIC
-if mustBuildExtensions:
- buildExtensionsNupic()
-setupNupic()
+if __name__ == '__main__':
+ setup = Setup()