Repository: intel/ipex-llm
Pull request: Support adding customized jars after pip install; fix release related
## What changes were proposed in this pull request?

Support adding customized jars after pip install.
Fix release issues.
Add docs.

## How was this patch tested?

Jenkins and manual test.

## Related links or issues (optional)
fixed #1989 



Problem statement:
support loading custom jars and python-api.zip for python via pip install
When user define new criterions, layers and wrap it in python, it requires loading external jar/python-api.zip when jupyter/BigDL starts up.  But pip install doesn't allow loading customized jars/zip now. External jars can only be loaded using script to launch jupyter. 

We need 

- Support loading custom jars and python-api.zip after pip installation of BigDL
- It's better to have external jars/zip loaded dynamically with a command (which can be executed inside jupyter) or an api.  It won't be convinient if it requires frequent install/uninstall.

Hints from issue discussion:

Commit message: support add jars

Patch to review:
diff --git a/pyspark/bigdl/util/common.py b/pyspark/bigdl/util/common.py
index 384b8c7427b..f60565f744c 100644
--- a/pyspark/bigdl/util/common.py
+++ b/pyspark/bigdl/util/common.py
@@ -39,6 +39,7 @@
     long = int
     unicode = str
 
+
 class SingletonMixin(object):
     _lock = threading.RLock()
     _instance = None
@@ -52,6 +53,7 @@ def instance(cls,
                     cls._instance = cls(bigdl_type)
         return cls._instance
 
+
 class JavaCreator(SingletonMixin):
     __creator_class="com.intel.analytics.bigdl.python.api.PythonBigDL"
 
@@ -114,11 +116,31 @@ def __str__(self):
         return "Evaluated result: %s, total_num: %s, method: %s" % (
             self.result, self.total_num, self.method)
 
+
 def get_dtype(bigdl_type):
     # Always return float32 for now
     return "float32"
 
 
+class Configuration(object):
+    __extra_jars = []
+
+    @staticmethod
+    def add_extra_jars(jars):
+        """
+        Add extra jars to classpath
+        :param jars: a string or a list of strings of jar paths
+        """
+        import six
+        if isinstance(jars, six.string_types):
+            jars = [jars]
+        Configuration.__extra_jars += jars
+
+    @staticmethod
+    def get_extra_jars():
+        return Configuration.__extra_jars
+
+
 class JActivity(object):
 
     def __init__(self, value):
@@ -358,6 +380,7 @@ def __str__(self):
     def __repr__(self):
         return "Sample: features: %s, labels: %s" % (self.features, self.labels)
 
+
 class RNG():
     """
     generate tensor data with seed
@@ -398,6 +421,7 @@ def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"):
     """
     callBigDlFunc(bigdl_type, "redirectSparkLogs", log_path)
 
+
 def show_bigdl_info_logs(bigdl_type="float"):
     """
     Set BigDL log level to INFO.
@@ -461,7 +485,10 @@ def create_spark_conf():
     sparkConf = SparkConf()
     sparkConf.setAll(bigdl_conf.items())
     if not is_spark_below_2_2():
-        extend_spark_driver_cp(sparkConf, get_bigdl_classpath())
+        for path in get_bigdl_classpath():
+            extend_spark_driver_cp(sparkConf, path)
+        for jar in Configuration.get_extra_jars():
+            extend_spark_driver_cp(sparkConf, jar)
 
     # add content in PYSPARK_FILES in spark.submit.pyFiles
     # This is a workaround for current Spark on k8s
@@ -506,6 +533,7 @@ def get_spark_sql_context(sc):
     else:
         return SQLContext(sc)  # Compatible with Spark1.5.1
 
+
 def callBigDlFunc(bigdl_type, name, *args):
     """ Call API in PythonBigDL """
     jinstance = JavaCreator.instance(bigdl_type=bigdl_type).value
@@ -513,6 +541,7 @@ def callBigDlFunc(bigdl_type, name, *args):
     api = getattr(jinstance, name)
     return callJavaFunc(sc, api, *args)
 
+
 def _java2py(sc, r, encoding="bytes"):
     if isinstance(r, JavaObject):
         clsName = r.getClass().getSimpleName()
@@ -647,6 +676,7 @@ def get_activation_by_name(activation_name, activation_id=None):
         activation.set_name(activation_id)
     return activation
 
+
 def _test():
     import doctest
     from pyspark import SparkContext
diff --git a/pyspark/bigdl/util/engine.py b/pyspark/bigdl/util/engine.py
index 2b278cd3edd..54448f646b2 100644
--- a/pyspark/bigdl/util/engine.py
+++ b/pyspark/bigdl/util/engine.py
@@ -18,6 +18,8 @@
 import os
 import glob
 import warnings
+from bigdl.util.common import Configuration
+
 
 def exist_pyspark():
     # check whether pyspark package exists
@@ -27,6 +29,7 @@ def exist_pyspark():
     except ImportError:
         return False
 
+
 def check_spark_source_conflict(spark_home, pyspark_path):
     # check if both spark_home env var and pyspark package exist
     # trigger a warning if two spark sources don't match
@@ -38,6 +41,7 @@ def check_spark_source_conflict(spark_home, pyspark_path):
                       "For example, you can unset SPARK_HOME and use pyspark only."
         warnings.warn(warning_msg)
 
+
 def __prepare_spark_env():
     spark_home = os.environ.get('SPARK_HOME', None)
     if exist_pyspark():
@@ -83,13 +87,13 @@ def get_bigdl_classpath():
     """
     Get and return the jar path for bigdl if exists.
     """
-    if(os.getenv("BIGDL_CLASSPATH")):
-        return os.environ["BIGDL_CLASSPATH"]
+    if os.getenv("BIGDL_CLASSPATH"):
+        return [os.environ["BIGDL_CLASSPATH"]]
     jar_dir = os.path.abspath(__file__ + "/../../")
     jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar"))
     if jar_paths:
         assert len(jar_paths) == 1, "Expecting one jar: %s" % len(jar_paths)
-        return jar_paths[0]
+        return [jar_paths[0]] + Configuration.get_extra_jars()
     return ""
 
 


Code review comments:
diff_hunk: @@ -83,13 +87,13 @@ def get_bigdl_classpath():
     """
     Get and return the jar path for bigdl if exists.
     """
-    if(os.getenv("BIGDL_CLASSPATH")):
-        return os.environ["BIGDL_CLASSPATH"]
+    if os.getenv("BIGDL_CLASSPATH"):
+        return [os.environ["BIGDL_CLASSPATH"]]
     jar_dir = os.path.abspath(__file__ + "/../../")
     jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar"))
     if jar_paths:
         assert len(jar_paths) == 1, "Expecting one jar: %s" % len(jar_paths)
-        return jar_paths[0]
+        return [jar_paths[0]] + Configuration.get_extra_jars()
path: pyspark/bigdl/util/engine.py
text: @user1:
This change should be move to 
```
   if bigdl_classpath and is_spark_below_2_2():
         append_path("SPARK_CLASSPATH", bigdl_classpath)
```
?