Repository: intel/e2eAIOK (language: python)

Issue:
Refine DLRM



Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index e4ab0a75b..2de2cf7f8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
 **/*.json
 **/*.txt
-**/*.pyc
\ No newline at end of file
+**/*.pyc
+**/__pycache__
+**/.ipynb_checkpoints
diff --git a/RecDP b/RecDP
deleted file mode 160000
index 09bb4f0dc..000000000
--- a/RecDP
+++ /dev/null
@@ -1,1 +0,0 @@
-Subproject commit 09bb4f0dcc85505cdf2b7eac33bc234a63e9654e
diff --git a/RecDP/LICENSE b/RecDP/LICENSE
new file mode 100644
index 000000000..47eb519be
--- /dev/null
+++ b/RecDP/LICENSE
@@ -0,0 +1,1957 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+--------------------------------------------------------------------------------
+
+src/plasma/fling.cc and src/plasma/fling.h: Apache 2.0
+
+Copyright 2013 Sharvil Nanavati
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+src/plasma/thirdparty/dlmalloc.c: CC0
+
+This is a version (aka dlmalloc) of malloc/free/realloc written by
+Doug Lea and released to the public domain, as explained at
+http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
+comments, complaints, performance data, etc to dl@cs.oswego.edu
+
+--------------------------------------------------------------------------------
+
+src/plasma/common.cc (some portions)
+
+Copyright (c) Austin Appleby (aappleby (AT) gmail)
+
+Some portions of this file are derived from code in the MurmurHash project
+
+All code is released to the public domain. For business purposes, Murmurhash is
+under the MIT license.
+
+https://sites.google.com/site/murmurhash/
+
+--------------------------------------------------------------------------------
+
+src/arrow/util (some portions): Apache 2.0, and 3-clause BSD
+
+Some portions of this module are derived from code in the Chromium project,
+copyright (c) Google inc and (c) The Chromium Authors and licensed under the
+Apache 2.0 License or the under the 3-clause BSD license:
+
+  Copyright (c) 2013 The Chromium Authors. All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are
+  met:
+
+     * Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+     * Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following disclaimer
+  in the documentation and/or other materials provided with the
+  distribution.
+     * Neither the name of Google Inc. nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from Daniel Lemire's FrameOfReference project.
+
+https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp
+
+Copyright: 2013 Daniel Lemire
+Home page: http://lemire.me/en/
+Project page: https://github.com/lemire/FrameOfReference
+License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from the TensorFlow project
+
+Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the NumPy project.
+
+https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910
+
+https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c
+
+Copyright (c) 2005-2017, NumPy Developers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+       copyright notice, this list of conditions and the following
+       disclaimer in the documentation and/or other materials provided
+       with the distribution.
+
+    * Neither the name of the NumPy Developers nor the names of any
+       contributors may be used to endorse or promote products derived
+       from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the Boost project
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the FlatBuffers project
+
+Copyright 2014 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the tslib project
+
+Copyright 2015 Microsoft Corporation. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the jemalloc project
+
+https://github.com/jemalloc/jemalloc
+
+Copyright (C) 2002-2017 Jason Evans <jasone@canonware.com>.
+All rights reserved.
+Copyright (C) 2007-2012 Mozilla Foundation.  All rights reserved.
+Copyright (C) 2009-2017 Facebook, Inc.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+1. Redistributions of source code must retain the above copyright notice(s),
+   this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice(s),
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
+EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+--------------------------------------------------------------------------------
+
+This project includes code from the Go project, BSD 3-clause license + PATENTS
+weak patent termination clause
+(https://github.com/golang/go/blob/master/PATENTS).
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the hs2client
+
+https://github.com/cloudera/hs2client
+
+Copyright 2016 Cloudera Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+The script ci/scripts/util_wait_for_it.sh has the following license
+
+Copyright (c) 2016 Giles Hall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The script r/configure has the following license (MIT)
+
+Copyright (c) 2017, Jeroen Ooms and Jim Hester
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and
+cpp/src/arrow/util/logging-test.cc are adapted from
+Ray Project (https://github.com/ray-project/ray) (Apache 2.0).
+
+Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h,
+cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h,
+cpp/src/arrow/vendored/datetime/tz.cpp are adapted from
+Howard Hinnant's date library (https://github.com/HowardHinnant/date)
+It is licensed under MIT license.
+
+The MIT License (MIT)
+Copyright (c) 2015, 2016, 2017 Howard Hinnant
+Copyright (c) 2016 Adrian Colomitchi
+Copyright (c) 2017 Florian Dang
+Copyright (c) 2017 Paul Thompson
+Copyright (c) 2018 Tomasz Kamiński
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The file cpp/src/arrow/util/utf8.h includes code adapted from the page
+  https://bjoern.hoehrmann.de/utf-8/decoder/dfa/
+with the following license (MIT)
+
+Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The file cpp/src/arrow/vendored/string_view.hpp has the following license
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The file cpp/src/arrow/vendored/variant.hpp has the following license
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/xxhash/ have the following license
+(BSD 2-Clause License)
+
+xxHash Library
+Copyright (c) 2012-2014, Yann Collet
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You can contact the author at :
+- xxHash homepage: http://www.xxhash.com
+- xxHash source repository : https://github.com/Cyan4973/xxHash
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/double-conversion/ have the following license
+(BSD 3-Clause License)
+
+Copyright 2006-2011, the V8 project authors. All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/uriparser/ have the following license
+(BSD 3-Clause License)
+
+uriparser - RFC 3986 URI parsing library
+
+Copyright (C) 2007, Weijia Song <songweijia@gmail.com>
+Copyright (C) 2007, Sebastian Pipping <sebastian@pipping.org>
+All rights reserved.
+
+Redistribution  and use in source and binary forms, with or without
+modification,  are permitted provided that the following conditions
+are met:
+
+    * Redistributions   of  source  code  must  retain  the   above
+      copyright  notice, this list of conditions and the  following
+      disclaimer.
+
+    * Redistributions  in  binary  form must  reproduce  the  above
+      copyright  notice, this list of conditions and the  following
+      disclaimer   in  the  documentation  and/or  other  materials
+      provided with the distribution.
+
+    * Neither  the name of the <ORGANIZATION> nor the names of  its
+      contributors  may  be  used to endorse  or  promote  products
+      derived  from  this software without specific  prior  written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT  NOT
+LIMITED  TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND  FITNESS
+FOR  A  PARTICULAR  PURPOSE ARE DISCLAIMED. IN NO EVENT  SHALL  THE
+COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL,    SPECIAL,   EXEMPLARY,   OR   CONSEQUENTIAL   DAMAGES
+(INCLUDING,  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES;  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT  LIABILITY,  OR  TORT (INCLUDING  NEGLIGENCE  OR  OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+The files under dev/tasks/conda-recipes have the following license
+
+BSD 3-clause license
+Copyright (c) 2015-2018, conda-forge
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/utf8cpp/ have the following license
+
+Copyright 2006 Nemanja Trifunovic
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from Apache Kudu.
+
+ * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake
+
+Copyright: 2016 The Apache Software Foundation.
+Home page: https://kudu.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from Apache Impala (incubating), formerly
+Impala. The Impala code and rights were donated to the ASF as part of the
+Incubator process after the initial code imports into Apache Parquet.
+
+Copyright: 2012 Cloudera, Inc.
+Copyright: 2016 The Apache Software Foundation.
+Home page: http://impala.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from Apache Aurora.
+
+* dev/release/{release,changelog,release-candidate} are based on the scripts from
+  Apache Aurora
+
+Copyright: 2016 The Apache Software Foundation.
+Home page: https://aurora.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from the Google styleguide.
+
+* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide.
+
+Copyright: 2009 Google Inc. All rights reserved.
+Homepage: https://github.com/google/styleguide
+License: 3-clause BSD
+
+--------------------------------------------------------------------------------
+
+This project includes code from Snappy.
+
+* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code
+  from Google's Snappy project.
+
+Copyright: 2009 Google Inc. All rights reserved.
+Homepage: https://github.com/google/snappy
+License: 3-clause BSD
+
+--------------------------------------------------------------------------------
+
+This project includes code from the manylinux project.
+
+* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py,
+  requirements.txt} are based on code from the manylinux project.
+
+Copyright: 2016 manylinux
+Homepage: https://github.com/pypa/manylinux
+License: The MIT License (MIT)
+
+--------------------------------------------------------------------------------
+
+This project includes code from the cymove project:
+
+* python/pyarrow/includes/common.pxd includes code from the cymove project
+
+The MIT License (MIT)
+Copyright (c) 2019 Omer Ozarslan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+OR OTHER DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The projects includes code from the Ursabot project under the dev/archery
+directory.
+
+License: BSD 2-Clause
+
+Copyright 2019 RStudio, Inc.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project include code from CMake.
+
+* cpp/cmake_modules/FindGTest.cmake is based on code from CMake.
+
+Copyright: Copyright 2000-2019 Kitware, Inc. and Contributors
+Homepage: https://gitlab.kitware.com/cmake/cmake
+License: 3-clause BSD
+
+--------------------------------------------------------------------------------
+
+This project include code from mingw-w64.
+
+* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5
+
+Copyright (c) 2009 - 2013 by the mingw-w64 project
+Homepage: https://mingw-w64.org
+License: Zope Public License (ZPL) Version 2.1.
+
+---------------------------------------------------------------------------------
+
+This project include code from Google's Asylo project.
+
+* cpp/src/arrow/result.h is based on status_or.h
+
+Copyright (c)  Copyright 2017 Asylo authors
+Homepage: https://asylo.dev/
+License: Apache 2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from Google's protobuf project
+
+* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN
+
+Copyright 2008 Google Inc.  All rights reserved.
+Homepage: https://developers.google.com/protocol-buffers/
+License:
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Code generated by the Protocol Buffer compiler is owned by the owner
+of the input file used when generating it.  This code is not
+standalone and requires a support library to be linked with it.  This
+support library is itself covered by the above license.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency LLVM is statically linked in certain binary
+distributions. LLVM has the following license:
+
+==============================================================================
+LLVM Release License
+==============================================================================
+University of Illinois/NCSA
+Open Source License
+
+Copyright (c) 2003-2018 University of Illinois at Urbana-Champaign.
+All rights reserved.
+
+Developed by:
+
+    LLVM Team
+
+    University of Illinois at Urbana-Champaign
+
+    http://llvm.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal with
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimers.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimers in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the names of the LLVM Team, University of Illinois at
+      Urbana-Champaign, nor the names of its contributors may be used to
+      endorse or promote products derived from this Software without specific
+      prior written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+SOFTWARE.
+
+==============================================================================
+Copyrights and Licenses for Third Party Software Distributed with LLVM:
+==============================================================================
+The LLVM software contains code written by third parties.  Such software will
+have its own individual LICENSE.TXT file in the directory in which it appears.
+This file will describe the copyrights, license, and restrictions which apply
+to that code.
+
+The disclaimer of warranty in the University of Illinois Open Source License
+applies to all code in the LLVM Distribution, and nothing in any of the
+other licenses gives permission to use the names of the LLVM Team or the
+University of Illinois to endorse or promote products derived from this
+Software.
+
+The following pieces of software have additional or alternate copyrights,
+licenses, and/or restrictions:
+
+Program             Directory
+-------             ---------
+Google Test         llvm/utils/unittest/googletest
+OpenBSD regex       llvm/lib/Support/{reg*, COPYRIGHT.regex}
+pyyaml tests        llvm/test/YAMLParser/{*.data, LICENSE.TXT}
+ARM contributions   llvm/lib/Target/ARM/LICENSE.TXT
+md5 contributions   llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency gRPC is statically linked in certain binary
+distributions, like the python wheels. gRPC has the following license:
+
+Copyright 2014 gRPC authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency Apache Thrift is statically linked in certain binary
+distributions, like the python wheels. Apache Thrift has the following license:
+
+Apache Thrift
+Copyright (C) 2006 - 2019, The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency Apache ORC is statically linked in certain binary
+distributions, like the python wheels. Apache ORC has the following license:
+
+Apache ORC
+Copyright 2013-2019 The Apache Software Foundation
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).
+
+This product includes software developed by Hewlett-Packard:
+(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency zstd is statically linked in certain binary
+distributions, like the python wheels. ZSTD has the following license:
+
+BSD License
+
+For Zstandard software
+
+Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+ * Neither the name Facebook nor the names of its contributors may be used to
+   endorse or promote products derived from this software without specific
+   prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency lz4 is statically linked in certain binary
+distributions, like the python wheels. lz4 has the following license:
+
+LZ4 Library
+Copyright (c) 2011-2016, Yann Collet
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency Brotli is statically linked in certain binary
+distributions, like the python wheels. Brotli has the following license:
+
+Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency rapidjson is statically linked in certain binary
+distributions, like the python wheels. rapidjson and its dependencies have the
+following licenses:
+
+Tencent is pleased to support the open source community by making RapidJSON
+available.
+
+Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip.
+All rights reserved.
+
+If you have downloaded a copy of the RapidJSON binary from Tencent, please note
+that the RapidJSON binary is licensed under the MIT License.
+If you have downloaded a copy of the RapidJSON source code from Tencent, please
+note that RapidJSON source code is licensed under the MIT License, except for
+the third-party components listed below which are subject to different license
+terms.  Your integration of RapidJSON into your own projects may require
+compliance with the MIT License, as well as the other licenses applicable to
+the third-party components included within RapidJSON. To avoid the problematic
+JSON license in your own projects, it's sufficient to exclude the
+bin/jsonchecker/ directory, as it's the only code under the JSON license.
+A copy of the MIT License is included in this file.
+
+Other dependencies and licenses:
+
+    Open Source Software Licensed Under the BSD License:
+    --------------------------------------------------------------------
+
+    The msinttypes r29
+    Copyright (c) 2006-2013 Alexander Chemeris
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+    * Neither the name of  copyright holder nor the names of its contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
+    EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+    DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR
+    ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+    LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+    OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+    DAMAGE.
+
+    Open Source Software Licensed Under the JSON License:
+    --------------------------------------------------------------------
+
+    json.org
+    Copyright (c) 2002 JSON.org
+    All Rights Reserved.
+
+    JSON_checker
+    Copyright (c) 2002 JSON.org
+    All Rights Reserved.
+
+
+    Terms of the JSON License:
+    ---------------------------------------------------
+
+    Permission is hereby granted, free of charge, to any person obtaining a
+    copy of this software and associated documentation files (the "Software"),
+    to deal in the Software without restriction, including without limitation
+    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+    and/or sell copies of the Software, and to permit persons to whom the
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    The Software shall be used for Good, not Evil.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+
+    Terms of the MIT License:
+    --------------------------------------------------------------------
+
+    Permission is hereby granted, free of charge, to any person obtaining a
+    copy of this software and associated documentation files (the "Software"),
+    to deal in the Software without restriction, including without limitation
+    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+    and/or sell copies of the Software, and to permit persons to whom the
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included
+    in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency snappy is statically linked in certain binary
+distributions, like the python wheels. snappy has the following license:
+
+Copyright 2011, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of Google Inc. nor the names of its contributors may be
+      used to endorse or promote products derived from this software without
+      specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+===
+
+Some of the benchmark data in testdata/ is licensed differently:
+
+ - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and
+   is licensed under the Creative Commons Attribution 3.0 license
+   (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/
+   for more information.
+
+ - kppkn.gtb is taken from the Gaviota chess tablebase set, and
+   is licensed under the MIT License. See
+   https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1
+   for more information.
+
+ - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper
+   “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA
+   Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro,
+   which is licensed under the CC-BY license. See
+   http://www.ploscompbiol.org/static/license for more ifnormation.
+
+ - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project
+   Gutenberg. The first three have expired copyrights and are in the public
+   domain; the latter does not have expired copyright, but is still in the
+   public domain according to the license information
+   (http://www.gutenberg.org/ebooks/53).
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency gflags is statically linked in certain binary
+distributions, like the python wheels. gflags has the following license:
+
+Copyright (c) 2006, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency glog is statically linked in certain binary
+distributions, like the python wheels. glog has the following license:
+
+Copyright (c) 2008, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+A function gettimeofday in utilities.cc is based on
+
+http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd
+
+The license of this code is:
+
+Copyright (c) 2003-2008, Jouni Malinen <j@w1.fi> and contributors
+All Rights Reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the name(s) of the above-listed copyright holder(s) nor the
+   names of its contributors may be used to endorse or promote products
+   derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency re2 is statically linked in certain binary
+distributions, like the python wheels. re2 has the following license:
+
+Copyright (c) 2009 The RE2 Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its contributors
+      may be used to endorse or promote products derived from this
+      software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency c-ares is statically linked in certain binary
+distributions, like the python wheels. c-ares has the following license:
+
+# c-ares license
+
+Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS
+file.
+
+Copyright 1998 by the Massachusetts Institute of Technology.
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted, provided that
+the above copyright notice appear in all copies and that both that copyright
+notice and this permission notice appear in supporting documentation, and that
+the name of M.I.T. not be used in advertising or publicity pertaining to
+distribution of the software without specific, written prior permission.
+M.I.T. makes no representations about the suitability of this software for any
+purpose.  It is provided "as is" without express or implied warranty.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency zlib is redistributed as a dynamically linked shared
+library in certain binary distributions, like the python wheels. In the future
+this will likely change to static linkage. zlib has the following license:
+
+zlib.h -- interface of the 'zlib' general purpose compression library
+  version 1.2.11, January 15th, 2017
+
+  Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Jean-loup Gailly        Mark Adler
+  jloup@gzip.org          madler@alumni.caltech.edu
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency openssl is redistributed as a dynamically linked shared
+library in certain binary distributions, like the python wheels. openssl
+preceding version 3 has the following license:
+
+  LICENSE ISSUES
+  ==============
+
+  The OpenSSL toolkit stays under a double license, i.e. both the conditions of
+  the OpenSSL License and the original SSLeay license apply to the toolkit.
+  See below for the actual license texts.
+
+  OpenSSL License
+  ---------------
+
+/* ====================================================================
+ * Copyright (c) 1998-2019 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * (eay@cryptsoft.com).  This product includes software written by Tim
+ * Hudson (tjh@cryptsoft.com).
+ *
+ */
+
+ Original SSLeay License
+ -----------------------
+
+/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
+ * All rights reserved.
+ *
+ * This package is an SSL implementation written
+ * by Eric Young (eay@cryptsoft.com).
+ * The implementation was written so as to conform with Netscapes SSL.
+ *
+ * This library is free for commercial and non-commercial use as long as
+ * the following conditions are aheared to.  The following conditions
+ * apply to all code found in this distribution, be it the RC4, RSA,
+ * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
+ * included with this distribution is covered by the same copyright terms
+ * except that the holder is Tim Hudson (tjh@cryptsoft.com).
+ *
+ * Copyright remains Eric Young's, and as such any Copyright notices in
+ * the code are not to be removed.
+ * If this package is used in a product, Eric Young should be given attribution
+ * as the author of the parts of the library used.
+ * This can be in the form of a textual message at program startup or
+ * in documentation (online or textual) provided with the package.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    "This product includes cryptographic software written by
+ *     Eric Young (eay@cryptsoft.com)"
+ *    The word 'cryptographic' can be left out if the rouines from the library
+ *    being used are not cryptographic related :-).
+ * 4. If you include any Windows specific code (or a derivative thereof) from
+ *    the apps directory (application code) you must include an acknowledgement:
+ *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The licence and distribution terms for any publically available version or
+ * derivative of this code cannot be changed.  i.e. this code cannot simply be
+ * copied and put under another distribution licence
+ * [including the GNU Public Licence.]
+ */
+
+--------------------------------------------------------------------------------
+
+This project includes code from the rtools-backports project.
+
+* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code
+  from the rtools-backports project.
+
+Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms.
+All rights reserved.
+Homepage: https://github.com/r-windows/rtools-backports
+License: 3-clause BSD
+
+--------------------------------------------------------------------------------
+
+Some code from pandas has been adapted for the pyarrow codebase. pandas is
+available under the 3-clause BSD license, which follows:
+
+pandas license
+==============
+
+Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team
+All rights reserved.
+
+Copyright (c) 2008-2011 AQR Capital Management, LLC
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+       copyright notice, this list of conditions and the following
+       disclaimer in the documentation and/or other materials provided
+       with the distribution.
+
+    * Neither the name of the copyright holder nor the names of any
+       contributors may be used to endorse or promote products derived
+       from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+Some bits from DyND, in particular aspects of the build system, have been
+adapted from libdynd and dynd-python under the terms of the BSD 2-clause
+license
+
+The BSD 2-Clause License
+
+    Copyright (C) 2011-12, Dynamic NDArray Developers
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+
+        * Redistributions of source code must retain the above copyright
+           notice, this list of conditions and the following disclaimer.
+
+        * Redistributions in binary form must reproduce the above
+           copyright notice, this list of conditions and the following
+           disclaimer in the documentation and/or other materials provided
+           with the distribution.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Dynamic NDArray Developers list:
+
+ * Mark Wiebe
+ * Continuum Analytics
+
+--------------------------------------------------------------------------------
+
+Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted
+for PyArrow. Ibis is released under the Apache License, Version 2.0.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the autobrew project.
+
+* r/tools/autobrew and dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
+  are based on code from the autobrew project.
+
+Copyright (c) 2019, Jeroen Ooms
+License: MIT
+Homepage: https://github.com/jeroen/autobrew
+
+--------------------------------------------------------------------------------
+
+dev/tasks/homebrew-formulae/apache-arrow.rb has the following license:
+
+BSD 2-Clause License
+
+Copyright (c) 2009-present, Homebrew contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+
+cpp/src/arrow/vendored/base64.cpp has the following license
+
+ZLIB License
+
+Copyright (C) 2004-2017 René Nyffenegger
+
+This source code is provided 'as-is', without any express or implied
+warranty. In no event will the author be held liable for any damages arising
+from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose, including
+commercial applications, and to alter it and redistribute it freely, subject to
+the following restrictions:
+
+1. The origin of this source code must not be misrepresented; you must not
+   claim that you wrote the original source code. If you use this source code
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+
+2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original source code.
+
+3. This notice may not be removed or altered from any source distribution.
+
+René Nyffenegger rene.nyffenegger@adp-gmbh.ch
+
+--------------------------------------------------------------------------------
+
+The file cpp/src/arrow/vendored/optional.hpp has the following license
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/RecDP/MANIFEST.in b/RecDP/MANIFEST.in
new file mode 100644
index 000000000..d22d80ed2
--- /dev/null
+++ b/RecDP/MANIFEST.in
@@ -0,0 +1,2 @@
+include ScalaProcessUtils/built/30/*.jar
+include ScalaProcessUtils/built/31/*.jar
diff --git a/RecDP/README.md b/RecDP/README.md
new file mode 100644
index 000000000..f0e87504d
--- /dev/null
+++ b/RecDP/README.md
@@ -0,0 +1,65 @@
+# RecDP
+
+# INTRODUCTION
+RecDP is a Data Process python module, specifically designed for Recommender System.
+* Easy-to-use – Wrap often used operations with simple APIs.
+* Collaborative pipeline with spark - provide stableness and scalability of handling huge dataset with spark as underlying distributed data process engine.
+* Optimized Performance - 1) Adaptive dataframe plan decision making; 2) Intel-OAP accelerator extensions (SIMD, Cache, Native).
+* Feature Engineer oriented – advanced feature engineering functions (target encoding) 
+
+# Getting Start
+## install with pip (require preinstall spark)
+```
+# default version is working with spark 3.1
+pip install pyrecdp
+```
+
+## install with spark preinstalled docker img
+```
+docker run --network host -w /home/vmagent/app/ -it xuechendi/recdp_spark3.1 /bin/bash
+pip install pyrecdp
+```
+
+## examples
+[More examples](tests/)
+
+categorify a source data
+* convert 'language' column from 'text' to 'unique_integral_id'
+* codes link: [tests/test_categorify.py](tests/test_categorify.py)
+``` python
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+proc = DataProcessor(spark, path_prefix, cur_folder)
+proc.reset_ops([Categorify(['language'])])
+df = proc.transform(df)
+```
+
+sort a list by frequency
+* when each cell data is a list, organize this list to unique_value order by frequency
+* codes link: [tests/test_sortArrayByFrequency.py](tests/test_sortArrayByFrequency.py)
+``` python
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+proc = DataProcessor(spark, path_prefix, cur_folder)
+# group langugage by hour of day
+df = df.groupby('dt_hour').agg(f.collect_list("language").alias("language_list"))
+# to sort language by its showing frequency in this hour
+df = df.withColumn("sorted_langugage", f.expr(f"sortStringArrayByFrequency(language_list)"))
+df = proc.transform(df)
+```
+![image](https://user-images.githubusercontent.com/4355494/144941079-9a06fc88-38fe-454a-b33a-b559edc636de.png)
+
+
+## use cases
+* Recsys2021 example [url](https://github.com/oap-project/recdp/blob/master/examples/notebooks/recsys2021/final_submission_feature_engineering.ipynb)
+* Recsys2020 example [url](https://github.com/oap-project/recdp/blob/master/examples/notebooks/recsys2020/recsys2020_feature_engineering.ipynb)
+* Recsys2020 multiitem-categorify example(support for Analytics Zoo Friesian) [url](https://github.com/oap-project/recdp/blob/master/examples/notebooks/recsys2020/recsys_for_friesian_integration.ipynb)
+* DLRM example [url](https://github.com/oap-project/recdp/blob/master/examples/notebooks/dlrm/DLRM_Performance.ipynb)
+* DIEN example [url](https://github.com/oap-project/recdp/blob/master/examples/notebooks/dien/dien_data_process.ipynb)
+
+## LICENSE
+* Apache 2.0
+
+## Dependency
+* Spark 3.x
+* python 3.*
diff --git a/RecDP/SECURITY.md b/RecDP/SECURITY.md
new file mode 100644
index 000000000..eb482d909
--- /dev/null
+++ b/RecDP/SECURITY.md
@@ -0,0 +1,12 @@
+# Security Policy
+
+## Report a Vulnerability
+
+Please report security issues or vulnerabilities to the [Intel® Security Center].
+
+For more information on how Intel® works to resolve security issues, see
+[Vulnerability Handling Guidelines].
+
+[Intel® Security Center]:https://www.intel.com/security
+
+[Vulnerability Handling Guidelines]:https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html
diff --git a/RecDP/ScalaProcessUtils/built/30/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar b/RecDP/ScalaProcessUtils/built/30/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar
new file mode 100644
index 000000000..f13fccb0e
Binary files /dev/null and b/RecDP/ScalaProcessUtils/built/30/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar differ
diff --git a/RecDP/ScalaProcessUtils/built/31/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar b/RecDP/ScalaProcessUtils/built/31/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar
new file mode 100644
index 000000000..d1413f79c
Binary files /dev/null and b/RecDP/ScalaProcessUtils/built/31/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar differ
diff --git a/RecDP/ScalaProcessUtils/pom.xml b/RecDP/ScalaProcessUtils/pom.xml
new file mode 100644
index 000000000..79e78a6cb
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/pom.xml
@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.intel.recdp</groupId>
+  <artifactId>recdp-scala-extensions</artifactId>
+  <version>0.1.0</version>
+  <packaging>jar</packaging>
+
+  <licenses>
+    <license>
+      <name>The Apache Software License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+    </license>
+  </licenses>
+
+  <profiles>
+    <profile>
+      <id>spark-3.0</id>
+      <properties>
+        <spark.version>3.0.0</spark.version>
+        <exclude.files>**/3.1/org/apache/spark/sql/api/CodegenSeparator.scala</exclude.files>
+      </properties>
+    </profile>
+    <profile>
+      <id>spark-3.1</id>
+      <properties>
+        <spark.version>3.1.0</spark.version>
+        <exclude.files>**/3.0/org/apache/spark/sql/api/CodegenSeparator.scala</exclude.files>
+      </properties>
+    </profile>
+  </profiles>
+
+  <properties>
+    <spark.version>3.1.0</spark.version>
+    <scala.binary.version>2.12</scala.binary.version>
+    <scala.version>2.12.8</scala.version>
+    <exclude.files>**/3.0/org/apache/spark/sql/api/CodegenSeparator.scala</exclude.files>
+    <jars.target.dir>${project.build.directory}/scala-${scala.binary.version}/jars</jars.target.dir>
+  </properties>
+  <dependencies>
+    <!-- Prevent our dummy JAR from being included in Spark distributions or uploaded to YARN -->
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-sql_${scala.binary.version}</artifactId>
+      <version>${spark.version}</version>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>net.alchim31.maven</groupId>
+        <artifactId>scala-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>scala-compile-first</id>
+            <phase>process-resources</phase>
+            <goals>
+              <goal>add-source</goal>
+              <goal>compile</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>scala-test-compile</id>
+            <phase>process-test-resources</phase>
+            <goals>
+              <goal>testCompile</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <excludes>
+            <exclude>${exclude.files}</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.0</version>
+        <executions>
+          <execution>
+            <phase>compile</phase>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <excludes>
+            <exclude>${exclude.files}</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>3.0.0</version>
+        <configuration>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/3.0/org/apache/spark/sql/api/CodegenSeparator.scala b/RecDP/ScalaProcessUtils/src/main/scala/3.0/org/apache/spark/sql/api/CodegenSeparator.scala
new file mode 100644
index 000000000..fbf473a09
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/3.0/org/apache/spark/sql/api/CodegenSeparator.scala
@@ -0,0 +1,104 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import scala.util.Try
+
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql._
+import org.apache.spark.sql.api.java.{UDF0, UDF1}
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.expressions.{Expression, Literal, ScalaUDF}
+import org.apache.spark.sql.expressions.{SparkUserDefinedFunction}
+import org.apache.spark.sql.types.{DataType, IntegerType, LongType, StringType}
+
+class CodegenFallbackUDF(
+    function: AnyRef,
+    dataType: DataType,
+    children: Seq[Expression],
+    inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil,
+    udfName: Option[String] = None,
+    nullable: Boolean = true,
+    udfDeterministic: Boolean = true
+  ) extends ScalaUDF(function, dataType, children, Nil, udfName = udfName) with CodegenFallback {
+    override def doGenCode(
+      ctx: CodegenContext,
+      ev: ExprCode): ExprCode = {
+        throw new NotImplementedError("CodeGen is not supported in CodegenFallbackUDF")
+      }
+  }
+
+case class DummyUDF() extends UDF0[String] {
+  def call() = ""
+}
+
+case class DummyUDFForString() extends UDF1[String, String] {
+  def call(x: String) = x
+}
+
+case class DummyUDFForInteger() extends UDF1[Int, Int] {
+  def call(x: Int) = x
+}
+
+case class DummyUDFForLong() extends UDF1[Long, Long] {
+  def call(x: Long) = x
+}
+
+class CodegenSeparator(
+    name: String,
+    f: AnyRef,
+    dataType: DataType
+    ) extends SparkUserDefinedFunction(f, dataType, Nil) {
+  def this() {
+    this("CodegenSeparator", () => (DummyUDF()).asInstanceOf[UDF0[Any]].call(), StringType)
+  }
+  override def createScalaUDF(exprs: Seq[Expression]): ScalaUDF = new CodegenFallbackUDF(f, dataType, exprs, udfName = Some(name))
+}
+
+class CodegenSeparator0(
+    name: String,
+    f: AnyRef,
+    dataType: DataType
+    ) extends SparkUserDefinedFunction(f, dataType, Nil) {
+  def this() {
+    this("CodegenSeparator", (DummyUDFForInteger()).asInstanceOf[UDF1[Int, Int]].call(_: Int), IntegerType)
+  }
+  override def createScalaUDF(exprs: Seq[Expression]): ScalaUDF = new CodegenFallbackUDF(f, dataType, exprs, udfName = Some(name))
+}
+
+class CodegenSeparator1(
+    name: String,
+    f: AnyRef,
+    dataType: DataType
+    ) extends SparkUserDefinedFunction(f, dataType, Nil) {
+  def this() {
+    this("CodegenSeparator", (DummyUDFForString()).asInstanceOf[UDF1[String, String]].call(_: String), StringType)
+  }
+  override def createScalaUDF(exprs: Seq[Expression]): ScalaUDF = new CodegenFallbackUDF(f, dataType, exprs, udfName = Some(name))
+}
+
+class CodegenSeparator2(
+    name: String,
+    f: AnyRef,
+    dataType: DataType
+    ) extends SparkUserDefinedFunction(f, dataType, Nil) {
+  def this() {
+    this("CodegenSeparator", (DummyUDFForLong()).asInstanceOf[UDF1[Long, Long]].call(_: Long), LongType)
+  }
+  override def createScalaUDF(exprs: Seq[Expression]): ScalaUDF = new CodegenFallbackUDF(f, dataType, exprs, udfName = Some(name))
+}
+
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/3.1/org/apache/spark/sql/api/CodegenSeparator.scala b/RecDP/ScalaProcessUtils/src/main/scala/3.1/org/apache/spark/sql/api/CodegenSeparator.scala
new file mode 100644
index 000000000..6b55f3ee3
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/3.1/org/apache/spark/sql/api/CodegenSeparator.scala
@@ -0,0 +1,105 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import scala.util.Try
+
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql._
+import org.apache.spark.sql.api.java.{UDF0, UDF1}
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.expressions.{Expression, Literal, ScalaUDF}
+import org.apache.spark.sql.expressions.{SparkUserDefinedFunction}
+import org.apache.spark.sql.types.{DataType, IntegerType, LongType, StringType}
+
+class CodegenFallbackUDF(
+    function: AnyRef,
+    dataType: DataType,
+    children: Seq[Expression],
+    inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil,
+    outputEncoder: Option[ExpressionEncoder[_]] = None,
+    udfName: Option[String] = None,
+    nullable: Boolean = true,
+    udfDeterministic: Boolean = true
+  ) extends ScalaUDF(function, dataType, children, Nil, udfName = udfName) with CodegenFallback {
+    override def doGenCode(
+      ctx: CodegenContext,
+      ev: ExprCode): ExprCode = {
+        throw new NotImplementedError("CodeGen is not supported in CodegenFallbackUDF")
+      }
+  }
+
+case class DummyUDF() extends UDF0[String] {
+  def call() = ""
+}
+
+case class DummyUDFForString() extends UDF1[String, String] {
+  def call(x: String) = x
+}
+
+case class DummyUDFForInteger() extends UDF1[Int, Int] {
+  def call(x: Int) = x
+}
+
+case class DummyUDFForLong() extends UDF1[Long, Long] {
+  def call(x: Long) = x
+}
+
+class CodegenSeparator(
+    name: String,
+    f: AnyRef,
+    dataType: DataType
+    ) extends SparkUserDefinedFunction(f, dataType, Nil) {
+  def this() {
+    this("CodegenSeparator", () => (DummyUDF()).asInstanceOf[UDF0[Any]].call(), StringType)
+  }
+  override def createScalaUDF(exprs: Seq[Expression]): ScalaUDF = new CodegenFallbackUDF(f, dataType, exprs, udfName = Some(name))
+}
+
+class CodegenSeparator0(
+    name: String,
+    f: AnyRef,
+    dataType: DataType
+    ) extends SparkUserDefinedFunction(f, dataType, Nil) {
+  def this() {
+    this("CodegenSeparator", (DummyUDFForInteger()).asInstanceOf[UDF1[Int, Int]].call(_: Int), IntegerType)
+  }
+  override def createScalaUDF(exprs: Seq[Expression]): ScalaUDF = new CodegenFallbackUDF(f, dataType, exprs, udfName = Some(name))
+}
+
+class CodegenSeparator1(
+    name: String,
+    f: AnyRef,
+    dataType: DataType
+    ) extends SparkUserDefinedFunction(f, dataType, Nil) {
+  def this() {
+    this("CodegenSeparator", (DummyUDFForString()).asInstanceOf[UDF1[String, String]].call(_: String), StringType)
+  }
+  override def createScalaUDF(exprs: Seq[Expression]): ScalaUDF = new CodegenFallbackUDF(f, dataType, exprs, udfName = Some(name))
+}
+
+class CodegenSeparator2(
+    name: String,
+    f: AnyRef,
+    dataType: DataType
+    ) extends SparkUserDefinedFunction(f, dataType, Nil) {
+  def this() {
+    this("CodegenSeparator", (DummyUDFForLong()).asInstanceOf[UDF1[Long, Long]].call(_: Long), LongType)
+  }
+  override def createScalaUDF(exprs: Seq[Expression]): ScalaUDF = new CodegenFallbackUDF(f, dataType, exprs, udfName = Some(name))
+}
+
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/com/intel/recdp/ScalaUDFTest.scala b/RecDP/ScalaProcessUtils/src/main/scala/com/intel/recdp/ScalaUDFTest.scala
new file mode 100644
index 000000000..693d1104f
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/com/intel/recdp/ScalaUDFTest.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intel.recdp
+
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql._
+import org.apache.spark.sql.api.java.UDF1
+import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType}
+import scala.collection.mutable.WrappedArray
+
+class ScalaUDFTestInt extends UDF1[Integer, Integer] {
+  def call(x: Integer): Integer = {
+    x + 1
+  }
+}
+
+class ScalaUDFTestStr extends UDF1[String, String] {
+  def call(x: String) = {
+    x.split('\t').mkString("_")
+  }
+}
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/com/intel/recdp/SortByFrequency.scala b/RecDP/ScalaProcessUtils/src/main/scala/com/intel/recdp/SortByFrequency.scala
new file mode 100644
index 000000000..eb881e255
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/com/intel/recdp/SortByFrequency.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intel.recdp
+
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql._
+import org.apache.spark.sql.api.java.UDF1
+import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType}
+import scala.collection.mutable.WrappedArray
+
+class SortIntArrayByFrequency extends UDF1[WrappedArray[Integer], Array[Integer]] {
+  def call(x: WrappedArray[Integer]) = {
+    if (x != null) {
+      x.toArray.groupBy(identity).toArray.sortBy(-_._2.size).map(_._1)
+    } else {
+      Array[Integer]()
+    }
+  }
+}
+
+class SortStringArrayByFrequency extends UDF1[WrappedArray[String], Array[String]] {
+  def call(x: WrappedArray[String]) = {
+    if (x != null) {
+      x.toArray.groupBy(identity).toArray.sortBy(-_._2.size).map(_._1)
+    } else {
+      Array[String]()
+    }
+  }
+}
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/com/intel/recdp/utils.scala b/RecDP/ScalaProcessUtils/src/main/scala/com/intel/recdp/utils.scala
new file mode 100644
index 000000000..7fab25e72
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/com/intel/recdp/utils.scala
@@ -0,0 +1,13 @@
+package com.intel.recdp
+
+import java.util.UUID.randomUUID
+import scala.collection.JavaConverters._
+
+object Utils {
+  def UUID(): String = {
+    randomUUID().toString()
+  }
+  def convertToScalaArray(x: Any): Array[String] = {
+    x.asInstanceOf[java.util.ArrayList[String]].asScala.toArray
+  }
+}
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/Categorify.scala b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/Categorify.scala
new file mode 100644
index 000000000..25e38662b
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/Categorify.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import scala.util.Try
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+
+object Categorify {
+  def categorify(sc: JavaSparkContext, df: DataFrame, col_name: String, dict_df: DataFrame): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col", "dict_col_id").collect().map( row => 
+      (row(0).asInstanceOf[String], row(1).asInstanceOf[Int])
+    ).toMap
+    val broadcast_handler = sc.broadcast(broadcast_data)
+    val categorifyUDF = udf((x: String) => {
+      val broadcasted = broadcast_handler.value
+      if (broadcasted == null || broadcasted.isEmpty || x == null) {
+        None
+      } else if (broadcasted.contains(x)) {
+        Some(broadcasted(x))
+      } else {
+        None
+      }
+    })
+    return df.withColumn(col_name, categorifyUDF(col(col_name)))
+  }
+}
\ No newline at end of file
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyBroadcast.scala b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyBroadcast.scala
new file mode 100644
index 000000000..e6f93899f
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyBroadcast.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types.{IntegerType, StringType}
+
+object CategorifyBroadcast {
+  def broadcast(sc: JavaSparkContext, dict_df: DataFrame): Broadcast[Map[String, Int]] = {
+    val broadcast_data = dict_df.select("dict_col", "dict_col_id").collect().map( row => 
+      (row(0).asInstanceOf[String], row(1).asInstanceOf[Int])
+    ).toMap
+    sc.broadcast(broadcast_data)
+  }
+}
\ No newline at end of file
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyByFreqForArray.scala b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyByFreqForArray.scala
new file mode 100644
index 000000000..7b1984e09
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyByFreqForArray.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import scala.collection.mutable.WrappedArray
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+
+object CategorifyByFreqForArray {
+  def categorify(sc: JavaSparkContext, df: DataFrame, col_name: String, dict_df: DataFrame): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col", "dict_col_id").collect().map( row => 
+      (row(0).asInstanceOf[String], row(1).asInstanceOf[Int])
+    ).toMap
+    val broadcast_handler = sc.broadcast(broadcast_data)
+    val categorifyUDF = udf((x_l: WrappedArray[String]) => {
+      val broadcasted = broadcast_handler.value
+      if (broadcasted == null || broadcasted.isEmpty || x_l == null) {
+        None
+      } else {
+        var min_val: Option[Int] = None
+        x_l.foreach(x => {
+          if (x != null && broadcasted.contains(x) && (min_val == None || broadcasted(x) < min_val.get)) {
+            min_val = Some(broadcasted(x))
+          }
+        })
+        min_val
+      }
+    })
+    return df.withColumn(col_name, categorifyUDF(col(col_name)))
+  }
+}
\ No newline at end of file
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyForArray.scala b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyForArray.scala
new file mode 100644
index 000000000..9ac5f8b44
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyForArray.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import scala.collection.mutable.WrappedArray
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+
+object CategorifyForArray {
+  def categorify(sc: JavaSparkContext, df: DataFrame, col_name: String, dict_df: DataFrame): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col", "dict_col_id").collect().map( row => 
+      (row(0).asInstanceOf[String], row(1).asInstanceOf[Int])
+    ).toMap
+    val broadcast_handler = sc.broadcast(broadcast_data)
+    val categorifyUDF = udf((x_l: WrappedArray[String]) => {
+      val broadcasted = broadcast_handler.value
+      if (broadcasted == null || broadcasted.isEmpty || x_l == null) {
+        Array[Int]()
+      } else {
+        x_l.toArray.map(x => if (x != null && broadcasted.contains(x)) broadcasted(x) else 0)
+      }
+    })
+    return df.withColumn(col_name, categorifyUDF(col(col_name)))
+  }
+}
\ No newline at end of file
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyForMultiLevelArray.scala b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyForMultiLevelArray.scala
new file mode 100644
index 000000000..90ee7b782
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/CategorifyForMultiLevelArray.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import scala.collection.mutable.WrappedArray
+import scala.reflect.{ClassTag, ClassManifest}
+import scala.reflect.runtime.universe.TypeTag
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+import com.intel.recdp._
+
+object CategorifyForMultiLevelArray {
+  def process[T : ClassTag](sc: JavaSparkContext, df: DataFrame, col_name: String, dict_df: DataFrame, j_sep_list: Any): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col", "dict_col_id").collect().map( row => 
+      if (row(0) != null) {
+        (row(0).toString, row(1).asInstanceOf[Int])
+      } else {
+        (null, row(1).asInstanceOf[Int])
+      }
+    ).toMap
+    val broadcast_handler = sc.broadcast(broadcast_data)
+    val sep_list = Utils.convertToScalaArray(j_sep_list)
+    val categorifyUDF = udf((x_l: WrappedArray[String]) => {
+      val broadcasted = broadcast_handler.value
+      def get_mapped(x_l: String, sep_id: Int): String = {
+        if (sep_id >= sep_list.size || !x_l.contains(sep_list(sep_id))) {
+          //System.out.println(s"sep is ${sep_id} and ${sep_list.toList} and x_l is ${x_l}")
+          if (x_l != null && broadcasted.contains(x_l)) broadcasted(x_l).toString
+          else "0"
+        } else {
+          x_l.split(sep_list(sep_id).toCharArray()(0)).map(x => get_mapped(x, sep_id + 1)).mkString(sep_list(sep_id))
+        }
+      }
+
+      if (broadcasted == null || broadcasted.isEmpty || x_l == null) {
+        Array[String]()
+      } else {
+        x_l.toArray.map(x => get_mapped(x, 0))
+      }
+    })
+    return df.withColumn(col_name, categorifyUDF(col(col_name)))
+  }
+
+  def categorify(sc: JavaSparkContext, df: DataFrame, col_name: String, dict_df: DataFrame, j_sep_list: Any): DataFrame = {
+    dict_df.dtypes.foreach {
+      case (dict_col_name, col_type) => if (dict_col_name == "dict_col") {
+        if (col_type == "StringType") {
+          return process[String](sc, df, col_name, dict_df, j_sep_list)
+        } else if (col_type == "IntegerType") {
+          return process[Int](sc, df, col_name, dict_df, j_sep_list)
+        } else if (col_type == "FloatType") {
+          return process[Float](sc, df, col_name, dict_df, j_sep_list)
+        } else {
+          throw new NotImplementedError(s"${col_type} is currently not supported")
+        }
+      }
+    }
+    df
+  }
+}
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/NegativeFeature.scala b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/NegativeFeature.scala
new file mode 100644
index 000000000..8cabfc04b
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/NegativeFeature.scala
@@ -0,0 +1,97 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import scala.util.Random
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types.{IntegerType, StringType}
+
+// This operator aims to add negative sample as new row
+object NegativeFeature {
+  def process_string(sc: JavaSparkContext, df: DataFrame, tgt_name: String, col_name: String, dict_df: DataFrame, neg_cnt: Int): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col").collect().map( row => 
+      row(0).asInstanceOf[String]
+    )
+    val broadcasted = sc.broadcast(broadcast_data)
+    val addNegativeFeatureUDF = udf((asin: String) => {
+      val r = new Random
+      val item_list = broadcasted.value
+      val num_items = item_list.size
+      var asin_neg = asin
+      do {
+          val asin_neg_index: Int = r.nextInt(num_items - 1)
+          asin_neg = item_list(asin_neg_index)
+      } while (asin_neg == asin)
+      asin_neg
+    })
+    return df.withColumn(tgt_name, addNegativeFeatureUDF(col(col_name)))
+  }
+  def process_int(sc: JavaSparkContext, df: DataFrame, tgt_name: String, col_name: String, dict_df: DataFrame, neg_cnt: Int): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col").collect().map( row => 
+      row(0).asInstanceOf[Int]
+    )
+    val broadcasted = sc.broadcast(broadcast_data)
+    val addNegativeFeatureUDF = udf((asin: Int) => {
+      val r = new Random
+      val item_list = broadcasted.value
+      val num_items = item_list.size
+      var asin_neg = asin
+      do {
+          val asin_neg_index: Int = r.nextInt(num_items - 1)
+          asin_neg = item_list(asin_neg_index)
+      } while (asin_neg == asin)
+      asin_neg
+    })
+    return df.withColumn(tgt_name, addNegativeFeatureUDF(col(col_name)))
+  }
+  def process_float(sc: JavaSparkContext, df: DataFrame, tgt_name: String, col_name: String, dict_df: DataFrame, neg_cnt: Int): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col").collect().map( row => 
+      row(0).asInstanceOf[Float]
+    )
+    val broadcasted = sc.broadcast(broadcast_data)
+    val addNegativeFeatureUDF = udf((asin: Float) => {
+      val r = new Random
+      val item_list = broadcasted.value
+      val num_items = item_list.size
+      var asin_neg = asin
+      do {
+          val asin_neg_index: Int = r.nextInt(num_items - 1)
+          asin_neg = item_list(asin_neg_index)
+      } while (asin_neg == asin)
+      asin_neg
+    })
+    return df.withColumn(tgt_name, addNegativeFeatureUDF(col(col_name)))
+  }
+  def add(sc: JavaSparkContext, df: DataFrame, tgt_name: String, col_name: String, dict_df: DataFrame, neg_cnt: Int): DataFrame = {
+    dict_df.dtypes.foreach {
+      case (dict_col_name, col_type) => if (dict_col_name == "dict_col") {
+        if (col_type == "StringType") {
+          return process_string(sc, df, tgt_name, col_name, dict_df, neg_cnt)
+        } else if (col_type == "IntegerType") {
+          return process_int(sc, df, tgt_name, col_name, dict_df, neg_cnt)
+        } else if (col_type == "FloatType") {
+          return process_float(sc, df, tgt_name, col_name, dict_df, neg_cnt)
+        } else {
+          throw new NotImplementedError(s"${col_type} is currently not supported")
+        }
+      }
+    }
+    df
+  }
+}
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/NegativeFeatureForArray.scala b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/NegativeFeatureForArray.scala
new file mode 100644
index 000000000..544a7fb35
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/NegativeFeatureForArray.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import scala.collection.mutable.WrappedArray
+import scala.collection.mutable.ArrayBuffer
+import scala.util.Random
+import scala.reflect.{ClassTag, ClassManifest}
+import scala.reflect.runtime.universe.TypeTag
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types.{IntegerType, StringType}
+
+// This operator aims to add negative sample as new row
+object NegativeFeatureForArray {
+  def process[T : ClassTag](sc: JavaSparkContext, df: DataFrame, tgt_name: String, col_name: String, dict_df: DataFrame, neg_cnt: Int): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col").collect().map( row => 
+      row(0).asInstanceOf[T]
+    )
+    val broadcasted = sc.broadcast(broadcast_data)
+    val addNegativeFeatureUDF = if (neg_cnt == 1) {
+      udf((hist_asin: WrappedArray[Any]) => {
+        val r = new Random
+        val item_list = broadcasted.value
+        val num_items = item_list.size
+        if (hist_asin == null) {
+          Array[String]()
+        } else {
+          hist_asin.toArray.map(asin_any => {
+            val asin = asin_any.asInstanceOf[T]
+            var asin_neg = asin
+            do {
+                val asin_neg_index: Int = r.nextInt(num_items - 1)
+                asin_neg = item_list(asin_neg_index)
+            } while (asin_neg == asin)
+            asin_neg.toString
+          })
+        }
+      })
+    } else {
+      udf((hist_asin: WrappedArray[Any]) => {
+        val r = new Random
+        val item_list = broadcasted.value
+        val num_items = item_list.size
+        if (hist_asin == null) {
+          Array[String]()
+        } else {
+          hist_asin.toArray.map(asin_any => {
+            val asin = asin_any.asInstanceOf[T]
+            var smp_cnt = 0
+            var res = ArrayBuffer[String]()
+            do {
+              var asin_neg = asin
+              do {
+                  val asin_neg_index: Int = r.nextInt(num_items - 1)
+                  asin_neg = item_list(asin_neg_index)
+              } while (asin_neg == asin)
+              res += asin_neg.toString
+              smp_cnt += 1
+            } while (smp_cnt < neg_cnt)
+            res.mkString("|")
+          })
+        }
+      })
+    }
+    return df.withColumn(tgt_name, addNegativeFeatureUDF(col(col_name)))
+  }
+  def add(sc: JavaSparkContext, df: DataFrame, tgt_name: String, col_name: String, dict_df: DataFrame, neg_cnt: Int): DataFrame = {
+    dict_df.dtypes.foreach {
+      case (dict_col_name, col_type) => if (dict_col_name == "dict_col") {
+        if (col_type == "StringType") {
+          return process[String](sc, df, tgt_name, col_name, dict_df, neg_cnt)
+        } else if (col_type == "IntegerType") {
+          return process[Int](sc, df, tgt_name, col_name, dict_df, neg_cnt)
+        } else if (col_type == "FloatType") {
+          return process[Float](sc, df, tgt_name, col_name, dict_df, neg_cnt)
+        } else {
+          throw new NotImplementedError(s"${col_type} is currently not supported")
+        }
+      }
+    }
+    df
+  }
+}
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/NegativeSample.scala b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/NegativeSample.scala
new file mode 100644
index 000000000..8807698e7
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/NegativeSample.scala
@@ -0,0 +1,103 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import scala.util.Random
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types.{IntegerType, StringType}
+
+// This operator aims to add negative sample as new row
+object NegativeSample {
+  def process_string(sc: JavaSparkContext, df: DataFrame, col_name: String, dict_df: DataFrame, neg_cnt: Int): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col").collect().map( row => 
+      row(0).asInstanceOf[String]
+    )
+    val broadcasted = sc.broadcast(broadcast_data)
+    val addNegativeSampleUDF = udf((asin: String) => {
+      val r = new Random
+      val item_list = broadcasted.value
+      val num_items = item_list.size
+      var asin_neg = asin
+      do {
+          val asin_neg_index: Int = r.nextInt(num_items - 1)
+          asin_neg = item_list(asin_neg_index)
+      } while (asin_neg == asin)
+      Array[String](asin_neg, asin)
+    })
+    var new_df = df.withColumn(col_name, addNegativeSampleUDF(col(col_name)))
+    new_df = new_df.select(col("*"), posexplode(col(col_name))).drop(col_name).withColumnRenamed("col", col_name)
+    return new_df
+  }
+  def process_int(sc: JavaSparkContext, df: DataFrame, col_name: String, dict_df: DataFrame, neg_cnt: Int): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col").collect().map( row => 
+      row(0).asInstanceOf[Int]
+    )
+    val broadcasted = sc.broadcast(broadcast_data)
+    val addNegativeSampleUDF = udf((asin: Int) => {
+      val r = new Random
+      val item_list = broadcasted.value
+      val num_items = item_list.size
+      var asin_neg = asin
+      do {
+          val asin_neg_index: Int = r.nextInt(num_items - 1)
+          asin_neg = item_list(asin_neg_index)
+      } while (asin_neg == asin)
+      Array[Int](asin_neg, asin)
+    })
+    var new_df = df.withColumn(col_name, addNegativeSampleUDF(col(col_name)))
+    new_df = new_df.select(col("*"), posexplode(col(col_name))).drop(col_name).withColumnRenamed("col", col_name)
+    return new_df
+  }
+  def process_float(sc: JavaSparkContext, df: DataFrame, col_name: String, dict_df: DataFrame, neg_cnt: Int): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col").collect().map( row => 
+      row(0).asInstanceOf[Float]
+    )
+    val broadcasted = sc.broadcast(broadcast_data)
+    val addNegativeSampleUDF = udf((asin: Float) => {
+      val r = new Random
+      val item_list = broadcasted.value
+      val num_items = item_list.size
+      var asin_neg = asin
+      do {
+          val asin_neg_index: Int = r.nextInt(num_items - 1)
+          asin_neg = item_list(asin_neg_index)
+      } while (asin_neg == asin)
+      Array[Float](asin_neg, asin)
+    })
+    var new_df = df.withColumn(col_name, addNegativeSampleUDF(col(col_name)))
+    new_df = new_df.select(col("*"), posexplode(col(col_name))).drop(col_name).withColumnRenamed("col", col_name)
+    return new_df
+  }
+  def add(sc: JavaSparkContext, df: DataFrame, col_name: String, dict_df: DataFrame, neg_cnt: Int): DataFrame = {
+    dict_df.dtypes.foreach {
+      case (dict_col_name, col_type) => if (dict_col_name == "dict_col") {
+        if (col_type == "StringType") {
+          return process_string(sc, df, col_name, dict_df, neg_cnt)
+        } else if (col_type == "IntegerType") {
+          return process_int(sc, df, col_name, dict_df, neg_cnt)
+        } else if (col_type == "FloatType") {
+          return process_float(sc, df, col_name, dict_df, neg_cnt)
+        } else {
+          throw new NotImplementedError(s"${col_type} is currently not supported")
+        }
+      }
+    }
+    df
+  }
+}
diff --git a/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/ScalaDFTest.scala b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/ScalaDFTest.scala
new file mode 100644
index 000000000..86fcb4576
--- /dev/null
+++ b/RecDP/ScalaProcessUtils/src/main/scala/org/apache/spark/sql/api/ScalaDFTest.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api
+
+import scala.util.Random
+
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types.{IntegerType, StringType}
+
+// This operator aims to add negative sample as new row
+object ScalaDFTest {
+  def scala_test_int(sc: JavaSparkContext, df: DataFrame, col_name: String): DataFrame = {
+    val addScalaDFTestUDF = udf((x: Int) => {
+      x + 1
+    })
+    df.withColumn(col_name, addScalaDFTestUDF(col(col_name)))
+  }
+  def scala_test_str(sc: JavaSparkContext, df: DataFrame, col_name: String): DataFrame = {
+    val addScalaDFTestUDF = udf((x: String) => {
+      x.split('\t').mkString("_")
+    })
+    df.withColumn(col_name, addScalaDFTestUDF(col(col_name)))
+  }
+  def scala_test_broadcast(sc: JavaSparkContext, df: DataFrame, col_name: String, dict_df: DataFrame): DataFrame = {
+    val broadcast_data = dict_df.select("dict_col", "dict_col_id").collect().map( row => 
+      (row(0).asInstanceOf[String], row(1).asInstanceOf[Int])
+    ).toMap
+    val broadcast_handler = sc.broadcast(broadcast_data)
+    val categorifyUDF = udf((x: String) => {
+      val broadcasted = broadcast_handler.value
+      if (broadcasted == null || broadcasted.isEmpty || x == null) {
+        None
+      } else if (broadcasted.contains(x)) {
+        Some(broadcasted(x))
+      } else {
+        None
+      }
+    })
+    df.withColumn(col_name, categorifyUDF(col(col_name)))
+  }
+}
\ No newline at end of file
diff --git a/RecDP/__init__.py b/RecDP/__init__.py
new file mode 100644
index 000000000..4b329f59e
--- /dev/null
+++ b/RecDP/__init__.py
@@ -0,0 +1,1 @@
+__all__ = ['pyrecdp']
diff --git a/RecDP/examples/notebooks/dien/dien_data_process.ipynb b/RecDP/examples/notebooks/dien/dien_data_process.ipynb
new file mode 100644
index 000000000..4b4ecd894
--- /dev/null
+++ b/RecDP/examples/notebooks/dien/dien_data_process.ipynb
@@ -0,0 +1,356 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "spark.stop()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import init\n",
+    "\n",
+    "import findspark\n",
+    "findspark.init()\n",
+    "\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pyspark.sql import *\n",
+    "from pyspark import *\n",
+    "import pyspark.sql.functions as f\n",
+    "from pyspark.sql.types import *\n",
+    "from timeit import default_timer as timer\n",
+    "import logging\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.utils import *\n",
+    "\n",
+    "def load_csv(spark, path):\n",
+    "    review_id_field = StructField('reviewer_id', StringType())\n",
+    "    asin_field = StructField('asin', StringType())\n",
+    "    overall_field = StructField('overall', FloatType())\n",
+    "    unix_time_field = StructField('unix_review_time', IntegerType())\n",
+    "    reviews_info_schema = StructType(\n",
+    "        [review_id_field, asin_field, overall_field, unix_time_field])\n",
+    "\n",
+    "    category_field = StructField('category', StringType())\n",
+    "    item_info_schema = StructType([asin_field, category_field])\n",
+    "\n",
+    "    reviews_info_df = spark.read.schema(reviews_info_schema).option('sep', '\\t').csv(path + \"/reviews-info\")\n",
+    "    item_info_df = spark.read.schema(item_info_schema).option('sep', '\\t').csv(path + \"/item-info\")\n",
+    "\n",
+    "    return reviews_info_df, item_info_df\n",
+    "\n",
+    "\n",
+    "def collapse_by_hist(df, item_info_df, proc, output_name, min_num_hist = 0, max_num_hist = 100):\n",
+    "    op_model_merge = ModelMerge([{'col_name': 'asin', 'dict': item_info_df}])\n",
+    "    op_collapse_by_hist = CollapseByHist(['asin', 'category'], by = ['reviewer_id'], orderBy = ['unix_review_time', 'asin'], minNumHist = min_num_hist, maxNumHist = 100)\n",
+    "    proc.reset_ops([op_model_merge, op_collapse_by_hist])\n",
+    "    \n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, output_name)\n",
+    "    t2 = timer()\n",
+    "    print(f\"Merge with category and collapse hist took {t2 - t1} secs\")\n",
+    "    return df\n",
+    "\n",
+    "\n",
+    "def get_dict_for_asin(df, proc):\n",
+    "    dict_dfs = []\n",
+    "    dict_names = ['asin']\n",
+    "    dict_dfs = [{'col_name': name, 'dict': df.select(spk_func.col(name).alias('dict_col'))} for name in dict_names]\n",
+    "    return dict_dfs\n",
+    "\n",
+    "\n",
+    "def add_negative_sample(df, item_info_df, dict_dfs, proc, output_name):\n",
+    "    # add negative_sample as new row\n",
+    "    op_negative_sample = NegativeSample(['asin'], dict_dfs)\n",
+    "    op_drop_category = DropFeature(['category'])\n",
+    "    op_add_category = ModelMerge([{'col_name': 'asin', 'dict': item_info_df}])\n",
+    "    op_select = SelectFeature(['pos', 'reviewer_id', 'asin', 'category', 'hist_asin', 'hist_category'])\n",
+    "    proc.reset_ops([op_negative_sample, op_drop_category, op_add_category, op_select])\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, output_name)\n",
+    "    t2 = timer()    \n",
+    "    print(f\"add_negative_sample took {t2 - t1} secs\")\n",
+    "    return df\n",
+    "\n",
+    "\n",
+    "def add_negative_hist_cols(df, item_info_df, dict_dfs, proc, output_name):\n",
+    "    # add negative_sample as new row\n",
+    "    op_columns_sample = NegativeFeature({'noclk_hist_asin': 'hist_asin'}, dict_dfs, doSplit=True, sep='\\x02', negCnt = 5)\n",
+    "    proc.reset_ops([op_columns_sample])\n",
+    "    t1 = timer()\n",
+    "\n",
+    "    df = proc.transform(df, output_name)\n",
+    "    t2 = timer()    \n",
+    "    print(f\"add_negative_hist_cols took {t2 - t1} secs\")\n",
+    "    return df\n",
+    "\n",
+    "\n",
+    "def load_processed_csv(spark, data_dir):\n",
+    "    label_field = StructField('pos', IntegerType())\n",
+    "    review_id_field = StructField('reviewer_id', StringType())\n",
+    "    asin_field = StructField('asin', StringType())\n",
+    "    category_field = StructField('category', StringType())\n",
+    "    hist_asin_field = StructField('hist_asin', StringType())\n",
+    "    hist_category_field = StructField('hist_category', StringType())\n",
+    "    csv_schema = StructType(\n",
+    "        [label_field, review_id_field, asin_field, category_field, hist_asin_field, hist_category_field])\n",
+    "\n",
+    "    return spark.read.schema(csv_schema).option('sep', '\\t').csv(data_dir)\n",
+    "\n",
+    "\n",
+    "def categorify_dien_data(df, user_df, asin_df, cat_df, asin_cat_df, proc, output_name):\n",
+    "    df = df.select('pos', 'reviewer_id', 'asin', 'category', 'hist_asin', 'hist_category',\n",
+    "                   'noclk_hist_asin', f.expr('noclk_hist_asin as noclk_hist_category'))\n",
+    "\n",
+    "    dict_dfs = []\n",
+    "    dict_dfs.append({'col_name': 'reviewer_id', 'dict': user_df})\n",
+    "    dict_dfs.append({'col_name': 'asin', 'dict': asin_df})\n",
+    "    dict_dfs.append({'col_name': 'category', 'dict': cat_df})\n",
+    "    dict_dfs.append({'col_name': 'hist_asin', 'dict': asin_df})\n",
+    "    dict_dfs.append({'col_name': 'hist_category', 'dict': cat_df})\n",
+    "    dict_dfs.append({'col_name': 'noclk_hist_asin', 'dict': asin_df})\n",
+    "    dict_dfs.append({'col_name': 'noclk_hist_category', 'dict': asin_cat_df})\n",
+    "\n",
+    "    op_categorify = Categorify(['reviewer_id', 'asin', 'category'], dict_dfs=dict_dfs)\n",
+    "    op_categorify_2 = Categorify(['hist_asin', 'hist_category'], dict_dfs=dict_dfs, doSplit=True, sep='\\x02')\n",
+    "    op_categorify_3 = Categorify(['noclk_hist_asin', 'noclk_hist_category'], dict_dfs=dict_dfs, doSplit=True, multiLevelSplit=True, multiLevelSep=['|'])\n",
+    "    op_fillna = FillNA(['asin', 'category'], 0)\n",
+    "    proc.reset_ops([op_categorify, op_categorify_2, op_categorify_3, op_fillna])\n",
+    "\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, output_name)\n",
+    "    t2 = timer()\n",
+    "    print(f\"categorify took {t2 - t1} secs\")\n",
+    "    return df\n",
+    "\n",
+    "\n",
+    "def save_to_voc(df, proc, fmt, cols, default_name, default_v, output_name):\n",
+    "    import pickle\n",
+    "    col_name = ''\n",
+    "    dtypes_list = []\n",
+    "    if isinstance(cols, list):\n",
+    "        col_name = cols[0]\n",
+    "        dtypes_list = df.select(*cols).dtypes\n",
+    "    elif isinstance(cols, str):\n",
+    "        col_name = cols\n",
+    "        dtypes_list = df.select(cols).dtypes\n",
+    "    else:\n",
+    "        raise ValueError(\"save_to_voc expects cols as String or list of String\")\n",
+    "\n",
+    "    to_select = []\n",
+    "    if len(dtypes_list) == 1 and 'array' not in dtypes_list[0][1]:\n",
+    "        to_select.append(f.col(dtypes_list[0][0]))\n",
+    "        dict_df = df.select(*to_select)\n",
+    "    else:\n",
+    "        for name, dtype in dtypes_list:\n",
+    "            if 'array' in dtype:\n",
+    "                to_select.append(f.col(name))\n",
+    "            else:\n",
+    "                to_select.append(f.array(f.col(name)))\n",
+    "\n",
+    "        dict_df = df.withColumn(col_name, f.array_union(*to_select))\n",
+    "        dict_df = dict_df.select(f.explode(f.col(col_name)).alias(col_name))\n",
+    "    if fmt == 'pkl':\n",
+    "        dict_df = dict_df.filter(f\"{col_name} is not null\").groupBy(col_name).count().orderBy(f.desc('count')).select(col_name)\n",
+    "        collected = [row[col_name] for row in dict_df.collect()]\n",
+    "\n",
+    "        voc = {}\n",
+    "        voc[default_name] = default_v\n",
+    "        voc.update(dict((col_id, col_idx) for (col_id, col_idx) in zip(collected, range(1, len(collected) + 1))))\n",
+    "        pickle.dump(voc, open(proc.current_path + f'/{output_name}.pkl', \"wb\"), protocol=0)\n",
+    "    else:\n",
+    "        op_gen_dict = GenerateDictionary([f\"{col_name}\"])\n",
+    "        proc.reset_ops([op_gen_dict])\n",
+    "        dict_dfs = proc.generate_dicts(dict_df)\n",
+    "        dict_df = dict_dfs[0]['dict']\n",
+    "        if fmt == 'adv_pkl':\n",
+    "            voc_count = dict_df.count()\n",
+    "            pickle.dump(voc_count, open(proc.current_path + f'/{output_name}.pkl', \"wb\"), protocol=0)\n",
+    "\n",
+    "    return dict_df\n",
+    "\n",
+    "\n",
+    "def save_to_uid_voc(df, proc, fmt = 'pkl'):\n",
+    "    # saving (using python)\n",
+    "    # build uid_dict, mid_dict and cat_dict\n",
+    "    t1 = timer()\n",
+    "    dict_df = save_to_voc(df, proc, fmt, ['reviewer_id'], 'A1Y6U82N6TYZPI', 0, 'uid_voc')\n",
+    "    t2 = timer()\n",
+    "    print(f\"save_to_uid_voc took {t2 - t1} secs\")\n",
+    "    return dict_df\n",
+    "    \n",
+    "\n",
+    "def save_to_mid_voc(df, proc, fmt = 'pkl'):\n",
+    "    t1 = timer()\n",
+    "    dict_df = save_to_voc(df, proc, fmt, ['hist_asin', 'asin'], 'default_mid', 0, 'mid_voc')\n",
+    "    t2 = timer()\n",
+    "    print(f\"save_to_mid_voc took {t2 - t1} secs\")\n",
+    "    return dict_df\n",
+    "    \n",
+    "    \n",
+    "def save_to_cat_voc(df, proc, fmt = 'pkl'):\n",
+    "    t1 = timer()\n",
+    "    dict_df = save_to_voc(df, proc, fmt, ['hist_category', 'category'], 'default_cat', 0, 'cat_voc')\n",
+    "    t2 = timer()\n",
+    "    print(f\"save_to_cat_voc took {t2 - t1} secs\")\n",
+    "    return dict_df\n",
+    "    \n",
+    "    \n",
+    "def save_to_local_splitByUser(df, proc, output_name):\n",
+    "    t1 = timer()\n",
+    "    if 'noclk_hist_asin' in df.columns:\n",
+    "        dict_df = df.select('pos', 'reviewer_id', 'asin', 'category',\n",
+    "                            f.expr(\"concat_ws('\\x02', hist_asin)\"),\n",
+    "                            f.expr(\"concat_ws('\\x02', hist_category)\"),\n",
+    "                            f.expr(\"concat_ws('\\x02', noclk_hist_asin)\"),\n",
+    "                            f.expr(\"concat_ws('\\x02', noclk_hist_category)\"))\n",
+    "        collected = [[c1, c2, c3, c4, c5, c6, c7, c8] for (c1, c2, c3, c4, c5, c6, c7, c8) in dict_df.collect()]\n",
+    "    else:\n",
+    "        dict_df = df.select('pos', 'reviewer_id', 'asin', 'category',\n",
+    "                            f.expr(\"concat_ws('\\x02', hist_asin)\"),\n",
+    "                            f.expr(\"concat_ws('\\x02', hist_category)\"))\n",
+    "        collected = [[c1, c2, c3, c4, c5, c6] for (c1, c2, c3, c4, c5, c6) in dict_df.collect()]\n",
+    "    user_map = {}\n",
+    "    for items in collected:\n",
+    "        if items[1] not in user_map:\n",
+    "            user_map[items[1]] = []\n",
+    "        user_map[items[1]].append(items)\n",
+    "    with open(proc.current_path + f\"/{output_name}\", 'w') as fp:\n",
+    "        for user, r in user_map.items():\n",
+    "            positive_sorted = sorted(r, key=lambda x: x[0])\n",
+    "            for items in positive_sorted:\n",
+    "                print('\\t'.join([str(x) for x in items]), file=fp)\n",
+    "\n",
+    "    t2 = timer()\n",
+    "    print(f\"save_to_local_splitByUser took {t2 - t1} secs\")\n",
+    "    return dict_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "option = '--basic'\n",
+    "\n",
+    "if option == '--basic':\n",
+    "    fmt = 'pkl'\n",
+    "elif option == '--advanced':\n",
+    "    fmt = 'adv_pkl'\n",
+    "else:\n",
+    "    raise NotImplementedError(f'{option} is not recognized.')\n",
+    "\n",
+    "path_prefix = \"file://\"\n",
+    "current_path = \"/home/vmagent/app/recdp/examples/python_tests/dien/output/\"\n",
+    "original_folder = \"/home/vmagent/app/recdp/examples/python_tests/dien/j2c_test/\"\n",
+    "scala_udf_jars = \"/home/vmagent/app/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar\"\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "t0 = timer()\n",
+    "spark = SparkSession.builder.master('local[104]')\\\n",
+    "        .appName(\"dien_data_process\")\\\n",
+    "        .config(\"spark.driver.memory\", \"480G\")\\\n",
+    "        .config(\"spark.driver.memoryOverhead\", \"20G\")\\\n",
+    "        .config(\"spark.executor.cores\", \"104\")\\\n",
+    "        .config(\"spark.driver.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "        .getOrCreate()\n",
+    "spark.sparkContext.setLogLevel(\"ERROR\")\n",
+    "    \n",
+    "# 1.1 prepare dataFrames\n",
+    "# 1.2 create RecDP DataProcessor\n",
+    "proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity=\"1200GB\", spark_mode='local')\n",
+    "\n",
+    "# ===============================================\n",
+    "# basic: Do categorify for all columns\n",
+    "reviews_info_df, item_info_df = load_csv(spark, path_prefix + original_folder)\n",
+    "\n",
+    "# 1. join records with its category and then collapse history \n",
+    "df = reviews_info_df\n",
+    "dict_dfs = get_dict_for_asin(df, proc)\n",
+    "df = collapse_by_hist(df, item_info_df, proc, \"collapsed\", min_num_hist = 2, max_num_hist = 100)\n",
+    "\n",
+    "# 2. add negative sample to records\n",
+    "df = add_negative_sample(df, item_info_df, dict_dfs, proc, \"records_with_negative_sample\")\n",
+    "\n",
+    "\n",
+    "if fmt == 'adv_pkl':\n",
+    "    dict_dfs[0]['col_name'] = 'hist_asin'\n",
+    "    df = add_negative_hist_cols(df, item_info_df, dict_dfs, proc, \"records_with_negative_hists\")\n",
+    "\n",
+    "uid_dict_df = save_to_uid_voc(df, proc, fmt = fmt)\n",
+    "mid_dict_df = save_to_mid_voc(df, proc, fmt = fmt)\n",
+    "cat_dict_df = save_to_cat_voc(df, proc, fmt = fmt)\n",
+    "if fmt == 'adv_pkl':\n",
+    "    # for create noclk_hist_category, we should create a dict to mapping from asin to its cat_id\n",
+    "    asin_df = item_info_df.withColumnRenamed('asin', 'dict_col')\n",
+    "    cat_dict_for_merge_df = cat_dict_df.withColumnRenamed('dict_col', 'category')\n",
+    "    op_asin_to_cat_id = ModelMerge([{'col_name': 'category', 'dict': cat_dict_for_merge_df}])\n",
+    "    op_select = SelectFeature(['dict_col', 'dict_col_id', 'count'])\n",
+    "    proc.reset_ops([op_asin_to_cat_id, op_select])\n",
+    "    asin_cat_df = proc.apply(asin_df)\n",
+    "\n",
+    "    df = categorify_dien_data(df, uid_dict_df, mid_dict_df, cat_dict_df, asin_cat_df, proc, \"local_train_splitByUser.parquet\")\n",
+    "    df = save_to_local_splitByUser(df, proc, 'local_train_splitByUser')\n",
+    "\n",
+    "    test_df = load_processed_csv(spark, path_prefix + original_folder + \"/local_test_splitByUser\")\n",
+    "    test_df = add_negative_hist_cols(test_df, item_info_df, dict_dfs, proc, \"test_records_with_negative_hists\")\n",
+    "    test_df = categorify_dien_data(test_df, uid_dict_df, mid_dict_df, cat_dict_df, asin_cat_df, proc, \"local_test_splitByUser.parquet\")\n",
+    "    test_df = save_to_local_splitByUser(test_df, proc, 'local_test_splitByUser')\n",
+    "else:\n",
+    "    train_df = save_to_local_splitByUser(df, proc, 'local_train_splitByUser')\n",
+    "t1 = timer()\n",
+    "\n",
+    "print(f\"Total process time is {(t1 - t0)} secs\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "* recdp-scala-extension is enabled\n",
+    "* per core memory size is 4.500 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+    "* \n",
+    "* Merge with category and collapse hist took 274.15958739898633 secs\n",
+    "* Generate Dictionary took 2.550 secs\n",
+    "* add_negative_sample took 9.529635827988386 secs\n",
+    "* save_to_uid_voc took 10.749223954975605 secs\n",
+    "* save_to_mid_voc took 13.278015322983265 secs\n",
+    "* save_to_cat_voc took 1.4347403410356492 secs\n",
+    "* save_to_local_train_splitByUser took 29.36967344605364 secs"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/notebooks/dien/download_dataset b/RecDP/examples/notebooks/dien/download_dataset
new file mode 100755
index 000000000..4603db2e9
--- /dev/null
+++ b/RecDP/examples/notebooks/dien/download_dataset
@@ -0,0 +1,17 @@
+#!/usr/bin/bash
+
+mkdir raw_data
+
+if [ ! -f "data.tar.gz" ]; then
+        wget https://zenodo.org/record/3463683/files/data.tar.gz
+fi
+
+tar -jxvf data.tar.gz
+mv data/local_test_splitByUser raw_data/
+rm -r data
+
+wget http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/reviews_Books.json.gz
+wget http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/meta_Books.json.gz
+gunzip reviews_Books.json.gz
+gunzip meta_Books.json.gz
+mv *json raw_data
diff --git a/RecDP/examples/notebooks/dien/init.py b/RecDP/examples/notebooks/dien/init.py
new file mode 100644
index 000000000..e0981db4f
--- /dev/null
+++ b/RecDP/examples/notebooks/dien/init.py
@@ -0,0 +1,8 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
+import findspark
+findspark.init()
diff --git a/RecDP/examples/notebooks/dien/json_2_csv_convert.ipynb b/RecDP/examples/notebooks/dien/json_2_csv_convert.ipynb
new file mode 100644
index 000000000..ab39e7a3e
--- /dev/null
+++ b/RecDP/examples/notebooks/dien/json_2_csv_convert.ipynb
@@ -0,0 +1,145 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import init\n",
+    "\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pyspark.sql import *\n",
+    "from pyspark import *\n",
+    "import pyspark.sql.functions as f\n",
+    "from pyspark.sql.types import *\n",
+    "from timeit import default_timer as timer\n",
+    "import logging\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.utils import *\n",
+    "import shutil\n",
+    "\n",
+    "def load_csv(spark, path):\n",
+    "    review_id_field = StructField('reviewerID', StringType())\n",
+    "    asin_field = StructField('asin', StringType())\n",
+    "    overall_field = StructField('overall', FloatType())\n",
+    "    unix_time_field = StructField('unixReviewTime', IntegerType())\n",
+    "    reviews_info_schema = StructType(\n",
+    "        [review_id_field, asin_field, overall_field, unix_time_field])\n",
+    "\n",
+    "    category_field = StructField('categories', StringType())\n",
+    "    item_info_schema = StructType([asin_field, category_field])\n",
+    "\n",
+    "    reviews_info_df = spark.read.schema(reviews_info_schema).option('sep', '\\t').csv(path + \"/reviews-info\")\n",
+    "    item_info_df = spark.read.schema(item_info_schema).option('sep', '\\t').csv(path + \"/item-info\")\n",
+    "\n",
+    "    return reviews_info_df, item_info_df\n",
+    "\n",
+    "def process_meta(file, o_path):\n",
+    "    fi = open(file, \"r\")\n",
+    "    fo = open(\"%s/item-info\" % o_path, \"w\")\n",
+    "    for line in fi:\n",
+    "        obj = eval(line)\n",
+    "        cat = obj[\"categories\"][0][-1]\n",
+    "        print(obj[\"asin\"] + \"\\t\" + cat, file=fo)\n",
+    "\n",
+    "\n",
+    "def list_dir(path):   \n",
+    "    source_path_dict = {}\n",
+    "    dirs = os.listdir(path)\n",
+    "    for files in dirs:\n",
+    "        try:\n",
+    "            sub_dirs = os.listdir(path + \"/\" + files)\n",
+    "            for file_name in sub_dirs:\n",
+    "                if (file_name.endswith('parquet') or file_name.endswith('csv')):\n",
+    "                    source_path_dict[files] = os.path.join(\n",
+    "                        path, files, file_name)\n",
+    "        except:\n",
+    "            source_path_dict[files] = os.path.join(path, files)\n",
+    "    return source_path_dict\n",
+    "\n",
+    "\n",
+    "def result_rename_or_convert(fpath):\n",
+    "    source_path_dict = list_dir(fpath)\n",
+    "    fix = \"-spark\"\n",
+    "    try:\n",
+    "        os.rename(source_path_dict[\"reviews-info\" + fix], fpath + 'reviews-info')\n",
+    "        shutil.rmtree(source_path_dict[\"reviews-info\" + fix], ignore_errors=True)\n",
+    "        # os.rename(source_path_dict[\"item-info\" + fix], fpath + 'item-info')\n",
+    "    except:\n",
+    "        pass\n",
+    "\n",
+    "def compare_with_expected(spark, path, records_df, item_info_df):\n",
+    "    records_expected_df, item_info_expected_df = load_csv(spark, path)\n",
+    "    cmp_res_records = records_expected_df.join(records_df, ['reviewerID', 'asin', 'overall', 'unixReviewTime'], 'anti')\n",
+    "    cmp_res_items = item_info_expected_df.join(item_info_df, ['categories', 'asin'], 'anti')\n",
+    "    error_parsed_records_len = cmp_res_records.count()\n",
+    "    error_parsed_items_len = cmp_res_items.count()\n",
+    "    print(f\"records_df error_parsed_len is {error_parsed_records_len}, example as below:\")\n",
+    "    cmp_res_records.show()\n",
+    "    print(f\"item_info_df error_parsed_len is {error_parsed_items_len}, example as below:\")\n",
+    "    cmp_res_items.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path_prefix = \"file://\"\n",
+    "current_path = \"/home/vmagent/app/recdp/examples/python_tests/dien/output/\"\n",
+    "original_folder = \"/home/vmagent/app/recdp/examples/python_tests/dien/\"\n",
+    "\n",
+    "scala_udf_jars = \"/home/vmagent/app/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar\"\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "t0 = timer()\n",
+    "spark = SparkSession.builder.master('local[104]')\\\n",
+    "    .appName(\"dien_data_ingestion\")\\\n",
+    "    .config(\"spark.driver.memory\", \"480G\")\\\n",
+    "    .config(\"spark.driver.memoryOverhead\", \"20G\")\\\n",
+    "    .config(\"spark.executor.cores\", \"104\")\\\n",
+    "    .config(\"spark.driver.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .getOrCreate()\n",
+    "\n",
+    "# 1.1 prepare dataFrames\n",
+    "# 1.2 create RecDP DataProcessor\n",
+    "proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity=\"1200GB\", spark_mode='local')\n",
+    "\n",
+    "records_df = spark.read.json(\"%s/%s/raw_data/reviews_Books.json\" % (path_prefix, original_folder))\n",
+    "records_df = records_df.select('reviewerID', 'asin', 'overall', 'unixReviewTime')\n",
+    "records_df.repartition(1).write.format(\"csv\").option('sep', '\\t').mode(\"overwrite\").save(\"%s/%s/j2c_test/reviews-info-spark\" % (path_prefix, original_folder))\n",
+    "process_meta('%s/raw_data/meta_Books.json' % original_folder, \"%s/j2c_test\" % original_folder)\n",
+    "result_rename_or_convert(\"%s/j2c_test/\" % (original_folder))\n",
+    "\n",
+    "t1 = timer()\n",
+    "\n",
+    "print(f\"Total process time is {(t1 - t0)} secs\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/notebooks/dlrm/DLRM_Performance-nativesql.ipynb b/RecDP/examples/notebooks/dlrm/DLRM_Performance-nativesql.ipynb
new file mode 100644
index 000000000..a05afe1dd
--- /dev/null
+++ b/RecDP/examples/notebooks/dlrm/DLRM_Performance-nativesql.ipynb
@@ -0,0 +1,1136 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "spark.stop()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## DLRM Categorify"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/mnt/nvme2/chendi/BlueWhale/recdp\n"
+     ]
+    }
+   ],
+   "source": [
+    "###### Start spark job ######\n",
+    "import init\n",
+    "\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.utils import *\n",
+    "\n",
+    "import logging\n",
+    "from timeit import default_timer as timer\n",
+    "import os\n",
+    "from pyspark import *\n",
+    "from pyspark.sql import *\n",
+    "from pyspark.sql.functions import *\n",
+    "from pyspark.sql.types import *\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Define Schema\n",
+    "LABEL_COL = 0\n",
+    "INT_COLS = list(range(1, 14))\n",
+    "CAT_COLS = list(range(14, 40))\n",
+    "label_fields = [StructField('_c%d' % LABEL_COL, IntegerType())]\n",
+    "int_fields = [StructField('_c%d' % i, IntegerType()) for i in INT_COLS]\n",
+    "str_fields = [StructField('_c%d' % i, StringType()) for i in CAT_COLS]\n",
+    "schema = StructType(label_fields + int_fields + str_fields)\n",
+    "\n",
+    "to_be_categorified = [23, 35, 14, 33]\n",
+    "\n",
+    "def categorifyAllFeatures(df, proc, output_name=\"categorified\", gen_dict=False, enable_freqlimit=False):\n",
+    "    dict_dfs = []\n",
+    "    to_categorify_cols = ['_c%d' % i for i in CAT_COLS]\n",
+    "    #to_categorify_cols = ['_c%d' % i for i in to_be_categorified]\n",
+    "    if gen_dict:\n",
+    "        # only call below function when target dicts were not pre-prepared        \n",
+    "        op_gen_dict = GenerateDictionary(to_categorify_cols, isParquet=False)\n",
+    "        proc.reset_ops([op_gen_dict])\n",
+    "        t1 = timer()\n",
+    "        dict_dfs = proc.generate_dicts(df)\n",
+    "        t2 = timer()\n",
+    "        print(\"Generate Dictionary took %.3f\" % (t2 - t1))\n",
+    "    else:\n",
+    "        # or we can simply load from pre-gened\n",
+    "        dict_dfs = [{'col_name': name, 'dict': proc.spark.read.format(\"arrow\").load(\n",
+    "            \"%s/%s/%s/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in to_categorify_cols]    \n",
+    "\n",
+    "    if enable_freqlimit:\n",
+    "        dict_dfs = [{'col_name': dict_df['col_name'], 'dict': dict_df['dict'].filter('count >= 15')} for dict_df in dict_dfs]\n",
+    "    else:\n",
+    "        dict_dfs = [{'col_name': dict_df['col_name'], 'dict': dict_df['dict'].filter('dict_col_id > 0')} for dict_df in dict_dfs]\n",
+    "\n",
+    "    # start to do categorify\n",
+    "    op_categorify = Categorify(to_categorify_cols, dict_dfs=dict_dfs)\n",
+    "    op_fillna_for_categorified = FillNA(to_categorify_cols, 0)\n",
+    "    proc.append_ops([op_categorify, op_fillna_for_categorified])\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name, df_cnt = 4373472329)\n",
+    "    t2 = timer()\n",
+    "    print(\"Categorify took %.3f\" % (t2 - t1))\n",
+    "    \n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "recdp-scala-extension is enabled\n",
+      "per core memory size is 5.500 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+      "do bhj to _c19\n",
+      "do bhj to _c30\n",
+      "do bhj to _c26\n",
+      "do bhj to _c32\n",
+      "do bhj to _c39\n",
+      "do bhj to _c22\n",
+      "do bhj to _c38\n",
+      "do bhj to _c29\n",
+      "do bhj to _c31\n",
+      "do bhj to _c21\n",
+      "do bhj to _c27\n",
+      "do bhj to _c20\n",
+      "do bhj to _c17\n",
+      "do bhj to _c28\n",
+      "do bhj to _c37\n",
+      "do bhj to _c16\n",
+      "do bhj to _c18\n",
+      "do bhj to _c15\n",
+      "do bhj to _c25\n",
+      "do bhj to _c36\n",
+      "do bhj to _c24\n",
+      "do bhj to _c34\n",
+      "do bhj to _c23\n",
+      "do bhj to _c35\n",
+      "do bhj to _c14\n",
+      "do bhj to _c33\n",
+      "Categorify took 1157.936\n"
+     ]
+    }
+   ],
+   "source": [
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/dlrm_nativesql/\"\n",
+    "csv_folder = \"/dlrm/csv_raw_data\"\n",
+    "parquet_folder = \"/dlrm_nativesql/dlrm_parquet\"\n",
+    "#path = os.path.join(path_prefix, file)\n",
+    "\n",
+    "scala_udf_jars = \"/mnt/nvme2/chendi/BlueWhale/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar\"\n",
+    "native_sql_path = \"/mnt/nvme2/chendi/intel-bigdata/OAP/native-sql-engine/native-sql-engine/core/target/spark-columnar-core-1.2.0-snapshot-jar-with-dependencies.jar\"\n",
+    "native_arrow_datasource_path = \"/mnt/nvme2/chendi/intel-bigdata/OAP/native-sql-engine/arrow-data-source/standard/target/spark-arrow-datasource-standard-1.2.0-snapshot-jar-with-dependencies.jar\"\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "t0 = timer()\n",
+    "spark = SparkSession\\\n",
+    "    .builder\\\n",
+    "    .master('yarn')\\\n",
+    "    .appName(\"DLRM_wo_frequence_limit\")\\\n",
+    "    .config(\"spark.executor.memory\", \"6g\")\\\n",
+    "    .config(\"spark.executor.memoryOverhead\", \"16g\")\\\n",
+    "    .config(\"spark.driver.memory\", \"60g\")\\\n",
+    "    .config(\"spark.driver.maxResultSize\", \"16G\")\\\n",
+    "    .config(\"spark.executor.extraJavaOptions\", \"-XX:MaxDirectMemorySize=25G\")\\\n",
+    "    .config(\"spark.memory.offHeap.use\", \"true\")\\\n",
+    "    .config(\"spark.memory.offHeap.size\", \"16G\")\\\n",
+    "    .config(\"spark.executor.cores\", \"4\")\\\n",
+    "    .config(\"spark.cleaner.periodicGC.interval\", \"10min\")\\\n",
+    "    .config(\"spark.driver.extraClassPath\", f\"{scala_udf_jars}:{native_sql_path}:{native_arrow_datasource_path}\")\\\n",
+    "    .config(\"spark.executor.extraClassPath\", f\"{scala_udf_jars}:{native_sql_path}:{native_arrow_datasource_path}\")\\\n",
+    "    .config(\"spark.executorEnv.LD_LIBRARY_PATH\", \"/usr/local/lib64/\")\\\n",
+    "    .config(\"spark.sql.extensions\", \"com.intel.oap.ColumnarPlugin, com.intel.oap.spark.sql.ArrowWriteExtension\")\\\n",
+    "    .config(\"spark.shuffle.manager\", \"org.apache.spark.shuffle.sort.ColumnarShuffleManager\")\\\n",
+    "    .config(\"spark.oap.sql.columnar.arrowudf\", \"true\")\\\n",
+    "    .config(\"spark.sql.adaptive.enabled\", \"false\")\\\n",
+    "    .getOrCreate()\n",
+    "\n",
+    "files = [\"day_%d\" % i for i in range(0, 24)]\n",
+    "#files = [\"day_0\"]\n",
+    "file_names = [os.path.join(path_prefix, parquet_folder, filename) for filename in files]\n",
+    "\n",
+    "proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity=\"1200GB\", enable_gazelle=True)\n",
+    "#df = spark.read.format(\"arrow\").option(\"originalFormat\", \"csv\").option('sep', '\\t').load(file_names)\n",
+    "df = spark.read.format(\"arrow\").load(file_names)\n",
+    "df = categorifyAllFeatures(df, proc, output_name=\"dlrm_categorified\", gen_dict=False, enable_freqlimit=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "-RECORD 0--------\n",
+      " _c0  | 0        \n",
+      " _c1  | 31       \n",
+      " _c2  | 70       \n",
+      " _c3  | 47       \n",
+      " _c4  | 47       \n",
+      " _c5  | null     \n",
+      " _c6  | 5        \n",
+      " _c7  | 0        \n",
+      " _c8  | 0        \n",
+      " _c9  | 44       \n",
+      " _c10 | 1        \n",
+      " _c11 | null     \n",
+      " _c12 | 10       \n",
+      " _c13 | 47       \n",
+      " _c14 | 6f341f8a \n",
+      " _c15 | 000877b3 \n",
+      " _c16 | ae0e09c7 \n",
+      " _c17 | 5e9ea730 \n",
+      " _c18 | 1315f676 \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 66dfe839 \n",
+      " _c21 | 59903d33 \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | 61aa5d8e \n",
+      " _c24 | b0464cac \n",
+      " _c25 | 12716184 \n",
+      " _c26 | a77a4a56 \n",
+      " _c27 | afed79a4 \n",
+      " _c28 | b867d0bc \n",
+      " _c29 | 83f41762 \n",
+      " _c30 | 1f7fc70b \n",
+      " _c31 | d9f758ff \n",
+      " _c32 | 1889c49d \n",
+      " _c33 | d2611cf6 \n",
+      " _c34 | ac6f605a \n",
+      " _c35 | cfa55246 \n",
+      " _c36 | 1bbb04d6 \n",
+      " _c37 | 64332f55 \n",
+      " _c38 | ff654802 \n",
+      " _c39 | ed10571d \n",
+      "-RECORD 1--------\n",
+      " _c0  | 0        \n",
+      " _c1  | 5        \n",
+      " _c2  | null     \n",
+      " _c3  | null     \n",
+      " _c4  | null     \n",
+      " _c5  | null     \n",
+      " _c6  | null     \n",
+      " _c7  | null     \n",
+      " _c8  | 2        \n",
+      " _c9  | 0        \n",
+      " _c10 | null     \n",
+      " _c11 | null     \n",
+      " _c12 | null     \n",
+      " _c13 | null     \n",
+      " _c14 | 8ae77c56 \n",
+      " _c15 | 4fb7dc21 \n",
+      " _c16 | 10448b24 \n",
+      " _c17 | 6521d620 \n",
+      " _c18 | 1b892b7d \n",
+      " _c19 | 919c68e4 \n",
+      " _c20 | 5e6216de \n",
+      " _c21 | 21aa6787 \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | ae9f2761 \n",
+      " _c24 | 0447d6af \n",
+      " _c25 | 12716184 \n",
+      " _c26 | 6397311a \n",
+      " _c27 | be4ee537 \n",
+      " _c28 | c90efd7d \n",
+      " _c29 | 4cdc3efa \n",
+      " _c30 | d20856aa \n",
+      " _c31 | 7232d217 \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | 49c539f2 \n",
+      " _c34 | 1e0728aa \n",
+      " _c35 | cd34fc44 \n",
+      " _c36 | 2415abbe \n",
+      " _c37 | b3116c88 \n",
+      " _c38 | ff654802 \n",
+      " _c39 | 2ccea557 \n",
+      "-RECORD 2--------\n",
+      " _c0  | 0        \n",
+      " _c1  | 1        \n",
+      " _c2  | 854      \n",
+      " _c3  | 7        \n",
+      " _c4  | 172      \n",
+      " _c5  | 19       \n",
+      " _c6  | 0        \n",
+      " _c7  | 0        \n",
+      " _c8  | 184      \n",
+      " _c9  | 43       \n",
+      " _c10 | 0        \n",
+      " _c11 | 7        \n",
+      " _c12 | 531      \n",
+      " _c13 | 8        \n",
+      " _c14 | 9f2b1967 \n",
+      " _c15 | 5391300e \n",
+      " _c16 | bafcb47b \n",
+      " _c17 | e3150ee2 \n",
+      " _c18 | ce5b610b \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | d935d694 \n",
+      " _c21 | 038e402d \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | b7cbb5a4 \n",
+      " _c24 | 0bd5c94b \n",
+      " _c25 | 8c1feb42 \n",
+      " _c26 | a77a4a56 \n",
+      " _c27 | null     \n",
+      " _c28 | fff9eb21 \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | a1eb1511 \n",
+      " _c32 | 34879027 \n",
+      " _c33 | df681ec4 \n",
+      " _c34 | 6fbb0b21 \n",
+      " _c35 | c792887d \n",
+      " _c36 | null     \n",
+      " _c37 | 636c3c07 \n",
+      " _c38 | d5578a54 \n",
+      " _c39 | 962813c6 \n",
+      "-RECORD 3--------\n",
+      " _c0  | 0        \n",
+      " _c1  | 1        \n",
+      " _c2  | 907      \n",
+      " _c3  | 1        \n",
+      " _c4  | 1        \n",
+      " _c5  | 3        \n",
+      " _c6  | 1        \n",
+      " _c7  | 0        \n",
+      " _c8  | 283      \n",
+      " _c9  | 6        \n",
+      " _c10 | 1        \n",
+      " _c11 | 1        \n",
+      " _c12 | 512      \n",
+      " _c13 | 1        \n",
+      " _c14 | cc78cbfd \n",
+      " _c15 | 3f269ed4 \n",
+      " _c16 | a52db7de \n",
+      " _c17 | f2463ffb \n",
+      " _c18 | 664ff944 \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 0ba0a6d5 \n",
+      " _c21 | 2699ebb0 \n",
+      " _c22 | e25a4c11 \n",
+      " _c23 | 2d3dd5b6 \n",
+      " _c24 | 4e36298d \n",
+      " _c25 | 0b4ca1bb \n",
+      " _c26 | a77a4a56 \n",
+      " _c27 | null     \n",
+      " _c28 | f6d5a217 \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | aeea8379 \n",
+      " _c34 | 83792acc \n",
+      " _c35 | 8ff3921c \n",
+      " _c36 | null     \n",
+      " _c37 | cd3fd35f \n",
+      " _c38 | 991321ea \n",
+      " _c39 | b757e957 \n",
+      "-RECORD 4--------\n",
+      " _c0  | 0        \n",
+      " _c1  | 20       \n",
+      " _c2  | 509      \n",
+      " _c3  | 3        \n",
+      " _c4  | null     \n",
+      " _c5  | null     \n",
+      " _c6  | 0        \n",
+      " _c7  | 0        \n",
+      " _c8  | 1        \n",
+      " _c9  | 3        \n",
+      " _c10 | 0        \n",
+      " _c11 | null     \n",
+      " _c12 | 2        \n",
+      " _c13 | 3        \n",
+      " _c14 | cb8850c0 \n",
+      " _c15 | 365d9e66 \n",
+      " _c16 | 6f49d5b1 \n",
+      " _c17 | f2463ffb \n",
+      " _c18 | c6fc10d3 \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 4c6fab27 \n",
+      " _c21 | f25f9a3a \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | 308f8759 \n",
+      " _c24 | 8f45b485 \n",
+      " _c25 | 067ba591 \n",
+      " _c26 | 405ca118 \n",
+      " _c27 | null     \n",
+      " _c28 | cb81aca6 \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | d2f45646 \n",
+      " _c34 | 8d1bbadb \n",
+      " _c35 | d7a9a047 \n",
+      " _c36 | null     \n",
+      " _c37 | a9e944a7 \n",
+      " _c38 | 991321ea \n",
+      " _c39 | 2ccea557 \n",
+      "-RECORD 5--------\n",
+      " _c0  | 0        \n",
+      " _c1  | 11       \n",
+      " _c2  | 1725     \n",
+      " _c3  | null     \n",
+      " _c4  | 835      \n",
+      " _c5  | 76       \n",
+      " _c6  | 0        \n",
+      " _c7  | 0        \n",
+      " _c8  | 2        \n",
+      " _c9  | 2        \n",
+      " _c10 | 0        \n",
+      " _c11 | 6        \n",
+      " _c12 | 4021     \n",
+      " _c13 | null     \n",
+      " _c14 | 265366bf \n",
+      " _c15 | 20fd45c1 \n",
+      " _c16 | 676ca702 \n",
+      " _c17 | 80b7260b \n",
+      " _c18 | 4f59463a \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | bde8e288 \n",
+      " _c21 | ab96c6b2 \n",
+      " _c22 | 7cee8453 \n",
+      " _c23 | e63d98b4 \n",
+      " _c24 | c939136f \n",
+      " _c25 | 34b40558 \n",
+      " _c26 | a77a4a56 \n",
+      " _c27 | null     \n",
+      " _c28 | 9f42eb42 \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | 47849e55 \n",
+      " _c34 | 73b3f46d \n",
+      " _c35 | d994ba60 \n",
+      " _c36 | null     \n",
+      " _c37 | 26689d6c \n",
+      " _c38 | ff654802 \n",
+      " _c39 | b757e957 \n",
+      "-RECORD 6--------\n",
+      " _c0  | 0        \n",
+      " _c1  | 13       \n",
+      " _c2  | 315      \n",
+      " _c3  | 4        \n",
+      " _c4  | 32       \n",
+      " _c5  | 11       \n",
+      " _c6  | 1        \n",
+      " _c7  | 0        \n",
+      " _c8  | 30       \n",
+      " _c9  | 21       \n",
+      " _c10 | 1        \n",
+      " _c11 | 9        \n",
+      " _c12 | 387      \n",
+      " _c13 | 32       \n",
+      " _c14 | 1a6c0e94 \n",
+      " _c15 | bac367d1 \n",
+      " _c16 | 593d32dc \n",
+      " _c17 | 3f85ecae \n",
+      " _c18 | 0801cd98 \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | c0b58ca1 \n",
+      " _c21 | 5b88fd2a \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | 93928b8e \n",
+      " _c24 | 750506a2 \n",
+      " _c25 | 04048bcc \n",
+      " _c26 | a77a4a56 \n",
+      " _c27 | be4ee537 \n",
+      " _c28 | 9541fe96 \n",
+      " _c29 | 4cdc3efa \n",
+      " _c30 | d20856aa \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | 1e0fd6f1 \n",
+      " _c34 | 35d12d61 \n",
+      " _c35 | 182d3c23 \n",
+      " _c36 | 32ddcfd7 \n",
+      " _c37 | 2ff98b0c \n",
+      " _c38 | 991321ea \n",
+      " _c39 | ed10571d \n",
+      "-RECORD 7--------\n",
+      " _c0  | 0        \n",
+      " _c1  | 3        \n",
+      " _c2  | 608      \n",
+      " _c3  | 6        \n",
+      " _c4  | 32       \n",
+      " _c5  | null     \n",
+      " _c6  | 1        \n",
+      " _c7  | 0        \n",
+      " _c8  | 1447     \n",
+      " _c9  | 32       \n",
+      " _c10 | 1        \n",
+      " _c11 | null     \n",
+      " _c12 | 187      \n",
+      " _c13 | 6        \n",
+      " _c14 | 265366bf \n",
+      " _c15 | db8548d7 \n",
+      " _c16 | 94c145d7 \n",
+      " _c17 | f2463ffb \n",
+      " _c18 | b196af28 \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 93d3654d \n",
+      " _c21 | ab96c6b2 \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | e63d98b4 \n",
+      " _c24 | c939136f \n",
+      " _c25 | d07399fc \n",
+      " _c26 | a77a4a56 \n",
+      " _c27 | null     \n",
+      " _c28 | dd7f4897 \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 108a0699 \n",
+      " _c33 | 47849e55 \n",
+      " _c34 | 73b3f46d \n",
+      " _c35 | d994ba60 \n",
+      " _c36 | null     \n",
+      " _c37 | c1f4d749 \n",
+      " _c38 | ff654802 \n",
+      " _c39 | 2ba8d787 \n",
+      "-RECORD 8--------\n",
+      " _c0  | 0        \n",
+      " _c1  | 1        \n",
+      " _c2  | 101      \n",
+      " _c3  | 3        \n",
+      " _c4  | 3        \n",
+      " _c5  | 2        \n",
+      " _c6  | 0        \n",
+      " _c7  | 0        \n",
+      " _c8  | 38       \n",
+      " _c9  | 4        \n",
+      " _c10 | 0        \n",
+      " _c11 | 2        \n",
+      " _c12 | 7041     \n",
+      " _c13 | 3        \n",
+      " _c14 | 733da960 \n",
+      " _c15 | 15e3350d \n",
+      " _c16 | 22d4f667 \n",
+      " _c17 | 6faef306 \n",
+      " _c18 | 4ff7b61d \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | cc649556 \n",
+      " _c21 | e1908b8d \n",
+      " _c22 | 46dedfa6 \n",
+      " _c23 | 55a8e1a9 \n",
+      " _c24 | 1b4ee2a7 \n",
+      " _c25 | 524821ab \n",
+      " _c26 | bf78d0d4 \n",
+      " _c27 | be4ee537 \n",
+      " _c28 | 9bfa8220 \n",
+      " _c29 | 4cdc3efa \n",
+      " _c30 | 1f7fc70b \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | 66b2fc1d \n",
+      " _c34 | e74c2d19 \n",
+      " _c35 | abd790f5 \n",
+      " _c36 | a0d71226 \n",
+      " _c37 | b40e8c15 \n",
+      " _c38 | 991321ea \n",
+      " _c39 | b757e957 \n",
+      "-RECORD 9--------\n",
+      " _c0  | 0        \n",
+      " _c1  | 3        \n",
+      " _c2  | 703      \n",
+      " _c3  | 0        \n",
+      " _c4  | 100      \n",
+      " _c5  | 2        \n",
+      " _c6  | 1        \n",
+      " _c7  | 0        \n",
+      " _c8  | 7        \n",
+      " _c9  | 47       \n",
+      " _c10 | 1        \n",
+      " _c11 | 2        \n",
+      " _c12 | 465      \n",
+      " _c13 | 38       \n",
+      " _c14 | 265366bf \n",
+      " _c15 | b1feb7c7 \n",
+      " _c16 | e668e35c \n",
+      " _c17 | f2463ffb \n",
+      " _c18 | c6fc10d3 \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 12d2971b \n",
+      " _c21 | ab96c6b2 \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | e63d98b4 \n",
+      " _c24 | c939136f \n",
+      " _c25 | 73be1f84 \n",
+      " _c26 | a77a4a56 \n",
+      " _c27 | null     \n",
+      " _c28 | 5cbc7f6a \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 108a0699 \n",
+      " _c33 | 47849e55 \n",
+      " _c34 | 73b3f46d \n",
+      " _c35 | d994ba60 \n",
+      " _c36 | null     \n",
+      " _c37 | fab13581 \n",
+      " _c38 | 991321ea \n",
+      " _c39 | 2ba8d787 \n",
+      "-RECORD 10-------\n",
+      " _c0  | 0        \n",
+      " _c1  | 3        \n",
+      " _c2  | 102      \n",
+      " _c3  | null     \n",
+      " _c4  | null     \n",
+      " _c5  | null     \n",
+      " _c6  | 0        \n",
+      " _c7  | 0        \n",
+      " _c8  | 1        \n",
+      " _c9  | 3        \n",
+      " _c10 | 0        \n",
+      " _c11 | null     \n",
+      " _c12 | 11260    \n",
+      " _c13 | null     \n",
+      " _c14 | 6b0d5f8c \n",
+      " _c15 | a91214a3 \n",
+      " _c16 | f01c55b2 \n",
+      " _c17 | e6b5b2ee \n",
+      " _c18 | c6fc10d3 \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 1b066087 \n",
+      " _c21 | 8f551379 \n",
+      " _c22 | 54fc547f \n",
+      " _c23 | 541b7eda \n",
+      " _c24 | 57606b4c \n",
+      " _c25 | 12716184 \n",
+      " _c26 | e07cc34d \n",
+      " _c27 | 814b2075 \n",
+      " _c28 | eea7f013 \n",
+      " _c29 | 2514f91d \n",
+      " _c30 | 753da5f3 \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | 4e4d8890 \n",
+      " _c34 | c298a8bf \n",
+      " _c35 | 06ddca3e \n",
+      " _c36 | 1e64bf16 \n",
+      " _c37 | 609998d1 \n",
+      " _c38 | 337b81aa \n",
+      " _c39 | 962813c6 \n",
+      "-RECORD 11-------\n",
+      " _c0  | 0        \n",
+      " _c1  | null     \n",
+      " _c2  | 143      \n",
+      " _c3  | null     \n",
+      " _c4  | 151      \n",
+      " _c5  | null     \n",
+      " _c6  | null     \n",
+      " _c7  | 0        \n",
+      " _c8  | -1       \n",
+      " _c9  | 0        \n",
+      " _c10 | null     \n",
+      " _c11 | null     \n",
+      " _c12 | 12405    \n",
+      " _c13 | null     \n",
+      " _c14 | 9bfd9a76 \n",
+      " _c15 | eeee1e9f \n",
+      " _c16 | 5cbd1d25 \n",
+      " _c17 | 6521d620 \n",
+      " _c18 | 3a57336e \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 4f3aef12 \n",
+      " _c21 | 775cf4c3 \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | b5d95a08 \n",
+      " _c24 | 38a9b7e4 \n",
+      " _c25 | 7c8554e3 \n",
+      " _c26 | a77a4a56 \n",
+      " _c27 | null     \n",
+      " _c28 | 1ee2fe9b \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | 628f1b8d \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | 4fce9c23 \n",
+      " _c34 | 63cd17c8 \n",
+      " _c35 | f6a4e544 \n",
+      " _c36 | null     \n",
+      " _c37 | 54b00be7 \n",
+      " _c38 | 991321ea \n",
+      " _c39 | b757e957 \n",
+      "-RECORD 12-------\n",
+      " _c0  | 0        \n",
+      " _c1  | 31       \n",
+      " _c2  | null     \n",
+      " _c3  | 13       \n",
+      " _c4  | null     \n",
+      " _c5  | null     \n",
+      " _c6  | 0        \n",
+      " _c7  | 0        \n",
+      " _c8  | 0        \n",
+      " _c9  | 13       \n",
+      " _c10 | 0        \n",
+      " _c11 | null     \n",
+      " _c12 | 4        \n",
+      " _c13 | 13       \n",
+      " _c14 | aa674078 \n",
+      " _c15 | 9a6c1869 \n",
+      " _c16 | 00d3882a \n",
+      " _c17 | f2463ffb \n",
+      " _c18 | 04a824be \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | beaac782 \n",
+      " _c21 | 17ae60ec \n",
+      " _c22 | e0d906cb \n",
+      " _c23 | b6824f8e \n",
+      " _c24 | db2aa19d \n",
+      " _c25 | 12716184 \n",
+      " _c26 | e07cc34d \n",
+      " _c27 | be4ee537 \n",
+      " _c28 | fe7e6ccb \n",
+      " _c29 | 4cdc3efa \n",
+      " _c30 | d20856aa \n",
+      " _c31 | a1eb1511 \n",
+      " _c32 | 0a1d513f \n",
+      " _c33 | 3b59136e \n",
+      " _c34 | 7cd8a300 \n",
+      " _c35 | 7e4e37ec \n",
+      " _c36 | f01bf1f8 \n",
+      " _c37 | 00c44a0c \n",
+      " _c38 | 57e36578 \n",
+      " _c39 | b757e957 \n",
+      "-RECORD 13-------\n",
+      " _c0  | 0        \n",
+      " _c1  | 5        \n",
+      " _c2  | 1143     \n",
+      " _c3  | 15       \n",
+      " _c4  | 52       \n",
+      " _c5  | 10       \n",
+      " _c6  | 1        \n",
+      " _c7  | 0        \n",
+      " _c8  | 8        \n",
+      " _c9  | 21       \n",
+      " _c10 | 1        \n",
+      " _c11 | 8        \n",
+      " _c12 | 1339     \n",
+      " _c13 | 15       \n",
+      " _c14 | 9004d890 \n",
+      " _c15 | df1b84fe \n",
+      " _c16 | 5cfbbadd \n",
+      " _c17 | f2463ffb \n",
+      " _c18 | 8884f6ae \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 660e2e35 \n",
+      " _c21 | 8670cd10 \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | 4ecd1b37 \n",
+      " _c24 | ed0ce343 \n",
+      " _c25 | 04048bcc \n",
+      " _c26 | a77a4a56 \n",
+      " _c27 | null     \n",
+      " _c28 | d96198f0 \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | c599b801 \n",
+      " _c34 | 7e98b867 \n",
+      " _c35 | e7220716 \n",
+      " _c36 | null     \n",
+      " _c37 | 4d260962 \n",
+      " _c38 | ff654802 \n",
+      " _c39 | b757e957 \n",
+      "-RECORD 14-------\n",
+      " _c0  | 0        \n",
+      " _c1  | null     \n",
+      " _c2  | 191      \n",
+      " _c3  | null     \n",
+      " _c4  | null     \n",
+      " _c5  | null     \n",
+      " _c6  | null     \n",
+      " _c7  | 0        \n",
+      " _c8  | 1        \n",
+      " _c9  | 0        \n",
+      " _c10 | null     \n",
+      " _c11 | null     \n",
+      " _c12 | 331181   \n",
+      " _c13 | null     \n",
+      " _c14 | 265366bf \n",
+      " _c15 | 20fd45c1 \n",
+      " _c16 | 84a4f066 \n",
+      " _c17 | f2463ffb \n",
+      " _c18 | 6ca919cf \n",
+      " _c19 | 919c68e4 \n",
+      " _c20 | 2f45348c \n",
+      " _c21 | ab96c6b2 \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | e63d98b4 \n",
+      " _c24 | c939136f \n",
+      " _c25 | 6b362a22 \n",
+      " _c26 | 00c5ffb7 \n",
+      " _c27 | null     \n",
+      " _c28 | 9f42eb42 \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | 7232d217 \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | 47849e55 \n",
+      " _c34 | 73b3f46d \n",
+      " _c35 | d994ba60 \n",
+      " _c36 | null     \n",
+      " _c37 | 5311c994 \n",
+      " _c38 | ff654802 \n",
+      " _c39 | b757e957 \n",
+      "-RECORD 15-------\n",
+      " _c0  | 0        \n",
+      " _c1  | 2        \n",
+      " _c2  | 127      \n",
+      " _c3  | 2        \n",
+      " _c4  | 20       \n",
+      " _c5  | 1        \n",
+      " _c6  | 0        \n",
+      " _c7  | 0        \n",
+      " _c8  | 0        \n",
+      " _c9  | 8        \n",
+      " _c10 | 0        \n",
+      " _c11 | 1        \n",
+      " _c12 | 9970     \n",
+      " _c13 | 2        \n",
+      " _c14 | 1785706a \n",
+      " _c15 | 9dc0382f \n",
+      " _c16 | 6d58a594 \n",
+      " _c17 | f2463ffb \n",
+      " _c18 | 729e35ab \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | eb970b33 \n",
+      " _c21 | 8d84792c \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | 834c90fd \n",
+      " _c24 | bfe0118a \n",
+      " _c25 | 95facdf2 \n",
+      " _c26 | 00c5ffb7 \n",
+      " _c27 | 8f3bcfaf \n",
+      " _c28 | 0ef5cd17 \n",
+      " _c29 | 10dd3744 \n",
+      " _c30 | 1f7fc70b \n",
+      " _c31 | a1eb1511 \n",
+      " _c32 | 34879027 \n",
+      " _c33 | 059f37a6 \n",
+      " _c34 | 8e997398 \n",
+      " _c35 | ee9ca70c \n",
+      " _c36 | 8a4f79ee \n",
+      " _c37 | 8e953d9f \n",
+      " _c38 | 991321ea \n",
+      " _c39 | ed10571d \n",
+      "-RECORD 16-------\n",
+      " _c0  | 0        \n",
+      " _c1  | null     \n",
+      " _c2  | 1        \n",
+      " _c3  | null     \n",
+      " _c4  | 102      \n",
+      " _c5  | 10       \n",
+      " _c6  | null     \n",
+      " _c7  | 0        \n",
+      " _c8  | -1       \n",
+      " _c9  | 0        \n",
+      " _c10 | null     \n",
+      " _c11 | 1        \n",
+      " _c12 | 39816    \n",
+      " _c13 | null     \n",
+      " _c14 | 8dbec555 \n",
+      " _c15 | f260029e \n",
+      " _c16 | 8d2214b5 \n",
+      " _c17 | f2463ffb \n",
+      " _c18 | acf676aa \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 4c95fdce \n",
+      " _c21 | 73175208 \n",
+      " _c22 | 25dd8f9a \n",
+      " _c23 | 6bc41856 \n",
+      " _c24 | 17090bdd \n",
+      " _c25 | 12716184 \n",
+      " _c26 | a77a4a56 \n",
+      " _c27 | null     \n",
+      " _c28 | 4fef10d6 \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | a87307e5 \n",
+      " _c34 | fcfdafc7 \n",
+      " _c35 | d6e6670e \n",
+      " _c36 | null     \n",
+      " _c37 | 2405ffd1 \n",
+      " _c38 | 321935cd \n",
+      " _c39 | 2ccea557 \n",
+      "-RECORD 17-------\n",
+      " _c0  | 0        \n",
+      " _c1  | null     \n",
+      " _c2  | 280      \n",
+      " _c3  | null     \n",
+      " _c4  | null     \n",
+      " _c5  | null     \n",
+      " _c6  | 0        \n",
+      " _c7  | 0        \n",
+      " _c8  | -1       \n",
+      " _c9  | 3        \n",
+      " _c10 | 0        \n",
+      " _c11 | null     \n",
+      " _c12 | null     \n",
+      " _c13 | null     \n",
+      " _c14 | null     \n",
+      " _c15 | 148a6bdc \n",
+      " _c16 | f4ed0aaf \n",
+      " _c17 | c9dd6f0a \n",
+      " _c18 | c6fc10d3 \n",
+      " _c19 | 919c68e4 \n",
+      " _c20 | 2ce8fb03 \n",
+      " _c21 | 015442fe \n",
+      " _c22 | 25dd8f9a \n",
+      " _c23 | null     \n",
+      " _c24 | null     \n",
+      " _c25 | 12716184 \n",
+      " _c26 | 6397311a \n",
+      " _c27 | null     \n",
+      " _c28 | ba77787b \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | b8170bba \n",
+      " _c32 | 9512c20b \n",
+      " _c33 | null     \n",
+      " _c34 | null     \n",
+      " _c35 | null     \n",
+      " _c36 | null     \n",
+      " _c37 | 82f9a941 \n",
+      " _c38 | 321935cd \n",
+      " _c39 | 6c730e3e \n",
+      "-RECORD 18-------\n",
+      " _c0  | 0        \n",
+      " _c1  | 1        \n",
+      " _c2  | 31       \n",
+      " _c3  | null     \n",
+      " _c4  | 38       \n",
+      " _c5  | 2        \n",
+      " _c6  | null     \n",
+      " _c7  | 0        \n",
+      " _c8  | 2        \n",
+      " _c9  | 0        \n",
+      " _c10 | null     \n",
+      " _c11 | 1        \n",
+      " _c12 | 12265    \n",
+      " _c13 | null     \n",
+      " _c14 | c364de43 \n",
+      " _c15 | 91f3c5ec \n",
+      " _c16 | 4ae63b23 \n",
+      " _c17 | 6faef306 \n",
+      " _c18 | 664ff944 \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 5ad43e9c \n",
+      " _c21 | 775cf4c3 \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | 34ede241 \n",
+      " _c24 | 38a9b7e4 \n",
+      " _c25 | c1ae4f81 \n",
+      " _c26 | bf78d0d4 \n",
+      " _c27 | null     \n",
+      " _c28 | b26ead5a \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | b6bc86c5 \n",
+      " _c32 | 139221a3 \n",
+      " _c33 | 66281ff5 \n",
+      " _c34 | 48c051b2 \n",
+      " _c35 | 02fd8626 \n",
+      " _c36 | null     \n",
+      " _c37 | 15559afb \n",
+      " _c38 | ff654802 \n",
+      " _c39 | 962813c6 \n",
+      "-RECORD 19-------\n",
+      " _c0  | 0        \n",
+      " _c1  | 1        \n",
+      " _c2  | 61       \n",
+      " _c3  | null     \n",
+      " _c4  | 0        \n",
+      " _c5  | 20       \n",
+      " _c6  | 13       \n",
+      " _c7  | 1        \n",
+      " _c8  | 2        \n",
+      " _c9  | 30       \n",
+      " _c10 | 1        \n",
+      " _c11 | 2        \n",
+      " _c12 | 1170     \n",
+      " _c13 | null     \n",
+      " _c14 | bc173037 \n",
+      " _c15 | 964910b2 \n",
+      " _c16 | d2114104 \n",
+      " _c17 | e6b5b2ee \n",
+      " _c18 | c6fc10d3 \n",
+      " _c19 | 6fcd6dcb \n",
+      " _c20 | 660e2e35 \n",
+      " _c21 | 8670cd10 \n",
+      " _c22 | 2e4e821f \n",
+      " _c23 | 8db71ef1 \n",
+      " _c24 | ed0ce343 \n",
+      " _c25 | f7ab09d9 \n",
+      " _c26 | f4fc3c5e \n",
+      " _c27 | null     \n",
+      " _c28 | d96198f0 \n",
+      " _c29 | null     \n",
+      " _c30 | null     \n",
+      " _c31 | a1eb1511 \n",
+      " _c32 | 139221a3 \n",
+      " _c33 | 5300bf1f \n",
+      " _c34 | 8a33c0cc \n",
+      " _c35 | 167376d2 \n",
+      " _c36 | null     \n",
+      " _c37 | 4d260962 \n",
+      " _c38 | ff654802 \n",
+      " _c39 | 2ccea557 \n",
+      "only showing top 20 rows\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "df = spark.read.format(\"arrow\").load(file_names)\n",
+    "#df = spark.read.format(\"arrow\").load(\"/dlrm_nativesql/dlrm_categorified\")\n",
+    "df.show(vertical=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "History Log\n",
+    "\n",
+    "* per core memory size is 5.000 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+    "* Generate Dictionary took 2373.706\n",
+    "* categorify threshold is 50.000 M rows, flush_threshold is 960.000 GB\n",
+    "* ('_c19', DataFrame[dict_col: string, dict_col_id: int], 3)\n",
+    "* ('_c30', DataFrame[dict_col: string, dict_col_id: int], 3)\n",
+    "* ('_c26', DataFrame[dict_col: string, dict_col_id: int], 10)\n",
+    "* ('_c32', DataFrame[dict_col: string, dict_col_id: int], 14)\n",
+    "* ('_c39', DataFrame[dict_col: string, dict_col_id: int], 36)\n",
+    "* ('_c22', DataFrame[dict_col: string, dict_col_id: int], 63)\n",
+    "* ('_c38', DataFrame[dict_col: string, dict_col_id: int], 108)\n",
+    "* ('_c29', DataFrame[dict_col: string, dict_col_id: int], 154)\n",
+    "* ('_c31', DataFrame[dict_col: string, dict_col_id: int], 976)\n",
+    "* ('_c21', DataFrame[dict_col: string, dict_col_id: int], 1543)\n",
+    "* ('_c27', DataFrame[dict_col: string, dict_col_id: int], 2208)\n",
+    "* ('_c20', DataFrame[dict_col: string, dict_col_id: int], 7122)\n",
+    "* ('_c17', DataFrame[dict_col: string, dict_col_id: int], 7424)\n",
+    "* ('_c28', DataFrame[dict_col: string, dict_col_id: int], 11938)\n",
+    "* ('_c37', DataFrame[dict_col: string, dict_col_id: int], 12973)\n",
+    "* ('_c16', DataFrame[dict_col: string, dict_col_id: int], 17295)\n",
+    "* ('_c18', DataFrame[dict_col: string, dict_col_id: int], 20265)\n",
+    "* ('_c15', DataFrame[dict_col: string, dict_col_id: int], 39060)\n",
+    "* ('_c25', DataFrame[dict_col: string, dict_col_id: int], 405282)\n",
+    "* ('_c36', DataFrame[dict_col: string, dict_col_id: int], 590151)\n",
+    "* ('_c24', DataFrame[dict_col: string, dict_col_id: int], 3067955)\n",
+    "* ('_c34', DataFrame[dict_col: string, dict_col_id: int], 40790947)\n",
+    "* ('_c23', DataFrame[dict_col: string, dict_col_id: int], 130229466)\n",
+    "* ('_c35', DataFrame[dict_col: string, dict_col_id: int], 187188509)\n",
+    "* ('_c14', DataFrame[dict_col: string, dict_col_id: int], 227605431)\n",
+    "* ('_c33', DataFrame[dict_col: string, dict_col_id: int], 292775613)\n",
+    "* _c19 will do bhj\n",
+    "* _c30 will do bhj\n",
+    "* _c26 will do bhj\n",
+    "* _c32 will do bhj\n",
+    "* _c39 will do bhj\n",
+    "* _c22 will do bhj\n",
+    "* _c38 will do bhj\n",
+    "* _c29 will do bhj\n",
+    "* _c31 will do bhj\n",
+    "* _c21 will do bhj\n",
+    "* _c27 will do bhj\n",
+    "* _c20 will do bhj\n",
+    "* _c17 will do bhj\n",
+    "* _c28 will do bhj\n",
+    "* _c37 will do bhj\n",
+    "* _c16 will do bhj\n",
+    "* _c18 will do bhj\n",
+    "* _c15 will do bhj\n",
+    "* _c25 will do bhj\n",
+    "* _c36 will do bhj\n",
+    "* _c24 will do bhj\n",
+    "* _c34 will do bhj\n",
+    "* etstimated_to_shuffle_size for _c23 is 749.453 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for _c35 is 725.014 GB, will do smj and spill to disk\n",
+    "* etstimated_to_shuffle_size for _c14 is 700.576 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for _c33 is 676.137 GB, will do smj and spill to disk\n",
+    "* do bhj to _c19\n",
+    "* do bhj to _c30\n",
+    "* do bhj to _c26\n",
+    "* do bhj to _c32\n",
+    "* do bhj to _c39\n",
+    "* do bhj to _c22\n",
+    "* do bhj to _c38\n",
+    "* do bhj to _c29\n",
+    "* do bhj to _c31\n",
+    "* do bhj to _c21\n",
+    "* do bhj to _c27\n",
+    "* do bhj to _c20\n",
+    "* do bhj to _c17\n",
+    "* do bhj to _c28\n",
+    "* do bhj to _c37\n",
+    "* do bhj to _c16\n",
+    "* do bhj to _c18\n",
+    "* do bhj to _c15\n",
+    "* do bhj to _c25\n",
+    "* do bhj to _c36\n",
+    "* do bhj to _c24\n",
+    "* do bhj to _c34\n",
+    "* do smj to _c23\n",
+    "* do smj to _c35\n",
+    "* do smj to _c14\n",
+    "* do smj to _c33\n",
+    "* Categorify took 12443.532"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/notebooks/dlrm/DLRM_Performance.ipynb b/RecDP/examples/notebooks/dlrm/DLRM_Performance.ipynb
new file mode 100644
index 000000000..0971ef8a1
--- /dev/null
+++ b/RecDP/examples/notebooks/dlrm/DLRM_Performance.ipynb
@@ -0,0 +1,331 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "spark.stop()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## DLRM Categorify"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/mnt/nvme2/chendi/BlueWhale/recdp\n"
+     ]
+    }
+   ],
+   "source": [
+    "###### Start spark job ######\n",
+    "import init\n",
+    "\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.utils import *\n",
+    "\n",
+    "import logging\n",
+    "from timeit import default_timer as timer\n",
+    "import os\n",
+    "from pyspark import *\n",
+    "from pyspark.sql import *\n",
+    "from pyspark.sql.functions import *\n",
+    "from pyspark.sql.types import *\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Define Schema\n",
+    "LABEL_COL = 0\n",
+    "INT_COLS = list(range(1, 14))\n",
+    "CAT_COLS = list(range(14, 40))\n",
+    "label_fields = [StructField('_c%d' % LABEL_COL, IntegerType())]\n",
+    "int_fields = [StructField('_c%d' % i, IntegerType()) for i in INT_COLS]\n",
+    "str_fields = [StructField('_c%d' % i, StringType()) for i in CAT_COLS]\n",
+    "schema = StructType(label_fields + int_fields + str_fields)\n",
+    "\n",
+    "to_be_categorified = [23, 35, 14, 33]\n",
+    "\n",
+    "def categorifyAllFeatures(df, proc, output_name=\"categorified\", gen_dict=False, enable_freqlimit=False):\n",
+    "    dict_dfs = []\n",
+    "    to_categorify_cols = ['_c%d' % i for i in CAT_COLS]\n",
+    "    #to_categorify_cols = ['_c%d' % i for i in to_be_categorified]\n",
+    "    if gen_dict:\n",
+    "        # only call below function when target dicts were not pre-prepared        \n",
+    "        op_gen_dict = GenerateDictionary(to_categorify_cols, isParquet=False)\n",
+    "        proc.reset_ops([op_gen_dict])\n",
+    "        t1 = timer()\n",
+    "        dict_dfs = proc.generate_dicts(df)\n",
+    "        t2 = timer()\n",
+    "        print(\"Generate Dictionary took %.3f\" % (t2 - t1))\n",
+    "    else:\n",
+    "        # or we can simply load from pre-gened\n",
+    "        dict_dfs = [{'col_name': name, 'dict': proc.spark.read.parquet(\n",
+    "            \"%s/%s/%s/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in to_categorify_cols]    \n",
+    "\n",
+    "    if enable_freqlimit:\n",
+    "        dict_dfs = [{'col_name': dict_df['col_name'], 'dict': dict_df['dict'].filter('count >= 15')} for dict_df in dict_dfs]\n",
+    "\n",
+    "    # start to do categorify\n",
+    "    op_categorify = Categorify(to_categorify_cols, dict_dfs=dict_dfs)\n",
+    "    op_fillna_for_categorified = FillNA(to_categorify_cols, 0)\n",
+    "    proc.append_ops([op_categorify, op_fillna_for_categorified])\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"Categorify took %.3f\" % (t2 - t1))\n",
+    "    \n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "per core memory size is 5.000 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+      "bhj total threshold is 67.109 M rows, one bhj threshold is 30.000 M rows, flush_threshold is 960.000 GB\n",
+      "('_c19', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 3)\n",
+      "('_c30', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 3)\n",
+      "('_c26', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 10)\n",
+      "('_c32', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 14)\n",
+      "('_c39', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 34)\n",
+      "('_c22', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 62)\n",
+      "('_c38', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 96)\n",
+      "('_c29', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 103)\n",
+      "('_c31', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 967)\n",
+      "('_c21', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 1381)\n",
+      "('_c27', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 2208)\n",
+      "('_c20', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 7104)\n",
+      "('_c17', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 7338)\n",
+      "('_c28', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 10666)\n",
+      "('_c37', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 12021)\n",
+      "('_c16', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 17138)\n",
+      "('_c18', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 20045)\n",
+      "('_c15', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 33822)\n",
+      "('_c25', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 245827)\n",
+      "('_c36', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 302515)\n",
+      "('_c24', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 582468)\n",
+      "('_c34', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 2675939)\n",
+      "('_c23', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 5554113)\n",
+      "('_c35', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 7156452)\n",
+      "('_c14', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 7912888)\n",
+      "('_c33', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 8165895)\n",
+      "_c19 will do bhj\n",
+      "_c30 will do bhj\n",
+      "_c26 will do bhj\n",
+      "_c32 will do bhj\n",
+      "_c39 will do bhj\n",
+      "_c22 will do bhj\n",
+      "_c38 will do bhj\n",
+      "_c29 will do bhj\n",
+      "_c31 will do bhj\n",
+      "_c21 will do bhj\n",
+      "_c27 will do bhj\n",
+      "_c20 will do bhj\n",
+      "_c17 will do bhj\n",
+      "_c28 will do bhj\n",
+      "_c37 will do bhj\n",
+      "_c16 will do bhj\n",
+      "_c18 will do bhj\n",
+      "_c15 will do bhj\n",
+      "_c25 will do bhj\n",
+      "_c36 will do bhj\n",
+      "_c24 will do bhj\n",
+      "_c34 will do bhj\n",
+      "_c23 will do bhj\n",
+      "_c35 will do bhj\n",
+      "_c14 will do bhj\n",
+      "_c33 will do bhj\n",
+      "do bhj to _c19\n",
+      "do bhj to _c30\n",
+      "do bhj to _c26\n",
+      "do bhj to _c32\n",
+      "do bhj to _c39\n",
+      "do bhj to _c22\n",
+      "do bhj to _c38\n",
+      "do bhj to _c29\n",
+      "do bhj to _c31\n",
+      "do bhj to _c21\n",
+      "do bhj to _c27\n",
+      "do bhj to _c20\n",
+      "do bhj to _c17\n",
+      "do bhj to _c28\n",
+      "do bhj to _c37\n",
+      "do bhj to _c16\n",
+      "do bhj to _c18\n",
+      "do bhj to _c15\n",
+      "do bhj to _c25\n",
+      "do bhj to _c36\n",
+      "do bhj to _c24\n",
+      "do bhj to _c34\n",
+      "do bhj to _c23\n",
+      "do bhj to _c35\n",
+      "do bhj to _c14\n",
+      "do bhj to _c33\n",
+      "Adding a CodegenSeparator to pure BHJ WSCG case\n",
+      "Categorify took 201.768\n"
+     ]
+    }
+   ],
+   "source": [
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/dlrm/\"\n",
+    "csv_folder = \"/dlrm/csv_raw_data\"\n",
+    "#path = os.path.join(path_prefix, file)\n",
+    "\n",
+    "scala_udf_jars = \"/mnt/nvme2/chendi/BlueWhale/recdp/ScalaProcessUtils/target/core-0.1.0-jar-with-dependencies.jar\"\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "t0 = timer()\n",
+    "spark = SparkSession\\\n",
+    "    .builder\\\n",
+    "    .master('yarn')\\\n",
+    "    .appName(\"DLRM\")\\\n",
+    "    .config(\"spark.executor.memory\", \"20g\")\\\n",
+    "    .config(\"spark.executor.memoryOverhead\", \"3g\")\\\n",
+    "    .config(\"spark.cleaner.periodicGC.interval\", \"10min\")\\\n",
+    "    .config(\"spark.driver.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.executor.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .getOrCreate()\n",
+    "\n",
+    "#files = [\"day_%d\" % i for i in range(0, 24)]\n",
+    "files = [\"day_0\"]\n",
+    "file_names = [os.path.join(path_prefix, csv_folder, filename) for filename in files]\n",
+    "\n",
+    "proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity=\"1200GB\")\n",
+    "df = spark.read.schema(schema).option('sep', '\\t').csv(file_names)\n",
+    "#df = spark.read.parquet(\"/dlrm/categorified_stage1\")\n",
+    "df = categorifyAllFeatures(df, proc, output_name=\"dlrm_categorified\", gen_dict=False, enable_freqlimit=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "History Log\n",
+    "\n",
+    "* per core memory size is 5.000 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+    "* Generate Dictionary took 2373.706\n",
+    "* categorify threshold is 50.000 M rows, flush_threshold is 960.000 GB\n",
+    "* ('_c19', DataFrame[dict_col: string, dict_col_id: int], 3)\n",
+    "* ('_c30', DataFrame[dict_col: string, dict_col_id: int], 3)\n",
+    "* ('_c26', DataFrame[dict_col: string, dict_col_id: int], 10)\n",
+    "* ('_c32', DataFrame[dict_col: string, dict_col_id: int], 14)\n",
+    "* ('_c39', DataFrame[dict_col: string, dict_col_id: int], 36)\n",
+    "* ('_c22', DataFrame[dict_col: string, dict_col_id: int], 63)\n",
+    "* ('_c38', DataFrame[dict_col: string, dict_col_id: int], 108)\n",
+    "* ('_c29', DataFrame[dict_col: string, dict_col_id: int], 154)\n",
+    "* ('_c31', DataFrame[dict_col: string, dict_col_id: int], 976)\n",
+    "* ('_c21', DataFrame[dict_col: string, dict_col_id: int], 1543)\n",
+    "* ('_c27', DataFrame[dict_col: string, dict_col_id: int], 2208)\n",
+    "* ('_c20', DataFrame[dict_col: string, dict_col_id: int], 7122)\n",
+    "* ('_c17', DataFrame[dict_col: string, dict_col_id: int], 7424)\n",
+    "* ('_c28', DataFrame[dict_col: string, dict_col_id: int], 11938)\n",
+    "* ('_c37', DataFrame[dict_col: string, dict_col_id: int], 12973)\n",
+    "* ('_c16', DataFrame[dict_col: string, dict_col_id: int], 17295)\n",
+    "* ('_c18', DataFrame[dict_col: string, dict_col_id: int], 20265)\n",
+    "* ('_c15', DataFrame[dict_col: string, dict_col_id: int], 39060)\n",
+    "* ('_c25', DataFrame[dict_col: string, dict_col_id: int], 405282)\n",
+    "* ('_c36', DataFrame[dict_col: string, dict_col_id: int], 590151)\n",
+    "* ('_c24', DataFrame[dict_col: string, dict_col_id: int], 3067955)\n",
+    "* ('_c34', DataFrame[dict_col: string, dict_col_id: int], 40790947)\n",
+    "* ('_c23', DataFrame[dict_col: string, dict_col_id: int], 130229466)\n",
+    "* ('_c35', DataFrame[dict_col: string, dict_col_id: int], 187188509)\n",
+    "* ('_c14', DataFrame[dict_col: string, dict_col_id: int], 227605431)\n",
+    "* ('_c33', DataFrame[dict_col: string, dict_col_id: int], 292775613)\n",
+    "* _c19 will do bhj\n",
+    "* _c30 will do bhj\n",
+    "* _c26 will do bhj\n",
+    "* _c32 will do bhj\n",
+    "* _c39 will do bhj\n",
+    "* _c22 will do bhj\n",
+    "* _c38 will do bhj\n",
+    "* _c29 will do bhj\n",
+    "* _c31 will do bhj\n",
+    "* _c21 will do bhj\n",
+    "* _c27 will do bhj\n",
+    "* _c20 will do bhj\n",
+    "* _c17 will do bhj\n",
+    "* _c28 will do bhj\n",
+    "* _c37 will do bhj\n",
+    "* _c16 will do bhj\n",
+    "* _c18 will do bhj\n",
+    "* _c15 will do bhj\n",
+    "* _c25 will do bhj\n",
+    "* _c36 will do bhj\n",
+    "* _c24 will do bhj\n",
+    "* _c34 will do bhj\n",
+    "* etstimated_to_shuffle_size for _c23 is 749.453 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for _c35 is 725.014 GB, will do smj and spill to disk\n",
+    "* etstimated_to_shuffle_size for _c14 is 700.576 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for _c33 is 676.137 GB, will do smj and spill to disk\n",
+    "* do bhj to _c19\n",
+    "* do bhj to _c30\n",
+    "* do bhj to _c26\n",
+    "* do bhj to _c32\n",
+    "* do bhj to _c39\n",
+    "* do bhj to _c22\n",
+    "* do bhj to _c38\n",
+    "* do bhj to _c29\n",
+    "* do bhj to _c31\n",
+    "* do bhj to _c21\n",
+    "* do bhj to _c27\n",
+    "* do bhj to _c20\n",
+    "* do bhj to _c17\n",
+    "* do bhj to _c28\n",
+    "* do bhj to _c37\n",
+    "* do bhj to _c16\n",
+    "* do bhj to _c18\n",
+    "* do bhj to _c15\n",
+    "* do bhj to _c25\n",
+    "* do bhj to _c36\n",
+    "* do bhj to _c24\n",
+    "* do bhj to _c34\n",
+    "* do smj to _c23\n",
+    "* do smj to _c35\n",
+    "* do smj to _c14\n",
+    "* do smj to _c33\n",
+    "* Categorify took 12443.532"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/notebooks/dlrm/JoinFillNA_pyspark.md b/RecDP/examples/notebooks/dlrm/JoinFillNA_pyspark.md
new file mode 100644
index 000000000..443da4462
--- /dev/null
+++ b/RecDP/examples/notebooks/dlrm/JoinFillNA_pyspark.md
@@ -0,0 +1,717 @@
+```python
+spark.stop()
+```
+
+## DLRM Categorify Performance Analysis
+
+#### 1. FillNA performance
+1. Baseline: Read from HDFS and write back, took 12min 35s
+[Baseline](http://sr602:18080/history/application_1615165886357_0496/SQL/execution/?id=53) [code block](#FillNA-overhead-evaluation)
+
+2. Read from HDFS and do fillNA to all columns and write, took 11min 57s 
+[FillNAToAll](http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=107) [code block](#Step3:-Fill-NA)
+
+3. Read from HDFS do fillNA to c1 column and write, took  12min 52s
+[FillNAToOne](http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=215) [code block](#FillNA-overhead-evaluation)
+
+4. Read from HDFS do fillNA to c14-c39 column and write, took 12min 32s
+[FillNATo26](http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=216) [code block](#FillNA-overhead-evaluation)
+
+
+##### Conclusion
+* FillNA project has very small performance overhead scaling from one column to 39 columns
+
+#### 2. Categorify & FillNA performance
+1. Separate Join and FillNA to two phases, took 31min 49s + 11min 57s
+[step 1: Categorify](http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=105)
+[step 2: FillNA](http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=107)
+
+2. AllInOne as one phase, took 34min 24s
+[allInOne](http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=214)
+
+##### Conclusion
+* FillNA project inside WSCG brought overhead, should avoid that!!!!
+* Reason for that is due to w/ project BHJ WSCG need no memcpy, comparison as below
+* firstly we do fillNA in the same WSCG with categorify, which took 54min [Link](http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=161)
+* Since project should not be that heavy, we seperate categorify and FillNA to two stage, and it only took 34 min(24 + 10)
+[Link](http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=214)
+
+### Script
+
+
+```python
+###### Start spark job ######
+
+from DataProcessUtils.init_spark import *
+from DataProcessUtils.utils import *
+from RecsysSchema import RecsysSchema
+
+import logging
+from timeit import default_timer as timer
+import os
+from pyspark import *
+from pyspark.sql import *
+from pyspark.sql.functions import *
+from pyspark.sql.types import *
+import numpy as np
+import pandas as pd
+
+path_prefix = "hdfs://"
+csv_folder = "/dlrm/csv_raw_data/"
+parquet_folder = "/dlrm/parquet_raw_data/"
+file = "/dlrm/raw_data/day_0"
+#path = [os.path.join(path_prefix, folder, file) for file in files]
+csv_path = os.path.join(path_prefix, csv_folder)
+parquet_path = os.path.join(path_prefix, parquet_folder)
+#path = os.path.join(path_prefix, file)
+
+
+##### 1. Start spark and initialize data processor #####
+t0 = timer()
+spark = SparkSession\
+    .builder\
+    .master('yarn')\
+    .appName("DLRM_JOIN_FILLNA") \
+    .getOrCreate()
+
+# Enable Arrow-based columnar data transfers
+spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
+```
+
+### Convert CSV to parquet
+
+
+```python
+#LABEL_COL = 0
+#INT_COLS = list(range(1, 14))
+#CAT_COLS = list(range(14, 40))
+#label_fields = [StructField('_c%d' % LABEL_COL, IntegerType())]
+#int_fields = [StructField('_c%d' % i, IntegerType()) for i in INT_COLS]
+#str_fields = [StructField('_c%d' % i, StringType()) for i in CAT_COLS]
+#schema = StructType(label_fields + int_fields + str_fields)
+
+#files = ["day_%d" % i for i in range(0, 24)]
+#for file_name in files:
+#    df = spark.read.schema(schema).option('sep', '\t').csv(os.path.join(path_prefix, csv_folder, file_name))
+#    df.write.format('parquet').mode('overwrite').save(os.path.join(path_prefix, parquet_folder, file_name))
+```
+
+### Generate Dictionary for all Category columns
+* DLRM categorify applied to 26 columns, _c14 -> _c39, corresponding dictionary length and size listed as below
+
+| col_id | numRows  | broadcastsize | broadcast elapse time |
+| ------ | -------  | ----------- | ----------- |
+| _c14   | 7912888	| 108.4 MiB	  | 7 s   |
+| _c15   | 33822	| 35.8 MiB	  | 2 s   |
+| _c16   | 17138	| 2.3 KiB	  | 92 ms |
+| _c17   | 7338	    | 3.8 KiB	  | 0.1 s |
+| _c18   | 20045	| 4.2 MiB	  | 0.9 s |
+| _c19   | 3	    | 95.1 MiB	  | 6 s   |
+| _c20   | 7104	    | 191.0 KiB	  | 0.3 s |
+| _c21   | 1381	    | 978.0 B	  | 0.4 s |
+| _c22   | 62	    | 19.6 KiB	  | 0.6 s |
+| _c23   | 5554113	| 4.1 KiB	  | 88 ms |
+| _c24   | 582468	| 683.0 B	  | 0.6 s |
+| _c25   | 245827	| 7.9 MiB	  | 0.9 s |
+| _c26   | 10	    | 39.8 KiB	  | 0.6 s |
+| _c27   | 2208	    | 73.9 MiB	  | 5 s   |
+| _c28   | 10666	| 169.1 KiB	  | 0.6 s |
+| _c29   | 103	    | 222.0 B	  | 0.5 s |
+| _c30   | 3	    | 3.4 MiB	  | 0.9 s |
+| _c31   | 967	    | 3.7 KiB	  | 0.6 s |
+| _c32   | 14	    | 26.9 KiB	  | 0.5 s |
+| _c33   | 8165895	| 116.8 KiB	  | 0.6 s |
+| _c34   | 2675939	| 297.0 KiB	  | 0.6 s |
+| _c35   | 7156452	| 221.0 B	  | 0.5 s |
+| _c36   | 302515	| 115.3 KiB	  | 0.1 s |
+| _c37   | 12021	| 268.1 KiB	  | 0.6 s |
+| _c38   | 96	    | 105.1 MiB	  | 7 s   |
+| _c39   | 34	    | 503.2 KiB	  | 0.7 s |
+
+
+```python
+#files = [os.path.join(parquet_folder, "day_%d" % i) for i in range(0, 24)]
+#df = spark.read.parquet(*files)
+```
+
+
+```python
+#def gen_dict_df(input_df, i):
+#    singular_df = input_df.select(i)
+#    singular_df = singular_df.filter("%s is not null" % i)
+#    singular_df = singular_df.groupBy(i).count()
+#    singular_df = singular_df.withColumnRenamed('count', 'model_count').withColumnRenamed(i, 'data')
+#    return singular_df
+```
+
+
+```python
+#cols = ['_c%d' % i for i in CAT_COLS]
+#for col in cols:
+#    col_df = gen_dict_df(df, col).filter("model_count >= 15")  # frequency_limit
+#    col_df.cache()
+#    col_df.write.format('parquet').mode(
+#                'overwrite').save("/dlrm/models/%s/" % col)
+#    print("%s saved, total numRows is %d" % (col, col_df.count()))
+```
+
+### Categorify for all category cols, start from here!
+
+
+```python
+import pyspark.sql.functions as f
+def get_mapping_udf(broadcast_data, default=None):
+    first_value = next(iter(broadcast_data.values()))
+    if not isinstance(first_value, int) and not isinstance(first_value, str) and not isinstance(first_value, float):
+        raise NotImplementedError
+
+    broadcast_dict = spark.sparkContext.broadcast(
+        broadcast_data)
+
+    def get_mapped(x):
+        broadcast_data = broadcast_dict.value
+        if x in broadcast_data:
+            return broadcast_data[x]
+        else:
+            return default
+
+    # switch return type
+    if isinstance(first_value, int):
+        return udf(get_mapped, IntegerType())
+    if isinstance(first_value, str):
+        return udf(get_mapped, StringType())
+    if isinstance(first_value, float):
+        return udf(get_mapped, FloatType())
+    
+def categorify_with_udf(df, dict_df, i):
+    sorted_data = dict_df.orderBy('model_count').collect()
+    dict_data = dict((row['data'], idx) for (row, idx) in zip(
+        sorted_data, range(0, len(sorted_data))))
+    udf_impl = get_mapping_udf(dict_data)
+    df = df.withColumn(i, udf_impl(f.col(i)))
+    return df
+
+def categorify_with_join(df, dict_df, i):
+    dict_df = dict_df.withColumn('id', row_number().over(
+        Window.orderBy(desc('model_count')))).withColumn('id', f.col('id') - 1).select('data', 'id')
+    df = df.join(dict_df.hint('shuffle_hash'), f.col(i) == dict_df.data, 'left')\
+           .withColumn(i, dict_df.id).drop("id", "data")
+    return df
+
+def categorify_with_bhj(df, dict_df, i):
+    dict_df = dict_df.withColumn('id', row_number().over(
+        Window.orderBy(desc('model_count')))).withColumn('id', f.col('id') - 1).select('data', 'id')
+    df = df.join(dict_df.hint('broadcast'), f.col(i) == dict_df.data, 'left')\
+           .withColumn(i, dict_df.id).drop("id", "data")
+    return df
+
+def categorify_strategy_decision_maker(dict_dfs):
+    small_cols = []
+    long_cols = []
+    huge_cols = []
+    for (col, dict_df) in dict_dfs:
+        if (dict_df.count() > 1000000000):
+            huge_cols.append(col)
+        elif (dict_df.count() > 1000000000):
+            long_cols.append(col)
+        else:
+            small_cols.append(col)
+    return {'short_dict': small_cols, 'long_dict': long_cols, 'huge_dict': huge_cols}
+
+def categorify(df, dict_dfs):
+    strategy = categorify_strategy_decision_maker(dict_dfs)
+    # for short dict, we will do bhj
+    for (col, dict_df) in dict_dfs:
+        if col in strategy['short_dict']:
+            df = categorify_with_bhj(df, dict_df, col)    
+    # for long dict, we will do shj all along    
+    for (col, dict_df) in dict_dfs:
+        if col in strategy['long_dict']:
+            df = categorify_with_bhj(df, dict_df, col)
+    # for huge dict, we will do shj seperately
+    for (col, dict_df) in dict_dfs:
+        if col in strategy['huge_dict']:
+            df = categorify_with_join(df, dict_df, col)
+            tmp_path = "/dlrm/tmp/%s/" % uuid.uuid1()
+            df.write.format('parquet').mode('overwrite').save(tmp_path)
+            df = spark.read.parquet(tmp_path)
+    return df
+
+```
+
+### Step 1: Load all pre-generated dicts and original data
+
+
+```python
+import uuid
+
+files = [os.path.join(parquet_folder, "day_%d" % i) for i in range(0, 24)]
+df = spark.read.parquet(*files)
+LABEL_COL = 0
+INT_COLS = list(range(1, 14))
+CAT_COLS = list(range(14, 40))
+cols = ['_c%d' % i for i in CAT_COLS]
+models_folder = "/dlrm/models/"
+dict_dfs = [(col, spark.read.parquet(os.path.join(models_folder, col))) for col in cols]
+```
+
+### Step 2 Categorify and FillNA - All In One
+* We tried to do short dict / long dict and FillNA seperately, 
+* noticed this data is not very memory intensive, so decided to do All In One
+
+##### Notice ! Now we verified option 1 performed better
+
+###### option 1: seperate categorify and fillNA in two stages ######
+
+
+```python
+df = categorify(df, dict_dfs)
+cols = ['_c%d' % i for i in INT_COLS + CAT_COLS]
+df= df.repartition(2000)
+df = df.fillna(0, cols)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# save to: /dlrm/tmp/d684684e-a970-11eb-b161-a4bf0121f496
+# spark plan: http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=162
+```
+
+    {'short_dict': ['_c14', '_c15', '_c16', '_c17', '_c18', '_c19', '_c20', '_c21', '_c22', '_c23', '_c24', '_c25', '_c26', '_c27', '_c28', '_c29', '_c30', '_c31', '_c32', '_c33', '_c34', '_c35', '_c36', '_c37', '_c38', '_c39'], 'long_dict': [], 'huge_dict': []}
+    CPU times: user 282 ms, sys: 180 ms, total: 462 ms
+    Wall time: 34min 24s
+
+
+###### option 2: categorify and fillNA in same WSCG stages ######
+
+
+```python
+df = categorify(df, dict_dfs)
+cols = ['_c%d' % i for i in INT_COLS + CAT_COLS]
+print("FillNA to below cols:")
+print(cols)
+df = df.fillna(0, cols)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# save to: /dlrm/tmp/e2b4b890-a965-11eb-b161-a4bf0121f496
+# spark plan: http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=161
+```
+
+    {'short_dict': ['_c14', '_c15', '_c16', '_c17', '_c18', '_c19', '_c20', '_c21', '_c22', '_c23', '_c24', '_c25', '_c26', '_c27', '_c28', '_c29', '_c30', '_c31', '_c32', '_c33', '_c34', '_c35', '_c36', '_c37', '_c38', '_c39'], 'long_dict': [], 'huge_dict': []}
+    FillNA to below cols:
+    ['_c1', '_c2', '_c3', '_c4', '_c5', '_c6', '_c7', '_c8', '_c9', '_c10', '_c11', '_c12', '_c13', '_c14', '_c15', '_c16', '_c17', '_c18', '_c19', '_c20', '_c21', '_c22', '_c23', '_c24', '_c25', '_c26', '_c27', '_c28', '_c29', '_c30', '_c31', '_c32', '_c33', '_c34', '_c35', '_c36', '_c37', '_c38', '_c39']
+    CPU times: user 463 ms, sys: 302 ms, total: 765 ms
+    Wall time: 54min 24s
+
+
+
+```python
+###### option 3: using stringIndexer for categorify and fillNA in two stages ######
+from pyspark.ml.feature import StringIndexer
+from pyspark.ml import Pipeline
+
+def categorify(df, dict_dfs):
+    strategy = categorify_strategy_decision_maker(dict_dfs)
+    indexers = []
+    # for short dict, we will do bhj
+    for (col, dict_df) in dict_dfs:
+        if col in strategy['short_dict'] or col in strategy['long_dict']:
+            indexer = StringIndexer(inputCol=col, outputCol=col+"_index").fit(df)
+            indexers.append(indexer)
+    # for huge dict, we will do shj seperately
+    for (col, dict_df) in dict_dfs:
+        if col in strategy['huge_dict']:
+            df = categorify_with_join(df, dict_df, col)
+            tmp_path = "/dlrm/tmp/%s/" % uuid.uuid1()
+            df.write.format('parquet').mode('overwrite').save(tmp_path)
+            df = spark.read.parquet(tmp_path)
+    pipeline = Pipeline(stages=indexers)
+    df = pipeline.fit(df).transform(df)
+    return df
+
+df = categorify(df, dict_dfs)
+cols = ['_c%d' % i for i in INT_COLS + CAT_COLS]
+#df= df.repartition(2000)
+df = df.fillna(0, cols)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# save to: /dlrm/tmp/d684684e-a970-11eb-b161-a4bf0121f496
+# spark plan: http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=162
+```
+
+## Below is why we decided to perform 'All In One'
+### Step 1: Categorify
+#### option 1: use python UDF to do categorify, took 1h 26min 36s  -> Bad
+#### option 2: use Join to do categorify, took 28min and 18 min (tried bhj to all 26 dict, took 31min 49s) -> Looks much better
+* First round do BHJ to small dict, took 28min
+* Second round do BHJ to long dict, took 18 min
+* Then tried to do above BHJ at once, took 31min 49s
+
+#### option 3: use scala UDF to do categorify, still wip -> similiar to StringIndexer
+#### option 4: use Join to do categorify(BHJ + SHJ) -> very stupid idea
+* SHJ will need to do repartition by join key, materialize every round
+* SHJ should only apply to huge dict (which BHJ will definitely go OOM case)
+
+### Step 2: FillNA
+* Took about 11min 57s
+
+### Step 2: Do categorify for all short disk columns (Categorify part I)
+
+
+```python
+%time categorify_strategy = categorify_strategy_decision_maker(dict_dfs)
+print(categorify_strategy)
+```
+
+    CPU times: user 14.7 ms, sys: 6.78 ms, total: 21.5 ms
+    Wall time: 20.6 s
+    {'short_dict': ['_c15', '_c16', '_c17', '_c18', '_c19', '_c20', '_c21', '_c22', '_c24', '_c25', '_c26', '_c27', '_c28', '_c29', '_c30', '_c31', '_c32', '_c36', '_c37', '_c38', '_c39'], 'long_dict': ['_c14', '_c23', '_c33', '_c34', '_c35'], 'huge_dict': []}
+
+
+
+```python
+###### Option 1: Do Python UDF to all short dist columns ######
+###### Comment: Python UDF performed very bad!!!!! ######
+
+def categorify(df, dict_dfs):
+    for (col, dict_df) in dict_dfs:
+        df = categorify_with_udf(df, dict_df, col)
+    return df
+
+cols = categorify_strategy['short_dict']
+models_folder = "/dlrm/models/"
+dict_dfs = [(col, spark.read.parquet(os.path.join(models_folder, col))) for col in cols]
+df = categorify(df, dict_dfs)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# spark plan: http://sr602:18080/history/application_1615165886357_0493/SQL/execution/?id=70
+```
+
+    CPU times: user 766 ms, sys: 288 ms, total: 1.05 s
+    Wall time: 1h 26min 36s
+
+
+
+```python
+###### Option 2: part 1 Do BHJ to all short dist columns ######
+###### Comment: Since dict are short, BHJ did pretty well ! ######
+
+cols = categorify_strategy['short_dict']
+models_folder = "/dlrm/models/"
+dict_dfs = [(col, spark.read.parquet(os.path.join(models_folder, col))) for col in cols]
+df = categorify(df, dict_dfs)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# spark plan: http://sr602:18080/history/application_1615165886357_0493/SQL/execution/?id=71
+```
+
+    CPU times: user 164 ms, sys: 78.4 ms, total: 243 ms
+    Wall time: 28min 36s
+
+
+
+```python
+###### Option 2: part 2 try to do bhj to long dict ######
+###### Comment: Since shj need to do repartition based on join key,  ######
+######          and long dict seems fit memory well, use BHJ as well ######
+
+def categorify(df, dict_dfs):
+    for (col, dict_df) in dict_dfs:
+        df = categorify_with_bhj(df, dict_df, col)
+    return df
+df = spark.read.parquet("/dlrm/tmp/7538e986-a8f6-11eb-914a-a4bf0121f496")
+cols = categorify_strategy['long_dict']
+models_folder = "/dlrm/models/"
+dict_dfs = [(col, spark.read.parquet(os.path.join(models_folder, col))) for col in cols]
+df = categorify(df, dict_dfs)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# saved to /dlrm/tmp/7ab036be-a944-11eb-8f99-a4bf0121f496
+# spark plan: http://sr602:18080/history/application_1615165886357_0496/SQL/execution/?id=52
+```
+
+    CPU times: user 134 ms, sys: 119 ms, total: 253 ms
+    Wall time: 18min
+
+
+
+```python
+###### option 2 (part 1 & part 2) ######
+df = categorify(df, dict_dfs)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# spark plan: http://sr602:8088/proxy/application_1615165886357_0497/SQL/execution/?id=105
+```
+
+    {'short_dict': ['_c14', '_c15', '_c16', '_c17', '_c18', '_c19', '_c20', '_c21', '_c22', '_c23', '_c24', '_c25', '_c26', '_c27', '_c28', '_c29', '_c30', '_c31', '_c32', '_c33', '_c34', '_c35', '_c36', '_c37', '_c38', '_c39'], 'long_dict': [], 'huge_dict': []}
+    CPU times: user 240 ms, sys: 191 ms, total: 432 ms
+    Wall time: 31min 49s
+
+
+
+```python
+###### Option 3: Can we do with scala UDFs ######
+###### Comment:  This should be a similiar approach as StringIndexer did ######
+
+
+```
+
+### Step3: Fill NA
+
+
+```python
+#read, fillNAToAllColumns and write
+df = spark.read.parquet("/dlrm/tmp/7ab036be-a944-11eb-8f99-a4bf0121f496")
+cols = ['_c%d' % i for i in INT_COLS + CAT_COLS]
+#print(cols)
+df = df.fillna(0, cols)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# spark plan: http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=107
+```
+
+    CPU times: user 108 ms, sys: 73.9 ms, total: 182 ms
+    Wall time: 11min 57s
+
+
+### FillNA overhead evaluation
+
+
+```python
+# baseline: simply read and write
+df = spark.read.parquet("/dlrm/tmp/7ab036be-a944-11eb-8f99-a4bf0121f496")
+cols = ['_c%d' % i for i in INT_COLS + CAT_COLS]
+#print(cols)
+df = df.select(*cols)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# spark plan: http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=108
+```
+
+    CPU times: user 108 ms, sys: 83.9 ms, total: 192 ms
+    Wall time: 12min 35s
+
+
+
+```python
+# fillNA to one column
+df = spark.read.parquet("/dlrm/tmp/7ab036be-a944-11eb-8f99-a4bf0121f496")
+cols = ['_c%d' % i for i in [1]]
+#print(cols)
+df = df.fillna(0, cols)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# spark plan: http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=215
+```
+
+    CPU times: user 122 ms, sys: 69 ms, total: 191 ms
+    Wall time: 12min 52s
+
+
+
+```python
+# fillNA to all categorify columns
+df = spark.read.parquet("/dlrm/tmp/7ab036be-a944-11eb-8f99-a4bf0121f496")
+cols = ['_c%d' % i for i in CAT_COLS]
+#print(cols)
+df = df.fillna(0, cols)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# save to: 
+# spark plan: http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=215
+```
+
+    CPU times: user 117 ms, sys: 68.7 ms, total: 185 ms
+    Wall time: 12min 32s
+
+
+### Redundant, skip belows
+
+
+```python
+#def assign_id_with_window(df):
+#    windowed = Window.partitionBy('column_id').orderBy(desc('count'))
+#    return (df
+#            .withColumn('id', row_number().over(windowed))
+#            .withColumnRenamed('count', 'model_count'))
+#
+#def get_column_counts_with_frequency_limit(df, frequency_limit = None):
+#    cols = ['_c%d' % i for i in CAT_COLS]
+#    df = (df
+#        .select(posexplode(array(*cols)))
+#        .withColumnRenamed('pos', 'column_id')
+#        .withColumnRenamed('col', 'data')
+#        .filter('data is not null')
+#        .groupBy('column_id', 'data')
+#        .count())
+#    if frequency_limit:
+#        exclude = []
+#        default_limit = frequency_limit
+#        if default_limit:
+#            remain = [x - CAT_COLS[0] for x in CAT_COLS if x not in exclude]
+#            df = df.filter((~col('column_id').isin(remain)) | (col('count') >= default_limit))
+#    return df
+#
+#def get_column_models(combined_model):
+#    for i in CAT_COLS:
+#        model = (combined_model
+#            .filter('column_id == %d' % (i - CAT_COLS[0]))
+#            .drop('column_id'))
+#        yield i, model
+#col_counts = get_column_counts_with_frequency_limit(df, 15)
+#combined_model_df = assign_id_with_window(col_counts)
+#column_models_df_dict = get_column_models(combined_model_df)
+#for i, column_models_df in column_models_df_dict:
+#    column_models_df.show()
+```
+
+
+```python
+cols = ['_c%d' % i for i in [14, 15, 16]]
+new_df = df.select(posexplode(array(*cols)))
+df.select(*cols).show()
+new_df.show()
+```
+
+    +--------+--------+--------+
+    |    _c14|    _c15|    _c16|
+    +--------+--------+--------+
+    |62770d79|e21f5d58|afea442f|
+    |e5f3fd8d|a0aaffa6|6faa15d5|
+    |62770d79|ad984203|62bec60d|
+    |    null|710103fd|c73d2eb5|
+    |02e197c5|c2ced437|a2427619|
+    |8a2b1e43|3fa554c6|0b8e4616|
+    |a80f39e1|79782afd|ddb2d2e1|
+    |a46c5216|ddc72fb0|49c7ebf8|
+    |9318c40b|53c06361|fea787e5|
+    |62770d79|ad984203|ddd956c1|
+    |ad98e872|3dbb483e|6faa15d5|
+    |09fd7fc8|d92abd7e|2826bc68|
+    |359aaecc|f501e8d6|43504368|
+    |7ffd46c3|710103fd|a1407382|
+    |9a8cb066|7a06385f|417e6103|
+    |77519961|a0d14bda|b69d8ea9|
+    |447072d2|831049b5|02c63370|
+    |3a20c9b6|c2ae8fa1|6faa15d5|
+    |e5f3fd8d|a15d1051|72181f31|
+    |072027fa|21789080|3223c131|
+    +--------+--------+--------+
+    only showing top 20 rows
+    
+    +---+--------+
+    |pos|     col|
+    +---+--------+
+    |  0|62770d79|
+    |  1|e21f5d58|
+    |  2|afea442f|
+    |  0|e5f3fd8d|
+    |  1|a0aaffa6|
+    |  2|6faa15d5|
+    |  0|62770d79|
+    |  1|ad984203|
+    |  2|62bec60d|
+    |  0|    null|
+    |  1|710103fd|
+    |  2|c73d2eb5|
+    |  0|02e197c5|
+    |  1|c2ced437|
+    |  2|a2427619|
+    |  0|8a2b1e43|
+    |  1|3fa554c6|
+    |  2|0b8e4616|
+    |  0|a80f39e1|
+    |  1|79782afd|
+    +---+--------+
+    only showing top 20 rows
+    
+
+
+
+```python
+###### option 1: seperate categorify and fillNA in two stages ######
+df = categorify(df, dict_dfs)
+cols = ['_c%d' % i for i in INT_COLS + CAT_COLS]
+df = df.fillna(0, cols)
+%time df.write.format('parquet').mode('overwrite').save("/dlrm/tmp/%s/" % uuid.uuid1())
+# save to: /dlrm/tmp/d684684e-a970-11eb-b161-a4bf0121f496
+# spark plan: http://sr602:18080/history/application_1615165886357_0497/SQL/execution/?id=162
+```
+
+
+    ---------------------------------------------------------------------------
+
+    Py4JJavaError                             Traceback (most recent call last)
+
+    <timed eval> in <module>
+
+
+    /hadoop/spark/python/pyspark/sql/readwriter.py in save(self, path, format, mode, partitionBy, **options)
+        825             self._jwrite.save()
+        826         else:
+    --> 827             self._jwrite.save(path)
+        828 
+        829     @since(1.4)
+
+
+    /hadoop/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py in __call__(self, *args)
+       1303         answer = self.gateway_client.send_command(command)
+       1304         return_value = get_return_value(
+    -> 1305             answer, self.gateway_client, self.target_id, self.name)
+       1306 
+       1307         for temp_arg in temp_args:
+
+
+    /hadoop/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
+        129     def deco(*a, **kw):
+        130         try:
+    --> 131             return f(*a, **kw)
+        132         except py4j.protocol.Py4JJavaError as e:
+        133             converted = convert_exception(e.java_exception)
+
+
+    /hadoop/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
+        326                 raise Py4JJavaError(
+        327                     "An error occurred while calling {0}{1}{2}.\n".
+    --> 328                     format(target_id, ".", name), value)
+        329             else:
+        330                 raise Py4JError(
+
+
+    Py4JJavaError: An error occurred while calling o1648.save.
+    : org.apache.spark.SparkException: Job aborted.
+    	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:226)
+    	at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:178)
+    	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)
+    	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)
+    	at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)
+    	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)
+    	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)
+    	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
+    	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)
+    	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)
+    	at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122)
+    	at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121)
+    	at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:944)
+    	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
+    	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
+    	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
+    	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
+    	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
+    	at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:944)
+    	at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:396)
+    	at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:380)
+    	at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:269)
+    	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+    	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
+    	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
+    	at java.lang.reflect.Method.invoke(Method.java:498)
+    	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
+    	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
+    	at py4j.Gateway.invoke(Gateway.java:282)
+    	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
+    	at py4j.commands.CallCommand.execute(CallCommand.java:79)
+    	at py4j.GatewayConnection.run(GatewayConnection.java:238)
+    	at java.lang.Thread.run(Thread.java:748)
+    Caused by: org.apache.spark.SparkException: Job 366 cancelled because killed via the Web UI
+    	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2023)
+    	at org.apache.spark.scheduler.DAGScheduler.handleJobCancellation(DAGScheduler.scala:1919)
+    	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleStageCancellation$1(DAGScheduler.scala:1908)
+    	at scala.runtime.java8.JFunction1$mcVI$sp.apply(JFunction1$mcVI$sp.java:23)
+    	at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
+    	at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
+    	at scala.collection.mutable.ArrayOps$ofInt.foreach(ArrayOps.scala:246)
+    	at org.apache.spark.scheduler.DAGScheduler.handleStageCancellation(DAGScheduler.scala:1901)
+    	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2166)
+    	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2152)
+    	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2141)
+    	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
+
+
diff --git a/RecDP/examples/notebooks/dlrm/dlrm_dicts_viewer.ipynb b/RecDP/examples/notebooks/dlrm/dlrm_dicts_viewer.ipynb
new file mode 100644
index 000000000..b06d6753a
--- /dev/null
+++ b/RecDP/examples/notebooks/dlrm/dlrm_dicts_viewer.ipynb
@@ -0,0 +1,90 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "###### Start spark job ######\n",
+    "import init\n",
+    "\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.utils import *\n",
+    "\n",
+    "import logging\n",
+    "from timeit import default_timer as timer\n",
+    "import os\n",
+    "from pyspark import *\n",
+    "from pyspark.sql import *\n",
+    "from pyspark.sql.functions import *\n",
+    "from pyspark.sql.types import *\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "spark = SparkSession\\\n",
+    "    .builder\\\n",
+    "    .master('local[1]')\\\n",
+    "    .appName(\"DLRM_viewer\")\\\n",
+    "    .getOrCreate()\n",
+    "\n",
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/dlrm/\"\n",
+    "dicts_path = \"dicts\"\n",
+    "to_be_categorified = [23, 35, 14, 33]\n",
+    "to_categorify_cols = ['_c%d' % i for i in to_be_categorified]\n",
+    "\n",
+    "dict_dfs = [{'col_name': name, 'dict': spark.read.parquet(\n",
+    "            \"%s/%s/%s/%s\" % (path_prefix, current_path, dicts_path, name))} for name in to_categorify_cols]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[('dict_col', 'string'), ('dict_col_id', 'int')]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dict_dfs[0]['dict'].dtypes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/notebooks/dlrm/init.py b/RecDP/examples/notebooks/dlrm/init.py
new file mode 100644
index 000000000..e0981db4f
--- /dev/null
+++ b/RecDP/examples/notebooks/dlrm/init.py
@@ -0,0 +1,8 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
+import findspark
+findspark.init()
diff --git a/RecDP/examples/notebooks/haystack_sod/gen_sod.py b/RecDP/examples/notebooks/haystack_sod/gen_sod.py
new file mode 100644
index 000000000..66cdfba17
--- /dev/null
+++ b/RecDP/examples/notebooks/haystack_sod/gen_sod.py
@@ -0,0 +1,57 @@
+import init
+
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+
+import logging
+from timeit import default_timer as timer
+import os
+from pyspark import *
+from pyspark.sql import *
+from pyspark.sql.functions import *
+from pyspark.sql.types import *
+import numpy as np
+import pandas as pd
+
+def main():
+    path_prefix = "file://"
+    current_path = "/home/vmagent/app/recdp/examples/python_tests/haystack_sod/"
+    data_folder = "stack-overflow/"
+    #data_folder = ""
+
+    scala_udf_jars = "/home/vmagent/app/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+
+    ##### 1. Start spark and initialize data processor #####
+    spark = SparkSession.builder.master('local[*]')\
+        .appName("HAYSTACK_SOD")\
+        .config("spark.driver.memory", "400G")\
+        .config("spark.driver.memoryOverhead", "80G")\
+        .config("spark.executor.cores", "80")\
+        .getOrCreate()
+
+    #files = ["day_%d" % i for i in range(0, 24)]
+    SO_FILE = {'question': 'Questions.csv', 'answer': 'Answers.csv'}
+    file_names = dict((key, path_prefix + os.path.join(current_path, data_folder, filename)) for key, filename in SO_FILE.items())
+
+    proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity="1200GB", spark_mode='local')
+    t0 = timer()
+    print(file_names['answer'])
+    answers_pdf = pd.read_csv(file_names['answer'], encoding="ISO-8859-1")
+    answer_df = spark.createDataFrame(answers_pdf)
+    answer_df.printSchema()
+    #answer_df.show(1, truncate=False, vertical=True)
+    t1 = timer()
+    print(f"Load Answer.csv took {(t1 - t0)} secs")
+
+    # Process answer
+    t0 = timer()
+    answer_df = answer_df.drop('OwnerUserId').drop('CreationDate').drop('Id')
+    top_answers = answer_df.groupby("ParentId").agg(f.max('Score').alias('Score'))
+top_answers.reset_index(drop=True, inplace=True)
+top_answers.rename(columns={"ParentId": "Id", "Body":"TopAnswer-html"}, inplace=True)
+top_answers.drop(columns=['Score'], inplace=True)
+    t1 = timer()
+    print(f"Load Answer.csv took {(t1 - t0)} secs")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/RecDP/examples/notebooks/haystack_sod/init.py b/RecDP/examples/notebooks/haystack_sod/init.py
new file mode 100644
index 000000000..a51233f58
--- /dev/null
+++ b/RecDP/examples/notebooks/haystack_sod/init.py
@@ -0,0 +1,6 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
diff --git a/RecDP/examples/notebooks/operations/init.py b/RecDP/examples/notebooks/operations/init.py
new file mode 100644
index 000000000..e0981db4f
--- /dev/null
+++ b/RecDP/examples/notebooks/operations/init.py
@@ -0,0 +1,8 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
+import findspark
+findspark.init()
diff --git a/RecDP/examples/notebooks/recsys2020/RecsysSchema.py b/RecDP/examples/notebooks/recsys2020/RecsysSchema.py
new file mode 100644
index 000000000..8447f50be
--- /dev/null
+++ b/RecDP/examples/notebooks/recsys2020/RecsysSchema.py
@@ -0,0 +1,105 @@
+from pyspark.sql.types import *
+
+
+class RecsysSchema:
+    def __init__(self):
+        self.string_cols1 = [
+            'text_tokens',
+            'hashtags',  # Tweet Features
+            'tweet_id',       #
+            'present_media',          #
+            'present_links',          #
+            'present_domains',        #
+            'tweet_type',     #
+            'language',       #
+        ]
+        self.int_cols1 = [
+            'tweet_timestamp',
+        ]
+        self.string_cols2 = [
+            'engaged_with_user_id',
+        ]
+        self.int_cols2 = [
+            'engaged_with_user_follower_count',  # Engaged With User Features
+            'engaged_with_user_following_count',      #
+        ]
+        self.bool_cols1 = [
+            'engaged_with_user_is_verified',          #
+        ]
+        self.int_cols3 = [
+            'engaged_with_user_account_creation',
+        ]
+        self.string_cols3 = [
+            'engaging_user_id',
+        ]
+        self.int_cols4 = [
+            'enaging_user_follower_count',  # Engaging User Features
+            'enaging_user_following_count',      #
+        ]
+        self.bool_cols2 = [
+            'enaging_user_is_verified',
+        ]
+        self.int_cols5 = [
+            'enaging_user_account_creation',
+        ]
+        self.bool_cols3 = [
+            'engagee_follows_engager',  # Engagement Features
+
+        ]
+        self.float_cols = [
+            'reply_timestamp',  # Target Reply
+            'retweet_timestamp',  # Target Retweet
+            'retweet_with_comment_timestamp',  # Target Retweet with comment
+            'like_timestamp',  # Target Like
+        ]
+
+        # After some conversion
+        self.int_cols6 = [
+            'tweet_timestamp',
+            'engaged_with_user_follower_count',  # Engaged With User Features
+            'engaged_with_user_following_count',      #
+            'engaged_with_user_account_creation',
+            'enaging_user_follower_count',  # Engaging User Features
+            'enaging_user_following_count',           #
+            'enaging_user_account_creation',
+        ]
+
+    def toStructType(self):
+        str_fields1 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols1]
+        int_fields1 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols1]
+        str_fields2 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols2]
+        int_fields2 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols2]
+        bool_fields1 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols1]
+        int_fields3 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols3]
+        str_fields3 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols3]
+        int_fields4 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols4]
+        bool_fields2 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols2]
+        int_fields5 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols5]
+        bool_fields3 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols3]
+        float_fields = [StructField('%s' % i, FloatType())
+                        for i in self.float_cols]
+        return StructType(
+            str_fields1
+            + int_fields1
+            + str_fields2
+            + int_fields2
+            + bool_fields1
+            + int_fields3
+            + str_fields3
+            + int_fields4
+            + bool_fields2
+            + int_fields5
+            + bool_fields3
+            + float_fields
+        )
diff --git a/RecDP/examples/notebooks/recsys2020/init.py b/RecDP/examples/notebooks/recsys2020/init.py
new file mode 100644
index 000000000..e0981db4f
--- /dev/null
+++ b/RecDP/examples/notebooks/recsys2020/init.py
@@ -0,0 +1,8 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
+import findspark
+findspark.init()
diff --git a/RecDP/examples/notebooks/recsys2020/recsys2020_feature_engineering.ipynb b/RecDP/examples/notebooks/recsys2020/recsys2020_feature_engineering.ipynb
new file mode 100644
index 000000000..19379561b
--- /dev/null
+++ b/RecDP/examples/notebooks/recsys2020/recsys2020_feature_engineering.ipynb
@@ -0,0 +1,623 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "spark.stop()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/mnt/nvme2/chendi/BlueWhale/recdp\n"
+     ]
+    }
+   ],
+   "source": [
+    "#!/env/bin/python\n",
+    "\n",
+    "import init\n",
+    "\n",
+    "import findspark\n",
+    "findspark.init()\n",
+    "\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pyspark.sql import *\n",
+    "from pyspark import *\n",
+    "import pyspark.sql.functions as f\n",
+    "from timeit import default_timer as timer\n",
+    "import logging\n",
+    "from RecsysSchema import RecsysSchema\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.encoder import *\n",
+    "from pyrecdp.utils import *\n",
+    "import hashlib\n",
+    "\n",
+    "\n",
+    "def decodeBertTokenizerAndExtractFeatures(df, proc, output_name):\n",
+    "    # modification on original feature\n",
+    "    op_fillna_str = FillNA(\n",
+    "        ['present_domains', 'present_links', 'hashtags', 'present_media', 'tweet_id'], \"\")\n",
+    "    op_fillna_num = FillNA(['reply_timestamp', 'retweet_timestamp',\n",
+    "                        'retweet_with_comment_timestamp', 'like_timestamp'], 0)\n",
+    "    \n",
+    "    op_feature_modification_type_convert = FeatureModification(cols=['tweet_timestamp',\n",
+    "                                                                     'engaged_with_user_follower_count',\n",
+    "                                                                     'engaged_with_user_following_count',\n",
+    "                                                                     'engaged_with_user_account_creation',\n",
+    "                                                                     'engaging_user_follower_count',\n",
+    "                                                                     'engaging_user_following_count',\n",
+    "                                                                     'engaging_user_account_creation',\n",
+    "                                                                     'reply_timestamp',\n",
+    "                                                                     'retweet_timestamp',\n",
+    "                                                                     'retweet_with_comment_timestamp',\n",
+    "                                                                     'like_timestamp'], op='toInt')\n",
+    "    \n",
+    "    op_feature_target_classify = FeatureModification(cols={\n",
+    "        \"reply_timestamp\": \"f.when(f.col('reply_timestamp') > 0, 1).otherwise(0)\",\n",
+    "        \"retweet_timestamp\": \"f.when(f.col('retweet_timestamp') > 0, 1).otherwise(0)\",\n",
+    "        \"retweet_with_comment_timestamp\": \"f.when(f.col('retweet_with_comment_timestamp') > 0, 1).otherwise(0)\",\n",
+    "        \"like_timestamp\": \"f.when(f.col('like_timestamp') > 0, 1).otherwise(0)\"}, op='inline')\n",
+    "    \n",
+    "    # adding new features\n",
+    "    op_feature_to_be_categorified = FeatureAdd(\n",
+    "        cols={\"present_domains_indicator\": \"f.col('present_domains')\",\\\n",
+    "              \"present_links_indicator\": \"f.col('present_links')\",\\\n",
+    "              \"hashtags_indicator\": \"f.col('hashtags')\",\\\n",
+    "              \"language_indicator\": \"f.col('language')\",\\\n",
+    "              \"tweet_id_indicator\": \"f.col('tweet_id')\",\\\n",
+    "              \"present_media_indicator\": \"f.col('present_media')\",\\\n",
+    "              \"tweet_type_indicator\": \"f.col('tweet_type')\",\\\n",
+    "              \"engaged_with_user_id_indicator\": \"f.col('engaged_with_user_id')\",\\\n",
+    "              \"engaging_user_id_indicator\": \"f.col('engaging_user_id')\"\n",
+    "             }, \n",
+    "        op='inline')\n",
+    "    \n",
+    "    op_feature_from_original = FeatureAdd(\n",
+    "        cols={\"has_photo\": \"f.col('present_media').contains('Photo').cast(t.IntegerType())\",\n",
+    "              \"has_video\": \"f.col('present_media').contains('Vedio').cast(t.IntegerType())\",\n",
+    "              \"has_gif\": \"f.col('present_media').contains('GIF').cast(t.IntegerType())\",             \n",
+    "              \"a_ff_rate\": \"f.col('engaged_with_user_following_count')/f.col('engaged_with_user_follower_count')\",\n",
+    "              \"b_ff_rate\": \"f.col('engaging_user_following_count') /f.col('engaging_user_follower_count')\",\n",
+    "              \"dt_dow\": \"f.dayofweek(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())\",\n",
+    "              \"dt_hour\": \"f.hour(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())\",  \n",
+    "              \"dt_minute\": \"f.minute(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())\",\n",
+    "              \"dt_second\": \"f.second(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())\",\n",
+    "              'present_domains_indicator': \"f.concat_ws('_', f.slice(f.split(f.col('present_domains_indicator'),'\\t'), 1, 2))\",\n",
+    "              'len_hashtags': \"f.when(f.col('hashtags') == '', f.lit(0)).otherwise(f.size(f.split(f.col('hashtags'), '\\t')))\",\n",
+    "              'len_domains': \"f.when(f.col('present_domains') == '', f.lit(0)).otherwise(f.size(f.split(f.col('present_domains'), '\\t')))\",\n",
+    "              'len_links': \"f.when(f.col('present_links') == '', f.lit(0)).otherwise(f.size(f.split(f.col('present_links'), '\\t')))\",\n",
+    "              'engage_time': \"f.least(f.col('reply_timestamp'), f.col('retweet_timestamp'), f.col('retweet_with_comment_timestamp'), f.col('like_timestamp'))\"   \n",
+    "        }, op='inline')\n",
+    "    op_fillna_tweet_timestamp = FillNA(['tweet_timestamp'], -1)\n",
+    "    ops = [op_fillna_str, op_fillna_num,\n",
+    "           op_feature_modification_type_convert, op_feature_target_classify,\n",
+    "           op_feature_to_be_categorified, op_feature_from_original, op_fillna_tweet_timestamp]\n",
+    "\n",
+    "    ########## Tweet token feature engineering ##########\n",
+    "    from transformers import BertTokenizer\n",
+    "    tokenizer = BertTokenizer.from_pretrained(\n",
+    "        'bert-base-multilingual-cased', do_lower_case=False)\n",
+    "\n",
+    "    # define UDF\n",
+    "    tokenizer_decode = f.udf(lambda x: tokenizer.decode(\n",
+    "        [int(n) for n in x.split('\\t')]))\n",
+    "    format_url = f.udf(lambda x: x.replace(\n",
+    "        'https : / / t. co / ', 'https://t.co/').replace('@ ', '@'))\n",
+    "\n",
+    "    def extract_hash(text, split_text='@', no=0):\n",
+    "        text = text.lower()\n",
+    "        uhash = ''\n",
+    "        text_split = text.split('@')\n",
+    "        if len(text_split) > (no+1):\n",
+    "            text_split = text_split[no+1].split(' ')\n",
+    "            cl_loop = True\n",
+    "            uhash += clean_text(text_split[0])\n",
+    "            while cl_loop:\n",
+    "                if len(text_split) > 1:\n",
+    "                    if text_split[1] in ['_']:\n",
+    "                        uhash += clean_text(text_split[1]) + \\\n",
+    "                            clean_text(text_split[2])\n",
+    "                        text_split = text_split[2:]\n",
+    "                    else:\n",
+    "                        cl_loop = False\n",
+    "                else:\n",
+    "                    cl_loop = False\n",
+    "        hash_object = hashlib.md5(uhash.encode('utf-8'))\n",
+    "        return hash_object.hexdigest()\n",
+    "\n",
+    "    def clean_text(text):\n",
+    "        if len(text) > 1:\n",
+    "            if text[-1] in ['!', '?', ':', ';', '.', ',']:\n",
+    "                return(text[:-1])\n",
+    "        return(text)\n",
+    "    \n",
+    "    # udf defines upon tweet\n",
+    "    to_notsign = f.udf(lambda x: x.replace('\\[CLS\\] RT @', ''))\n",
+    "    count_space = f.udf(lambda x: x.count(' '))\n",
+    "    count_text_length = f.udf(lambda x: len(x))\n",
+    "    user_defined_hash = f.udf(lambda x: extract_hash(x, split_text='RT @', no=0))\n",
+    "    count_at = f.udf(lambda x: x.count('@'))\n",
+    "    user_define_hash_1 = f.udf(lambda x: extract_hash(x))\n",
+    "    user_define_hash_2 = f.udf(lambda x: extract_hash(x, no=1))\n",
+    "\n",
+    "    # decode\n",
+    "    op_feature_modification_tokenizer_decode = FeatureAdd(cols={'tweet': 'text_tokens'}, udfImpl=tokenizer_decode)\n",
+    "    op_feature_modification_format_url = FeatureModification(cols=['tweet'], udfImpl=format_url)\n",
+    "\n",
+    "    # adding new features\n",
+    "    op_feature_add_tweet_indicator = FeatureAdd(cols={\"tweet_indicator\": \"f.col('tweet')\"}, op='inline')\n",
+    "    op_feature_add_tweet_nortsign = FeatureAdd(cols={'tweet_nortsign': 'tweet'}, udfImpl=to_notsign)\n",
+    "    op_feature_add_count_words = FeatureAdd(cols={'count_words': 'tweet'}, udfImpl=count_space)\n",
+    "    op_feature_add_count_char = FeatureAdd(cols={'count_char': 'tweet'}, udfImpl=count_text_length)\n",
+    "    op_feature_add_tw_uhash = FeatureAdd(cols={'tw_uhash': 'tweet'}, udfImpl=user_defined_hash)\n",
+    "    op_feature_add_tw_hash = FeatureAdd(cols={'tw_hash': \"f.hash(f.col('tweet'))%1000000000\"}, op='inline')\n",
+    "    # features upon tweet_nortsign\n",
+    "    op_feature_add_count_at = FeatureAdd(cols={'count_ats': 'tweet_nortsign'}, udfImpl=count_at)\n",
+    "    op_feature_add_tw_uhash0 = FeatureAdd(cols={'tw_hash0': 'tweet_nortsign'}, udfImpl=user_define_hash_1)\n",
+    "    op_feature_add_tw_uhash1 = FeatureAdd(cols={'tw_hash1': 'tweet_nortsign'}, udfImpl=user_define_hash_2)\n",
+    "\n",
+    "    ops += [op_feature_modification_tokenizer_decode, op_feature_modification_format_url,\n",
+    "            op_feature_add_tweet_indicator,\n",
+    "            op_feature_add_tweet_nortsign, op_feature_add_count_words, op_feature_add_count_char,\n",
+    "            op_feature_add_tw_uhash, op_feature_add_tw_hash, op_feature_add_count_at,\n",
+    "            op_feature_add_tw_uhash0, op_feature_add_tw_uhash1]\n",
+    "    proc.reset_ops(ops)\n",
+    "\n",
+    "    # execute\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"BertTokenizer decode and feature extacting took %.3f\" % (t2 - t1))\n",
+    "\n",
+    "    return df\n",
+    "\n",
+    "def splitByDate(df, proc, train_output, test_output, numFolds=5):\n",
+    "    # 1.1 get timestamp range\n",
+    "    import datetime\n",
+    "    min_timestamp = df.select('tweet_timestamp').agg({'tweet_timestamp': 'min'}).collect()[0]['min(tweet_timestamp)']\n",
+    "    max_timestamp = df.select('tweet_timestamp').agg({'tweet_timestamp': 'max'}).collect()[0]['max(tweet_timestamp)']\n",
+    "    seconds_in_day = 3600 * 24\n",
+    "\n",
+    "    print(\n",
+    "        \"min_timestamp is %s, max_timestamp is %s, 20 days max is %s\" % (\n",
+    "            datetime.datetime.fromtimestamp(min_timestamp).strftime('%Y-%m-%d %H:%M:%S'),\n",
+    "            datetime.datetime.fromtimestamp(max_timestamp).strftime('%Y-%m-%d %H:%M:%S'),\n",
+    "            datetime.datetime.fromtimestamp(min_timestamp + 20 * seconds_in_day).strftime('%Y-%m-%d %H:%M:%S')\n",
+    "        ))\n",
+    "\n",
+    "    time_range_split = {\n",
+    "        'train': (min_timestamp, seconds_in_day * 18 + min_timestamp),\n",
+    "        'test': (seconds_in_day * 18 + min_timestamp, max_timestamp)\n",
+    "    }\n",
+    "\n",
+    "    print(time_range_split)\n",
+    "\n",
+    "    # 1.2 save ranged data for train\n",
+    "    # filtering out train range data and save\n",
+    "    train_start, train_end = time_range_split['train']\n",
+    "    test_start, test_end = time_range_split['test']\n",
+    "    t1 = timer()\n",
+    "    train_df = df.filter(\n",
+    "        (f.col('tweet_timestamp') >= f.lit(train_start)) & (f.col('tweet_timestamp') < f.lit(train_end)))\n",
+    "    train_df = train_df.withColumn(\"fold\", f.round(f.rand(seed=42)*numFolds))\n",
+    "    train_df.write.format('parquet').mode('overwrite').save(proc.path_prefix + proc.current_path + train_output)\n",
+    "    t2 = timer()\n",
+    "    print(\"split to train took %.3f\" % (t2 - t1))\n",
+    "    \n",
+    "    t1 = timer()\n",
+    "    test_df = df.filter(\n",
+    "        (f.col('tweet_timestamp') >= f.lit(test_start)) & (f.col('tweet_timestamp') < f.lit(test_end)))\n",
+    "    test_df.write.format('parquet').mode('overwrite').save(proc.path_prefix + proc.current_path + test_output)\n",
+    "    t2 = timer()\n",
+    "    print(\"split to test took %.3f\" % (t2 - t1))\n",
+    "    \n",
+    "    return (proc.spark.read.parquet(proc.path_prefix + proc.current_path + train_output),\n",
+    "            proc.spark.read.parquet(proc.path_prefix + proc.current_path + test_output))\n",
+    "\n",
+    "\n",
+    "def categorifyFeatures(df, proc, output_name=\"train_with_categorified_features\", gen_dict=True, sampleRatio=1):\n",
+    "    if gen_dict:\n",
+    "        # only call below function when target dicts were not pre-prepared\n",
+    "        op_multiItems = GenerateDictionary(\n",
+    "            ['present_domains_indicator', 'present_links_indicator', 'hashtags_indicator'], doSplit=True)\n",
+    "        op_singleItems = GenerateDictionary(\n",
+    "            ['tweet_id_indicator', 'language_indicator', \n",
+    "             {'src_cols': ['engaged_with_user_id_indicator', 'engaging_user_id_indicator'], 'col_name': 'user_id'}])\n",
+    "        op_tweet = GenerateDictionary(\n",
+    "            ['tweet_indicator'], doSplit=True, withCount=True, sep=' ')\n",
+    "        \n",
+    "        proc.reset_ops([op_multiItems, op_singleItems, op_tweet])\n",
+    "        t1 = timer()\n",
+    "        dict_dfs = proc.generate_dicts(df)\n",
+    "        t2 = timer()\n",
+    "        print(\"Generate Dictionary took %.3f\" % (t2 - t1))\n",
+    "    else:\n",
+    "        # or we can simply load from pre-gened\n",
+    "        dict_names = ['hashtags_indicator', 'language_indicator', 'present_domains_indicator',\n",
+    "                      'present_links_indicator', 'tweet_id_indicator', 'user_id_indicator', 'tweet_indicator']\n",
+    "        dict_dfs = [{'col_name': name, 'dict': proc.spark.read.parquet(\n",
+    "            \"%s/%s/%s/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in dict_names]\n",
+    "\n",
+    "    # pre-defined dict\n",
+    "    # pre-define\n",
+    "    media = {\n",
+    "        '': 0,\n",
+    "        'GIF': 1,\n",
+    "        'GIF_GIF': 2,\n",
+    "        'GIF_Photo': 3,\n",
+    "        'GIF_Video': 4,\n",
+    "        'Photo': 5,\n",
+    "        'Photo_GIF': 6,\n",
+    "        'Photo_Photo': 7,\n",
+    "        'Photo_Video': 8,\n",
+    "        'Video': 9,\n",
+    "        'Video_GIF': 10,\n",
+    "        'Video_Photo': 11,\n",
+    "        'Video_Video': 12\n",
+    "    }\n",
+    "\n",
+    "    tweet_type = {'Quote': 0, 'Retweet': 1, 'TopLevel': 2}\n",
+    "\n",
+    "    media_df = proc.spark.createDataFrame(convert_to_spark_dict(media))\n",
+    "    tweet_type_df = proc.spark.createDataFrame(\n",
+    "        convert_to_spark_dict(tweet_type))\n",
+    "\n",
+    "    dict_dfs.append({'col_name': 'present_media_indicator', 'dict': media_df})\n",
+    "    dict_dfs.append({'col_name': 'tweet_type_indicator', 'dict': tweet_type_df})\n",
+    "\n",
+    "    for i in dict_dfs:\n",
+    "        dict_name = i['col_name']\n",
+    "        dict_df = i['dict']\n",
+    "        print(\"%s has numRows as %d\" % (dict_name, dict_df.count()))\n",
+    "\n",
+    "    ###### 2. define operations and append them to data processor ######\n",
+    "\n",
+    "    # 1. define operations\n",
+    "    # 1.1 filter on tweet dict\n",
+    "    i = 0\n",
+    "    for dict_df in dict_dfs:\n",
+    "        if dict_df['col_name'] == 'tweet':\n",
+    "            tweet_dict_df = dict_df['dict']\n",
+    "            df_cnt = tweet_dict_df.count()\n",
+    "            freqRange = [2, df_cnt * 0.9]\n",
+    "            tweet_dict_df = tweet_dict_df.filter((f.col('count') <= f.lit(freqRange[1])) & (f.col('count') >= f.lit(freqRange[0])))\n",
+    "            dict_dfs[i]['dict'] = tweet_dict_df\n",
+    "        i += 1      \n",
+    "\n",
+    "    # 1.3 categorify\n",
+    "    # since language dict is small, we may use udf to make partition more even\n",
+    "    op_categorify_1 = Categorify(\n",
+    "        ['present_domains_indicator', 'present_links_indicator', 'hashtags_indicator'], dict_dfs=dict_dfs, doSplit=True, keepMostFrequent=True)\n",
+    "    op_categorify_2 = Categorify(['language_indicator', 'present_media_indicator', 'tweet_type_indicator'], dict_dfs=dict_dfs)\n",
+    "    op_categorify_3 = Categorify([{'engaged_with_user_id_indicator': 'user_id'}, {'engaging_user_id_indicator': 'user_id'}], dict_dfs=dict_dfs)\n",
+    "    op_categorify_4 = Categorify(['tweet_id_indicator'], dict_dfs=dict_dfs)\n",
+    "    op_categorify_5 = Categorify(['tweet_indicator'], dict_dfs=dict_dfs, doSplit=True, sep=' ', doSortForArray=True)\n",
+    "    #### below are features upon categorified tweet\n",
+    "    op_feature_add_tw_word = FeatureAdd({\n",
+    "        'tw_first_word': \"f.col('tweet_indicator').getItem(0)\",\n",
+    "        'tw_second_word': \"f.col('tweet_indicator').getItem(1)\",\n",
+    "        'tw_last_word': \"f.col('tweet_indicator').getItem(f.size(f.col('tweet_indicator')) - 1)\",\n",
+    "        'tw_llast_word': \"f.col('tweet_indicator').getItem(f.size(f.col('tweet_indicator')) - 1)\",\n",
+    "        'tw_len': \"f.size(f.col('tweet_indicator'))\"\n",
+    "    }, op='inline')\n",
+    "    ops = [op_categorify_1, op_categorify_2, op_categorify_3, op_categorify_4, op_categorify_5, op_feature_add_tw_word]\n",
+    "    proc.reset_ops(ops)\n",
+    "\n",
+    "    t1 = timer()\n",
+    "    if sampleRatio < 1 and sampleRatio > 0:\n",
+    "        df = df.sample(sampleRatio)\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"categorify took %.3f\" % (t2 - t1))    \n",
+    "    return df, dict_dfs\n",
+    "\n",
+    "\n",
+    "def encodingFeatures(df, proc, output_name, gen_dict, sampleRatio=1):   \n",
+    "    targets = ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp']\n",
+    "    y_mean_all = []\n",
+    "    \n",
+    "    t1 = timer()\n",
+    "    if gen_dict:\n",
+    "        for tgt in targets:\n",
+    "            tmp = df.groupBy().mean(tgt).collect()[0]\n",
+    "            y_mean = tmp[f\"avg({tgt})\"]\n",
+    "            y_mean_all.append(y_mean)\n",
+    "        schema = t.StructType([t.StructField(tgt, t.FloatType(), True) for tgt in targets])\n",
+    "        y_mean_all_df = proc.spark.createDataFrame([tuple(y_mean_all)], schema)\n",
+    "        y_mean_all_df.write.format(\"parquet\").mode(\"overwrite\").save(\n",
+    "            \"%s/%s/%s/targets_mean\" % (proc.path_prefix, proc.current_path, proc.dicts_path))\n",
+    "    y_mean_all_df = proc.spark.read.parquet(\n",
+    "        \"%s/%s/%s/targets_mean\" % (proc.path_prefix, proc.current_path, proc.dicts_path))\n",
+    "\n",
+    "    te_features = [\n",
+    "        'present_media',\n",
+    "        'tweet_type',\n",
+    "        'language',\n",
+    "        'engaged_with_user_id',\n",
+    "        'engaging_user_id',\n",
+    "        ['present_domains','language','engagee_follows_engager','tweet_type','present_media','engaged_with_user_is_verified'],\n",
+    "        ['engaged_with_user_id','tweet_type','language'],\n",
+    "        ['tw_first_word','tweet_type','language'],\n",
+    "        ['tw_last_word','tweet_type','language'],\n",
+    "        ['tw_hash0','tweet_type','language'],\n",
+    "        ['tw_hash1','tweet_type','language'],\n",
+    "        ['tw_uhash','tweet_type','language'],\n",
+    "        ['tw_hash'],\n",
+    "        ['present_media','tweet_type','language','engaged_with_user_is_verified','engaging_user_is_verified','engagee_follows_engager'],\n",
+    "        ['present_domains','present_media','tweet_type','language'],\n",
+    "        ['present_links','present_media','tweet_type','language'],\n",
+    "        ['hashtags','present_media','tweet_type','language']\n",
+    "        \n",
+    "    ]\n",
+    "    ce_features = ['present_media', 'tweet_type', 'language', 'engaged_with_user_id', 'engaging_user_id']\n",
+    "    fe_features = ['present_media', 'tweet_type', 'language', 'engaged_with_user_id', 'engaging_user_id']\n",
+    "    encoding_features = [(\"TE\", te_features), (\"CE\", ce_features), (\"FE\", fe_features)]\n",
+    "\n",
+    "    te_train_dfs = []\n",
+    "    te_test_dfs = []\n",
+    "    ce_train_dfs = []\n",
+    "    ce_test_dfs = []\n",
+    "    fe_train_dfs = []\n",
+    "    fe_test_dfs = []\n",
+    "    for feature_type, features in encoding_features:\n",
+    "        for c in features:\n",
+    "            target_tmp = targets\n",
+    "            out_name = \"\"\n",
+    "            out_col_list = []\n",
+    "            for tgt in target_tmp:\n",
+    "                if isinstance(c, list):\n",
+    "                    out_col_list.append(f'G{feature_type}_' + '_'.join(c) + f'_{tgt}')\n",
+    "                    out_name = f'G{feature_type}_' + '_'.join(c)\n",
+    "                else:\n",
+    "                    out_col_list.append(f'{feature_type}_{c}_{tgt}')\n",
+    "                    out_name = f'{feature_type}_{c}'\n",
+    "            if gen_dict:\n",
+    "                start = timer()\n",
+    "                if feature_type == 'TE':\n",
+    "                    encoder = TargetEncoder(proc, c, target_tmp, out_col_list, out_name, out_dtype=t.FloatType(), y_mean_list=y_mean_all)\n",
+    "                    te_train_df, te_test_df = encoder.transform(df)\n",
+    "                    te_train_dfs.append({'col_name': ['fold'] + (c if isinstance(c, list) else [c]), 'dict': te_train_df})\n",
+    "                    te_test_dfs.append({'col_name': c, 'dict': te_test_df})\n",
+    "                    print(f\"generating target encoding for %s upon %s took %.1f seconds\"%(str(c), str(target_tmp), timer()-start))\n",
+    "\n",
+    "                elif feature_type == 'CE':\n",
+    "                    encoder = CountEncoder(proc, c, target_tmp, out_col_list, out_name)  \n",
+    "                    ce_train_df, ce_test_df = encoder.transform(df)\n",
+    "                    ce_train_dfs.append({'col_name': c if isinstance(c, list) else [c], 'dict': ce_train_df})\n",
+    "                    ce_test_dfs.append({'col_name': c, 'dict': ce_test_df})  \n",
+    "                    print(f\"generating count encoding for %s upon %s took %.1f seconds\"%(str(c), str(target_tmp), timer()-start))\n",
+    "\n",
+    "                elif feature_type == 'FE':\n",
+    "                    # For frequency encoding, we don't need to merge with train data\n",
+    "                    encoder = FrequencyEncoder(proc, c, target_tmp, out_col_list, out_name) \n",
+    "                    fe_train_df, fe_test_df = encoder.transform(df)\n",
+    "                    fe_train_dfs.append({'col_name': c if isinstance(c, list) else [c], 'dict': fe_train_df})\n",
+    "                    print(f\"generating frequency encoding for %s upon %s took %.1f seconds\"%(str(c), str(target_tmp), timer()-start))\n",
+    "\n",
+    "            else:\n",
+    "                te_train_path = \"%s/%s/%s/train/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name)\n",
+    "                te_test_path = \"%s/%s/%s/test/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name) \n",
+    "                if feature_type == 'TE':\n",
+    "                    te_train_dfs.append({'col_name': ['fold'] + (c if isinstance(c, list) else [c]), 'dict': proc.spark.read.parquet(te_train_path)})\n",
+    "                    te_test_dfs.append({'col_name': c, 'dict': proc.spark.read.parquet(te_test_path)})\n",
+    "                if feature_type == 'CE':\n",
+    "                    ce_train_dfs.append({'col_name': c if isinstance(c, list) else [c], 'dict': proc.spark.read.parquet(te_train_path)})\n",
+    "                    ce_test_dfs.append({'col_name': c, 'dict': proc.spark.read.parquet(te_test_path)})\n",
+    "                if feature_type == 'FE':\n",
+    "                    # For frequency encoding, we don't need to merge with train data\n",
+    "                    fe_train_dfs.append({'col_name': c if isinstance(c, list) else [c], 'dict': proc.spark.read.parquet(te_train_path)})\n",
+    "\n",
+    "    t2 = timer()\n",
+    "    print(\"Generate encoding feature totally took %.3f\" % (t2 - t1))\n",
+    "\n",
+    "    # merge dicts to original table\n",
+    "    if sampleRatio < 1 and sampleRatio > 0:\n",
+    "        df = df.sample(sampleRatio)\n",
+    "    i = 3\n",
+    "    for train_dfs in [te_train_dfs, ce_train_dfs, fe_train_dfs]:\n",
+    "        op_merge_to_train = ModelMerge(train_dfs)\n",
+    "        proc.reset_ops([op_merge_to_train])\n",
+    "        i -= 1\n",
+    "        _output_name = output_name if i == 0 else f\"{output_name}_{i}\"\n",
+    "\n",
+    "        t1 = timer()\n",
+    "        df = proc.transform(df, name=_output_name)\n",
+    "        t2 = timer()\n",
+    "        print(\"encodingFeatures took %.3f\" % (t2 - t1))\n",
+    "    \n",
+    "    return (df, te_train_dfs, te_test_dfs, y_mean_all_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/recsys2020_example/\"\n",
+    "original_folder = \"/recsys2021_0608/\"\n",
+    "dicts_folder = \"recsys_dicts/\"\n",
+    "recsysSchema = RecsysSchema()\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "scala_udf_jars = \"/mnt/nvme2/chendi/BlueWhale/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar\"\n",
+    "\n",
+    "t0 = timer()\n",
+    "spark = SparkSession.builder.master('yarn')\\\n",
+    "    .appName(\"Recsys2021_data_process\")\\\n",
+    "    .config(\"spark.executor.memory\", \"20g\")\\\n",
+    "    .config(\"spark.executor.memoryOverhead\", \"10g\")\\\n",
+    "    .config(\"spark.sql.broadcastTimeout\", \"7200\")\\\n",
+    "    .config(\"spark.cleaner.periodicGC.interval\", \"10min\")\\\n",
+    "    .config(\"spark.executorEnv.HF_DATASETS_OFFLINE\", \"1\")\\\n",
+    "    .config(\"spark.executorEnv.TRANSFORMERS_OFFLINE\", \"1\")\\\n",
+    "    .config(\"spark.driver.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.executor.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .getOrCreate()\n",
+    "\n",
+    "schema = recsysSchema.toStructType()\n",
+    "\n",
+    "# 1.1 prepare dataFrames\n",
+    "# 1.2 create RecDP DataProcessor\n",
+    "proc = DataProcessor(spark, path_prefix,\n",
+    "                     current_path=current_path, dicts_path=dicts_folder, shuffle_disk_capacity=\"1200GB\")\n",
+    "df = spark.read.parquet(path_prefix + original_folder)\n",
+    "df = df.withColumnRenamed('enaging_user_following_count', 'engaging_user_following_count')\n",
+    "df = df.withColumnRenamed('enaging_user_is_verified', 'engaging_user_is_verified')\n",
+    "\n",
+    "# fast test, comment for full dataset\n",
+    "# df.sample(0.01).write.format(\"parquet\").mode(\"overwrite\").save(\"%s/sample_0_0_1\" % current_path)\n",
+    "# df = spark.read.parquet(\"%s/sample_0_0_1\" % current_path)\n",
+    "\n",
+    "# ===============================================\n",
+    "# decode tweet_tokens\n",
+    "df = decodeBertTokenizerAndExtractFeatures(df, proc, output_name=\"decoded_with_extracted_features\")\n",
+    "\n",
+    "# ===============================================\n",
+    "# splitting and sampling\n",
+    "df, test_df = splitByDate(df, proc, train_output=\"train\", test_output=\"test\", numFolds=5)\n",
+    "\n",
+    "# ===============================================\n",
+    "# generate dictionary for categorify indexing\n",
+    "df, dict_dfs = categorifyFeatures(df, proc, output_name=\"train_with_categorified_features\", gen_dict=True, sampleRatio=0.03)\n",
+    "\n",
+    "# ===============================================\n",
+    "# encoding features\n",
+    "df, te_train_dfs, te_test_dfs, y_mean_all_df = encodingFeatures(df, proc, output_name=\"train_with_features_sample_0_0_3\", gen_dict=True, sampleRatio=0.03)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Output with sample as 0.01\n",
+    "\n",
+    "* recdp-scala-extension is enabled\n",
+    "* per core memory size is 5.000 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+    "* BertTokenizer decode and feature extacting took 242.857\n",
+    "* min_timestamp is 2021-02-04 08:00:00, max_timestamp is 2021-02-25 07:59:59, 20 days max is 2021-02-24 08:00:00\n",
+    "* {'train': (1612396800, 1613952000), 'test': (1613952000, 1614211199)}\n",
+    "* split to train took 15.447\n",
+    "* split to test took 6.934\n",
+    "* Generate Dictionary took 37.361\n",
+    "* present_domains_indicator has numRows as 80174\n",
+    "* present_links_indicator has numRows as 497560\n",
+    "* hashtags_indicator has numRows as 494988\n",
+    "* tweet_id_indicator has numRows as 4708592\n",
+    "* language_indicator has numRows as 66\n",
+    "* user_id has numRows as 5065850\n",
+    "* tweet_indicator has numRows as 8332709\n",
+    "* present_media_indicator has numRows as 13\n",
+    "* tweet_type_indicator has numRows as 3\n",
+    "* do scala_udf to present_domains_indicator\n",
+    "* do scala_udf to hashtags_indicator\n",
+    "* do scala_udf to present_links_indicator\n",
+    "* do bhj to tweet_type_indicator\n",
+    "* do bhj to present_media_indicator\n",
+    "* do bhj to language_indicator\n",
+    "* Adding a CodegenSeparator to pure BHJ WSCG case\n",
+    "* do bhj to engaged_with_user_id_indicator\n",
+    "* do bhj to engaging_user_id_indicator\n",
+    "* Adding a CodegenSeparator to pure BHJ WSCG case\n",
+    "* do bhj to tweet_id_indicator\n",
+    "* Adding a CodegenSeparator to pure BHJ WSCG case\n",
+    "* do scala_udf to tweet_indicator\n",
+    "* categorify took 281.024\n",
+    "* generating target encoding for present_media upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 3.5 seconds\n",
+    "* generating target encoding for tweet_type upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 1.5 seconds\n",
+    "* generating target encoding for language upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 2.9 seconds\n",
+    "* generating target encoding for engaged_with_user_id upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 7.9 seconds\n",
+    "* generating target encoding for engaging_user_id upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 5.7 seconds\n",
+    "* generating target encoding for ['present_domains', 'language', 'engagee_follows_engager', 'tweet_type', 'present_media', 'engaged_with_user_is_verified'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 6.0 seconds\n",
+    "* generating target encoding for ['engaged_with_user_id', 'tweet_type', 'language'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 7.0 seconds\n",
+    "* generating target encoding for ['tw_first_word', 'tweet_type', 'language'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 3.8 seconds\n",
+    "* generating target encoding for ['tw_last_word', 'tweet_type', 'language'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 6.3 seconds\n",
+    "* generating target encoding for ['tw_hash0', 'tweet_type', 'language'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 5.2 seconds\n",
+    "* generating target encoding for ['tw_hash1', 'tweet_type', 'language'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 3.7 seconds\n",
+    "* generating target encoding for ['tw_uhash', 'tweet_type', 'language'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 4.4 seconds\n",
+    "* generating target encoding for ['tw_hash'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 5.2 seconds\n",
+    "* generating target encoding for ['present_media', 'tweet_type', 'language', 'engaged_with_user_is_verified', 'engaging_user_is_verified', 'engagee_follows_engager'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 4.5 seconds\n",
+    "* generating target encoding for ['present_domains', 'present_media', 'tweet_type', 'language'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 6.0 seconds\n",
+    "* generating target encoding for ['present_links', 'present_media', 'tweet_type', 'language'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 3.7 seconds\n",
+    "* generating target encoding for ['hashtags', 'present_media', 'tweet_type', 'language'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 4.7 seconds\n",
+    "* generating count encoding for present_media upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 1.5 seconds\n",
+    "* generating count encoding for tweet_type upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 1.4 seconds\n",
+    "* generating count encoding for language upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 2.2 seconds\n",
+    "* generating count encoding for engaged_with_user_id upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 3.0 seconds\n",
+    "* generating count encoding for engaging_user_id upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 3.0 seconds\n",
+    "* generating frequency encoding for present_media upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 1.2 seconds\n",
+    "* generating frequency encoding for tweet_type upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 1.1 seconds\n",
+    "* generating frequency encoding for language upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 1.2 seconds\n",
+    "* generating frequency encoding for engaged_with_user_id upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 1.8 seconds\n",
+    "* generating frequency encoding for engaging_user_id upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 1.7 seconds\n",
+    "* Generate encoding feature totally took 101.488\n",
+    "* do bhj to ['fold', 'tweet_type']\n",
+    "* do bhj to ['fold', 'present_media']\n",
+    "* do bhj to ['fold', 'language']\n",
+    "* do bhj to ['fold', 'tw_first_word', 'tweet_type', 'language']\n",
+    "* do bhj to ['fold', 'present_media', 'tweet_type', 'language', 'engaged_with_user_is_verified', 'engaging_user_is_verified', 'engagee_follows_engager']\n",
+    "* do bhj to ['fold', 'present_domains', 'present_media', 'tweet_type', 'language']\n",
+    "* do bhj to ['fold', 'present_domains', 'language', 'engagee_follows_engager', 'tweet_type', 'present_media', 'engaged_with_user_is_verified']\n",
+    "* do bhj to ['fold', 'tw_hash1', 'tweet_type', 'language']\n",
+    "* do bhj to ['fold', 'present_links', 'present_media', 'tweet_type', 'language']\n",
+    "* do bhj to ['fold', 'hashtags', 'present_media', 'tweet_type', 'language']\n",
+    "* do bhj to ['fold', 'tw_hash0', 'tweet_type', 'language']\n",
+    "* do bhj to ['fold', 'tw_uhash', 'tweet_type', 'language']\n",
+    "* do bhj to ['fold', 'engaged_with_user_id']\n",
+    "* do bhj to ['fold', 'engaged_with_user_id', 'tweet_type', 'language']\n",
+    "* do bhj to ['fold', 'engaging_user_id']\n",
+    "* do bhj to ['fold', 'tw_last_word', 'tweet_type', 'language']\n",
+    "* do bhj to ['fold', 'tw_hash']\n",
+    "* encodingFeatures took 52.923\n",
+    "* do bhj to ['tweet_type']\n",
+    "* do bhj to ['present_media']\n",
+    "* do bhj to ['language']\n",
+    "* do bhj to ['engaged_with_user_id']\n",
+    "* do bhj to ['engaging_user_id']\n",
+    "* encodingFeatures took 17.919\n",
+    "* do bhj to ['tweet_type']\n",
+    "* do bhj to ['present_media']\n",
+    "* do bhj to ['language']\n",
+    "* do bhj to ['engaged_with_user_id']\n",
+    "* do bhj to ['engaging_user_id']\n",
+    "* encodingFeatures took 19.710"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/notebooks/recsys2020/recsys_for_friesian_integration.ipynb b/RecDP/examples/notebooks/recsys2020/recsys_for_friesian_integration.ipynb
new file mode 100644
index 000000000..60fc578a5
--- /dev/null
+++ b/RecDP/examples/notebooks/recsys2020/recsys_for_friesian_integration.ipynb
@@ -0,0 +1,178 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "spark.stop()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#!/env/bin/python\n",
+    "\n",
+    "import init\n",
+    "\n",
+    "import findspark\n",
+    "findspark.init()\n",
+    "\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pyspark.sql import *\n",
+    "from pyspark import *\n",
+    "import pyspark.sql.functions as f\n",
+    "from timeit import default_timer as timer\n",
+    "import logging\n",
+    "from RecsysSchema import RecsysSchema\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.utils import *\n",
+    "import hashlib\n",
+    "\n",
+    "def categorifyAllFeatures(df, proc, output_name=\"categorified\", gen_dicts=False):\n",
+    "    # 1. define operations\n",
+    "    # 1.1 fill na and features\n",
+    "    op_fillna_str = FillNA(\n",
+    "        ['present_domains', 'present_links', 'hashtags'], \"\")\n",
+    "\n",
+    "    # 1.3 categorify\n",
+    "    # since language dict is small, we may use udf to make partition more even\n",
+    "    #'present_domains', 'present_links', \n",
+    "    op_categorify_multi = Categorify(\n",
+    "        ['present_domains', 'present_links', 'hashtags'], gen_dicts=gen_dicts, doSplit=True, keepMostFrequent=True)\n",
+    "    op_fillna_for_categorified = FillNA(['present_domains', 'present_links', 'hashtags'], -1)\n",
+    "\n",
+    "    # transform\n",
+    "    proc.append_ops([op_fillna_str, op_categorify_multi, op_fillna_for_categorified])\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"Data Process 1 and udf categorify took %.3f\" % (t2 - t1))\n",
+    "\n",
+    "    return df\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/recsys2020_0608_categorify_example_1/\"\n",
+    "original_folder = \"/recsys2021_0608/\"\n",
+    "dicts_folder = \"recsys_dicts/\"\n",
+    "recsysSchema = RecsysSchema()\n",
+    "\n",
+    "scala_udf_jars = \"/mnt/nvme2/chendi/BlueWhale/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar\"\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "t0 = timer()\n",
+    "spark = SparkSession.builder.master('yarn')\\\n",
+    "    .appName(\"Recsys2020_data_process\")\\\n",
+    "    .config(\"spark.executor.memory\", \"30g\")\\\n",
+    "    .config(\"spark.executor.memoryOverhead\", \"5g\")\\\n",
+    "    .config(\"spark.driver.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.executor.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.executor.cores\", \"4\")\\\n",
+    "    .getOrCreate()\n",
+    "\n",
+    "schema = recsysSchema.toStructType()\n",
+    "\n",
+    "# 1.1 prepare dataFrames\n",
+    "# 1.2 create RecDP DataProcessor\n",
+    "proc = DataProcessor(spark, path_prefix,\n",
+    "                     current_path=current_path, dicts_path=dicts_folder, shuffle_disk_capacity=\"1200GB\")\n",
+    "\n",
+    "# ===============================================\n",
+    "# basic: Do categorify for all columns\n",
+    "df = spark.read.parquet(path_prefix + original_folder)\n",
+    "\n",
+    "# rename firstly\n",
+    "df = df.withColumnRenamed('enaging_user_following_count', 'engaging_user_following_count')\n",
+    "df = df.withColumnRenamed('enaging_user_is_verified', 'engaging_user_is_verified')\n",
+    "df = categorifyAllFeatures(df, proc, gen_dicts=True)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "* per core memory size is 7.500 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+    "* hashtags has numRows as 6541935\n",
+    "* language has numRows as 66\n",
+    "* present_domains has numRows as 896783\n",
+    "* present_links has numRows as 14804300\n",
+    "* tweet_id has numRows as 279597527\n",
+    "* user_id has numRows as 42850559\n",
+    "* present_media has numRows as 13\n",
+    "* tweet_type has numRows as 3\n",
+    "* categorify threshold is 50.000 M rows, flush_threshold is 960.000 GB\n",
+    "* ('tweet_type', DataFrame[dict_col: string, dict_col_id: bigint], 3)\n",
+    "* ('present_media', DataFrame[dict_col: string, dict_col_id: bigint], 13)\n",
+    "* ('language', DataFrame[dict_col: string, dict_col_id: int], 66)\n",
+    "* ('present_domains', DataFrame[dict_col: string, dict_col_id: int], 896783)\n",
+    "* ('hashtags', DataFrame[dict_col: string, dict_col_id: int], 6541935)\n",
+    "* ('present_links', DataFrame[dict_col: string, dict_col_id: int], 14804300)\n",
+    "* ('user_id', DataFrame[dict_col: string, dict_col_id: int], 42850559)\n",
+    "* ('tweet_id', DataFrame[dict_col: string, dict_col_id: int], 279597527)\n",
+    "* tweet_type will do udf\n",
+    "* present_media will do udf\n",
+    "* language will do udf\n",
+    "* present_domains will do udf\n",
+    "* hashtags will do udf\n",
+    "* present_links will do udf\n",
+    "* etstimated_to_shuffle_size for user_id is 81.653 GB, will do shj\n",
+    "* etstimated_to_shuffle_size for tweet_id is 78.153 GB, will do shj\n",
+    "* Data Process 1 and udf categorify took 799.987\n",
+    "\n",
+    "### After switching to use scala udf, process time will be longer while we don't need to assign bigger memoryOverHead to spark\n",
+    "\n",
+    "* recdp-scala-extension is enabled\n",
+    "* per core memory size is 4.500 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+    "* hashtags has numRows as 6541935\n",
+    "* present_domains has numRows as 896783\n",
+    "* present_links has numRows as 14804300\n",
+    "* present_media has numRows as 13\n",
+    "* tweet_type has numRows as 3\n",
+    "* bhj total threshold is 60.398 M rows, one bhj threshold is 30.000 M rows, flush_threshold is 960.000 GB\n",
+    "* ('present_domains', DataFrame[dict_col: string, dict_col_id: int, count: bigint], 896783)\n",
+    "* ('hashtags', DataFrame[dict_col: string, dict_col_id: int, count: bigint], 6541935)\n",
+    "* ('present_links', DataFrame[dict_col: string, dict_col_id: int, count: bigint], 14804300)\n",
+    "* present_domains will do udf\n",
+    "* hashtags will do udf\n",
+    "* present_links will do udf\n",
+    "* Data Process 1 and udf categorify took 1045.646"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/notebooks/recsys2021/RecsysSchema.py b/RecDP/examples/notebooks/recsys2021/RecsysSchema.py
new file mode 100644
index 000000000..8447f50be
--- /dev/null
+++ b/RecDP/examples/notebooks/recsys2021/RecsysSchema.py
@@ -0,0 +1,105 @@
+from pyspark.sql.types import *
+
+
+class RecsysSchema:
+    def __init__(self):
+        self.string_cols1 = [
+            'text_tokens',
+            'hashtags',  # Tweet Features
+            'tweet_id',       #
+            'present_media',          #
+            'present_links',          #
+            'present_domains',        #
+            'tweet_type',     #
+            'language',       #
+        ]
+        self.int_cols1 = [
+            'tweet_timestamp',
+        ]
+        self.string_cols2 = [
+            'engaged_with_user_id',
+        ]
+        self.int_cols2 = [
+            'engaged_with_user_follower_count',  # Engaged With User Features
+            'engaged_with_user_following_count',      #
+        ]
+        self.bool_cols1 = [
+            'engaged_with_user_is_verified',          #
+        ]
+        self.int_cols3 = [
+            'engaged_with_user_account_creation',
+        ]
+        self.string_cols3 = [
+            'engaging_user_id',
+        ]
+        self.int_cols4 = [
+            'enaging_user_follower_count',  # Engaging User Features
+            'enaging_user_following_count',      #
+        ]
+        self.bool_cols2 = [
+            'enaging_user_is_verified',
+        ]
+        self.int_cols5 = [
+            'enaging_user_account_creation',
+        ]
+        self.bool_cols3 = [
+            'engagee_follows_engager',  # Engagement Features
+
+        ]
+        self.float_cols = [
+            'reply_timestamp',  # Target Reply
+            'retweet_timestamp',  # Target Retweet
+            'retweet_with_comment_timestamp',  # Target Retweet with comment
+            'like_timestamp',  # Target Like
+        ]
+
+        # After some conversion
+        self.int_cols6 = [
+            'tweet_timestamp',
+            'engaged_with_user_follower_count',  # Engaged With User Features
+            'engaged_with_user_following_count',      #
+            'engaged_with_user_account_creation',
+            'enaging_user_follower_count',  # Engaging User Features
+            'enaging_user_following_count',           #
+            'enaging_user_account_creation',
+        ]
+
+    def toStructType(self):
+        str_fields1 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols1]
+        int_fields1 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols1]
+        str_fields2 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols2]
+        int_fields2 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols2]
+        bool_fields1 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols1]
+        int_fields3 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols3]
+        str_fields3 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols3]
+        int_fields4 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols4]
+        bool_fields2 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols2]
+        int_fields5 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols5]
+        bool_fields3 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols3]
+        float_fields = [StructField('%s' % i, FloatType())
+                        for i in self.float_cols]
+        return StructType(
+            str_fields1
+            + int_fields1
+            + str_fields2
+            + int_fields2
+            + bool_fields1
+            + int_fields3
+            + str_fields3
+            + int_fields4
+            + bool_fields2
+            + int_fields5
+            + bool_fields3
+            + float_fields
+        )
diff --git a/RecDP/examples/notebooks/recsys2021/final_submission_feature_engineering.ipynb b/RecDP/examples/notebooks/recsys2021/final_submission_feature_engineering.ipynb
new file mode 100644
index 000000000..ba32f013c
--- /dev/null
+++ b/RecDP/examples/notebooks/recsys2021/final_submission_feature_engineering.ipynb
@@ -0,0 +1,759 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#!/env/bin/python\n",
+    "\n",
+    "import init\n",
+    "\n",
+    "import findspark\n",
+    "findspark.init()\n",
+    "\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pyspark.sql import *\n",
+    "from pyspark import *\n",
+    "import pyspark.sql.functions as f\n",
+    "import pyspark.sql.types as t\n",
+    "from timeit import default_timer as timer\n",
+    "import logging\n",
+    "from RecsysSchema import RecsysSchema\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.encoder import *\n",
+    "from pyrecdp.utils import *\n",
+    "import hashlib\n",
+    "\n",
+    "target_list = [\n",
+    " 'reply_timestamp',\n",
+    " 'retweet_timestamp',\n",
+    " 'retweet_with_comment_timestamp',\n",
+    " 'like_timestamp'\n",
+    "]\n",
+    "\n",
+    "final_feature_list = [\n",
+    " 'engaged_with_user_follower_count',\n",
+    " 'engaged_with_user_following_count',\n",
+    " 'engaged_with_user_is_verified',\n",
+    " 'engaging_user_follower_count',\n",
+    " 'engaging_user_following_count',\n",
+    " 'engaging_user_is_verified',\n",
+    " 'has_photo',\n",
+    " 'has_video',\n",
+    " 'has_gif',\n",
+    " 'a_ff_rate',\n",
+    " 'b_ff_rate', \n",
+    " 'dt_hour',\n",
+    " 'dt_dow',\n",
+    " 'has_mention',  \n",
+    " 'mentioned_bucket_id',    \n",
+    " 'mentioned_count',    \n",
+    " 'most_used_word_bucket_id',\n",
+    " 'second_used_word_bucket_id',\n",
+    " 'TE_tweet_type_reply_timestamp',\n",
+    " 'TE_tweet_type_retweet_timestamp',\n",
+    " 'TE_dt_dow_retweet_timestamp',\n",
+    " 'TE_most_used_word_bucket_id_reply_timestamp',\n",
+    " 'TE_most_used_word_bucket_id_retweet_timestamp',\n",
+    " 'TE_most_used_word_bucket_id_retweet_with_comment_timestamp',\n",
+    " 'TE_most_used_word_bucket_id_like_timestamp',\n",
+    " 'TE_second_used_word_bucket_id_reply_timestamp',\n",
+    " 'TE_second_used_word_bucket_id_retweet_timestamp',\n",
+    " 'TE_second_used_word_bucket_id_retweet_with_comment_timestamp',\n",
+    " 'TE_second_used_word_bucket_id_like_timestamp',\n",
+    " 'TE_mentioned_bucket_id_retweet_timestamp',\n",
+    " 'TE_mentioned_bucket_id_retweet_with_comment_timestamp',\n",
+    " 'TE_mentioned_bucket_id_like_timestamp',\n",
+    " 'TE_mentioned_bucket_id_reply_timestamp',\n",
+    " 'TE_language_reply_timestamp',\n",
+    " 'TE_language_retweet_timestamp',\n",
+    " 'TE_language_retweet_with_comment_timestamp',\n",
+    " 'TE_language_like_timestamp',\n",
+    " 'TE_mentioned_count_reply_timestamp',\n",
+    " 'TE_mentioned_count_retweet_timestamp',\n",
+    " 'TE_mentioned_count_retweet_with_comment_timestamp',\n",
+    " 'TE_mentioned_count_like_timestamp',\n",
+    " 'TE_engaged_with_user_id_reply_timestamp',\n",
+    " 'TE_engaged_with_user_id_retweet_timestamp',\n",
+    " 'TE_engaged_with_user_id_retweet_with_comment_timestamp',\n",
+    " 'TE_engaged_with_user_id_like_timestamp',\n",
+    " 'GTE_language_engaged_with_user_id_reply_timestamp',\n",
+    " 'GTE_language_engaged_with_user_id_retweet_timestamp',\n",
+    " 'GTE_language_engaged_with_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_language_engaged_with_user_id_like_timestamp',\n",
+    " 'GTE_tweet_type_engaged_with_user_id_reply_timestamp',\n",
+    " 'GTE_tweet_type_engaged_with_user_id_retweet_timestamp',\n",
+    " 'GTE_tweet_type_engaged_with_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_tweet_type_engaged_with_user_id_like_timestamp',\n",
+    " 'GTE_has_mention_engaging_user_id_reply_timestamp',\n",
+    " 'GTE_has_mention_engaging_user_id_retweet_timestamp',\n",
+    " 'GTE_has_mention_engaging_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_has_mention_engaging_user_id_like_timestamp',\n",
+    " 'GTE_mentioned_bucket_id_engaging_user_id_reply_timestamp',\n",
+    " 'GTE_mentioned_bucket_id_engaging_user_id_retweet_timestamp',\n",
+    " 'GTE_mentioned_bucket_id_engaging_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_mentioned_bucket_id_engaging_user_id_like_timestamp',\n",
+    " 'GTE_language_engaging_user_id_reply_timestamp',\n",
+    " 'GTE_language_engaging_user_id_retweet_timestamp',\n",
+    " 'GTE_language_engaging_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_language_engaging_user_id_like_timestamp',\n",
+    " 'GTE_tweet_type_engaging_user_id_reply_timestamp',\n",
+    " 'GTE_tweet_type_engaging_user_id_retweet_timestamp',\n",
+    " 'GTE_tweet_type_engaging_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_tweet_type_engaging_user_id_like_timestamp',\n",
+    " 'GTE_dt_dow_engaged_with_user_id_reply_timestamp',\n",
+    " 'GTE_dt_dow_engaged_with_user_id_retweet_timestamp',\n",
+    " 'GTE_dt_dow_engaged_with_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_dt_dow_engaged_with_user_id_like_timestamp',\n",
+    " 'GTE_mentioned_count_engaging_user_id_reply_timestamp',\n",
+    " 'GTE_mentioned_count_engaging_user_id_retweet_timestamp',\n",
+    " 'GTE_mentioned_count_engaging_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_mentioned_count_engaging_user_id_like_timestamp',\n",
+    " 'GTE_dt_hour_engaged_with_user_id_reply_timestamp',\n",
+    " 'GTE_dt_hour_engaged_with_user_id_retweet_timestamp',\n",
+    " 'GTE_dt_hour_engaged_with_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_dt_hour_engaged_with_user_id_like_timestamp',\n",
+    " 'GTE_dt_dow_engaging_user_id_reply_timestamp',\n",
+    " 'GTE_dt_dow_engaging_user_id_retweet_timestamp',\n",
+    " 'GTE_dt_dow_engaging_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_dt_dow_engaging_user_id_like_timestamp',\n",
+    " 'GTE_dt_hour_engaging_user_id_reply_timestamp',\n",
+    " 'GTE_dt_hour_engaging_user_id_retweet_timestamp',\n",
+    " 'GTE_dt_hour_engaging_user_id_retweet_with_comment_timestamp',\n",
+    " 'GTE_dt_hour_engaging_user_id_like_timestamp'\n",
+    "]\n",
+    "\n",
+    "def decodeBertTokenizerAndExtractFeatures(df, proc, output_name):\n",
+    "    from transformers import BertTokenizer\n",
+    "    tokenizer = BertTokenizer.from_pretrained(\n",
+    "        'bert-base-multilingual-cased', do_lower_case=False)\n",
+    "\n",
+    "    # define UDF\n",
+    "    tokenizer_decode = f.udf(lambda x: tokenizer.decode(\n",
+    "        [int(n) for n in x.split('\\t')]))\n",
+    "    format_url = f.udf(lambda x: x.replace(\n",
+    "        'https : / / t. co / ', 'https://t.co/').replace('@ ', '@'))\n",
+    "\n",
+    "    # define decode udf operations\n",
+    "    op_feature_modification_tokenizer_decode = FeatureAdd(\n",
+    "        cols={'tweet': 'text_tokens'}, udfImpl=tokenizer_decode)\n",
+    "    op_feature_modification_format_url = FeatureModification(\n",
+    "        cols=['tweet'], udfImpl=format_url)\n",
+    "    \n",
+    "    op_feature_target_classify = FeatureModification(cols={\n",
+    "        \"reply_timestamp\": \"f.when(f.col('reply_timestamp') > 0, 1).otherwise(0)\",\n",
+    "        \"retweet_timestamp\": \"f.when(f.col('retweet_timestamp') > 0, 1).otherwise(0)\",\n",
+    "        \"retweet_with_comment_timestamp\": \"f.when(f.col('retweet_with_comment_timestamp') > 0, 1).otherwise(0)\",\n",
+    "        \"like_timestamp\": \"f.when(f.col('like_timestamp') > 0, 1).otherwise(0)\"}, op='inline')\n",
+    "    \n",
+    "    # define new features\n",
+    "    op_feature_from_original = FeatureAdd(\n",
+    "        cols={\"has_photo\": \"f.col('present_media').contains('Photo').cast(t.IntegerType())\",\n",
+    "              \"has_video\": \"f.col('present_media').contains('Vedio').cast(t.IntegerType())\",\n",
+    "              \"has_gif\": \"f.col('present_media').contains('GIF').cast(t.IntegerType())\",             \n",
+    "              \"a_ff_rate\": \"f.col('engaged_with_user_following_count')/f.col('engaged_with_user_follower_count')\",\n",
+    "              \"b_ff_rate\": \"f.col('engaging_user_following_count') /f.col('engaging_user_follower_count')\",\n",
+    "              \"dt_dow\": \"f.dayofweek(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())\",\n",
+    "              \"dt_hour\": \"f.hour(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())\",           \n",
+    "              \"mention\": \"f.regexp_extract(f.col('tweet'), r'[^RT]\\s@(\\S+)', 1)\",\n",
+    "              \"has_mention\": \"(f.col('mention')!= '').cast(t.IntegerType())\"\n",
+    "        }, op='inline')\n",
+    "\n",
+    "    # execute\n",
+    "    proc.reset_ops([op_feature_modification_tokenizer_decode,\n",
+    "                    op_feature_modification_format_url,\n",
+    "                    op_feature_target_classify,\n",
+    "                    op_feature_from_original])\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"BertTokenizer decode and feature extacting took %.3f\" % (t2 - t1))\n",
+    "\n",
+    "    return df\n",
+    "\n",
+    "def categorifyFeatures(df, proc, output_name, gen_dict, sampleRatio=1):\n",
+    "    # 1. prepare dictionary\n",
+    "    dict_dfs = []\n",
+    "    if gen_dict:\n",
+    "        # only call below function when target dicts were not pre-prepared\n",
+    "        op_gen_dict_multiItems = GenerateDictionary(['tweet'], doSplit=True, sep=' ', bucketSize=100)\n",
+    "        op_gen_dict_singleItems = GenerateDictionary(['mention'], bucketSize=100)\n",
+    "        proc.reset_ops([op_gen_dict_multiItems, op_gen_dict_singleItems])\n",
+    "        t1 = timer()\n",
+    "        dict_dfs = proc.generate_dicts(df)\n",
+    "        t2 = timer()\n",
+    "        print(\"Generate Dictionary took %.3f\" % (t2 - t1))\n",
+    "    else:\n",
+    "        dict_names = ['tweet', 'mention']\n",
+    "        dict_dfs = [{'col_name': name, 'dict': proc.spark.read.parquet(\n",
+    "            \"%s/%s/%s/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in dict_names]\n",
+    "    # 2. since we need both mentioned_bucket_id and mentioned_count, add two mention id dict_dfs\n",
+    "    for dict_df in dict_dfs:\n",
+    "        if dict_df['col_name'] == 'mention':\n",
+    "            dict_dfs.append({'col_name': 'mentioned_bucket_id', 'dict':dict_df['dict']})\n",
+    "            dict_dfs.append({'col_name': 'mentioned_count', 'dict':dict_df['dict'].drop('dict_col_id').withColumnRenamed('count', 'dict_col_id')})\n",
+    "    op_feature_add = FeatureAdd({\"mentioned_bucket_id\": \"f.col('mention')\", \"mentioned_count\": \"f.col('mention')\"}, op='inline')\n",
+    "    \n",
+    "    # 3. categorify\n",
+    "    op_categorify_multiItems = Categorify([{'bucketized_tweet_word': 'tweet'}], dict_dfs=dict_dfs, doSplit=True, sep=' ')\n",
+    "    op_categorify_singleItem = Categorify(['mentioned_bucket_id', 'mentioned_count'], dict_dfs=dict_dfs)\n",
+    "    proc.reset_ops([op_feature_add, op_categorify_multiItems, op_categorify_singleItem])\n",
+    "    \n",
+    "    # 4. get most and second used bucketized_tweet_word\n",
+    "    op_feature_add_sorted_bucketized_tweet_word = FeatureAdd(\n",
+    "        cols={'sorted_bucketized_tweet_word': \"f.expr('sortIntArrayByFrequency(bucketized_tweet_word)')\"}, op='inline')\n",
+    "    op_feature_add_convert = FeatureAdd(\n",
+    "        cols={'most_used_word_bucket_id': \"f.when(f.size(f.col('sorted_bucketized_tweet_word'))>0, f.col('sorted_bucketized_tweet_word').getItem(0)).otherwise(np.nan)\",\n",
+    "             'second_used_word_bucket_id': \"f.when(f.size(f.col('sorted_bucketized_tweet_word'))>1, f.col('sorted_bucketized_tweet_word').getItem(1)).otherwise(np.nan)\"}, op='inline')\n",
+    "    proc.append_ops([op_feature_add_sorted_bucketized_tweet_word, op_feature_add_convert])\n",
+    "\n",
+    "    # 5. transform\n",
+    "    t1 = timer()\n",
+    "    if sampleRatio != 1:\n",
+    "        df = df.sample(sampleRatio)\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"categorify and getMostAndSecondUsedWordBucketId took %.3f\" % (t2 - t1))\n",
+    "    return (df, dict_dfs)\n",
+    "\n",
+    "\n",
+    "def encodingFeatures(df, proc, output_name, gen_dict, sampleRatio=1):   \n",
+    "    targets = ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp']\n",
+    "    y_mean_all = []\n",
+    "    \n",
+    "    t1 = timer()\n",
+    "    if gen_dict:\n",
+    "        for tgt in targets:\n",
+    "            tmp = df.groupBy().mean(tgt).collect()[0]\n",
+    "            y_mean = tmp[f\"avg({tgt})\"]\n",
+    "            y_mean_all.append(y_mean)\n",
+    "        schema = t.StructType([t.StructField(tgt, t.FloatType(), True) for tgt in targets])\n",
+    "        y_mean_all_df = proc.spark.createDataFrame([tuple(y_mean_all)], schema)\n",
+    "        y_mean_all_df.write.format(\"parquet\").mode(\"overwrite\").save(\n",
+    "            \"%s/%s/%s/targets_mean\" % (proc.path_prefix, proc.current_path, proc.dicts_path))\n",
+    "    y_mean_all_df = proc.spark.read.parquet(\n",
+    "        \"%s/%s/%s/targets_mean\" % (proc.path_prefix, proc.current_path, proc.dicts_path))\n",
+    "\n",
+    "    features = [\n",
+    "            'engaged_with_user_id',\n",
+    "            'language',\n",
+    "            'dt_dow',\n",
+    "            'tweet_type',\n",
+    "            'most_used_word_bucket_id',\n",
+    "            'second_used_word_bucket_id',\n",
+    "            'mentioned_count',\n",
+    "            'mentioned_bucket_id',\n",
+    "            ['has_mention', 'engaging_user_id'],\n",
+    "            ['mentioned_count', 'engaging_user_id'],\n",
+    "            ['mentioned_bucket_id', 'engaging_user_id'],\n",
+    "            ['language', 'engaged_with_user_id'],\n",
+    "            ['language', 'engaging_user_id'],\n",
+    "            ['dt_dow', 'engaged_with_user_id'],\n",
+    "            ['dt_dow', 'engaging_user_id'],\n",
+    "            ['dt_hour', 'engaged_with_user_id'],\n",
+    "            ['dt_hour', 'engaging_user_id'],\n",
+    "            ['tweet_type', 'engaged_with_user_id'],\n",
+    "            ['tweet_type', 'engaging_user_id']\n",
+    "    ]\n",
+    "    excludes = {'dt_dow': ['reply_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'],\n",
+    "               'tweet_type': ['like_timestamp', 'retweet_with_comment_timestamp']\n",
+    "              }\n",
+    "\n",
+    "    te_train_dfs = []\n",
+    "    te_test_dfs = []\n",
+    "    for c in features:\n",
+    "        target_tmp = targets\n",
+    "        out_name = \"\"\n",
+    "        if str(c) in excludes:\n",
+    "            target_tmp = []\n",
+    "            for tgt in targets:\n",
+    "                if tgt not in excludes[c]:\n",
+    "                    target_tmp.append(tgt)\n",
+    "        out_col_list = []\n",
+    "        for tgt in target_tmp:\n",
+    "            if isinstance(c, list):\n",
+    "                out_col_list.append('GTE_'+'_'.join(c)+'_'+tgt)\n",
+    "                out_name = 'GTE_'+'_'.join(c)\n",
+    "            else:\n",
+    "                out_col_list.append(f'TE_{c}_{tgt}')\n",
+    "                out_name = f'TE_{c}'\n",
+    "        if gen_dict:\n",
+    "            start = timer()\n",
+    "            encoder = TargetEncoder(proc, c, target_tmp, out_col_list, out_name, out_dtype=t.FloatType(), y_mean_list=y_mean_all)\n",
+    "            te_train_df, te_test_df = encoder.transform(df)\n",
+    "            te_train_dfs.append({'col_name': ['fold'] + (c if isinstance(c, list) else [c]), 'dict': te_train_df})\n",
+    "            te_test_dfs.append({'col_name': c, 'dict': te_test_df})\n",
+    "            print(f\"generating target encoding for %s upon %s took %.1f seconds\"%(str(c), str(target_tmp), timer()-start))\n",
+    "        else:\n",
+    "            te_train_path = \"%s/%s/%s/train/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name)\n",
+    "            te_test_path = \"%s/%s/%s/test/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name)               \n",
+    "            te_train_dfs.append({'col_name': ['fold'] + (c if isinstance(c, list) else [c]), 'dict': proc.spark.read.parquet(te_train_path)})\n",
+    "            te_test_dfs.append({'col_name': c, 'dict': proc.spark.read.parquet(te_test_path)})\n",
+    "    t2 = timer()\n",
+    "    print(\"Generate encoding feature totally took %.3f\" % (t2 - t1))\n",
+    "\n",
+    "    # merge dicts to original table\n",
+    "    op_merge_to_train = ModelMerge(te_train_dfs)\n",
+    "    proc.reset_ops([op_merge_to_train])\n",
+    "    \n",
+    "    # select features\n",
+    "    op_select = SelectFeature(target_list + final_feature_list)\n",
+    "    proc.append_ops([op_select])\n",
+    "\n",
+    "    t1 = timer()\n",
+    "    if sampleRatio != 1:\n",
+    "        df = df.sample(sampleRatio)\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"encodingFeatures took %.3f\" % (t2 - t1))\n",
+    "    \n",
+    "    return (df, te_train_dfs, te_test_dfs, y_mean_all_df)\n",
+    "\n",
+    "\n",
+    "def splitByDate(df, proc, train_output, test_output, numFolds=5):\n",
+    "    # 1.1 get timestamp range\n",
+    "    import datetime\n",
+    "    min_timestamp = df.select('tweet_timestamp').agg({'tweet_timestamp': 'min'}).collect()[0]['min(tweet_timestamp)']\n",
+    "    max_timestamp = df.select('tweet_timestamp').agg({'tweet_timestamp': 'max'}).collect()[0]['max(tweet_timestamp)']\n",
+    "    seconds_in_day = 3600 * 24\n",
+    "\n",
+    "    print(\n",
+    "        \"min_timestamp is %s, max_timestamp is %s, 20 days max is %s\" % (\n",
+    "            datetime.datetime.fromtimestamp(min_timestamp).strftime('%Y-%m-%d %H:%M:%S'),\n",
+    "            datetime.datetime.fromtimestamp(max_timestamp).strftime('%Y-%m-%d %H:%M:%S'),\n",
+    "            datetime.datetime.fromtimestamp(min_timestamp + 20 * seconds_in_day).strftime('%Y-%m-%d %H:%M:%S')\n",
+    "        ))\n",
+    "\n",
+    "    time_range_split = {\n",
+    "        'train': (min_timestamp, seconds_in_day * 18 + min_timestamp),\n",
+    "        'test': (seconds_in_day * 18 + min_timestamp, max_timestamp)\n",
+    "    }\n",
+    "\n",
+    "    print(time_range_split)\n",
+    "\n",
+    "    # 1.2 save ranged data for train\n",
+    "    # filtering out train range data and save\n",
+    "    train_start, train_end = time_range_split['train']\n",
+    "    test_start, test_end = time_range_split['test']\n",
+    "    t1 = timer()\n",
+    "    train_df = df.filter(\n",
+    "        (f.col('tweet_timestamp') >= f.lit(train_start)) & (f.col('tweet_timestamp') < f.lit(train_end)))\n",
+    "    train_df = train_df.withColumn(\"fold\", f.round(f.rand(seed=42)*(numFolds-1)).cast(\"int\"))\n",
+    "    train_df.write.format('parquet').mode('overwrite').save(proc.path_prefix + proc.current_path + train_output)\n",
+    "    t2 = timer()\n",
+    "    print(\"split to train took %.3f\" % (t2 - t1))\n",
+    "    \n",
+    "    t1 = timer()\n",
+    "    test_df = df.filter(\n",
+    "        (f.col('tweet_timestamp') >= f.lit(test_start)) & (f.col('tweet_timestamp') < f.lit(test_end)))\n",
+    "    test_df.write.format('parquet').mode('overwrite').save(proc.path_prefix + proc.current_path + test_output)\n",
+    "    t2 = timer()\n",
+    "    print(\"split to test took %.3f\" % (t2 - t1))\n",
+    "    \n",
+    "    return (proc.spark.read.parquet(proc.path_prefix + proc.current_path + train_output),\n",
+    "            proc.spark.read.parquet(proc.path_prefix + proc.current_path + test_output))\n",
+    "\n",
+    "\n",
+    "def mergeFeaturesToTest(df, dict_dfs, te_test_dfs, y_mean_all_df, proc, output_name):\n",
+    "    # categorify test data with train generated dictionary\n",
+    "    # 1. since we need both mentioned_bucket_id and mentioned_count, add two mention id dict_dfs\n",
+    "    for dict_df in dict_dfs:\n",
+    "        if dict_df['col_name'] == 'mention':\n",
+    "            dict_dfs.append({'col_name': 'mentioned_bucket_id', 'dict':dict_df['dict']})\n",
+    "            dict_dfs.append({'col_name': 'mentioned_count', 'dict':dict_df['dict'].drop('dict_col_id').withColumnRenamed('count', 'dict_col_id')})\n",
+    "    op_feature_add = FeatureAdd({\"mentioned_bucket_id\": \"f.col('mention')\", \"mentioned_count\": \"f.col('mention')\"}, op='inline')\n",
+    "    \n",
+    "    # 2. categorify\n",
+    "    op_categorify_multiItems = Categorify([{'bucketized_tweet_word': 'tweet'}], dict_dfs=dict_dfs, doSplit=True, sep=' ')\n",
+    "    op_categorify_singleItem = Categorify(['mentioned_bucket_id', 'mentioned_count'], dict_dfs=dict_dfs)\n",
+    "    proc.reset_ops([op_feature_add, op_categorify_multiItems, op_categorify_singleItem])\n",
+    "    \n",
+    "    # 3. get most and second used bucketized_tweet_word\n",
+    "    op_feature_add_sorted_bucketized_tweet_word = FeatureAdd(\n",
+    "        cols={'sorted_bucketized_tweet_word': \"f.expr('sortIntArrayByFrequency(bucketized_tweet_word)')\"}, op='inline')\n",
+    "    op_feature_add_convert = FeatureAdd(\n",
+    "        cols={'most_used_word_bucket_id': \"f.when(f.size(f.col('sorted_bucketized_tweet_word'))>0, f.col('sorted_bucketized_tweet_word').getItem(0)).otherwise(np.nan)\",\n",
+    "             'second_used_word_bucket_id': \"f.when(f.size(f.col('sorted_bucketized_tweet_word'))>1, f.col('sorted_bucketized_tweet_word').getItem(1)).otherwise(np.nan)\"}, op='inline')\n",
+    "    proc.append_ops([op_feature_add_sorted_bucketized_tweet_word, op_feature_add_convert])\n",
+    "    \n",
+    "    # 4. merge dicts to original table\n",
+    "    op_merge_to_test = ModelMerge(te_test_dfs)\n",
+    "    proc.append_ops([op_merge_to_test])\n",
+    "        \n",
+    "    # 5. set null in encoding features to y_mean\n",
+    "    y_mean_all = y_mean_all_df.collect()[0]\n",
+    "    for tgt in target_list:\n",
+    "        to_fill_list = []\n",
+    "        for feature in final_feature_list:\n",
+    "            if 'TE_' in feature and tgt in feature:\n",
+    "                to_fill_list.append(feature)\n",
+    "        op_fill_na = FillNA(to_fill_list, y_mean_all[tgt])\n",
+    "        proc.append_ops([op_fill_na])\n",
+    "    \n",
+    "    # select features\n",
+    "    op_select = SelectFeature(target_list + final_feature_list)\n",
+    "    proc.append_ops([op_select])\n",
+    "\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"mergeFeaturesToTest took %.3f\" % (t2 - t1))\n",
+    "\n",
+    "\n",
+    "def get_encoding_features_dicts(proc):\n",
+    "    targets = ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp']\n",
+    "    y_mean_all = []\n",
+    "    y_mean_all_df = proc.spark.read.parquet(\n",
+    "        \"%s/%s/%s/targets_mean\" % (proc.path_prefix, proc.current_path, proc.dicts_path))\n",
+    "    features = [\n",
+    "            'engaged_with_user_id',\n",
+    "            'language',\n",
+    "            'dt_dow',\n",
+    "            'tweet_type',\n",
+    "            'most_used_word_bucket_id',\n",
+    "            'second_used_word_bucket_id',\n",
+    "            'mentioned_count',\n",
+    "            'mentioned_bucket_id',\n",
+    "            ['has_mention', 'engaging_user_id'],\n",
+    "            ['mentioned_count', 'engaging_user_id'],\n",
+    "            ['mentioned_bucket_id', 'engaging_user_id'],\n",
+    "            ['language', 'engaged_with_user_id'],\n",
+    "            ['language', 'engaging_user_id'],\n",
+    "            ['dt_dow', 'engaged_with_user_id'],\n",
+    "            ['dt_dow', 'engaging_user_id'],\n",
+    "            ['dt_hour', 'engaged_with_user_id'],\n",
+    "            ['dt_hour', 'engaging_user_id'],\n",
+    "            ['tweet_type', 'engaged_with_user_id'],\n",
+    "            ['tweet_type', 'engaging_user_id']\n",
+    "    ]\n",
+    "    excludes = {'dt_dow': ['reply_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'],\n",
+    "               'tweet_type': ['like_timestamp', 'retweet_with_comment_timestamp']\n",
+    "              }\n",
+    "\n",
+    "    te_train_dfs = []\n",
+    "    te_test_dfs = []\n",
+    "    for c in features:\n",
+    "        target_tmp = targets\n",
+    "        out_name = \"\"\n",
+    "        if str(c) in excludes:\n",
+    "            target_tmp = []\n",
+    "            for tgt in targets:\n",
+    "                if tgt not in excludes[c]:\n",
+    "                    target_tmp.append(tgt)\n",
+    "        out_col_list = []\n",
+    "        for tgt in target_tmp:\n",
+    "            if isinstance(c, list):\n",
+    "                out_col_list.append('GTE_'+'_'.join(c)+'_'+tgt)\n",
+    "                out_name = 'GTE_'+'_'.join(c)\n",
+    "            else:\n",
+    "                out_col_list.append(f'TE_{c}_{tgt}')\n",
+    "                out_name = f'TE_{c}'\n",
+    "        te_train_path = \"%s/%s/%s/train/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name)\n",
+    "        te_test_path = \"%s/%s/%s/test/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name)\n",
+    "        te_train_dfs.append({'col_name': ['fold'] + (c if isinstance(c, list) else [c]), 'dict': proc.spark.read.parquet(te_train_path)})\n",
+    "        te_test_dfs.append({'col_name': c, 'dict': proc.spark.read.parquet(te_test_path)})\n",
+    "        \n",
+    "    return (te_train_dfs, te_test_dfs, y_mean_all_df)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/recsys2021_0608_example/\"\n",
+    "original_folder = \"/recsys2021_0608/\"\n",
+    "dicts_folder = \"recsys_dicts/\"\n",
+    "recsysSchema = RecsysSchema()\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "scala_udf_jars = \"/mnt/nvme2/chendi/BlueWhale/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar\"\n",
+    "\n",
+    "t0 = timer()\n",
+    "spark = SparkSession.builder.master('yarn')\\\n",
+    "    .appName(\"Recsys2021_data_process\")\\\n",
+    "    .config(\"spark.executor.memory\", \"20g\")\\\n",
+    "    .config(\"spark.executor.memoryOverhead\", \"10g\")\\\n",
+    "    .config(\"spark.sql.broadcastTimeout\", \"7200\")\\\n",
+    "    .config(\"spark.cleaner.periodicGC.interval\", \"10min\")\\\n",
+    "    .config(\"spark.executorEnv.HF_DATASETS_OFFLINE\", \"1\")\\\n",
+    "    .config(\"spark.executorEnv.TRANSFORMERS_OFFLINE\", \"1\")\\\n",
+    "    .config(\"spark.driver.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.executor.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .getOrCreate()\n",
+    "\n",
+    "schema = recsysSchema.toStructType()\n",
+    "\n",
+    "# 1.1 prepare dataFrames\n",
+    "# 1.2 create RecDP DataProcessor\n",
+    "proc = DataProcessor(spark, path_prefix,\n",
+    "                     current_path=current_path, dicts_path=dicts_folder, shuffle_disk_capacity=\"1200GB\")\n",
+    "df = spark.read.parquet(path_prefix + original_folder)\n",
+    "df = df.withColumnRenamed('enaging_user_following_count', 'engaging_user_following_count')\n",
+    "df = df.withColumnRenamed('enaging_user_is_verified', 'engaging_user_is_verified')\n",
+    "\n",
+    "# # fast test, comment for full dataset\n",
+    "# df.sample(0.01).write.format(\"parquet\").mode(\"overwrite\").save(\"%s/sample_0_0_1\" % current_path)\n",
+    "# df = spark.read.parquet(\"%s/sample_0_0_1\" % current_path)\n",
+    "\n",
+    "# ===============================================\n",
+    "# decode tweet_tokens\n",
+    "df = decodeBertTokenizerAndExtractFeatures(df, proc, output_name=\"decoded_with_extracted_features\")\n",
+    "\n",
+    "# ===============================================\n",
+    "# splitting and sampling\n",
+    "df, test_df = splitByDate(df, proc, train_output=\"train\", test_output=\"test\", numFolds=5)\n",
+    "\n",
+    "# ===============================================\n",
+    "# generate dictionary for categorify indexing\n",
+    "df, dict_dfs = categorifyFeatures(df, proc, output_name=\"train_with_categorified_features\", gen_dict=True, sampleRatio=1)\n",
+    "\n",
+    "# ===============================================\n",
+    "# encoding features\n",
+    "df, te_train_dfs, te_test_dfs, y_mean_all_df = encodingFeatures(df, proc, output_name=\"train_with_features_sample_0_0_3\", gen_dict=True, sampleRatio=0.03)\n",
+    "\n",
+    "# ===============================================\n",
+    "# adding features to test\n",
+    "### Below codes is used to prepare for mergeFeaturesToTest for test separately ###\n",
+    "# test_df = spark.read.parquet(\"/recsys2021_0608_example/test\")\n",
+    "# dict_names = ['tweet', 'mention']\n",
+    "# dict_dfs = [{'col_name': name, 'dict': spark.read.parquet(\n",
+    "#     \"%s/%s/%s/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in dict_names]\n",
+    "# te_train_dfs, te_test_dfs, y_mean_all_df = get_encoding_features_dicts(proc)\n",
+    "##################################################################################\n",
+    "test_df = mergeFeaturesToTest(test_df, dict_dfs, te_test_dfs, y_mean_all_df, proc, output_name=\"test_with_features\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "* per core memory size is 5.000 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+    "* None of PyTorch, TensorFlow >= 2.0, or Flax have been found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.\n",
+    "* BertTokenizer decode and feature extacting took 2992.285\n",
+    "\n",
+    "======\n",
+    "\n",
+    "* min_timestamp is 2021-02-04 08:00:00, max_timestamp is 2021-02-25 07:59:59, 20 days max is 2021-02-24 08:00:00\n",
+    "* {'train': (1612396800, 1613952000), 'test': (1613952000, 1614211199)}\n",
+    "* split to train took 595.243\n",
+    "* split to test took 311.072\n",
+    "\n",
+    "======\n",
+    "\n",
+    "* Generate Dictionary took 847.997\n",
+    "\n",
+    "======\n",
+    "\n",
+    "* bhj total threshold is 67.109 M rows, one bhj threshold is 30.000 M rows, flush_threshold is 960.000 GB\n",
+    "* ('tweet', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 148528557)\n",
+    "* etstimated_to_shuffle_size for tweet is 123.897 GB, will do smj\n",
+    "* do smj to bucketized_tweet_word\n",
+    "* bhj total threshold is 67.109 M rows, one bhj threshold is 30.000 M rows, flush_threshold is 960.000 GB\n",
+    "* ('mentioned_bucket_id', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 4891669)\n",
+    "* ('mentioned_count', DataFrame[dict_col: string, dict_col_id: bigint], 4891669)\n",
+    "* mentioned_bucket_id will do bhj\n",
+    "* mentioned_count will do bhj\n",
+    "* do bhj to mentioned_bucket_id\n",
+    "* do bhj to mentioned_count\n",
+    "* categorify and getMostAndSecondUsedWordBucketId took 3731.166\n",
+    "\n",
+    "======\n",
+    "\n",
+    "* generating target encoding for engaged_with_user_id upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 92.6 seconds\n",
+    "* generating target encoding for language upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 21.1 seconds\n",
+    "* generating target encoding for dt_dow upon ['retweet_timestamp'] took 11.7 seconds\n",
+    "* generating target encoding for tweet_type upon ['retweet_with_comment_timestamp', 'like_timestamp'] took 14.7 seconds\n",
+    "* generating target encoding for most_used_word_bucket_id upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 17.6 seconds\n",
+    "* generating target encoding for second_used_word_bucket_id upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 16.5 seconds\n",
+    "* generating target encoding for mentioned_count upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 20.6 seconds\n",
+    "* generating target encoding for mentioned_bucket_id upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 14.1 seconds\n",
+    "* generating target encoding for ['has_mention', 'engaging_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 120.0 seconds\n",
+    "* generating target encoding for ['mentioned_count', 'engaging_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 126.9 seconds\n",
+    "* generating target encoding for ['mentioned_bucket_id', 'engaging_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 119.3 seconds\n",
+    "* generating target encoding for ['language', 'engaged_with_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 126.6 seconds\n",
+    "* generating target encoding for ['language', 'engaging_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 144.0 seconds\n",
+    "* generating target encoding for ['dt_dow', 'engaged_with_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 136.2 seconds\n",
+    "* generating target encoding for ['dt_dow', 'engaging_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 157.3 seconds\n",
+    "* generating target encoding for ['dt_hour', 'engaged_with_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 153.2 seconds\n",
+    "* generating target encoding for ['dt_hour', 'engaging_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 177.4 seconds\n",
+    "* generating target encoding for ['tweet_type', 'engaged_with_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 118.2 seconds\n",
+    "* generating target encoding for ['tweet_type', 'engaging_user_id'] upon ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'] took 141.2 seconds\n",
+    "* Generate encoding feature totally took 1738.177\n",
+    "\n",
+    "======\n",
+    "\n",
+    "* bhj total threshold is 67.109 M rows, one bhj threshold is 30.000 M rows, flush_threshold is 960.000 GB\n",
+    "* (['fold', 'tweet_type'], DataFrame[tweet_type: string, fold: double, TE_tweet_type_retweet_with_comment_timestamp: float, TE_tweet_type_like_timestamp: float], 18)\n",
+    "* (['fold', 'dt_dow'], DataFrame[dt_dow: int, fold: double, TE_dt_dow_retweet_timestamp: float], 42)\n",
+    "* (['fold', 'most_used_word_bucket_id'], DataFrame[most_used_word_bucket_id: string, fold: double, TE_most_used_word_bucket_id_reply_timestamp: float, TE_most_used_word_bucket_id_retweet_timestamp: float, TE_most_used_word_bucket_id_retweet_with_comment_timestamp: float, TE_most_used_word_bucket_id_like_timestamp: float], 120)\n",
+    "* (['fold', 'second_used_word_bucket_id'], DataFrame[second_used_word_bucket_id: string, fold: double, TE_second_used_word_bucket_id_reply_timestamp: float, TE_second_used_word_bucket_id_retweet_timestamp: float, TE_second_used_word_bucket_id_retweet_with_comment_timestamp: float, TE_second_used_word_bucket_id_like_timestamp: float], 126)\n",
+    "* (['fold', 'mentioned_bucket_id'], DataFrame[mentioned_bucket_id: int, fold: double, TE_mentioned_bucket_id_reply_timestamp: float, TE_mentioned_bucket_id_retweet_timestamp: float, TE_mentioned_bucket_id_retweet_with_comment_timestamp: float, TE_mentioned_bucket_id_like_timestamp: float], 132)\n",
+    "* (['fold', 'language'], DataFrame[language: string, fold: double, TE_language_reply_timestamp: float, TE_language_retweet_timestamp: float, TE_language_retweet_with_comment_timestamp: float, TE_language_like_timestamp: float], 396)\n",
+    "* (['fold', 'mentioned_count'], DataFrame[mentioned_count: bigint, fold: double, TE_mentioned_count_reply_timestamp: float, TE_mentioned_count_retweet_timestamp: float, TE_mentioned_count_retweet_with_comment_timestamp: float, TE_mentioned_count_like_timestamp: float], 25968)\n",
+    "* (['fold', 'engaged_with_user_id'], DataFrame[engaged_with_user_id: string, fold: double, TE_engaged_with_user_id_reply_timestamp: float, TE_engaged_with_user_id_retweet_timestamp: float, TE_engaged_with_user_id_retweet_with_comment_timestamp: float, TE_engaged_with_user_id_like_timestamp: float], 68843219)\n",
+    "* (['fold', 'language', 'engaged_with_user_id'], DataFrame[language: string, engaged_with_user_id: string, fold: double, GTE_language_engaged_with_user_id_reply_timestamp: float, GTE_language_engaged_with_user_id_retweet_timestamp: float, GTE_language_engaged_with_user_id_retweet_with_comment_timestamp: float, GTE_language_engaged_with_user_id_like_timestamp: float], 89264853)\n",
+    "* (['fold', 'tweet_type', 'engaged_with_user_id'], DataFrame[tweet_type: string, engaged_with_user_id: string, fold: double, GTE_tweet_type_engaged_with_user_id_reply_timestamp: float, GTE_tweet_type_engaged_with_user_id_retweet_timestamp: float, GTE_tweet_type_engaged_with_user_id_retweet_with_comment_timestamp: float, GTE_tweet_type_engaged_with_user_id_like_timestamp: float], 95344319)\n",
+    "* (['fold', 'mentioned_bucket_id', 'engaging_user_id'], DataFrame[mentioned_bucket_id: int, engaging_user_id: string, fold: double, GTE_mentioned_bucket_id_engaging_user_id_reply_timestamp: float, GTE_mentioned_bucket_id_engaging_user_id_retweet_timestamp: float, GTE_mentioned_bucket_id_engaging_user_id_retweet_with_comment_timestamp: float, GTE_mentioned_bucket_id_engaging_user_id_like_timestamp: float], 129179090)\n",
+    "* (['fold', 'has_mention', 'engaging_user_id'], DataFrame[has_mention: int, engaging_user_id: string, fold: double, GTE_has_mention_engaging_user_id_reply_timestamp: float, GTE_has_mention_engaging_user_id_retweet_timestamp: float, GTE_has_mention_engaging_user_id_retweet_with_comment_timestamp: float, GTE_has_mention_engaging_user_id_like_timestamp: float], 131959138)\n",
+    "* (['fold', 'language', 'engaging_user_id'], DataFrame[language: string, engaging_user_id: string, fold: double, GTE_language_engaging_user_id_reply_timestamp: float, GTE_language_engaging_user_id_retweet_timestamp: float, GTE_language_engaging_user_id_retweet_with_comment_timestamp: float, GTE_language_engaging_user_id_like_timestamp: float], 149779869)\n",
+    "* (['fold', 'dt_dow', 'engaged_with_user_id'], DataFrame[dt_dow: int, engaged_with_user_id: string, fold: double, GTE_dt_dow_engaged_with_user_id_reply_timestamp: float, GTE_dt_dow_engaged_with_user_id_retweet_timestamp: float, GTE_dt_dow_engaged_with_user_id_retweet_with_comment_timestamp: float, GTE_dt_dow_engaged_with_user_id_like_timestamp: float], 155971686)\n",
+    "* (['fold', 'tweet_type', 'engaging_user_id'], DataFrame[tweet_type: string, engaging_user_id: string, fold: double, GTE_tweet_type_engaging_user_id_reply_timestamp: float, GTE_tweet_type_engaging_user_id_retweet_timestamp: float, GTE_tweet_type_engaging_user_id_retweet_with_comment_timestamp: float, GTE_tweet_type_engaging_user_id_like_timestamp: float], 162333565)\n",
+    "* (['fold', 'mentioned_count', 'engaging_user_id'], DataFrame[mentioned_count: bigint, engaging_user_id: string, fold: double, GTE_mentioned_count_engaging_user_id_reply_timestamp: float, GTE_mentioned_count_engaging_user_id_retweet_timestamp: float, GTE_mentioned_count_engaging_user_id_retweet_with_comment_timestamp: float, GTE_mentioned_count_engaging_user_id_like_timestamp: float], 167086112)\n",
+    "* (['fold', 'dt_hour', 'engaged_with_user_id'], DataFrame[dt_hour: int, engaged_with_user_id: string, fold: double, GTE_dt_hour_engaged_with_user_id_reply_timestamp: float, GTE_dt_hour_engaged_with_user_id_retweet_timestamp: float, GTE_dt_hour_engaged_with_user_id_retweet_with_comment_timestamp: float, GTE_dt_hour_engaged_with_user_id_like_timestamp: float], 201562399)\n",
+    "* (['fold', 'dt_dow', 'engaging_user_id'], DataFrame[dt_dow: int, engaging_user_id: string, fold: double, GTE_dt_dow_engaging_user_id_reply_timestamp: float, GTE_dt_dow_engaging_user_id_retweet_timestamp: float, GTE_dt_dow_engaging_user_id_retweet_with_comment_timestamp: float, GTE_dt_dow_engaging_user_id_like_timestamp: float], 247354193)\n",
+    "* (['fold', 'dt_hour', 'engaging_user_id'], DataFrame[dt_hour: int, engaging_user_id: string, fold: double, GTE_dt_hour_engaging_user_id_reply_timestamp: float, GTE_dt_hour_engaging_user_id_retweet_timestamp: float, GTE_dt_hour_engaging_user_id_retweet_with_comment_timestamp: float, GTE_dt_hour_engaging_user_id_like_timestamp: float], 337378319)\n",
+    "* ['fold', 'tweet_type'] will do bhj\n",
+    "* ['fold', 'dt_dow'] will do bhj\n",
+    "* ['fold', 'most_used_word_bucket_id'] will do bhj\n",
+    "* ['fold', 'second_used_word_bucket_id'] will do bhj\n",
+    "* ['fold', 'mentioned_bucket_id'] will do bhj\n",
+    "* ['fold', 'language'] will do bhj\n",
+    "* ['fold', 'mentioned_count'] will do bhj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'engaged_with_user_id'] is 3.328 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'language', 'engaged_with_user_id'] is 3.238 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'tweet_type', 'engaged_with_user_id'] is 3.148 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'mentioned_bucket_id', 'engaging_user_id'] is 3.058 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'has_mention', 'engaging_user_id'] is 2.968 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'language', 'engaging_user_id'] is 2.878 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'dt_dow', 'engaged_with_user_id'] is 2.788 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'tweet_type', 'engaging_user_id'] is 2.698 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'mentioned_count', 'engaging_user_id'] is 2.608 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'dt_hour', 'engaged_with_user_id'] is 2.518 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'dt_dow', 'engaging_user_id'] is 2.428 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['fold', 'dt_hour', 'engaging_user_id'] is 2.338 GB, will do smj\n",
+    "* do bhj to ['fold', 'tweet_type']\n",
+    "* do bhj to ['fold', 'dt_dow']\n",
+    "* do bhj to ['fold', 'most_used_word_bucket_id']\n",
+    "* do bhj to ['fold', 'second_used_word_bucket_id']\n",
+    "* do bhj to ['fold', 'mentioned_bucket_id']\n",
+    "* do bhj to ['fold', 'language']\n",
+    "* do bhj to ['fold', 'mentioned_count']\n",
+    "* do smj to ['fold', 'engaged_with_user_id']\n",
+    "* do smj to ['fold', 'language', 'engaged_with_user_id']\n",
+    "* do smj to ['fold', 'tweet_type', 'engaged_with_user_id']\n",
+    "* do smj to ['fold', 'mentioned_bucket_id', 'engaging_user_id']\n",
+    "* do smj to ['fold', 'has_mention', 'engaging_user_id']\n",
+    "* do smj to ['fold', 'language', 'engaging_user_id']\n",
+    "* do smj to ['fold', 'dt_dow', 'engaged_with_user_id']\n",
+    "* do smj to ['fold', 'tweet_type', 'engaging_user_id']\n",
+    "* do smj to ['fold', 'mentioned_count', 'engaging_user_id']\n",
+    "* do smj to ['fold', 'dt_hour', 'engaged_with_user_id']\n",
+    "* do smj to ['fold', 'dt_dow', 'engaging_user_id']\n",
+    "* do smj to ['fold', 'dt_hour', 'engaging_user_id']\n",
+    "* encodingFeatures took 362.135\n",
+    "\n",
+    "======\n",
+    "\n",
+    "* bhj total threshold is 67.109 M rows, one bhj threshold is 30.000 M rows, flush_threshold is 960.000 GB\n",
+    "* ('tweet', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 148528557)\n",
+    "* etstimated_to_shuffle_size for tweet is 20.076 GB, will do smj\n",
+    "* do smj to bucketized_tweet_word\n",
+    "* bhj total threshold is 67.109 M rows, one bhj threshold is 30.000 M rows, flush_threshold is 960.000 GB\n",
+    "* ('mentioned_bucket_id', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 4891669)\n",
+    "* ('mentioned_count', DataFrame[dict_col: string, dict_col_id: bigint], 4891669)\n",
+    "* ('mentioned_bucket_id', DataFrame[dict_col: string, count: bigint, dict_col_id: int], 4891669)\n",
+    "* ('mentioned_count', DataFrame[dict_col: string, dict_col_id: bigint], 4891669)\n",
+    "* mentioned_bucket_id will do bhj\n",
+    "* mentioned_count will do bhj\n",
+    "* mentioned_bucket_id will do bhj\n",
+    "* mentioned_count will do bhj\n",
+    "* do bhj to mentioned_bucket_id\n",
+    "* do bhj to mentioned_count\n",
+    "* do bhj to mentioned_bucket_id\n",
+    "* do bhj to mentioned_count\n",
+    "* bhj total threshold is 67.109 M rows, one bhj threshold is 30.000 M rows, flush_threshold is 960.000 GB\n",
+    "* ('tweet_type', DataFrame[tweet_type: string, TE_tweet_type_retweet_with_comment_timestamp: float, TE_tweet_type_like_timestamp: float], 3)\n",
+    "* ('dt_dow', DataFrame[dt_dow: int, TE_dt_dow_retweet_timestamp: float], 7)\n",
+    "* ('most_used_word_bucket_id', DataFrame[most_used_word_bucket_id: string, TE_most_used_word_bucket_id_reply_timestamp: float, TE_most_used_word_bucket_id_retweet_timestamp: float, TE_most_used_word_bucket_id_retweet_with_comment_timestamp: float, TE_most_used_word_bucket_id_like_timestamp: float], 20)\n",
+    "* ('second_used_word_bucket_id', DataFrame[second_used_word_bucket_id: string, TE_second_used_word_bucket_id_reply_timestamp: float, TE_second_used_word_bucket_id_retweet_timestamp: float, TE_second_used_word_bucket_id_retweet_with_comment_timestamp: float, TE_second_used_word_bucket_id_like_timestamp: float], 21)\n",
+    "* ('mentioned_bucket_id', DataFrame[mentioned_bucket_id: int, TE_mentioned_bucket_id_reply_timestamp: float, TE_mentioned_bucket_id_retweet_timestamp: float, TE_mentioned_bucket_id_retweet_with_comment_timestamp: float, TE_mentioned_bucket_id_like_timestamp: float], 22)\n",
+    "* ('language', DataFrame[language: string, TE_language_reply_timestamp: float, TE_language_retweet_timestamp: float, TE_language_retweet_with_comment_timestamp: float, TE_language_like_timestamp: float], 66)\n",
+    "* ('mentioned_count', DataFrame[mentioned_count: bigint, TE_mentioned_count_reply_timestamp: float, TE_mentioned_count_retweet_timestamp: float, TE_mentioned_count_retweet_with_comment_timestamp: float, TE_mentioned_count_like_timestamp: float], 4328)\n",
+    "* ('engaged_with_user_id', DataFrame[engaged_with_user_id: string, TE_engaged_with_user_id_reply_timestamp: float, TE_engaged_with_user_id_retweet_timestamp: float, TE_engaged_with_user_id_retweet_with_comment_timestamp: float, TE_engaged_with_user_id_like_timestamp: float], 22887349)\n",
+    "* (['language', 'engaged_with_user_id'], DataFrame[language: string, engaged_with_user_id: string, GTE_language_engaged_with_user_id_reply_timestamp: float, GTE_language_engaged_with_user_id_retweet_timestamp: float, GTE_language_engaged_with_user_id_retweet_with_comment_timestamp: float, GTE_language_engaged_with_user_id_like_timestamp: float], 35483505)\n",
+    "* (['tweet_type', 'engaged_with_user_id'], DataFrame[tweet_type: string, engaged_with_user_id: string, GTE_tweet_type_engaged_with_user_id_reply_timestamp: float, GTE_tweet_type_engaged_with_user_id_retweet_timestamp: float, GTE_tweet_type_engaged_with_user_id_retweet_with_comment_timestamp: float, GTE_tweet_type_engaged_with_user_id_like_timestamp: float], 36043484)\n",
+    "* (['has_mention', 'engaging_user_id'], DataFrame[has_mention: int, engaging_user_id: string, GTE_has_mention_engaging_user_id_reply_timestamp: float, GTE_has_mention_engaging_user_id_retweet_timestamp: float, GTE_has_mention_engaging_user_id_retweet_with_comment_timestamp: float, GTE_has_mention_engaging_user_id_like_timestamp: float], 47561701)\n",
+    "* (['mentioned_bucket_id', 'engaging_user_id'], DataFrame[mentioned_bucket_id: int, engaging_user_id: string, GTE_mentioned_bucket_id_engaging_user_id_reply_timestamp: float, GTE_mentioned_bucket_id_engaging_user_id_retweet_timestamp: float, GTE_mentioned_bucket_id_engaging_user_id_retweet_with_comment_timestamp: float, GTE_mentioned_bucket_id_engaging_user_id_like_timestamp: float], 52816686)\n",
+    "* (['language', 'engaging_user_id'], DataFrame[language: string, engaging_user_id: string, GTE_language_engaging_user_id_reply_timestamp: float, GTE_language_engaging_user_id_retweet_timestamp: float, GTE_language_engaging_user_id_retweet_with_comment_timestamp: float, GTE_language_engaging_user_id_like_timestamp: float], 62810211)\n",
+    "* (['tweet_type', 'engaging_user_id'], DataFrame[tweet_type: string, engaging_user_id: string, GTE_tweet_type_engaging_user_id_reply_timestamp: float, GTE_tweet_type_engaging_user_id_retweet_timestamp: float, GTE_tweet_type_engaging_user_id_retweet_with_comment_timestamp: float, GTE_tweet_type_engaging_user_id_like_timestamp: float], 63468214)\n",
+    "* (['dt_dow', 'engaged_with_user_id'], DataFrame[dt_dow: int, engaged_with_user_id: string, GTE_dt_dow_engaged_with_user_id_reply_timestamp: float, GTE_dt_dow_engaged_with_user_id_retweet_timestamp: float, GTE_dt_dow_engaged_with_user_id_retweet_with_comment_timestamp: float, GTE_dt_dow_engaged_with_user_id_like_timestamp: float], 69049586)\n",
+    "* (['mentioned_count', 'engaging_user_id'], DataFrame[mentioned_count: bigint, engaging_user_id: string, GTE_mentioned_count_engaging_user_id_reply_timestamp: float, GTE_mentioned_count_engaging_user_id_retweet_timestamp: float, GTE_mentioned_count_engaging_user_id_retweet_with_comment_timestamp: float, GTE_mentioned_count_engaging_user_id_like_timestamp: float], 92173751)\n",
+    "* (['dt_hour', 'engaged_with_user_id'], DataFrame[dt_hour: int, engaged_with_user_id: string, GTE_dt_hour_engaged_with_user_id_reply_timestamp: float, GTE_dt_hour_engaged_with_user_id_retweet_timestamp: float, GTE_dt_hour_engaged_with_user_id_retweet_with_comment_timestamp: float, GTE_dt_hour_engaged_with_user_id_like_timestamp: float], 102369370)\n",
+    "* (['dt_dow', 'engaging_user_id'], DataFrame[dt_dow: int, engaging_user_id: string, GTE_dt_dow_engaging_user_id_reply_timestamp: float, GTE_dt_dow_engaging_user_id_retweet_timestamp: float, GTE_dt_dow_engaging_user_id_retweet_with_comment_timestamp: float, GTE_dt_dow_engaging_user_id_like_timestamp: float], 116332109)\n",
+    "* (['dt_hour', 'engaging_user_id'], DataFrame[dt_hour: int, engaging_user_id: string, GTE_dt_hour_engaging_user_id_reply_timestamp: float, GTE_dt_hour_engaging_user_id_retweet_timestamp: float, GTE_dt_hour_engaging_user_id_retweet_with_comment_timestamp: float, GTE_dt_hour_engaging_user_id_like_timestamp: float], 192785849)\n",
+    "* tweet_type will do bhj\n",
+    "* dt_dow will do bhj\n",
+    "* most_used_word_bucket_id will do bhj\n",
+    "* second_used_word_bucket_id will do bhj\n",
+    "* mentioned_bucket_id will do bhj\n",
+    "* language will do bhj\n",
+    "* mentioned_count will do bhj\n",
+    "* engaged_with_user_id will do bhj\n",
+    "* etstimated_to_shuffle_size for ['language', 'engaged_with_user_id'] is 17.399 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['tweet_type', 'engaged_with_user_id'] is 16.897 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['has_mention', 'engaging_user_id'] is 16.396 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['mentioned_bucket_id', 'engaging_user_id'] is 15.894 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['language', 'engaging_user_id'] is 15.392 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['tweet_type', 'engaging_user_id'] is 14.890 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['dt_dow', 'engaged_with_user_id'] is 14.388 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['mentioned_count', 'engaging_user_id'] is 13.886 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['dt_hour', 'engaged_with_user_id'] is 13.384 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['dt_dow', 'engaging_user_id'] is 12.882 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for ['dt_hour', 'engaging_user_id'] is 12.380 GB, will do smj\n",
+    "* do bhj to tweet_type\n",
+    "* do bhj to dt_dow\n",
+    "* do bhj to most_used_word_bucket_id\n",
+    "* do bhj to second_used_word_bucket_id\n",
+    "* do bhj to mentioned_bucket_id\n",
+    "* do bhj to language\n",
+    "* do bhj to mentioned_count\n",
+    "* do bhj to engaged_with_user_id\n",
+    "* do smj to ['language', 'engaged_with_user_id']\n",
+    "* do smj to ['tweet_type', 'engaged_with_user_id']\n",
+    "* do smj to ['has_mention', 'engaging_user_id']\n",
+    "* do smj to ['mentioned_bucket_id', 'engaging_user_id']\n",
+    "* do smj to ['language', 'engaging_user_id']\n",
+    "* do smj to ['tweet_type', 'engaging_user_id']\n",
+    "* do smj to ['dt_dow', 'engaged_with_user_id']\n",
+    "* do smj to ['mentioned_count', 'engaging_user_id']\n",
+    "* do smj to ['dt_hour', 'engaged_with_user_id']\n",
+    "* do smj to ['dt_dow', 'engaging_user_id']\n",
+    "* do smj to ['dt_hour', 'engaging_user_id']\n",
+    "* mergeFeaturesToTest took 1259.446\n",
+    "\n",
+    "======\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/notebooks/recsys2021/init.py b/RecDP/examples/notebooks/recsys2021/init.py
new file mode 100644
index 000000000..e0981db4f
--- /dev/null
+++ b/RecDP/examples/notebooks/recsys2021/init.py
@@ -0,0 +1,8 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
+import findspark
+findspark.init()
diff --git a/RecDP/examples/notebooks/wnd/data/feature_description.py b/RecDP/examples/notebooks/wnd/data/feature_description.py
new file mode 100644
index 000000000..a52db4aad
--- /dev/null
+++ b/RecDP/examples/notebooks/wnd/data/feature_description.py
@@ -0,0 +1,138 @@
+LABEL_COLUMN = "label"
+
+DISPLAY_ID_COLUMN = 'display_id'
+
+IS_LEAK_COLUMN = 'is_leak'
+
+DISPLAY_ID_AND_IS_LEAK_ENCODED_COLUMN = 'display_ad_and_is_leak'
+
+CATEGORICAL_COLUMNS = [
+    'ad_id',
+    'campaign_id',
+    'doc_id',
+    'doc_event_id',
+    'ad_advertiser',
+    'doc_ad_source_id',
+    'doc_ad_publisher_id',
+    'doc_event_publisher_id',
+    'doc_event_source_id',
+    'event_country',
+    'event_country_state',
+    'event_geo_location',
+    'event_platform']
+
+DOC_CATEGORICAL_MULTIVALUED_COLUMNS = {
+}
+
+BOOL_COLUMNS = []
+
+INT_COLUMNS = [
+    'ad_views',
+    'doc_views',
+    'doc_event_days_since_published',
+    'doc_ad_days_since_published']
+
+FLOAT_COLUMNS_LOG_BIN_TRANSFORM = []
+FLOAT_COLUMNS_NO_TRANSFORM = [
+    'pop_ad_id',
+    'pop_document_id',
+    'pop_publisher_id',
+    'pop_advertiser_id',
+    'pop_campain_id',
+    'pop_source_id',
+    'doc_event_doc_ad_sim_categories',
+    'doc_event_doc_ad_sim_topics',
+    'doc_event_doc_ad_sim_entities',
+]
+FLOAT_COLUMNS_SIMPLE_BIN_TRANSFORM = []
+FLOAT_COLUMNS = FLOAT_COLUMNS_LOG_BIN_TRANSFORM + FLOAT_COLUMNS_SIMPLE_BIN_TRANSFORM + FLOAT_COLUMNS_NO_TRANSFORM
+
+REQUEST_SINGLE_HOT_COLUMNS = [
+    "doc_event_id",
+    "doc_id",
+    "doc_event_source_id",
+    "event_geo_location",
+    "event_country_state",
+    "doc_event_publisher_id",
+    "event_country",
+    "event_hour",
+    "event_platform",
+    "traffic_source",
+    "event_weekend",
+    "user_has_already_viewed_doc"]
+
+REQUEST_MULTI_HOT_COLUMNS = [
+    "doc_event_entity_id",
+    "doc_event_topic_id",
+    "doc_event_category_id"]
+
+REQUEST_NUMERIC_COLUMNS = [
+    "pop_document_id_conf",
+    "pop_publisher_id_conf",
+    "pop_source_id_conf",
+    "pop_entity_id_conf",
+    "pop_topic_id_conf",
+    "pop_category_id_conf",
+    "pop_document_id",
+    "pop_publisher_id",
+    "pop_source_id",
+    "pop_entity_id",
+    "pop_topic_id",
+    "pop_category_id",
+    "user_views",
+    "doc_views",
+    "doc_event_days_since_published",
+    "doc_event_hour"]
+
+ITEM_SINGLE_HOT_COLUMNS = [
+    "ad_id",
+    'campaign_id',
+    "doc_ad_source_id",
+    "ad_advertiser",
+    "doc_ad_publisher_id"]
+
+ITEM_MULTI_HOT_COLUMNS = [
+    "doc_ad_topic_id",
+    "doc_ad_entity_id",
+    "doc_ad_category_id"]
+
+ITEM_NUMERIC_COLUMNS = [
+    "pop_ad_id_conf",
+    "user_doc_ad_sim_categories_conf",
+    "user_doc_ad_sim_topics_conf",
+    "pop_advertiser_id_conf",
+    "pop_ad_id",
+    "pop_advertiser_id",
+    "pop_campain_id",
+    "user_doc_ad_sim_categories",
+    "user_doc_ad_sim_topics",
+    "user_doc_ad_sim_entities",
+    "doc_event_doc_ad_sim_categories",
+    "doc_event_doc_ad_sim_topics",
+    "doc_event_doc_ad_sim_entities",
+    "ad_views",
+    "doc_ad_days_since_published"]
+
+NV_TRAINING_COLUMNS = (
+        REQUEST_SINGLE_HOT_COLUMNS +
+        REQUEST_MULTI_HOT_COLUMNS +
+        REQUEST_NUMERIC_COLUMNS +
+        ITEM_SINGLE_HOT_COLUMNS +
+        ITEM_MULTI_HOT_COLUMNS +
+        ITEM_NUMERIC_COLUMNS)
+
+HASH_BUCKET_SIZES = {
+    'doc_event_id': 300000,
+    'ad_id': 250000,
+    'doc_id': 100000,
+    'doc_ad_source_id': 4000,
+    'doc_event_source_id': 4000,
+    'event_geo_location': 2500,
+    'ad_advertiser': 2500,
+    'event_country_state': 2000,
+    'doc_ad_publisher_id': 1000,
+    'doc_event_publisher_id': 1000,
+    'event_country': 300,
+    'event_platform': 4,
+    'campaign_id': 5000
+}
\ No newline at end of file
diff --git a/RecDP/examples/notebooks/wnd/data/tensorflow-hadoop-1.5.0.jar b/RecDP/examples/notebooks/wnd/data/tensorflow-hadoop-1.5.0.jar
new file mode 100644
index 000000000..e2a2155be
Binary files /dev/null and b/RecDP/examples/notebooks/wnd/data/tensorflow-hadoop-1.5.0.jar differ
diff --git a/RecDP/examples/notebooks/wnd/init.py b/RecDP/examples/notebooks/wnd/init.py
new file mode 100644
index 000000000..a6fc7ddd8
--- /dev/null
+++ b/RecDP/examples/notebooks/wnd/init.py
@@ -0,0 +1,8 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
+# import findspark
+# findspark.init()
diff --git a/RecDP/examples/notebooks/wnd/wnd_preproc.ipynb b/RecDP/examples/notebooks/wnd/wnd_preproc.ipynb
new file mode 100644
index 000000000..9b80845c4
--- /dev/null
+++ b/RecDP/examples/notebooks/wnd/wnd_preproc.ipynb
@@ -0,0 +1,907 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import init\n",
+    "\n",
+    "# import findspark\n",
+    "# findspark.init()\n",
+    "\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.utils import *\n",
+    "\n",
+    "from pyspark.sql.types import IntegerType, StringType, StructType, StructField, TimestampType, FloatType, ArrayType, DoubleType\n",
+    "import datetime\n",
+    "import hashlib\n",
+    "import math\n",
+    "import time\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import pyspark.sql.functions as F\n",
+    "import tensorflow as tf\n",
+    "from pyspark import TaskContext\n",
+    "from tensorflow_transform.tf_metadata import dataset_metadata\n",
+    "from tensorflow_transform.tf_metadata import dataset_schema\n",
+    "from tensorflow_transform.tf_metadata import metadata_io\n",
+    "from examples.notebooks.wnd.data.feature_description import LABEL_COLUMN, DISPLAY_ID_COLUMN, CATEGORICAL_COLUMNS, \\\n",
+    "    DOC_CATEGORICAL_MULTIVALUED_COLUMNS, BOOL_COLUMNS, INT_COLUMNS, FLOAT_COLUMNS, \\\n",
+    "    FLOAT_COLUMNS_LOG_BIN_TRANSFORM, FLOAT_COLUMNS_SIMPLE_BIN_TRANSFORM, FLOAT_COLUMNS_NO_TRANSFORM, HASH_BUCKET_SIZES\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "OUTPUT_BUCKET_FOLDER = \"/tmp/spark/preprocessed/recdp/\"\n",
+    "DATA_BUCKET_FOLDER = \"/outbrain/orig/\"\n",
+    "SPARK_TEMP_FOLDER = \"/tmp/spark/spark-temp/\"\n",
+    "TENSORFLOW_HADOOP = \"data/tensorflow-hadoop-1.5.0.jar\"\n",
+    "scala_udf_jars = \"/root/ht/ML/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar\"\n",
+    "\n",
+    "conf = SparkConf().setMaster('spark://sr112:7077').set('spark.executor.memory', '40g').set('spark.driver.memory', '200g').set('spark.executor.cores', '10')\n",
+    "conf.set(\"spark.jars\", TENSORFLOW_HADOOP)\n",
+    "conf.set(\"spark.driver.extraClassPath\", f\"{scala_udf_jars}\")\n",
+    "conf.set(\"spark.executor.extraClassPath\", f\"{scala_udf_jars}\")\n",
+    "\n",
+    "\n",
+    "sc = SparkContext(conf=conf)\n",
+    "spark = SparkSession(sc)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#######################  split train/test set #######################\n",
+    "events_schema = StructType(\n",
+    "    [StructField(\"display_id\", IntegerType(), True),\n",
+    "     StructField(\"uuid_event\", StringType(), True),\n",
+    "     StructField(\"document_id_event\", IntegerType(), True),\n",
+    "     StructField(\"timestamp_event\", IntegerType(), True),\n",
+    "     StructField(\"platform_event\", IntegerType(), True),\n",
+    "     StructField(\"geo_location_event\", StringType(), True)]\n",
+    ")\n",
+    "\n",
+    "events_df = spark.read.schema(events_schema) \\\n",
+    "    .options(header='true', inferschema='false', nullValue='\\\\N') \\\n",
+    "    .csv(DATA_BUCKET_FOLDER + \"events.csv\") \\\n",
+    "    .withColumn('day_event', (F.col('timestamp_event')/ 1000 / 60 / 60 / 24).cast(IntegerType())) \\\n",
+    "    .withColumn('event_country', F.substring('geo_location_event', 0, 2)) \\\n",
+    "    .withColumn('event_country_state', F.substring('geo_location_event', 0, 5)) \\\n",
+    "    .alias('events')\n",
+    "\n",
+    "print('Drop rows with empty \"geo_location\"...')\n",
+    "events_df = events_df.dropna(subset=\"geo_location_event\")\n",
+    "\n",
+    "print('Drop rows with empty \"platform\"...')\n",
+    "events_df = events_df.dropna(subset=\"platform_event\")\n",
+    "\n",
+    "promoted_content_schema = StructType(\n",
+    "    [StructField(\"ad_id\", IntegerType(), True),\n",
+    "     StructField(\"document_id_promo\", IntegerType(), True),\n",
+    "     StructField(\"campaign_id\", IntegerType(), True),\n",
+    "     StructField(\"advertiser_id\", IntegerType(), True)]\n",
+    ")\n",
+    "\n",
+    "promoted_content_df = spark.read.schema(promoted_content_schema) \\\n",
+    "    .options(header='true', inferschema='false', nullValue='\\\\N') \\\n",
+    "    .csv(DATA_BUCKET_FOLDER + \"promoted_content.csv\") \\\n",
+    "    .alias('promoted_content')\n",
+    "\n",
+    "clicks_train_schema = StructType(\n",
+    "    [StructField(\"display_id\", IntegerType(), True),\n",
+    "     StructField(\"ad_id\", IntegerType(), True),\n",
+    "     StructField(\"clicked\", IntegerType(), True)]\n",
+    ")\n",
+    "\n",
+    "clicks_train_df = spark.read.schema(clicks_train_schema) \\\n",
+    "    .options(header='true', inferschema='false', nullValue='\\\\N') \\\n",
+    "    .csv(DATA_BUCKET_FOLDER + \"clicks_train.csv\") \\\n",
+    "    .alias('clicks_train')\n",
+    "\n",
+    "documents_meta_schema = StructType(\n",
+    "    [StructField(\"document_id_doc\", IntegerType(), True),\n",
+    "     StructField(\"source_id\", IntegerType(), True),\n",
+    "     StructField(\"publisher_id\", IntegerType(), True),\n",
+    "     StructField(\"publish_time\", TimestampType(), True)]\n",
+    ")\n",
+    "\n",
+    "documents_meta_df = spark.read.schema(documents_meta_schema) \\\n",
+    "    .options(header='true', inferschema='false', nullValue='\\\\N') \\\n",
+    "    .csv(DATA_BUCKET_FOLDER + \"documents_meta.csv\") \\\n",
+    "    .alias('documents_meta')\n",
+    "\n",
+    "# Drop rows with empty \"source_id\"\n",
+    "documents_meta_df = documents_meta_df.dropna(subset=\"source_id\")\n",
+    "\n",
+    "source_publishers_df = documents_meta_df.select([\"source_id\", \"publisher_id\"]).dropDuplicates()\n",
+    "\n",
+    "# get list of source_ids without publisher_id\n",
+    "rows_no_pub = source_publishers_df.filter(\"publisher_id is NULL\")\n",
+    "source_ids_without_publisher = [row['source_id'] for row in rows_no_pub.collect()]\n",
+    "\n",
+    "# maximum value of publisher_id used so far\n",
+    "max_pub = max(source_publishers_df.select([\"publisher_id\"]).dropna().collect())['publisher_id']\n",
+    "\n",
+    "# rows filled with new publisher_ids\n",
+    "new_publishers = [(source, max_pub + 1 + nr) for nr, source in enumerate(source_ids_without_publisher)]\n",
+    "new_publishers_df = spark.createDataFrame(new_publishers, (\"source_id\", \"publisher_id\"))\n",
+    "\n",
+    "# old and new publishers merged\n",
+    "fixed_source_publishers_df = source_publishers_df.dropna().union(new_publishers_df)\n",
+    "\n",
+    "# update documents_meta with bew publishers\n",
+    "documents_meta_df = documents_meta_df.drop('publisher_id').join(fixed_source_publishers_df, on='source_id')\n",
+    "\n",
+    "documents_total = documents_meta_df.count()\n",
+    "\n",
+    "\n",
+    "events_joined_df = events_df.join(documents_meta_df\n",
+    "                                  .withColumnRenamed('source_id', 'source_id_doc_event')\n",
+    "                                  .withColumnRenamed('publisher_id', 'publisher_doc_event')\n",
+    "                                  .withColumnRenamed('publish_time', 'publish_time_doc_event')\n",
+    "                                  .withColumnRenamed('document_id_doc', 'document_id_doc_event'),\n",
+    "                                  on=F.col(\"document_id_event\") == F.col(\"document_id_doc_event\"), how='left').alias('events')\n",
+    "\n",
+    "clicks_train_joined_df = clicks_train_df \\\n",
+    "    .join(promoted_content_df, on='ad_id', how='left') \\\n",
+    "    .join(documents_meta_df, on=F.col(\"promoted_content.document_id_promo\") == F.col(\"documents_meta.document_id_doc\"), how='left') \\\n",
+    "    .join(events_joined_df, on='display_id', how='left')\n",
+    "\n",
+    "clicks_train_joined_df.createOrReplaceTempView('clicks_train_joined')\n",
+    "\n",
+    "\n",
+    "validation_display_ids_df = clicks_train_joined_df.select('display_id', 'day_event').distinct() \\\n",
+    "    .sampleBy(\"day_event\", fractions={0: 0.2, 1: 0.2, 2: 0.2, 3: 0.2, 4: 0.2,\n",
+    "                                      5: 0.2, 6: 0.2, 7: 0.2, 8: 0.2, 9: 0.2, 10: 0.2, 11: 1.0, 12: 1.0}, seed=0)\n",
+    "\n",
+    "valid_id = validation_display_ids_df.select('display_id').distinct().createOrReplaceTempView(\"validation_display_ids\")\n",
+    "\n",
+    "valid_set_df = spark.sql('''\n",
+    "SELECT * FROM clicks_train_joined t\n",
+    "WHERE EXISTS (SELECT display_id FROM validation_display_ids\n",
+    "WHERE display_id = t.display_id)''')\n",
+    "\n",
+    "s_time = time.time()\n",
+    "valid_set_df.write.format('parquet').mode('overwrite').save(OUTPUT_BUCKET_FOLDER + 'valid_set_df')\n",
+    "valid_set_df = spark.read.parquet(OUTPUT_BUCKET_FOLDER + 'valid_set_df')\n",
+    "print(f'valid_set_df time: {time.time() - s_time}')\n",
+    "\n",
+    "train_set_df = spark.sql('''\n",
+    "SELECT * FROM clicks_train_joined t\n",
+    "WHERE NOT EXISTS (SELECT display_id FROM validation_display_ids\n",
+    "WHERE display_id = t.display_id)''')\n",
+    "\n",
+    "s_time = time.time()\n",
+    "train_set_df.write.format('parquet').mode('overwrite').save(OUTPUT_BUCKET_FOLDER + 'train_set_df')\n",
+    "train_set_df = spark.read.parquet(OUTPUT_BUCKET_FOLDER + 'train_set_df')\n",
+    "print(f'train_set_df time: {time.time() - s_time}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "documents_categories_schema = StructType(\n",
+    "    [StructField(\"document_id_cat\", IntegerType(), True),\n",
+    "     StructField(\"category_id\", IntegerType(), True),\n",
+    "     StructField(\"confidence_level_cat\", FloatType(), True)]\n",
+    ")\n",
+    "\n",
+    "documents_categories_df = spark.read.schema(documents_categories_schema) \\\n",
+    "    .options(header='true', inferschema='false', nullValue='\\\\N') \\\n",
+    "    .csv(DATA_BUCKET_FOLDER + \"documents_categories.csv\") \\\n",
+    "    .alias('documents_categories')\n",
+    "\n",
+    "documents_topics_schema = StructType(\n",
+    "    [StructField(\"document_id_top\", IntegerType(), True),\n",
+    "     StructField(\"topic_id\", IntegerType(), True),\n",
+    "     StructField(\"confidence_level_top\", FloatType(), True)]\n",
+    ")\n",
+    "\n",
+    "documents_topics_df = spark.read.schema(documents_topics_schema) \\\n",
+    "    .options(header='true', inferschema='false', nullValue='\\\\N') \\\n",
+    "    .csv(DATA_BUCKET_FOLDER + \"documents_topics.csv\") \\\n",
+    "    .alias('documents_topics')\n",
+    "\n",
+    "documents_entities_schema = StructType(\n",
+    "    [StructField(\"document_id_ent\", IntegerType(), True),\n",
+    "     StructField(\"entity_id\", StringType(), True),\n",
+    "     StructField(\"confidence_level_ent\", FloatType(), True)]\n",
+    ")\n",
+    "\n",
+    "documents_entities_df = spark.read.schema(documents_entities_schema) \\\n",
+    "    .options(header='true', inferschema='false', nullValue='\\\\N') \\\n",
+    "    .csv(DATA_BUCKET_FOLDER + \"documents_entities.csv\") \\\n",
+    "    .alias('documents_entities')\n",
+    "\n",
+    "documents_categories_grouped_df = documents_categories_df.groupBy('document_id_cat') \\\n",
+    "    .agg(F.collect_list('category_id').alias('category_id_list'),\n",
+    "         F.collect_list('confidence_level_cat').alias('confidence_level_cat_list')) \\\n",
+    "    .alias('documents_categories_grouped').cache()\n",
+    "\n",
+    "documents_topics_grouped_df = documents_topics_df.groupBy('document_id_top') \\\n",
+    "    .agg(F.collect_list('topic_id').alias('topic_id_list'),\n",
+    "         F.collect_list('confidence_level_top').alias('confidence_level_top_list')) \\\n",
+    "    .alias('documents_topics_grouped').cache()\n",
+    "\n",
+    "documents_entities_grouped_df = documents_entities_df.groupBy('document_id_ent') \\\n",
+    "    .agg(F.collect_list('entity_id').alias('entity_id_list'),\n",
+    "         F.collect_list('confidence_level_ent').alias('confidence_level_ent_list')) \\\n",
+    "    .alias('documents_entities_grouped').cache()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "########################## generate dictionary ##################################\n",
+    "country_value_cat = events_df.select('event_country').groupBy('event_country').count().filter('event_country is not null and count >= 10')\n",
+    "\n",
+    "state_value_cal = events_df.select('event_country_state').groupBy('event_country_state').count().filter('event_country_state is not null and count >= 10')\n",
+    "\n",
+    "geo_location_value_cat = events_df.select('geo_location_event').groupBy('geo_location_event').count().filter('geo_location_event is not null and count >= 10')\n",
+    "\n",
+    "\n",
+    "# ### Average CTR by ad_id\n",
+    "ad_id_popularity_df = train_set_df \\\n",
+    "    .groupby('ad_id') \\\n",
+    "    .agg(F.sum('clicked').alias('clicks'),F.count('*').alias('views')) \\\n",
+    "    .withColumn('ctr', F.col('clicks') / F.col('views')) \\\n",
+    "    .filter('views > 5').select('ad_id', 'ctr', 'views')\n",
+    "\n",
+    "# ### Average CTR by document_id (promoted_content)\n",
+    "document_id_popularity_df = train_set_df \\\n",
+    "    .groupby('document_id_promo') \\\n",
+    "    .agg(F.sum('clicked').alias('clicks'), F.count('*').alias('views')) \\\n",
+    "    .withColumn('ctr', F.col('clicks') / F.col('views')) \\\n",
+    "    .filter('views > 5').select('document_id_promo', 'ctr', 'views')\n",
+    "\n",
+    "# ### Average CTR by source_id\n",
+    "source_id_popularity_df = train_set_df.select('clicked', 'source_id', 'ad_id') \\\n",
+    "    .groupby('source_id').agg(F.sum('clicked').alias('clicks'), F.count('*').alias('views')) \\\n",
+    "    .withColumn('ctr', F.col('clicks') / F.col('views')) \\\n",
+    "    .filter('views > 10 and source_id is not null').select('source_id', 'ctr', 'views')\n",
+    "\n",
+    "# ### Average CTR by publisher_id\n",
+    "publisher_popularity_df = train_set_df.select('clicked', 'publisher_id', 'ad_id') \\\n",
+    "    .groupby('publisher_id').agg(F.sum('clicked').alias('clicks'), F.count('*').alias('views')) \\\n",
+    "    .withColumn('ctr', F.col('clicks') / F.col('views')) \\\n",
+    "    .filter('views > 10 and publisher_id is not null').select('publisher_id', 'ctr', 'views')\n",
+    "\n",
+    "# ### Average CTR by advertiser_id\n",
+    "advertiser_id_popularity_df = train_set_df.select('clicked', 'advertiser_id', 'ad_id') \\\n",
+    "    .groupby('advertiser_id').agg(F.sum('clicked').alias('clicks'), F.count('*').alias('views')) \\\n",
+    "    .withColumn('ctr', F.col('clicks') / F.col('views')) \\\n",
+    "    .filter('views > 10 and advertiser_id is not null').select('advertiser_id', 'ctr', 'views')\n",
+    "\n",
+    "# ### Average CTR by campaign_id\n",
+    "campaign_id_popularity_df = train_set_df.select('clicked', 'campaign_id', 'ad_id') \\\n",
+    "    .groupby('campaign_id').agg(F.sum('clicked').alias('clicks'), F.count('*').alias('views')) \\\n",
+    "    .withColumn('ctr', F.col('clicks') / F.col('views')) \\\n",
+    "    .filter('views > 10 and campaign_id is not null').select('campaign_id', 'ctr', 'views')\n",
+    "\n",
+    "\n",
+    "categories_docs_counts = documents_categories_df.groupBy('category_id').count().rdd.collectAsMap()\n",
+    "\n",
+    "topics_docs_counts = documents_topics_df.groupBy('topic_id').count().rdd.collectAsMap()\n",
+    "\n",
+    "entities_docs_counts = documents_entities_df.groupBy('entity_id').count().rdd.collectAsMap()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "##################### udf used for feature engineering ###############################\n",
+    "def cosine_similarity_dicts(dict1, dict2):\n",
+    "    dict1_norm = math.sqrt(sum([v ** 2 for v in dict1.values()]))\n",
+    "    dict2_norm = math.sqrt(sum([v ** 2 for v in dict2.values()]))\n",
+    "\n",
+    "    sum_common_aspects = 0.0\n",
+    "    intersections = 0\n",
+    "    for key in dict1:\n",
+    "        if key in dict2:\n",
+    "            sum_common_aspects += dict1[key] * dict2[key]\n",
+    "            intersections += 1\n",
+    "\n",
+    "    return sum_common_aspects / (dict1_norm * dict2_norm), intersections\n",
+    "\n",
+    "\n",
+    "def cosine_similarity_doc_event_doc_ad_aspects(doc_event_aspect_ids, doc_event_aspects_confidence,\n",
+    "                                               doc_ad_aspect_ids, doc_ad_aspects_confidence,\n",
+    "                                               aspect_docs_counts):\n",
+    "    if doc_event_aspect_ids is None or len(doc_event_aspect_ids) == 0 \\\n",
+    "            or doc_ad_aspect_ids is None or len(doc_ad_aspect_ids) == 0:\n",
+    "        return None, None\n",
+    "\n",
+    "    doc_event_aspects = dict(zip(doc_event_aspect_ids, doc_event_aspects_confidence))\n",
+    "    doc_event_aspects_tfidf_confid = {}\n",
+    "    for key in doc_event_aspect_ids:\n",
+    "        tf = 1.0\n",
+    "        idf = math.log(math.log(documents_total / float(aspect_docs_counts[key])))\n",
+    "        confidence = doc_event_aspects[key]\n",
+    "        doc_event_aspects_tfidf_confid[key] = tf * idf * confidence\n",
+    "\n",
+    "    doc_ad_aspects = dict(zip(doc_ad_aspect_ids, doc_ad_aspects_confidence))\n",
+    "    doc_ad_aspects_tfidf_confid = {}\n",
+    "    for key in doc_ad_aspect_ids:\n",
+    "        tf = 1.0\n",
+    "        idf = math.log(math.log(documents_total / float(aspect_docs_counts[key])))\n",
+    "        confidence = doc_ad_aspects[key]\n",
+    "        doc_ad_aspects_tfidf_confid[key] = tf * idf * confidence\n",
+    "\n",
+    "    similarity, intersections = cosine_similarity_dicts(doc_event_aspects_tfidf_confid, doc_ad_aspects_tfidf_confid)\n",
+    "\n",
+    "    if intersections > 0:\n",
+    "        # P(A intersect B)_intersections = P(A)^intersections * P(B)^intersections\n",
+    "        random_error = math.pow(len(doc_event_aspect_ids) / float(len(aspect_docs_counts)),\n",
+    "                                intersections) * math.pow(len(doc_ad_aspect_ids) / float(len(aspect_docs_counts)),\n",
+    "                                                          intersections)\n",
+    "    else:\n",
+    "        # P(A not intersect B) = 1 - P(A intersect B)\n",
+    "        random_error = 1 - ((len(doc_event_aspect_ids) / float(len(aspect_docs_counts))) *\n",
+    "                            (len(doc_ad_aspect_ids) / float(len(aspect_docs_counts))))\n",
+    "\n",
+    "    confidence = 1.0 - random_error\n",
+    "\n",
+    "    return similarity, confidence\n",
+    "\n",
+    "def convert_odd_timestamp(timestamp_ms_relative):\n",
+    "    TIMESTAMP_DELTA = 1465876799998\n",
+    "    return datetime.datetime.fromtimestamp((int(timestamp_ms_relative) + TIMESTAMP_DELTA) // 1000)\n",
+    "\n",
+    "def timestamp_delta(df, publish_time, timestamp):\n",
+    "    def timestamp_delta_udf(publish_time, timestamp):\n",
+    "        if timestamp > -1:\n",
+    "            dt_timestamp_event = convert_odd_timestamp(timestamp)\n",
+    "            if publish_time is not None:\n",
+    "                delta_days = (dt_timestamp_event - publish_time).days\n",
+    "                if 0 <= delta_days <= 365 * 10:  # 10 years\n",
+    "                    return float(delta_days)\n",
+    "    udf_inter = F.udf(lambda publish_time, timestamp: timestamp_delta_udf(publish_time, timestamp), DoubleType())\n",
+    "    df = df.withColumn(publish_time + '_delta', udf_inter(publish_time, timestamp))\n",
+    "    return df\n",
+    "\n",
+    "# Setting Doc_event-doc_ad CB Similarity fields\n",
+    "def get_doc_event_doc_ad_cb_similarity_score_fn(df, doc_event_ids, doc_event_levels, doc_ad_ids, doc_ad_levels, cnt):\n",
+    "    udf_inter = F.udf(\n",
+    "        lambda doc_event_ids, doc_event_levels, doc_ad_ids, doc_ad_levels: \n",
+    "        cosine_similarity_doc_event_doc_ad_aspects(doc_event_ids, doc_event_levels, doc_ad_ids, doc_ad_levels, cnt)[0], DoubleType())\n",
+    "    df = df.withColumn(doc_event_ids + '_sim', udf_inter(doc_event_ids, doc_event_levels, doc_ad_ids, doc_ad_levels))\n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def enrich_df(df):\n",
+    "    df_enriched = df \\\n",
+    "        .join(documents_categories_grouped_df,\n",
+    "          on=F.col(\"document_id_promo\") == F.col(\"documents_categories_grouped.document_id_cat\"),\n",
+    "          how='left') \\\n",
+    "        .join(documents_topics_grouped_df,\n",
+    "          on=F.col(\"document_id_promo\") == F.col(\"documents_topics_grouped.document_id_top\"),\n",
+    "          how='left') \\\n",
+    "        .join(documents_entities_grouped_df,\n",
+    "          on=F.col(\"document_id_promo\") == F.col(\"documents_entities_grouped.document_id_ent\"),\n",
+    "          how='left') \\\n",
+    "        .join(documents_categories_grouped_df\n",
+    "          .withColumnRenamed('category_id_list', 'doc_event_category_id_list')\n",
+    "          .withColumnRenamed('confidence_level_cat_list', 'doc_event_confidence_level_cat_list')\n",
+    "          .alias('documents_event_categories_grouped'),\n",
+    "          on=F.col(\"document_id_event\") == F.col(\"documents_event_categories_grouped.document_id_cat\"),\n",
+    "          how='left') \\\n",
+    "        .join(documents_topics_grouped_df\n",
+    "          .withColumnRenamed('topic_id_list', 'doc_event_topic_id_list')\n",
+    "          .withColumnRenamed('confidence_level_top_list', 'doc_event_confidence_level_top_list')\n",
+    "          .alias('documents_event_topics_grouped'),\n",
+    "          on=F.col(\"document_id_event\") == F.col(\"documents_event_topics_grouped.document_id_top\"),\n",
+    "          how='left') \\\n",
+    "        .join(documents_entities_grouped_df\n",
+    "          .withColumnRenamed('entity_id_list', 'doc_event_entity_id_list')\n",
+    "          .withColumnRenamed('confidence_level_ent_list', 'doc_event_confidence_level_ent_list')\n",
+    "          .alias('documents_event_entities_grouped'),\n",
+    "          on=F.col(\"document_id_event\") == F.col(\"documents_event_entities_grouped.document_id_ent\"),\n",
+    "          how='left') \\\n",
+    "        .select('display_id', 'uuid_event', 'event_country', 'event_country_state', 'platform_event',\n",
+    "            'source_id_doc_event', 'publisher_doc_event', 'publish_time_doc_event',\n",
+    "            'publish_time', 'ad_id', 'document_id_promo', 'clicked',\n",
+    "            'geo_location_event', 'advertiser_id', 'publisher_id',\n",
+    "            'campaign_id', 'document_id_event',\n",
+    "            F.coalesce(\"doc_event_category_id_list\", F.array())\n",
+    "            .alias('doc_event_category_id_list'),\n",
+    "            F.coalesce(\"doc_event_confidence_level_cat_list\", F.array())\n",
+    "            .alias('doc_event_confidence_level_cat_list'),\n",
+    "            F.coalesce(\"doc_event_topic_id_list\", F.array())\n",
+    "            .alias('doc_event_topic_id_list'),\n",
+    "            F.coalesce(\"doc_event_confidence_level_top_list\", F.array())\n",
+    "            .alias('doc_event_confidence_level_top_list'),\n",
+    "            F.coalesce(\"doc_event_entity_id_list\", F.array())\n",
+    "            .alias('doc_event_entity_id_list'),\n",
+    "            F.coalesce(\"doc_event_confidence_level_ent_list\", F.array())\n",
+    "            .alias('doc_event_confidence_level_ent_list'),\n",
+    "            F.coalesce(\"source_id\", F.lit(-1)).alias('source_id'),\n",
+    "            F.coalesce(\"timestamp_event\", F.lit(-1)).alias('timestamp_event'),\n",
+    "            F.coalesce(\"category_id_list\", F.array()).alias('category_id_list'),\n",
+    "            F.coalesce(\"confidence_level_cat_list\", F.array())\n",
+    "            .alias('confidence_level_cat_list'),\n",
+    "            F.coalesce(\"topic_id_list\", F.array()).alias('topic_id_list'),\n",
+    "            F.coalesce(\"confidence_level_top_list\", F.array())\n",
+    "            .alias('confidence_level_top_list'),\n",
+    "            F.coalesce(\"entity_id_list\", F.array()).alias('entity_id_list'),\n",
+    "            F.coalesce(\"confidence_level_ent_list\", F.array())\n",
+    "            .alias('confidence_level_ent_list'))\n",
+    "    df_enriched = df_enriched.fillna(-1, subset=['source_id', 'timestamp_event'])\n",
+    "    return df_enriched\n",
+    "\n",
+    "train_set_enriched_df = enrich_df(train_set_df)\n",
+    "s_time = time.time()\n",
+    "train_set_enriched_df.write.format('parquet').mode('overwrite').save(OUTPUT_BUCKET_FOLDER + 'train_set_enriched_df')\n",
+    "train_set_enriched_df = spark.read.parquet(OUTPUT_BUCKET_FOLDER + 'train_set_enriched_df')\n",
+    "print(f'train_set_enriched_df time: {time.time() - s_time}')\n",
+    "\n",
+    "test_set_enriched_df = enrich_df(valid_set_df)\n",
+    "s_time = time.time()\n",
+    "test_set_enriched_df.write.format('parquet').mode('overwrite').save(OUTPUT_BUCKET_FOLDER + 'test_set_enriched_df')\n",
+    "test_set_enriched_df = spark.read.parquet(OUTPUT_BUCKET_FOLDER + 'test_set_enriched_df')\n",
+    "print(f'test_set_enriched_df time: {time.time() - s_time}')\n",
+    "\n",
+    "documents_categories_grouped_df.unpersist()\n",
+    "documents_topics_grouped_df.unpersist()\n",
+    "documents_entities_grouped_df.unpersist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def format_number(element, name):\n",
+    "    if name in BOOL_COLUMNS + CATEGORICAL_COLUMNS:\n",
+    "        return element.cast(\"int\")\n",
+    "    else:\n",
+    "        return element\n",
+    "\n",
+    "FEAT_CSV_ORDERED_COLUMNS = ['ad_views', 'campaign_id','doc_views',\n",
+    "                            'doc_event_days_since_published', 'doc_ad_days_since_published',\n",
+    "                            'pop_ad_id', 'pop_document_id', 'pop_publisher_id', 'pop_advertiser_id', 'pop_campain_id',\n",
+    "                            'pop_source_id',\n",
+    "                            'doc_event_doc_ad_sim_categories', 'doc_event_doc_ad_sim_topics',\n",
+    "                            'doc_event_doc_ad_sim_entities', 'ad_advertiser', 'doc_ad_publisher_id',\n",
+    "                            'doc_ad_source_id', 'doc_event_publisher_id', 'doc_event_source_id', 'event_country',\n",
+    "                            'event_country_state', 'event_geo_location', 'event_platform',\n",
+    "                            'traffic_source']\n",
+    "feature_vector_labels = ['ad_id_views', 'campaign_id','document_id_promo_views',\n",
+    "                            'publish_time_doc_event_delta', 'publish_time_delta', \n",
+    "                            'ad_id_ctr', 'document_id_promo_ctr', 'publisher_id_ctr', \n",
+    "                            'advertiser_id_ctr', 'campaign_id_ctr', 'source_id_ctr', \n",
+    "                            'doc_event_category_id_list_sim', 'doc_event_topic_id_list_sim',\n",
+    "                            'doc_event_entity_id_list_sim', \n",
+    "                            'advertiser_id', 'publisher_id', 'source_id', 'publisher_doc_event', 'source_id_doc_event', \n",
+    "                            'event_country_count', 'event_country_state_count', 'geo_location_event_count', 'platform_event', \n",
+    "                            'traffic_source']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def categorifyFeatures(df, proc, output_name=\"categorify\", gen_dict=False):\n",
+    "    categorify_cols = ['ad_id_views', 'document_id_promo_views',\n",
+    "                            'ad_id_ctr', 'document_id_promo_ctr', 'publisher_id_ctr', \n",
+    "                            'advertiser_id_ctr', 'campaign_id_ctr', 'source_id_ctr',\n",
+    "                            'event_country_count', 'event_country_state_count', 'geo_location_event_count']\n",
+    "    to_categorify_cols = ['ad_id', 'document_id_promo',\n",
+    "                            'ad_id', 'document_id_promo', 'publisher_id', \n",
+    "                            'advertiser_id', 'campaign_id', 'source_id',\n",
+    "                            'event_country', 'event_country_state', 'geo_location_event']\n",
+    "    \n",
+    "    # transform dict column name to match recdp\n",
+    "    dfs = [ad_id_popularity_df.select(F.col('ad_id').alias('dict_col'), F.col('views').alias('dict_col_id')), \n",
+    "            document_id_popularity_df.select(F.col('document_id_promo').alias('dict_col'), F.col('views').alias('dict_col_id')), \n",
+    "            ad_id_popularity_df.select(F.col('ad_id').alias('dict_col'), F.col('ctr').alias('dict_col_id')), \n",
+    "            document_id_popularity_df.select(F.col('document_id_promo').alias('dict_col'), F.col('ctr').alias('dict_col_id')), \n",
+    "            publisher_popularity_df.select(F.col('publisher_id').alias('dict_col'), F.col('ctr').alias('dict_col_id')), \n",
+    "            advertiser_id_popularity_df.select(F.col('advertiser_id').alias('dict_col'), F.col('ctr').alias('dict_col_id')), \n",
+    "            campaign_id_popularity_df.select(F.col('campaign_id').alias('dict_col'), F.col('ctr').alias('dict_col_id')), \n",
+    "            source_id_popularity_df.select(F.col('source_id').alias('dict_col'), F.col('ctr').alias('dict_col_id')), \n",
+    "            country_value_cat.select(F.col('event_country').alias('dict_col'), F.col('count').alias('dict_col_id')), \n",
+    "            state_value_cal.select(F.col('event_country_state').alias('dict_col'), F.col('count').alias('dict_col_id')), \n",
+    "            geo_location_value_cat.select(F.col('geo_location_event').alias('dict_col'), F.col('count').alias('dict_col_id'))]\n",
+    "\n",
+    "    dict_dfs = [{'col_name': name, 'dict': dfs[index]} for index, name in enumerate(categorify_cols)]\n",
+    "    # add new columns since recdp will overwrite cat columns\n",
+    "    for index, column in enumerate(categorify_cols):\n",
+    "        df = df.withColumn(column, F.col(to_categorify_cols[index]))\n",
+    "\n",
+    "    op_categorify = Categorify(categorify_cols, dict_dfs=dict_dfs)\n",
+    "    op_fillna = FillNA(['event_country_count', 'event_country_state_count', 'geo_location_event_count'], 0)\n",
+    "    op_feature_modify = FeatureModification(cols={\"platform_event\": \"f.col('platform_event') - 1\"} , op='inline')\n",
+    "    proc.reset_ops([op_categorify, op_fillna, op_feature_modify])\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"Categorify took %.3f\" % (t2 - t1))\n",
+    "\n",
+    "    return df\n",
+    "\n",
+    "def convertType(df, proc, output_name=\"convertType\"):\n",
+    "    op_feature_type_convert = FeatureModification(\n",
+    "        cols={\"campaign_id\": \"f.col('campaign_id').cast(spk_type.DoubleType())\", \n",
+    "             \"advertiser_id\": \"f.col('advertiser_id').cast(spk_type.DoubleType())\", \n",
+    "             \"source_id\": \"f.col('source_id').cast(spk_type.DoubleType())\", \n",
+    "             \"publisher_id\": \"f.col('publisher_id').cast(spk_type.DoubleType())\", \n",
+    "             \"source_id_doc_event\": \"f.col('source_id_doc_event').cast(spk_type.DoubleType())\", \n",
+    "             \"publisher_doc_event\": \"f.col('publisher_doc_event').cast(spk_type.DoubleType())\"\n",
+    "             }, op='inline')\n",
+    "    proc.reset_ops([op_feature_type_convert])\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"Convert type took %.3f\" % (t2 - t1))\n",
+    "    \n",
+    "    return df\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "################################## feature engineer with RecDP ################################\n",
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/wnd/\"\n",
+    "proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity=\"1200GB\")\n",
+    "\n",
+    "######################### trainset feature engineer ###############################\n",
+    "train_set_features_df = train_set_enriched_df\n",
+    "train_set_features_df = categorifyFeatures(train_set_features_df, proc, output_name=\"train_categorified\", gen_dict=False)\n",
+    "train_set_features_df = timestamp_delta(train_set_features_df, 'publish_time', 'timestamp_event')\n",
+    "train_set_features_df = timestamp_delta(train_set_features_df, 'publish_time_doc_event', 'timestamp_event')\n",
+    "\n",
+    "train_set_features_df = get_doc_event_doc_ad_cb_similarity_score_fn(\n",
+    "    train_set_features_df, 'doc_event_category_id_list', 'doc_event_confidence_level_cat_list', \n",
+    "    'category_id_list', 'confidence_level_cat_list', categories_docs_counts)\n",
+    "train_set_features_df = get_doc_event_doc_ad_cb_similarity_score_fn(\n",
+    "    train_set_features_df, 'doc_event_topic_id_list', 'doc_event_confidence_level_top_list',\n",
+    "    'topic_id_list', 'confidence_level_top_list', topics_docs_counts)\n",
+    "train_set_features_df = get_doc_event_doc_ad_cb_similarity_score_fn(\n",
+    "    train_set_features_df, 'doc_event_entity_id_list', 'doc_event_confidence_level_ent_list', \n",
+    "    'entity_id_list', 'confidence_level_ent_list', entities_docs_counts)\n",
+    "\n",
+    "train_set_features_df = convertType(train_set_features_df, proc, output_name=\"train_convert_type\")\n",
+    "\n",
+    "train_set_features_df = train_set_features_df \\\n",
+    "    .withColumn('traffic_source', F.lit(0).cast(DoubleType())) \\\n",
+    "    .withColumnRenamed('document_id_promo', 'document_id') \\\n",
+    "    .withColumnRenamed('clicked', 'label')\n",
+    "\n",
+    "train_set_features_df = train_set_features_df.fillna(0, subset=feature_vector_labels)\n",
+    "\n",
+    "train_feature_vectors_integral_csv_rdd_df = train_set_features_df.select(\n",
+    "    ['label'] + ['display_id'] + ['ad_id'] + [F.col('document_id').alias('doc_id')] + [F.col('document_id_event').alias('doc_event_id')] + [\n",
+    "        format_number(element, FEAT_CSV_ORDERED_COLUMNS[index]).alias(FEAT_CSV_ORDERED_COLUMNS[index]) for\n",
+    "        index, element in enumerate([F.col(column) for column in feature_vector_labels])]).replace(\n",
+    "    float('nan'), 0)\n",
+    "\n",
+    "\n",
+    "######################### testset feature engineer ###############################\n",
+    "test_set_features_df = test_set_enriched_df\n",
+    "test_set_features_df = categorifyFeatures(test_set_features_df, proc, output_name=\"test_categorified\", gen_dict=False)\n",
+    "test_set_features_df = timestamp_delta(test_set_features_df, 'publish_time', 'timestamp_event')\n",
+    "test_set_features_df = timestamp_delta(test_set_features_df, 'publish_time_doc_event', 'timestamp_event')\n",
+    "\n",
+    "test_set_features_df = get_doc_event_doc_ad_cb_similarity_score_fn(\n",
+    "    test_set_features_df, 'doc_event_category_id_list', 'doc_event_confidence_level_cat_list', \n",
+    "    'category_id_list', 'confidence_level_cat_list', categories_docs_counts)\n",
+    "test_set_features_df = get_doc_event_doc_ad_cb_similarity_score_fn(\n",
+    "    test_set_features_df, 'doc_event_topic_id_list', 'doc_event_confidence_level_top_list',\n",
+    "    'topic_id_list', 'confidence_level_top_list', topics_docs_counts)\n",
+    "test_set_features_df = get_doc_event_doc_ad_cb_similarity_score_fn(\n",
+    "    test_set_features_df, 'doc_event_entity_id_list', 'doc_event_confidence_level_ent_list', \n",
+    "    'entity_id_list', 'confidence_level_ent_list', entities_docs_counts)\n",
+    "\n",
+    "test_set_features_df = convertType(test_set_features_df, proc, output_name=\"test_convert_type\")\n",
+    "\n",
+    "test_set_features_df = test_set_features_df \\\n",
+    "    .withColumn('traffic_source', F.lit(0).cast(DoubleType())) \\\n",
+    "    .withColumnRenamed('document_id_promo', 'document_id') \\\n",
+    "    .withColumnRenamed('clicked', 'label')\n",
+    "\n",
+    "test_set_features_df = test_set_features_df.fillna(0, subset=feature_vector_labels)\n",
+    "\n",
+    "test_validation_feature_vectors_integral_csv_rdd_df = test_set_features_df.repartition(40,'display_id').orderBy('display_id').select(\n",
+    "    ['label'] + ['display_id'] + ['ad_id'] + [F.col('document_id').alias('doc_id')] + [F.col('document_id_event').alias('doc_event_id')] + [\n",
+    "        format_number(element, FEAT_CSV_ORDERED_COLUMNS[index]).alias(FEAT_CSV_ORDERED_COLUMNS[index]) for\n",
+    "        index, element in enumerate([F.col(column) for column in feature_vector_labels])]).replace(\n",
+    "    float('nan'), 0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def log2_1p(x):\n",
+    "    return np.log1p(x) / np.log(2.0)\n",
+    "\n",
+    "\n",
+    "# calculate min and max stats for the given dataframes all in one go\n",
+    "def compute_min_max_logs(df):\n",
+    "    print(str(datetime.datetime.now()) + '\\tComputing min and max')\n",
+    "    min_logs = {}\n",
+    "    max_logs = {}\n",
+    "    float_expr = []\n",
+    "    for name in FLOAT_COLUMNS_LOG_BIN_TRANSFORM + INT_COLUMNS:\n",
+    "        float_expr.append(F.min(name))\n",
+    "        float_expr.append(F.max(name))\n",
+    "    floatDf = all_df.agg(*float_expr).collect()\n",
+    "    print(floatDf)\n",
+    "    for name in FLOAT_COLUMNS_LOG_BIN_TRANSFORM:\n",
+    "        minAgg = floatDf[0][\"min(\" + name + \")\"]\n",
+    "        maxAgg = floatDf[0][\"max(\" + name + \")\"]\n",
+    "        min_logs[name + '_log_01scaled'] = log2_1p(minAgg * 1000)\n",
+    "        max_logs[name + '_log_01scaled'] = log2_1p(maxAgg * 1000)\n",
+    "    for name in INT_COLUMNS:\n",
+    "        minAgg = floatDf[0][\"min(\" + name + \")\"]\n",
+    "        maxAgg = floatDf[0][\"max(\" + name + \")\"]\n",
+    "        min_logs[name + '_log_01scaled'] = log2_1p(minAgg)\n",
+    "        max_logs[name + '_log_01scaled'] = log2_1p(maxAgg)\n",
+    "\n",
+    "    return min_logs, max_logs\n",
+    "\n",
+    "def log_and_norm(df, proc, output_name=\"log_norm\", gen_dict=False):\n",
+    "    log_cols = {col + '_log_01': f'f.log1p(\"{col}\")' for col in INT_COLUMNS}\n",
+    "    norm_cols = {col + 'scaled': f'(f.col(\"{col}\")-{min_logs[col+\"scaled\"]}) / ({max_logs[col+\"scaled\"]-min_logs[col+\"scaled\"]})' for col in log_cols.keys()}\n",
+    "    op_log = FeatureAdd(cols=log_cols, op='inline')\n",
+    "    op_norm = FeatureAdd(cols=norm_cols, op='inline')\n",
+    "    proc.reset_ops([op_log, op_norm])\n",
+    "\n",
+    "    op_fillna = FillNA(CATEGORICAL_COLUMNS, 0)\n",
+    "    proc.append_ops([op_fillna])\n",
+    "\n",
+    "    hash_bucket_cols = {col: f'f.col(\"{col}\") % {size}' for col, size in HASH_BUCKET_SIZES.items()}\n",
+    "    op_hash_bucket = FeatureModification(cols=hash_bucket_cols, op='inline')\n",
+    "    proc.append_ops([op_hash_bucket])\n",
+    "    \n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"Log and norm took %.3f\" % (t2 - t1))\n",
+    "\n",
+    "    for name, size in HASH_BUCKET_SIZES.items():\n",
+    "        df = df.withColumn(name, F.when(F.col(name)<0, F.col(name)+size).otherwise(F.col(name)))\n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_df = test_validation_feature_vectors_integral_csv_rdd_df.union(train_feature_vectors_integral_csv_rdd_df)\n",
+    "min_logs, max_logs = compute_min_max_logs(all_df)\n",
+    "\n",
+    "train_feature_norm = log_and_norm(train_feature_vectors_integral_csv_rdd_df, proc, output_name=\"log_norm_train\")\n",
+    "test_feature_norm = log_and_norm(test_validation_feature_vectors_integral_csv_rdd_df, proc, output_name=\"log_norm_test\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def make_spec(output_dir, batch_size=None):\n",
+    "    fixed_shape = [batch_size, 1] if batch_size is not None else []\n",
+    "    spec = {}\n",
+    "    spec[LABEL_COLUMN] = tf.io.FixedLenFeature(shape=fixed_shape, dtype=tf.int64, default_value=None)\n",
+    "    spec[DISPLAY_ID_COLUMN] = tf.io.FixedLenFeature(shape=fixed_shape, dtype=tf.int64, default_value=None)\n",
+    "    for name in BOOL_COLUMNS:\n",
+    "        spec[name] = tf.io.FixedLenFeature(shape=fixed_shape, dtype=tf.int64, default_value=None)\n",
+    "    for name in INT_COLUMNS:\n",
+    "        spec[name + '_log_01scaled'] = tf.io.FixedLenFeature(shape=fixed_shape, dtype=tf.float32, default_value=None)\n",
+    "    for name in BOOL_COLUMNS + CATEGORICAL_COLUMNS:\n",
+    "        spec[name] = tf.io.FixedLenFeature(shape=fixed_shape, dtype=tf.int64, default_value=None)\n",
+    "    metadata = dataset_metadata.DatasetMetadata(dataset_schema.from_feature_spec(spec))\n",
+    "    metadata_io.write_metadata(metadata, output_dir)\n",
+    "\n",
+    "\n",
+    "def create_tf_example_spark(df):\n",
+    "    result = {}\n",
+    "    result[LABEL_COLUMN] = tf.train.Feature(int64_list=tf.train.Int64List(value=df[LABEL_COLUMN].to_list()))\n",
+    "    result[DISPLAY_ID_COLUMN] = tf.train.Feature(int64_list=tf.train.Int64List(value=df[DISPLAY_ID_COLUMN].to_list()))\n",
+    "    for name in FLOAT_COLUMNS:\n",
+    "        value = df[name].to_list()\n",
+    "        result[name] = tf.train.Feature(float_list=tf.train.FloatList(value=value))\n",
+    "    for name in INT_COLUMNS:\n",
+    "        nn = name + '_log_01scaled'\n",
+    "        value = df[nn].to_list()\n",
+    "        result[nn] = tf.train.Feature(float_list=tf.train.FloatList(value=value))\n",
+    "    for name in BOOL_COLUMNS + CATEGORICAL_COLUMNS:\n",
+    "        value = df[name].to_list()\n",
+    "        result[name] = tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n",
+    "    tf_example = tf.train.Example(features=tf.train.Features(feature=result))\n",
+    "    return tf_example\n",
+    "\n",
+    "\n",
+    "def _transform_to_slices(rdds):\n",
+    "    taskcontext = TaskContext.get()\n",
+    "    partitionid = taskcontext.partitionId()\n",
+    "    csv = pd.DataFrame(list(rdds), columns=columns)\n",
+    "    num_rows = len(csv.index)\n",
+    "    examples = []\n",
+    "    for start_ind in range(0, num_rows, batch_size if batch_size is not None else 1):  # for each batch\n",
+    "        if start_ind + batch_size - 1 > num_rows:  # if we'd run out of rows\n",
+    "            csv_slice = csv.iloc[start_ind:]\n",
+    "            print(\"last Example has: \", len(csv_slice), partitionid)\n",
+    "            examples.append((csv_slice, len(csv_slice)))\n",
+    "            return examples\n",
+    "        else:\n",
+    "            csv_slice = csv.iloc[start_ind:start_ind + (batch_size if batch_size is not None else 1)]\n",
+    "        examples.append((csv_slice, len(csv_slice)))\n",
+    "    return examples\n",
+    "\n",
+    "\n",
+    "def _transform_to_tfrecords_from_slices(rdds):\n",
+    "    examples = []\n",
+    "    for slice in rdds:\n",
+    "        if len(slice[0]) != batch_size:\n",
+    "            print(\"slice size is not correct, dropping: \", len(slice[0]))\n",
+    "        else:\n",
+    "            examples.append(\n",
+    "                (bytearray((create_tf_example_spark(slice[0])).SerializeToString()), None))\n",
+    "    return examples\n",
+    "\n",
+    "\n",
+    "def _transform_to_tfrecords_from_reslice(rdds):\n",
+    "    examples = []\n",
+    "    all_dataframes = pd.DataFrame([])\n",
+    "    for slice in rdds:\n",
+    "        all_dataframes = all_dataframes.append(slice[0])\n",
+    "    num_rows = len(all_dataframes.index)\n",
+    "    examples = []\n",
+    "    for start_ind in range(0, num_rows, batch_size if batch_size is not None else 1):  # for each batch\n",
+    "        if start_ind + batch_size - 1 > num_rows:  # if we'd run out of rows\n",
+    "            csv_slice = all_dataframes.iloc[start_ind:]\n",
+    "            if TEST_SET_MODE:\n",
+    "                remain_len = batch_size - len(csv_slice)\n",
+    "                (m, n) = divmod(remain_len, len(csv_slice))\n",
+    "                print(\"remainder: \", len(csv_slice), remain_len, m, n)\n",
+    "                if m:\n",
+    "                    for i in range(m):\n",
+    "                        csv_slice = csv_slice.append(csv_slice)\n",
+    "                csv_slice = csv_slice.append(csv_slice.iloc[:n])\n",
+    "                print(\"after fill remainder: \", len(csv_slice))\n",
+    "                examples.append(\n",
+    "                    (bytearray((create_tf_example_spark(csv_slice)).SerializeToString()), None))\n",
+    "                return examples\n",
+    "            # drop the remainder\n",
+    "            print(\"dropping remainder: \", len(csv_slice))\n",
+    "            return examples\n",
+    "        else:\n",
+    "            csv_slice = all_dataframes.iloc[start_ind:start_ind + (batch_size if batch_size is not None else 1)]\n",
+    "            examples.append(\n",
+    "                (bytearray((create_tf_example_spark(csv_slice)).SerializeToString()), None))\n",
+    "    return examples"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "########################### convert data to TFRecods ######################\n",
+    "pd.set_option('display.max_columns', 1000)\n",
+    "evaluation = True\n",
+    "evaluation_verbose = False\n",
+    "LOCAL_DATA_TFRECORDS_DIR = \"/outbrain/tfrecords-test/recdp\"\n",
+    "train_output_string = '/train'\n",
+    "eval_output_string = '/eval'\n",
+    "\n",
+    "TEST_SET_MODE = False\n",
+    "\n",
+    "num_train_partitions = 40\n",
+    "num_valid_partitions = 40\n",
+    "batch_size = 4096\n",
+    "# write out tfrecords meta\n",
+    "make_spec(LOCAL_DATA_TFRECORDS_DIR + '/transformed_metadata', batch_size=batch_size)\n",
+    "\n",
+    "columns = train_feature_norm.columns\n",
+    "\n",
+    "TEST_SET_MODE = False\n",
+    "train_features = train_feature_norm.coalesce(30).rdd.mapPartitions(_transform_to_slices)\n",
+    "cached_train_features = train_features.cache()\n",
+    "train_full = cached_train_features.filter(lambda x: x[1] == batch_size)\n",
+    "# split out slies where we don't have a full batch so that we can reslice them so we only drop mininal rows\n",
+    "train_not_full = cached_train_features.filter(lambda x: x[1] < batch_size)\n",
+    "train_examples_full = train_full.mapPartitions(_transform_to_tfrecords_from_slices)\n",
+    "train_left = train_not_full.coalesce(1).mapPartitions(_transform_to_tfrecords_from_reslice)\n",
+    "all_train = train_examples_full.union(train_left)\n",
+    "\n",
+    "all_train.saveAsNewAPIHadoopFile(LOCAL_DATA_TFRECORDS_DIR + train_output_string,\n",
+    "                                 \"org.tensorflow.hadoop.io.TFRecordFileOutputFormat\",\n",
+    "                                 keyClass=\"org.apache.hadoop.io.BytesWritable\",\n",
+    "                                 valueClass=\"org.apache.hadoop.io.NullWritable\")\n",
+    "train_features.unpersist()\n",
+    "\n",
+    "\n",
+    "valid_features = test_feature_norm.coalesce(num_valid_partitions).rdd.mapPartitions(_transform_to_slices)\n",
+    "cached_valid_features = valid_features.cache()\n",
+    "valid_full = cached_valid_features.filter(lambda x: x[1] == batch_size)\n",
+    "valid_not_full = cached_valid_features.filter(lambda x: x[1] < batch_size)\n",
+    "valid_examples_full = valid_full.mapPartitions(_transform_to_tfrecords_from_slices)\n",
+    "valid_left = valid_not_full.coalesce(1).mapPartitions(_transform_to_tfrecords_from_reslice)\n",
+    "all_valid = valid_examples_full.union(valid_left)\n",
+    "\n",
+    "all_valid.saveAsNewAPIHadoopFile(LOCAL_DATA_TFRECORDS_DIR + eval_output_string,\n",
+    "                                 \"org.tensorflow.hadoop.io.TFRecordFileOutputFormat\",\n",
+    "                                 keyClass=\"org.apache.hadoop.io.BytesWritable\",\n",
+    "                                 valueClass=\"org.apache.hadoop.io.NullWritable\")\n",
+    "valid_features.unpersist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "spark.stop()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [conda env:spark]",
+   "language": "python",
+   "name": "conda-env-spark-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.0"
+  },
+  "metadata": {
+   "interpreter": {
+    "hash": "11fce9c02c04210b4e393e56cc8683f43e45b3c4b54df7b32543429b3d427785"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
\ No newline at end of file
diff --git a/RecDP/pyproject.toml b/RecDP/pyproject.toml
new file mode 100644
index 000000000..374b58cbf
--- /dev/null
+++ b/RecDP/pyproject.toml
@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"
diff --git a/RecDP/pyrecdp/__init__.py b/RecDP/pyrecdp/__init__.py
new file mode 100644
index 000000000..bf73a904c
--- /dev/null
+++ b/RecDP/pyrecdp/__init__.py
@@ -0,0 +1,1 @@
+__all__ = ["data_processor", "encoder", "init_spark", "utils"]
diff --git a/RecDP/pyrecdp/data_processor.py b/RecDP/pyrecdp/data_processor.py
new file mode 100644
index 000000000..edfd82345
--- /dev/null
+++ b/RecDP/pyrecdp/data_processor.py
@@ -0,0 +1,1151 @@
+import uuid
+from pyrecdp.utils import *
+from pyspark.ml.feature import *
+import pandas as pd
+import numpy as np
+import pyspark.sql.types as spk_type
+import pyspark.sql.types as t
+import pyspark.sql.functions as spk_func
+import pyspark.sql.functions as f
+from pyspark.sql import *
+from pyspark import *
+from pyspark.ml.feature import *
+import os
+import sys
+from timeit import default_timer as timer
+import logging
+import shutil
+import random
+
+
+class Operation:
+    '''
+    Base abstract class of operation
+
+    Args:
+
+        cols (list): columns used to generate new features or modify in place
+
+    Attributes:
+
+        op_name (str): operation name
+        cols (list): columns
+    '''
+
+    def __init__(self, cols):
+        self.op_name = "AbstractOperation"
+        self.cols = cols
+
+    def describe(self):
+        return "%s(%s)" % (self.op_name, ','.join(self.cols))
+
+    def collect(self, df):
+        raise NotImplementedError(self.op_name + "doesn't support collect")
+
+    def to_dict_dfs(self, df, spark = None, enable_gazelle=False):
+        raise NotImplementedError(self.op_name + "doesn't support to_dict_dfs")
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        return df
+
+    # some common method
+    def find_dict(self, name, dict_dfs):
+        for i in dict_dfs:
+            if isinstance(i, tuple):
+                for v in i:
+                    if isinstance(v, str):
+                        dict_name = v
+                    else:
+                        dict_df = v
+                if str(dict_name) == str(name):
+                    return dict_df
+            else:
+                dict_df = i['dict']
+                dict_name = i['col_name']
+                if str(dict_name) == str(name):
+                    return dict_df
+        return None
+
+    def get_colname_dict_as_tuple(self, i):
+        if isinstance(i, tuple):
+            for v in i:
+                if isinstance(v, str):
+                    dict_name = v
+                else:
+                    dict_df = v
+            return (dict_name, dict_df)
+        else:
+            dict_df = i['dict']
+            dict_name = i['col_name']
+            return (dict_name, dict_df)
+
+    def generate_dict_dfs(self, df, spark=None, cache_path=None, withCount=True, isParquet=True, enable_gazelle=False):
+        # at least we can cache here to make read much faster
+        # handle multiple columns issue
+        if isParquet == False and spark != None and cache_path != None:
+            cols = self.cols
+            # let's raise NotImplemented exception for union case and doSplit case
+            if self.doSplit:
+                raise NotImplementedError("csv optimization is not supporting for doSplit yet.")
+            for i in self.cols:
+                if isinstance(i, dict):
+                    raise NotImplementedError("csv optimization is not supporting for generate dictionary for multiple columns yet.")
+            df = (df
+                .select(spk_func.posexplode(spk_func.array(*cols)))
+                .withColumnRenamed('pos', 'column_id')
+                .withColumnRenamed('col', 'dict_col')
+                .filter('dict_col is not null')
+                .groupBy('column_id', 'dict_col')
+                .count())
+            windowed = Window.partitionBy('column_id').orderBy(spk_func.desc('count'))
+            df = df.withColumn('dict_col_id', spk_func.row_number().over(windowed))
+            if enable_gazelle:
+                df.write.format('arrow').mode('overwrite').save(cache_path)
+                df = spark.read.format("arrow").load(cache_path)
+            else:
+                df.write.format("parquet").mode("overwrite").save(cache_path)
+                df = spark.read.parquet(cache_path)
+            i = 0
+            for col_name in cols:
+                dict_df = df.filter('column_id == %d' % i).drop('column_id')
+                if withCount:
+                    self.dict_dfs.append({'col_name': col_name, 'dict': dict_df})
+                else:
+                    self.dict_dfs.append(
+                        {'col_name': col_name, 'dict': dict_df.drop('count')})
+                i += 1
+            return self.dict_dfs
+        # below is when input is parquet
+        for i in self.cols:
+            col_name = ""
+            if isinstance(i, dict):
+                # This is more those need to union columns
+                if 'col_name' in i:
+                    col_name = i['col_name']
+                else:
+                    col_name = 'union_dict'
+                src_cols = []
+                if 'src_cols' in i:
+                    src_cols.extend(i['src_cols'])
+                else:
+                    raise ValueError(
+                        "Union columns must has argument 'src_cols'")
+                first = True
+                dict_df = df
+                for _i in src_cols:
+                    if first:
+                        dict_df = df.select(spk_func.col(_i).alias('dict_col'))
+                        first = False
+                    else:
+                        dict_df = dict_df.union(
+                            df.select(spk_func.col(_i).alias('dict_col')))
+            else:  # if cols are simply col_name
+                col_name = i
+                dict_df = df.select(spk_func.col(i).alias('dict_col'))
+                if self.doSplit:
+                    dict_df = dict_df.select(
+                        spk_func.explode(spk_func.split(spk_func.col('dict_col'), self.sep))).withColumn('dict_col', spk_func.col('col'))
+            if self.bucketSize == -1:
+                if self.id_by_python_dict:
+                    dict_df = dict_df.distinct()
+                    dict_data = dict((row['dict_col'], 1) for row in dict_df.collect())
+                    for i, x in enumerate(dict_data):
+                        dict_data[x] = i
+                    dict_df = convert_to_spark_df(dict_data, df.spark)
+                elif self.id_by_count:
+                    dict_df = dict_df.groupBy('dict_col').count()
+                    dict_df = dict_df.withColumn('dict_col_id', spk_func.row_number().over(
+                        Window.orderBy(spk_func.desc('count')))).withColumn('dict_col_id', spk_func.col('dict_col_id') - 1).select('dict_col', 'dict_col_id', 'count')
+                else:
+                    dict_df = dict_df.withColumn("monotonically_increasing_id", spk_func.monotonically_increasing_id())
+                    dict_df = dict_df.groupBy('dict_col').agg(spk_func.min("monotonically_increasing_id").alias("monotonically_increasing_id"), spk_func.count("*").alias("count"))
+                    dict_df = dict_df.withColumn('dict_col_id', spk_func.row_number().over(
+                    Window.orderBy(spk_func.col('monotonically_increasing_id')))).withColumn('dict_col_id', spk_func.col('dict_col_id') - 1).select('dict_col', 'dict_col_id', 'count')
+
+            else:
+                # when user set a bucketSize, we will quantileDiscretizer in this case
+                dict_df = dict_df.groupBy('dict_col').count()
+                qd = QuantileDiscretizer(numBuckets=self.bucketSize, inputCol="count", outputCol='dict_col_id')
+                dict_df  = qd.fit(dict_df).transform(dict_df).withColumn('dict_col_id', spk_func.col('dict_col_id').cast(spk_type.IntegerType()))
+            if withCount:
+                self.dict_dfs.append({'col_name': col_name, 'dict': dict_df})
+            else:
+                self.dict_dfs.append(
+                    {'col_name': col_name, 'dict': dict_df.drop('count')})
+        return self.dict_dfs
+
+
+    def categorify_strategy_decision_maker(self, dict_dfs, df, df_cnt, per_core_memory_size, flush_threshold, enable_gazelle,estimated_bytes=20):
+        small_cols = []
+        long_cols = []
+        huge_cols = []
+        smj_cols = []
+        udf_cols = []
+        total_small_cols_num_rows = 0
+        total_estimated_shuffled_size = 0
+        df_estimzate_size_per_row = sum([get_estimate_size_of_dtype(dtype) for _, dtype in df.dtypes])
+        df_estimated_size = df_estimzate_size_per_row * df_cnt
+        # Below threshold is maximum BHJ numRows, 4 means maximum 25% memory to cache Broadcast data, and 20 means we estimate each row has 20 bytes.
+        if enable_gazelle:
+            threshold = per_core_memory_size / estimated_bytes
+            threshold_per_bhj = threshold if threshold <= 100000000 else 100000000
+        else:
+            threshold = per_core_memory_size / 4 / estimated_bytes
+            threshold_per_bhj = threshold if threshold <= 30000000 else 30000000
+        flush_threshold = flush_threshold * 0.8
+        # print("[DEBUG] bhj total threshold is %.3f M rows, one bhj threshold is %.3f M rows, flush_threshold is %.3f GB" % (threshold / 1000000, threshold_per_bhj/1000000, flush_threshold / 2**30))
+        dict_dfs_with_cnt = []
+        for i in dict_dfs:
+            col_name, dict_df = self.get_colname_dict_as_tuple(i)
+            found = False
+            if self.cols == None:
+                found = True
+            else:
+                for j in self.cols:
+                    _, col_name_from_cols = self.get_col_tgt_src(j)
+                    if col_name_from_cols == col_name:
+                        found = True
+                        break
+            if found == True:
+              dict_df_cnt = dict_df.count()
+              dict_dfs_with_cnt.append((col_name, dict_df, dict_df_cnt))
+        sorted_dict_dfs_with_cnt = [(col_name, dict_df, dict_df_cnt) for col_name, dict_df, dict_df_cnt in sorted(dict_dfs_with_cnt, key=lambda pair: pair[2])]
+        # for to_print in sorted_dict_dfs_with_cnt:
+        #    print(to_print)
+
+        sorted_cols = []
+        for (col_name, dict_df, dict_df_cnt) in sorted_dict_dfs_with_cnt:
+            sorted_cols.append(col_name)
+            if self.doSplit:
+                threshold_per_bhj = 30000000
+            if (dict_df_cnt > threshold_per_bhj or (total_small_cols_num_rows + dict_df_cnt) > threshold):
+                print(f"for {col_name} current dict_length {dict_df_cnt} exceeded threshold_per_bhj {threshold_per_bhj} or accumulated BHJ length {total_small_cols_num_rows + dict_df_cnt} exceeded estimated threshold {threshold}, will go smj")
+                if ((total_estimated_shuffled_size + df_estimated_size) > flush_threshold):
+                    # print("etstimated_to_shuffle_size for %s is %.3f GB, will do smj and spill to disk" % (str(col_name), df_estimated_size / 2**30))
+                    huge_cols.append(col_name)
+                    smj_cols.append(col_name)
+                    long_cols.append(col_name)
+                    total_estimated_shuffled_size = 0
+                else:
+                    # print("etstimated_to_shuffle_size for %s is %.3f GB, will do smj" % (str(col_name), df_estimated_size / 2**30))
+                    smj_cols.append(col_name)
+                    long_cols.append(col_name)
+                    # if accumulate shuffle capacity may exceed maximum shuffle disk size, we should use hdfs instead
+                    total_estimated_shuffled_size += df_estimated_size
+            else:
+                if self.doSplit:
+                    # print("%s will do udf" % (col_name))
+                    total_small_cols_num_rows += dict_df_cnt
+                    udf_cols.append(col_name)
+                else:
+                    # print("%s will do bhj" % (col_name))
+                    total_small_cols_num_rows += dict_df_cnt
+                    small_cols.append(col_name)
+            df_estimated_size -= df_cnt * 6
+        return (sorted_cols, {'short_dict': small_cols, 'long_dict': long_cols, 'huge_dict': huge_cols, 'udf': udf_cols, 'smj_dict': smj_cols})
+
+    def check_scala_extension(self, spark):
+        driverClassPath = spark.sparkContext.getConf().get('spark.driver.extraClassPath')
+        if driverClassPath == None or 'recdp' not in driverClassPath:
+            return False
+        else:
+            return True
+
+class FeatureModification(Operation):
+    '''
+    Operation to modify feature column in-place, support build-in function and udf
+
+    Args:
+
+        cols (list or dict): columns which are be modified
+        op (str): define modify function name
+        udfImpl (udf): if op is udf, udfImpl will be applied to specified cols
+    '''
+
+    def __init__(self, cols, op='udf', udfImpl=None):
+        self.op_name = "FeatureModification"
+        self.cols = cols
+        self.op = op
+        self.udf_impl = udfImpl
+
+    def describe(self):
+        if self.op == 'udf':
+            f_cols = ["udf(%s)" % x for x in self.cols]
+            return "%s(%s)" % (self.op_name, ','.join(f_cols))
+        elif self.op == 'inline':
+            f_cols = ["modify(%s, %s)" % (x, y)
+                      for (x, y) in self.cols.items()]
+            return "%s(%s)" % (self.op_name, ','.join(f_cols))
+        else:
+            f_cols = ["%s(%s)" % (self.op, x) for x in self.cols]
+            return "%s(%s)" % (self.op_name, ','.join(f_cols))
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        if self.op == 'udf':
+            for i in self.cols:
+                df = df.withColumn(i, self.udf_impl(spk_func.col(i)))
+        elif self.op == 'inline':
+            for i, func in self.cols.items():
+                df = df.withColumn(i, eval(func))
+        elif self.op == 'toInt':
+            for i in self.cols:
+                df = df.withColumn(i, spk_func.col(
+                    i).cast(spk_type.IntegerType()))
+        return df
+
+
+class FeatureAdd(Operation):
+    '''
+    Operation to add new feature column based on current columns, support build-in function and udf
+
+    Args:
+
+        cols (dict): new_col: col or inline_func create new features
+        op (str): define modify function name
+        udfImpl (udf): if op is udf, udfImpl will be applied to specified cols
+    '''
+
+    def __init__(self, cols, op='udf', udfImpl=None):
+        self.op_name = "FeatureAdd"
+        self.cols = cols
+        self.op = op
+        self.udf_impl = udfImpl
+
+    def describe(self):
+        if self.op == 'udf':
+            f_cols = ["new_feature(%s, udf(%s))" % (x, y)
+                      for (x, y) in self.cols.items()]
+            return "%s(%s)" % (self.op_name, ','.join(f_cols))
+        elif self.op == 'inline':
+            f_cols = ["new_feature(%s, %s)" % (x, y)
+                      for (x, y) in self.cols.items()]
+            return "%s(%s)" % (self.op_name, ','.join(f_cols))
+        else:
+            f_cols = ["new_feature(%s, %s(%s))" % (x, self.op, y)
+                      for (x, y) in self.cols.items()]
+            return "%s(%s)" % (self.op_name, ','.join(f_cols))
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        if self.op == 'udf':
+            for after, before in self.cols.items():
+                df = df.withColumn(after, self.udf_impl(spk_func.col(before)))
+        elif self.op == 'inline':
+            for after, before in self.cols.items():
+                try:
+                    df = df.withColumn(after, eval(before))
+                except Exception as e:
+                    print("[ERROR]: inline script is %s" % before)
+                    raise e
+        elif self.op == 'toInt':
+            for after, before in self.cols.items():
+                df = df.withColumn(after, spk_func.col(
+                    before).cast(spk_type.IntegerType()))
+        return df
+
+
+class FillNA(Operation):
+    '''
+    Operation to fillna to columns
+
+    Args:
+
+        cols (list): columns which are be modified
+        default: filled default value
+    '''
+
+    def __init__(self, cols, default):
+        self.op_name = "FillNA"
+        self.cols = cols
+        self.default = default
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        return df.fillna(self.default, [i for i in self.cols])
+
+
+class DropFeature(Operation):
+    '''
+    Operation to fillna to columns
+
+    Args:
+
+        cols (list): columns which are be modified
+        default: filled default value
+    '''
+
+    def __init__(self, cols):
+        self.op_name = "DropFeature"
+        self.cols = cols
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        for i in self.cols:
+            df = df.drop(i)
+        return df
+
+
+class Distinct(Operation):
+    '''
+    Operation to perform distinct to columns
+
+    Args:
+
+        cols (list): columns which are be modified
+        default: filled default value
+    '''
+
+    def __init__(self):
+        self.op_name = "Distinct"
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        return df.distinct()
+
+class SelectFeature(Operation):
+    '''
+    Operation to select and rename columns
+
+    Args:
+
+        cols (list): columns which are be modified
+        default: filled default value
+    '''
+
+    def __init__(self, cols=[]):
+        self.op_name = "SelectFeature"
+        self.cols = cols
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        to_select = []
+        for i in self.cols:
+            if isinstance(i, str):
+                to_select.append(i)
+            elif isinstance(i, tuple):
+                to_select.append("%s as %s" % (i[0], i[1]))
+
+        return df.selectExpr(*to_select)
+
+
+class Sort(Operation):
+    def __init__(self, args = []):
+        self.args = args
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        return df.orderBy(*self.args)
+
+class Categorify(Operation):
+    '''
+    Operation to categorify columns by distinct id
+
+    Args:
+
+        cols (list): columns which are be modified
+    '''
+
+    def __init__(self, cols, dict_dfs=None, gen_dicts=True, hint='auto', doSplit=False, sep='\t', doSortForArray=False, keepMostFrequent=False, saveTmpToDisk=False, multiLevelSplit=False, multiLevelSep=[],estimated_bytes=20):
+        self.op_name = "Categorify"
+        self.cols = cols
+        self.dict_dfs = dict_dfs
+        self.gen_dicts = gen_dicts
+        self.hint = hint
+        self.doSplit = doSplit
+        self.sep = sep
+        self.doSortForArray = doSortForArray
+        self.keepMostFrequent = keepMostFrequent
+        self.saveTmpToDisk = saveTmpToDisk
+        self.save_path_id = 0
+        self.multi_level_split = multiLevelSplit
+        if self.multi_level_split:
+            self.hint = 'udf'
+        self.multi_level_sep = multiLevelSep
+        self.strategy_type = {
+            'udf': 'udf', 'broadcast_join': 'short_dict', 'shuffle_join': 'huge_dict'}
+        self.estimated_bytes = estimated_bytes
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        strategy = {}
+        sorted_cols = []
+        if self.hint != "auto" and self.hint in self.strategy_type:
+            strategy[self.strategy_type[self.hint]] = []
+            for i in self.dict_dfs:
+                col_name, dict_df = self.get_colname_dict_as_tuple(i)
+                strategy[self.strategy_type[self.hint]].append(col_name)
+                sorted_cols.append(col_name)
+        else:
+            sorted_cols, strategy = self.categorify_strategy_decision_maker(self.dict_dfs, df, df_cnt, per_core_memory_size, flush_threshold, enable_gazelle,self.estimated_bytes)
+        
+        sorted_cols_pair = []
+        pri_key_loaded = []
+        for cn in sorted_cols:
+            for i in self.cols:
+                col_name, src_name = self.get_col_tgt_src(i)
+                if src_name == cn:
+                    if col_name not in pri_key_loaded:
+                        pri_key_loaded.append(col_name)
+                        sorted_cols_pair.append({col_name: src_name})
+
+
+        last_method = 'bhj'
+        # For now, we prepared dict_dfs and strategy
+        for i in sorted_cols_pair:
+            col_name, src_name = self.get_col_tgt_src(i)
+            dict_df = self.find_dict(src_name, self.dict_dfs)
+            if col_name != src_name and col_name not in df.columns:
+                df = df.withColumn(col_name, spk_func.col(src_name))
+            # for udf type, we will do udf to do user-defined categorify
+            if 'udf' in strategy and src_name in strategy['udf']:
+                last_method = 'udf'
+                if not enable_scala and not self.multi_level_split:
+                    df = self.categorify_with_udf(df, dict_df, col_name, spark)
+                else:
+                    df = self.categorify_with_scala_udf(df, dict_df, col_name, spark)
+        for i in sorted_cols_pair:
+            col_name, src_name = self.get_col_tgt_src(i)
+            dict_df = self.find_dict(src_name, self.dict_dfs)
+            if col_name != src_name and col_name not in df.columns:
+                df = df.withColumn(col_name, spk_func.col(src_name))
+            # for short dict, we will do bhj
+            if 'short_dict' in strategy and src_name in strategy['short_dict']:
+                last_method = 'bhj'
+                df = self.categorify_with_join(df, dict_df, col_name, spark, save_path, method="bhj")
+        for i in sorted_cols_pair:
+            col_name, src_name = self.get_col_tgt_src(i)
+            dict_df = self.find_dict(src_name, self.dict_dfs)
+            if col_name != src_name and col_name not in df.columns:
+                df = df.withColumn(col_name, spk_func.col(src_name))
+            # for long dict, we will do shj all along
+            if 'long_dict' in strategy and src_name in strategy['long_dict']:
+                if 'huge_dict' in strategy and src_name in strategy['huge_dict']:
+                    if 'smj_dict' in strategy and src_name in strategy['smj_dict']:
+                        last_method = 'smj'
+                        df = self.categorify_with_join(df, dict_df, col_name, spark, save_path, method="smj", saveTmpToDisk=True, enable_gazelle=enable_gazelle)
+                    else:
+                        last_method = 'shj'
+                        df = self.categorify_with_join(df, dict_df, col_name, spark, save_path, method="shj", saveTmpToDisk=True, enable_gazelle=enable_gazelle)
+                else:
+                    if 'smj_dict' in strategy and src_name in strategy['smj_dict']:
+                        last_method = 'smj'
+                        df = self.categorify_with_join(df, dict_df, col_name, spark, save_path, method="smj", saveTmpToDisk=False, enable_gazelle=enable_gazelle)
+                    else:
+                        last_method = 'shj'
+                        df = self.categorify_with_join(df, dict_df, col_name, spark, save_path, method="shj", saveTmpToDisk=False, enable_gazelle=enable_gazelle)
+        # when last_method is BHJ, we should add a separator for spark wscg optimization
+        if last_method == 'bhj' and not enable_gazelle and enable_scala:
+            vspark = str(spark.version)
+            if not (vspark.startswith("3.1") or vspark.startswith("3.0")):
+                return df
+            #print("Adding a CodegenSeparator to pure BHJ WSCG case")
+            found = False
+            for dname, dtype in df.dtypes:
+                if dtype == "string":
+                    df = df.withColumn(dname, spk_func.expr(f"CodegenSeparator1({dname})"))
+                    found = True
+                    break
+                if dtype == "int":
+                    df = df.withColumn(dname, spk_func.expr(f"CodegenSeparator0({dname})"))
+                    found = True
+                    break
+                if dtype == "bigint":
+                    df = df.withColumn(dname, spk_func.expr(f"CodegenSeparator2({dname})"))
+                    found = True
+                    break
+            if found == False:
+                df = df.withColumn("CodegenSeparator", spk_func.expr(f"CodegenSeparator())"))
+    
+        return df
+
+    def get_col_tgt_src(self, i):
+        if isinstance(i, str):
+            col_name = i
+            src_name = i
+            return (col_name, src_name)
+        elif isinstance(i, dict):
+            col_name = next(iter(i.keys()))
+            src_name = next(iter(i.values()))
+            return (col_name, src_name)
+
+    def get_mapping_udf(self, broadcast_data, spark, default=None):
+        broadcast_dict = spark.sparkContext.broadcast(
+            broadcast_data)
+        sep = self.sep
+        doSortForArray = self.doSortForArray
+        def largest_freq_encode(x):
+            broadcast_data = broadcast_dict.value
+            min_val = None
+            if x != '':
+                x_l = x.split(sep) if not isinstance(x, list) else x
+                for v in x_l:
+                    if v != '' and v in broadcast_data and (min_val == None or broadcast_data[v] < min_val):
+                        min_val = broadcast_data[v]
+            return min_val
+
+        def freq_encode(x):
+            broadcast_data = broadcast_dict.value
+            val = []
+            if x != '':
+                x_l = x.split(sep) if not isinstance(x, list) else x
+                for v in x_l:
+                    if v != '' and v in broadcast_data:
+                        val.append(broadcast_data[v])
+                if doSortForArray and len(val) > 0:
+                    val.sort()
+            return val
+
+        if self.keepMostFrequent:
+            return spk_func.udf(largest_freq_encode, spk_type.IntegerType())
+        else:
+            return spk_func.udf(freq_encode, spk_type.ArrayType(spk_type.IntegerType()))
+
+    def categorify_with_udf(self, df, dict_df, i, spark):
+        #print("do %s to %s" % ("python_udf", i))
+        dict_data = dict((row['dict_col'], row['dict_col_id']) for row in dict_df.collect())
+        udf_impl = self.get_mapping_udf(dict_data, spark)
+        df = df.withColumn(i, udf_impl(spk_func.col(i)))
+        return df
+
+    def categorify_with_scala_udf(self, df, dict_df, i, spark):
+        #print("do %s to %s" % ("scala_udf", i))
+        # call java to broadcast data and set broadcast handler to udf
+        if not self.check_scala_extension(spark):
+            raise ValueError("RecDP need to enable recdp-scala-extension to run categorify_with_udf, please config spark.driver.extraClassPath and spark.executor.extraClassPath for spark")
+
+        if self.doSplit == False:
+            df = jvm_categorify(spark, df, i, dict_df)
+        else:
+            if get_dtype(df, i) == 'string':
+                df = df.withColumn(i, spk_func.split(spk_func.col(i), self.sep))
+            if self.keepMostFrequent:
+                df = jvm_categorify_by_freq_for_array(spark, df, i, dict_df)
+            else:
+                if self.multi_level_split:
+                    df = jvm_categorify_for_multi_level_array(spark, df, i, dict_df, self.multi_level_sep)
+                else:
+                    df = jvm_categorify_for_array(spark, df, i, dict_df)
+                if self.doSortForArray:
+                    df = df.withColumn(i, spk_func.array_sort(spk_func.col(i)))
+        return df
+
+    def categorify_with_join(self, df, dict_df, i, spark, save_path, method='shj', saveTmpToDisk=False, enable_gazelle=False):
+        saveTmpToDisk = self.saveTmpToDisk or saveTmpToDisk
+        hint_type = "shuffle_hash"
+        if method == "bhj":
+            hint_type = "broadcast"
+        elif method == "smj":
+            hint_type = "merge"
+        #print("do %s to %s" % (method, i))
+        to_select = df.columns + ['dict_col_id']
+        if self.doSplit == False:
+            df = df.join(dict_df.hint(hint_type), spk_func.col(i) == dict_df.dict_col, 'left')\
+                .select(*to_select).drop(i).withColumnRenamed('dict_col_id', i)
+        else:
+            df = df.withColumn(
+                'row_id', spk_func.monotonically_increasing_id())
+            if get_dtype(df, i) == 'string':
+                df = df.withColumn(i, spk_func.split(spk_func.col(i), self.sep))
+            tmp_df = df.select('row_id', i).withColumn(i, spk_func.explode(spk_func.col(i)))
+            tmp_df = tmp_df.join(
+                dict_df.hint("shuffle_hash"),
+                spk_func.col(i) == dict_df.dict_col,
+                'left').filter(spk_func.col('dict_col_id').isNotNull())
+            tmp_df = tmp_df.select(
+                'row_id', spk_func.col('dict_col_id'))
+            if self.keepMostFrequent:
+                tmp_df = tmp_df.groupby('row_id').agg(spk_func.array_sort(
+                    spk_func.collect_list(spk_func.col('dict_col_id'))).getItem(0).alias('dict_col_id'))
+            elif self.doSortForArray:
+                tmp_df = tmp_df.groupby('row_id').agg(spk_func.array_sort(
+                    spk_func.collect_list(spk_func.col('dict_col_id'))).alias('dict_col_id'))
+            else:
+                tmp_df = tmp_df.groupby('row_id').agg(
+                    spk_func.collect_list(spk_func.col('dict_col_id')).alias('dict_col_id'))
+
+            df = df.join(tmp_df.hint(hint_type),
+                         'row_id', 'left').select(*to_select).drop(i).withColumnRenamed('dict_col_id', i)
+        # if self.saveTmpToDisk is True, we will save current df to hdfs or localFS instead of replying on Shuffle
+        if saveTmpToDisk:
+            cur_save_path = "%s_%d" % (save_path, self.save_path_id)
+            self.save_path_id += 1
+            if enable_gazelle:
+                df.write.format('arrow').mode('overwrite').save(save_path)
+                df = spark.read.format("arrow").load(save_path)
+            else:
+                df.write.format('parquet').mode('overwrite').save(cur_save_path)
+                df = spark.read.parquet(cur_save_path)
+        return df
+
+    def categorify_with_bhj(self, df, dict_df, i, spark):
+        if self.doSplit == False:
+            df = df.join(dict_df.hint('broadcast'), spk_func.col(i) == dict_df.dict_col, 'left')\
+                .withColumn(i, dict_df.dict_col_id).drop("dict_col_id", "dict_col")
+        else:
+            raise NotImplementedError(
+                "We should use udf to handle withSplit + small dict scenario")
+        return df
+
+
+class GenerateDictionary(Operation):
+    '''
+    Operation to generate dictionary based on single or multi-columns by distinct id
+
+    Args:
+
+        cols (list): columns which are be modified
+        doSplit (bool): If we need to split data
+        sep (str): split seperator
+    '''
+    # TODO: We should add an optimization for csv input
+
+    def __init__(self, cols, withCount=True, doSplit=False, sep='\t', isParquet=True, bucketSize=-1, id_by_count = True, id_by_python_dict = False):
+        self.op_name = "GenerateDictionary"
+        self.cols = cols
+        self.doSplit = doSplit
+        self.sep = sep
+        self.withCount = withCount
+        self.dict_dfs = []
+        self.isParquet = isParquet
+        self.bucketSize = bucketSize
+        self.id_by_count = id_by_count
+        self.id_by_python_dict = id_by_python_dict
+
+    def merge_dict(self, dict_dfs, to_merge_dict_dfs):
+        self.dict_dfs = []
+        for i in dict_dfs:
+            dict_df = i['dict']
+            dict_name = i['col_name']
+            to_merge = self.find_dict(dict_name, to_merge_dict_dfs)
+            if to_merge == None:
+                raise ValueError(
+                    "Expect '%s' in merge_dicts.to_merge_dict_dfs, while find none")
+            max_id_of_to_merge_row = to_merge.agg(
+                {'dict_col_id': 'max'}).collect()[0]
+            max_id_of_to_merge = max_id_of_to_merge_row['max(dict_col_id)']
+            fwd_dict_df = dict_df.join(to_merge, 'dict_col', 'anti').withColumn('dict_col_id', spk_func.row_number().over(
+                Window.orderBy('dict_col_id')) + max_id_of_to_merge)
+            self.dict_dfs.append(
+                {'col_name': dict_name, 'dict': to_merge.union(fwd_dict_df)})
+        return self.dict_dfs
+
+    def to_dict_dfs(self, df, spark = None, cache_path = None, enable_gazelle=False):
+        return self.generate_dict_dfs(df, spark, cache_path, self.withCount, self.isParquet, enable_gazelle=enable_gazelle)
+
+
+class ModelMerge(Operation):
+    '''
+    Operation to Merge Pre-Generated Model
+
+    Args:
+      example - dicts
+      [{'col_name': ['Id'], 'dict': top_answers_df}]
+    '''
+    # TODO: We should add an optimization for csv input
+
+    def __init__(self, dicts, saveTmpToDisk=False,estimated_bytes=20):
+        self.op_name = "ModelMerge"
+        self.dict_dfs = dicts
+        self.saveTmpToDisk = saveTmpToDisk
+        self.save_path_id = 0
+        self.cols = None
+        self.doSplit = False
+        self.estimated_bytes=estimated_bytes
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        if self.dict_dfs == None:
+            raise NotImplementedError("process %s ")
+        sorted_cols, strategy = self.categorify_strategy_decision_maker(self.dict_dfs, df, df_cnt, per_core_memory_size, flush_threshold, enable_gazelle,self.estimated_bytes)
+
+        # For now, we prepared dict_dfs and strategy
+        for col_name in sorted_cols:
+            dict_df = self.find_dict(col_name, self.dict_dfs) 
+            # for short dict, we will do bhj
+            if 'short_dict' in strategy and col_name in strategy['short_dict']:
+                df = self.merge_with_join(df, dict_df, col_name, spark, save_path, saveTmpToDisk=False, method='bhj', enable_gazelle=enable_gazelle)
+        for col_name in sorted_cols:
+            dict_df = self.find_dict(col_name, self.dict_dfs) 
+            # for huge dict, we will do shj seperately
+            if 'long_dict' in strategy and col_name in strategy['long_dict']:
+                if 'huge_dict' in strategy and col_name in strategy['huge_dict']:
+                    if 'smj_dict' in strategy and col_name in strategy['smj_dict']:
+                        df = self.merge_with_join(df, dict_df, col_name, spark, save_path, saveTmpToDisk=True, method='smj', enable_gazelle=enable_gazelle)
+                    else:
+                        df = self.merge_with_join(df, dict_df, col_name, spark, save_path, saveTmpToDisk=True, method='shj', enable_gazelle=enable_gazelle)
+                else:
+                    if 'smj_dict' in strategy and col_name in strategy['smj_dict']:
+                        df = self.merge_with_join(df, dict_df, col_name, spark, save_path, saveTmpToDisk=False, method='smj', enable_gazelle=enable_gazelle)
+                    else:
+                        df = self.merge_with_join(df, dict_df, col_name, spark, save_path, saveTmpToDisk=False, method='shj', enable_gazelle=enable_gazelle)
+        return df
+
+    def merge_with_join(self, df, dict_df, i, spark, save_path, saveTmpToDisk=False, method='shj', enable_gazelle=False):
+        saveTmpToDisk = self.saveTmpToDisk or saveTmpToDisk
+        hint_type = "shuffle_hash"
+        if method == "bhj":
+            hint_type = "broadcast"
+        elif method == "smj":
+            hint_type = "merge"
+        #print("do %s to %s" % (method, i))
+        df = df.join(dict_df.hint(hint_type), i, how='left')
+        # if self.saveTmpToDisk is True, we will save current df to hdfs or localFS instead of replying on Shuffle
+        if saveTmpToDisk:
+            cur_save_path = "%s_%d" % (save_path, self.save_path_id)
+            self.save_path_id += 1
+            if enable_gazelle:
+                df.write.format('arrow').mode('overwrite').save(save_path)
+                df = spark.read.format("arrow").load(save_path)
+            else:
+                df.write.format('parquet').mode('overwrite').save(cur_save_path)
+                df = spark.read.parquet(cur_save_path)
+        return df
+
+
+class NegativeSample(Operation):
+    def __init__(self, cols, dicts, negCnt = 1):
+        self.op_name = "NegativeSample"
+        self.cols = cols
+        self.dict_dfs = dicts
+        self.neg_cnt = negCnt
+        if len(cols) != len(dicts):
+            raise ValueError("NegativeSample expects input dicts has same size cols for mapping")
+        self.num_cols = len(cols)
+
+    def get_negative_sample_udf(self, spark, broadcast_data):
+        broadcast_movie_id_list = spark.sparkContext.broadcast(
+            broadcast_data)
+
+        def get_random_id(asin):
+            item_list = broadcast_movie_id_list.value
+            asin_total_len = len(item_list)
+            asin_neg = asin
+            while True:
+                asin_neg_index = random.randint(0, asin_total_len - 1)
+                asin_neg = item_list[asin_neg_index]
+                if asin_neg == None or asin_neg == asin:
+                    continue
+                else:
+                    break
+            return [asin_neg, asin]
+
+        return spk_func.udf(get_random_id, spk_type.ArrayType(spk_type.StringType()))
+
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        if enable_scala:
+            for i in range(0, self.num_cols):
+                df = jvm_get_negative_sample(spark, df, self.cols[i], self.get_colname_dict_as_tuple(self.dict_dfs[i])[1], self.neg_cnt)
+        else:
+            udf_list = []
+            for i in range(0, self.num_cols):
+                broadcasted_data = [row['dict_col'] for row in self.get_colname_dict_as_tuple(self.dict_dfs[i])[1].select("dict_col").collect()]
+                negative_sample = self.get_negative_sample_udf(spark, broadcasted_data)
+                udf_list.append(negative_sample)
+            for i in range(0, self.num_cols):
+                df = df.withColumn(self.cols[i], udf_list[i](spk_func.col(self.cols[i])))
+                df = df.select(spk_func.col("*"), spk_func.posexplode(spk_func.col(self.cols[i]))).drop(self.cols[i]).withColumnRenamed("col", self.cols[i])
+        return df
+
+
+class NegativeFeature(Operation):
+    def __init__(self, cols, dicts, doSplit=False, sep='\t', negCnt = 1):
+        self.op_name = "NegativeFeature"
+        self.cols = cols
+        self.dict_dfs = dicts
+        self.doSplit = doSplit
+        self.neg_cnt = negCnt
+        self.sep = sep
+        if len(cols) != len(dicts):
+            raise ValueError("NegativeFeature expects input dicts has same size cols for mapping")
+        self.num_cols = len(cols)
+
+    def get_negative_feature_udf(self, spark, broadcast_data):
+        broadcast_movie_id_list = spark.sparkContext.broadcast(
+            broadcast_data)
+
+        if self.doSplit:
+            def get_random_id(hist_asin):
+                item_list = broadcast_movie_id_list.value
+                asin_total_len = len(item_list)
+                res = []
+                asin_list = hist_asin.split(self.sep) if not isinstance(hist_asin, list) else hist_asin
+                for asin in asin_list:
+                    asin_neg = asin
+                    while True:
+                        asin_neg_index = random.randint(0, asin_total_len - 1)
+                        asin_neg = item_list[asin_neg_index]
+                        if asin_neg == None or asin_neg == asin:
+                            continue
+                        else:
+                            res.append(asin_neg)
+                            break
+                return res
+
+            return spk_func.udf(get_random_id, spk_type.ArrayType(spk_type.StringType()))
+        else:
+            def get_random_id(asin):
+                item_list = broadcast_movie_id_list.value
+                asin_total_len = len(item_list)
+                asin_neg = asin
+                while True:
+                    asin_neg_index = random.randint(0, asin_total_len - 1)
+                    asin_neg = item_list[asin_neg_index]
+                    if asin_neg == None or asin_neg == asin:
+                        continue
+                    else:
+                        break
+                return asin_neg
+
+            return spk_func.udf(get_random_id, spk_type.StringType())
+
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        if enable_scala:
+            for tgt, src in self.cols.items():
+                dict_df = self.find_dict(src, self.dict_dfs)
+                if self.doSplit:
+                    src_type = get_dtype(df, src)
+                    if src_type == 'string':
+                        df = df.withColumn(tgt, spk_func.split(spk_func.col(src), self.sep))
+                        df = jvm_get_negative_feature_for_array(spark, df, tgt, tgt, dict_df, self.neg_cnt)
+                    else:
+                        df = jvm_get_negative_feature_for_array(spark, df, tgt, src, dict_df, self.neg_cnt)
+                else:
+                    df = jvm_get_negative_feature(spark, df, tgt, src, dict_df, self.neg_cnt)
+        else:
+            for tgt, src in self.cols.items():
+                broadcasted_data = [row['dict_col'] for row in self.find_dict(src, self.dict_dfs)[1].select("dict_col").collect()]
+                negative_feature = self.get_negative_feature_udf(spark, broadcasted_data)
+                df = df.withColumn(tgt, negative_feature(spk_func.col(src)))
+        return df
+
+
+class ScalaDFTest(Operation):
+    def __init__(self, cols, dicts=None, method="int"):
+        self.op_name = "ScalaDFTest"
+        self.cols = cols
+        self.dict_dfs = dicts
+        if dicts != None and len(cols) != len(dicts):
+            raise ValueError("ScalaDFTest expects input dicts has same size cols for mapping")
+        self.num_cols = len(cols)
+        self.method = method
+
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        if self.method == "int":
+            for i in range(0, self.num_cols):
+                df = jvm_scala_df_test_int(spark, df, self.cols[i])
+        elif self.method == "str":
+            for i in range(0, self.num_cols):
+                df = jvm_scala_df_test_str(spark, df, self.cols[i])
+        elif self.method == "broadcast" and self.dict_dfs != None:
+            for i in range(0, self.num_cols):
+                df = jvm_scala_df_test_broadcast(spark, df, self.cols[i], self.get_colname_dict_as_tuple(self.dict_dfs[i])[1])
+        return df
+
+
+class CollapseByHist(Operation):
+    def __init__(self, cols = [], by = None, orderBy = None, minNumHist = 0, maxNumHist = -1):
+        self.op_name = "CollapseByHist"
+        self.cols = cols
+        self.by = by
+        self.orderBy = orderBy
+        self.minNumHist = minNumHist
+        self.maxNumHist = maxNumHist
+        if by == None or (isinstance(by, list) and len(by) == 0):
+            raise ValueError("CollapseByHist expects input by should not None or Empty")
+
+
+    def process(self, df, spark, df_cnt, enable_scala=True, save_path="", per_core_memory_size=0, flush_threshold = 0, enable_gazelle=False):
+        w = Window.partitionBy(self.by)
+        last_collapse_col = ""
+        if self.orderBy != None:
+            w = w.orderBy(self.orderBy)
+        df = df.withColumn('row_id', spk_func.row_number().over(w))
+        for c in self.cols:
+            df = df.withColumn(f'hist_{c}', spk_func.collect_list(c).over(Window.partitionBy(self.by)))
+            df = df.withColumn(f'hist_{c}', spk_func.when(spk_func.col(f'hist_{c}').isNull(), spk_func.array()).otherwise(spk_func.col(f'hist_{c}')))
+            last_collapse_col = f'hist_{c}'
+        if last_collapse_col == "":
+            if self.orderBy != None:
+                df = df.withColumn('row_max', spk_func.max(self.orderBy).over(Window.partitionBy(self.by)))
+                df = df.filter((df[self.orderBy] == df.row_max))
+                df = df.drop('row_id').drop('row_max')
+            else:
+                df = df.withColumn('row_cnt', spk_func.max('row_id').over(Window.partitionBy(self.by)))
+                df = df.filter((df.row_id == df.row_cnt) & (df.row_cnt > spk_func.lit(self.minNumHist)))
+                df = df.drop('row_id')
+        else:
+            df = df.withColumn('row_cnt', spk_func.size(spk_func.col(last_collapse_col)))
+            df = df.withColumn('row_cnt', spk_func.col('row_cnt').cast(spk_type.IntegerType()))
+            df = df.filter((df.row_id == df.row_cnt) & (df.row_cnt > spk_func.lit(self.minNumHist)))
+            for c in self.cols:
+                if self.maxNumHist != -1:
+                    df = df.withColumn('max_hist_len', spk_func.when(spk_func.col('row_cnt') > self.maxNumHist, self.maxNumHist).otherwise(spk_func.col('row_cnt')))
+                else:
+                    df = df.withColumn('max_hist_len', spk_func.col('row_cnt'))
+                df = df.withColumn(f'hist_{c}', spk_func.expr(f"slice(hist_{c}, 1, max_hist_len - 1)"))
+            df = df.drop('row_id').drop('row_cnt').drop('max_hist_len')
+        return df
+
+
+class DataProcessor:
+    def __init__(self, spark, path_prefix="hdfs://", current_path="", shuffle_disk_capacity="unlimited", dicts_path="dicts", spark_mode='local', enable_gazelle=False):
+        self.ops = []
+        self.spark = spark
+        self.uuid = uuid.uuid1()
+        self.tmp_id = 0
+        self.path_prefix = path_prefix
+        self.current_path = current_path
+        self.dicts_path = dicts_path
+        self.tmp_materialzed_list = []
+        self.per_core_memory_size = 0
+        self.enable_gazelle = enable_gazelle
+        self.spark_mode = spark_mode
+        fail_to_parse = True
+        if spark_mode == 'yarn' or spark_mode == 'standalone':
+            try:
+                memory_size = parse_size(spark.sparkContext.getConf().get('spark.executor.memory'))
+                memory_size += parse_size(spark.sparkContext.getConf().get('spark.executor.memoryOverhead'))
+                numCores = int(spark.sparkContext.getConf().get('spark.executor.cores'))
+                self.per_core_memory_size = memory_size / numCores
+                fail_to_parse = False
+            except:
+                pass
+        elif spark_mode == 'local' or fail_to_parse:
+            memory_size = parse_size(spark.sparkContext.getConf().get('spark.driver.memory'))
+            numCores = int(parse_cores_num(spark.sparkContext.getConf().get('spark.master')))
+            self.per_core_memory_size = memory_size / numCores
+            self.spark_mode = 'local'
+        if shuffle_disk_capacity == "unlimited":
+            self.flush_threshold = (2**63 - 1)
+        else:
+            self.flush_threshold = parse_size(shuffle_disk_capacity)
+        self.gateway = spark.sparkContext._gateway
+        self.enable_scala = self.registerScalaUDFs()
+        print("per core memory size is %.3f GB and shuffle_disk maximum capacity is %.3f GB" % (self.per_core_memory_size * 1.0/(2**30), self.flush_threshold * 1.0/(2**30)))
+
+    def __del__(self):
+        for tmp_file in self.tmp_materialzed_list:
+            shutil.rmtree(tmp_file, ignore_errors=True)
+
+    def registerScalaUDFs(self):
+        driverClassPath = self.spark.sparkContext.getConf().get('spark.driver.extraClassPath')
+        execClassPath = self.spark.sparkContext.getConf().get('spark.executor.extraClassPath')
+        if self.spark_mode == 'local':
+            if driverClassPath == None or 'recdp' not in driverClassPath:
+                return False
+        else: 
+            if driverClassPath == None or execClassPath == None or 'recdp' not in driverClassPath or 'recdp' not in execClassPath:
+                return False
+        print("recdp-scala-extension is enabled")
+        self.spark.udf.registerJavaFunction("sortStringArrayByFrequency","com.intel.recdp.SortStringArrayByFrequency")
+        self.spark.udf.registerJavaFunction("sortIntArrayByFrequency","com.intel.recdp.SortIntArrayByFrequency")
+        self.spark._jsparkSession.udf().register("CodegenSeparator", self.gateway.jvm.org.apache.spark.sql.api.CodegenSeparator())
+        self.spark._jsparkSession.udf().register("CodegenSeparator0", self.gateway.jvm.org.apache.spark.sql.api.CodegenSeparator0())
+        self.spark._jsparkSession.udf().register("CodegenSeparator1", self.gateway.jvm.org.apache.spark.sql.api.CodegenSeparator1())
+        self.spark._jsparkSession.udf().register("CodegenSeparator2", self.gateway.jvm.org.apache.spark.sql.api.CodegenSeparator2())
+        return True
+
+    def describe(self):
+        description = []
+        for op in self.ops:
+            description.append(op.describe())
+        print("DataProcessor current worflow is \n    ",
+              '\n    ->'.join(description))
+
+    def append_ops(self, ops):
+        self.ops += ops
+
+    def reset_ops(self, ops):
+        self.ops = ops
+
+    def get_tmp_cache_path(self):
+        tmp_id = self.tmp_id
+        self.tmp_id += 1
+        save_path = "%s/%s/tmp/%s-%s-%d" % (
+            self.path_prefix, self.current_path, "materialized_tmp", self.uuid, tmp_id)
+        self.tmp_materialzed_list.append(save_path)
+        return save_path
+
+    def materialize(self, df, df_name="materialized_tmp", method=1):
+        if method == 0:
+            return df.cache()
+        else:
+            save_path = ""
+            if df_name == "materialized_tmp":
+                save_path = self.get_tmp_cache_path()
+                self.tmp_materialzed_list.append(save_path)
+            else:
+                save_path = "%s/%s/%s" % (self.path_prefix,
+                                          self.current_path, df_name)
+            print(f"save data to {save_path}")
+            if self.enable_gazelle:
+                #df.write.format('parquet').mode('overwrite').save(save_path)
+                df.write.format('arrow').mode('overwrite').save(save_path)
+                return self.spark.read.format("arrow").load(save_path)
+            else:
+                df.write.format('parquet').mode('overwrite').save(save_path)
+                return self.spark.read.parquet(save_path)
+
+    def refine_op(self, op, df, save_path):
+        if isinstance(op, Categorify) and op.dict_dfs == None :
+            dict_dfs = []
+            if not op.gen_dicts:
+                dict_names = op.cols
+                dict_dfs = [{'col_name': name, 'dict': self.spark.read.parquet(
+                    "%s/%s/%s/%s" % (self.path_prefix, self.current_path, self.dicts_path, name))} for name in dict_names]
+            if op.gen_dicts:
+                dfs = []
+                op_gen_dicts = GenerateDictionary(op.cols, doSplit=op.doSplit, sep=op.sep)
+                dfs.extend(op_gen_dicts.to_dict_dfs(df, self.spark, save_path, self.enable_gazelle))
+                for dict_df in dfs:
+                    dict_dfs.append({'col_name': dict_df['col_name'], 'dict': self.materialize(
+                        dict_df['dict'], "%s/%s" % (self.dicts_path, dict_df['col_name']))})
+            op.dict_dfs = dict_dfs
+        return op
+
+    def apply(self, df, df_cnt = None):
+        if len(self.ops) > 0 and df_cnt == None:
+            df_cnt = df.count()
+        for op in self.ops:
+            save_path = self.get_tmp_cache_path()
+            op = self.refine_op(op, df, save_path)
+            df = op.process(df, self.spark, df_cnt, self.enable_scala, save_path=save_path, per_core_memory_size = self.per_core_memory_size, flush_threshold = self.flush_threshold, enable_gazelle=self.enable_gazelle)
+        self.ops = []
+        return df
+
+    def transform(self, df, name="materialized_tmp", df_cnt = None):
+        return self.materialize(self.apply(df, df_cnt), df_name=name)
+
+    def generate_dicts(self, df):
+        # flat ops to dfs
+        dfs = []
+        materialized_dfs = []
+        for op in self.ops:
+            if op.op_name != "GenerateDictionary":
+                raise NotImplementedError(
+                    "We haven't support apply generate_dict to not GenerateDictionary operator yet.")
+            save_path = self.get_tmp_cache_path()
+            dfs.extend(op.to_dict_dfs(df, self.spark, save_path, self.enable_gazelle))
+        for dict_df in dfs:
+            materialized_dfs.append({'col_name': dict_df['col_name'], 'dict': self.materialize(
+                dict_df['dict'], "%s/%s" % (self.dicts_path, dict_df['col_name']))})
+        return materialized_dfs
+
+    def merge_dicts(self, df, to_merge_dict_dfs):
+        # flat ops to dfs
+        dfs = []
+        materialized_dfs = []
+        for op in self.ops:
+            if op.op_name != "GenerateDictionary":
+                raise NotImplementedError(
+                    "We haven't support apply generate_dict to not GenerateDictionary operator yet.")
+            save_path = get_tmp_cache_path()
+            dfs.extend(op.merge_dict(op.to_dict_dfs(df, self.spark, save_path, self.enable_gazelle), to_merge_dict_dfs))
+        for dict_df in dfs:
+            materialized_dfs.append({'col_name': dict_df['col_name'], 'dict': self.materialize(
+                dict_df['dict'], "%s/%s_merged" % (self.dicts_path, dict_df['col_name']))})
+        return materialized_dfs
+
+    def get_sample(self, df, df_cnt = None, vertical = False, truncate = 50):
+        if df_cnt == None:
+            df_cnt = df.count()
+        for op in self.ops:
+            save_path = self.get_tmp_cache_path()
+            op = self.refine_op(op, df, save_path)
+            df = op.process(df, self.spark, df_cnt, self.enable_scala, save_path=save_path, per_core_memory_size = self.per_core_memory_size, flush_threshold = self.flush_threshold, enable_gazelle=self.enable_gazelle)
+        df.show(vertical = vertical, truncate = truncate)
diff --git a/RecDP/pyrecdp/encoder.py b/RecDP/pyrecdp/encoder.py
new file mode 100644
index 000000000..f179df488
--- /dev/null
+++ b/RecDP/pyrecdp/encoder.py
@@ -0,0 +1,194 @@
+import uuid
+from pyrecdp.utils import *
+from pyspark.ml.feature import *
+import pandas as pd
+import numpy as np
+import pyspark.sql.types as spk_type
+import pyspark.sql.types as t
+import pyspark.sql.functions as spk_func
+import pyspark.sql.functions as f
+from pyspark.sql import *
+from pyspark import *
+import os
+import sys
+from timeit import default_timer as timer
+import logging
+import shutil
+
+
+class Encoder:
+    def __init__(self, proc):
+        self.op_name = "Encoder"
+        self.uuid = proc.uuid
+        self.tmp_id = proc.tmp_id
+        self.path_prefix = proc.path_prefix
+        self.current_path = proc.current_path
+        self.dicts_path = proc.dicts_path
+        self.spark = proc.spark
+        self.tmp_materialzed_list = []
+
+    def transform(self, train, valid, train_only=True):
+        raise NotImplementedError("This is base Encoder class")
+
+    def materialize(self, df, df_name="materialized_tmp"):
+        tmp_id = self.tmp_id
+        self.tmp_id += 1
+        save_path = ""
+        if df_name == "materialized_tmp":
+            save_path = "%s/%s/tmp/%s-%s-%d" % (
+                self.path_prefix, self.current_path, df_name, self.uuid, tmp_id)
+            self.tmp_materialzed_list.append(save_path)
+        else:
+            save_path = "%s/%s/%s" % (self.path_prefix,
+                                      self.current_path, df_name)
+        df.write.format('parquet').mode('overwrite').save(save_path)
+        return self.spark.read.parquet(save_path)
+
+
+class TargetEncoder(Encoder):
+    def __init__(self, proc, x_col_list, y_col_list, out_col_list, out_name, out_dtype=None, y_mean_list=None, smooth=20, seed=42,threshold=0):
+        super().__init__(proc)
+        self.op_name = "TargetEncoder"
+        self.x_col_list = x_col_list
+        self.y_col_list = y_col_list
+        self.out_col_list = out_col_list
+        self.out_dtype = out_dtype
+        self.out_name = out_name
+        self.y_mean_list = y_mean_list
+        self.seed = seed
+        self.smooth = smooth
+        self.expected_list_size = len(y_col_list)
+        self.threshold = threshold
+        if len(self.out_col_list) < self.expected_list_size:
+            raise ValueError("TargetEncoder __init__, input out_col_list should be same size as y_col_list")      
+        if y_mean_list != None and len(self.y_mean_list) < self.expected_list_size:
+            raise ValueError("TargetEncoder __init__, input y_mean_list should be same size as y_col_list")        
+
+    def transform(self, df):
+        x_col = self.x_col_list
+        cols = ['fold', x_col] if isinstance(x_col, str) else ['fold'] + x_col
+        agg_per_fold = df.groupBy(cols)
+        agg_all = df.groupBy(x_col)
+
+        per_fold_list = []
+        all_list = []
+
+        for i in range(0, self.expected_list_size):
+            y_col = self.y_col_list[i]
+            per_fold_list.append(f.count(y_col).alias(f'count_{y_col}'))
+            per_fold_list.append(f.sum(y_col).alias(f'sum_{y_col}'))
+            all_list.append(f.count(y_col).alias(f'count_all_{y_col}'))
+            all_list.append(f.sum(y_col).alias(f'sum_all_{y_col}'))
+
+        agg_per_fold = agg_per_fold.agg(*per_fold_list)
+        agg_all = agg_all.agg(*all_list)
+        agg_per_fold = agg_per_fold.join(agg_all, x_col, 'left')
+
+        if self.threshold > 0:
+            agg_all = agg_all.where(f.col(f'count_all_{self.y_col_list[0]}')>self.threshold)
+
+        for i in range(0, self.expected_list_size):
+            y_col = self.y_col_list[i]
+            out_col = self.out_col_list[i]
+            out_dtype = self.out_dtype
+            y_mean = self.y_mean_list[i] if self.y_mean_list != None else None
+            
+            if y_mean is None:
+                y_mean = np.array(df.groupBy().mean(y_col).collect())[0][0]
+            mean = float(y_mean)
+            smooth = self.smooth
+
+            # print(agg_per_fold.dtypes)
+
+            # prepare for agg_per_fold
+            agg_per_fold = agg_per_fold.withColumn(
+                f'count_all_{y_col}', f.col(f'count_all_{y_col}')-f.col(f'count_{y_col}'))
+            agg_per_fold = agg_per_fold.withColumn(
+                f'sum_all_{y_col}', f.col(f'sum_all_{y_col}')-f.col(f'sum_{y_col}'))
+            agg_per_fold = agg_per_fold.withColumn(
+                out_col,
+                (f.col(f'sum_all_{y_col}') + f.lit(mean) * f.lit(smooth))/(f.col(f'count_all_{y_col}') + f.lit(smooth)))
+            agg_all = agg_all.withColumn(
+                out_col,
+                (f.col(f'sum_all_{y_col}') + f.lit(mean) * f.lit(smooth))/(f.col(f'count_all_{y_col}') + f.lit(smooth)))
+            if out_dtype is not None:
+                agg_per_fold = agg_per_fold.withColumn(
+                    out_col, f.col(out_col).cast(out_dtype))
+                agg_all = agg_all.withColumn(
+                    out_col, f.col(out_col).cast(out_dtype))
+            agg_per_fold = agg_per_fold.drop(
+                f'count_all_{y_col}', f'count_{y_col}', f'sum_all_{y_col}', f'sum_{y_col}')
+            agg_all = agg_all.drop(f'count_all_{y_col}', f'sum_all_{y_col}')
+        return (self.materialize(agg_per_fold, "%s/train/%s" % (self.dicts_path, self.out_name)),
+                self.materialize(agg_all, "%s/test/%s" % (self.dicts_path, self.out_name)))
+
+
+class CountEncoder(Encoder):
+    def __init__(self, proc, x_col_list, y_col_list, out_col_list, out_name, train_generate=True):
+        super().__init__(proc)
+        self.op_name = "CountEncoder"
+        self.x_col_list = x_col_list
+        self.y_col_list = y_col_list
+        self.out_col_list = out_col_list
+        self.out_name = out_name        
+        self.expected_list_size = len(y_col_list)
+        self.train_generate = train_generate
+        if len(self.out_col_list) < self.expected_list_size:
+            raise ValueError("CountEncoder __init__, input out_col_list should be same size as y_col_list")
+
+    def transform(self, df):
+        x_col = self.x_col_list
+        cols = [x_col] if isinstance(x_col, str) else x_col
+        agg_all = df.groupby(cols)
+
+        all_list = []
+
+        for i in range(0, self.expected_list_size):
+            y_col = self.y_col_list[i]
+            out_col = self.out_col_list[i]
+            all_list.append(f.count(y_col).alias(f'{out_col}'))
+
+        agg_all = agg_all.agg(*all_list)
+        for i in range(0, self.expected_list_size):
+            out_col = self.out_col_list[i]
+            agg_all = agg_all.withColumn(out_col, f.col(out_col).cast(spk_type.IntegerType()))
+
+        if self.train_generate:
+            return (self.materialize(agg_all, "%s/train/%s" % (self.dicts_path, self.out_name)),
+                    self.materialize(agg_all, "%s/test/%s" % (self.dicts_path, self.out_name)))
+        else:
+            return (self.materialize(agg_all, "%s/test/%s" % (self.dicts_path, self.out_name)))
+
+
+
+class FrequencyEncoder(Encoder):
+    def __init__(self, proc, x_col_list, y_col_list, out_col_list, out_name):
+        super().__init__(proc)
+        self.op_name = "FrequencyEncoder"
+        self.x_col_list = x_col_list
+        self.y_col_list = y_col_list
+        self.out_col_list = out_col_list
+        self.out_name = out_name        
+        self.expected_list_size = len(y_col_list)
+        if len(self.out_col_list) < self.expected_list_size:
+            raise ValueError("FrequencyEncoder __init__, input out_col_list should be same size as y_col_list")
+
+    def transform(self, df):
+        length_df = df.count()
+        x_col = self.x_col_list
+        cols = [x_col] if isinstance(x_col, str) else x_col
+        agg_all = df.groupby(cols)
+
+        all_list = []
+
+        for i in range(0, self.expected_list_size):
+            y_col = self.y_col_list[i]
+            out_col = self.out_col_list[i]
+            all_list.append(f.count(y_col).alias(out_col))
+        agg_all = agg_all.agg(*all_list)
+
+        for i in range(0, self.expected_list_size):
+            out_col = self.out_col_list[i]
+            agg_all = agg_all.withColumn(out_col, (f.col(out_col)*1.0/length_df).cast(spk_type.FloatType()))
+        return (self.materialize(agg_all, "%s/train/%s" % (self.dicts_path, self.out_name)),
+                None)
\ No newline at end of file
diff --git a/RecDP/pyrecdp/init_spark.py b/RecDP/pyrecdp/init_spark.py
new file mode 100644
index 000000000..bad961816
--- /dev/null
+++ b/RecDP/pyrecdp/init_spark.py
@@ -0,0 +1,5 @@
+try:
+    import findspark
+    findspark.init()
+except:
+    pass
\ No newline at end of file
diff --git a/RecDP/pyrecdp/utils.py b/RecDP/pyrecdp/utils.py
new file mode 100644
index 000000000..9c28e3043
--- /dev/null
+++ b/RecDP/pyrecdp/utils.py
@@ -0,0 +1,157 @@
+from pyrecdp.init_spark import *
+import os
+import re
+from pyspark import *
+from pyspark.sql import *
+from py4j.protocol import Py4JJavaError
+from py4j.java_gateway import JavaObject
+from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap
+import pyspark.sql.types as spk_type
+import pyspark.sql.functions as spk_func
+
+def convert_to_spark_dict(orig_dict, schema=['dict_col', 'dict_col_id']):
+    ret = []
+    for row_k, row_v in orig_dict.items():
+        ret.append({schema[0]: row_k, schema[1]: row_v})
+    return ret
+
+
+def convert_to_spark_df(orig_dict, spark):
+    df = spark.createDataFrame(convert_to_spark_dict(orig_dict))
+    df = df.withColumn('dict_col_id', spk_func.col('dict_col_id').cast(spk_type.IntegerType()))
+    return df
+
+def list_dir(path, only_get_one = True):
+    source_path_dict = {}
+    dirs = os.listdir(path)
+    for files in dirs:
+        try:
+            sub_dirs = os.listdir(path + "/" + files)
+            if not only_get_one:
+                source_path_dict[files] = []
+            for file_name in sub_dirs:
+                if (file_name.endswith('parquet') or file_name.endswith('csv')):
+                    if not only_get_one:
+                        source_path_dict[files].append(os.path.join(
+                            path, files, file_name))
+                    else:
+                        source_path_dict[files] = os.path.join(
+                            path, files, file_name)
+        except:
+            if not only_get_one:
+                source_path_dict[files] = [os.path.join(path, files)]
+            else:
+                source_path_dict[files] = os.path.join(path, files)
+    return source_path_dict
+
+
+def parse_size(size):
+    units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40, "K": 2**10, "M": 2**20, "G": 2**30, "T": 2**40}
+    size = size.upper()
+    number, unit = re.findall('(\d+)(\w*)', size)[0]
+    return int(float(number)*units[unit])
+
+
+def parse_cores_num(local_str):
+    number = re.findall('local\[(\d+)\]', local_str)
+    if len(number) > 0:
+        return int(number[0])
+    else:
+        return os.cpu_count()
+
+
+def get_estimate_size_of_dtype(dtype_name):
+    units = {'byte': 1, 'short': 2, 'int': 4, 'long': 8, 'float': 4, 'double': 8, 'string': 10}
+    return units[dtype_name] if dtype_name in units else 4
+
+
+def get_dtype(df, colname):
+    return [dtype for name, dtype in df.dtypes if name == colname][0]
+
+
+def _j4py(spark, r):
+    sc = spark.sparkContext
+    if isinstance(r, JavaObject):
+        clsName = r.getClass().getSimpleName()
+        # convert RDD into JavaRDD
+        if clsName != 'JavaRDD' and clsName.endswith("RDD"):
+            r = r.toJavaRDD()
+            clsName = 'JavaRDD'
+        elif clsName == 'JavaRDD':
+            jrdd = sc._jvm.SerDe.javaToPython(r)
+            return RDD(jrdd, sc)
+        elif clsName == 'DataFrame' or clsName == 'Dataset':
+            return DataFrame(r, SQLContext.getOrCreate(sc))
+        else:
+             raise NotImplementedError(f"can't convert {clsName} {r} to java")
+    else:
+        raise NotImplementedError(f"can't convert {r} to java")
+    return r
+
+
+def _py4j(obj, gateway = None):
+    """ Convert Python object into Java """
+    if isinstance(obj, DataFrame):
+        obj = obj._jdf
+    elif isinstance(obj, SparkContext):
+        obj = obj._jsc
+    elif isinstance(obj, (list, tuple)):
+        obj = ListConverter().convert([_py4j(x) for x in obj],
+                                      gateway._gateway_client)
+    elif isinstance(obj, JavaObject):
+        pass
+    elif isinstance(obj, (int, float, bool, bytes, str)):
+        pass
+    else:
+        raise NotImplementedError(f"can't convert {str(obj)} to python")
+    return obj    
+
+def jvm_categorify(spark, df, col_name, dict_df):
+    gateway = spark.sparkContext._gateway
+    _jcls = gateway.jvm.org.apache.spark.sql.api.Categorify
+    return _j4py(spark, _jcls.categorify(_py4j(spark.sparkContext), _py4j(df), _py4j(col_name), _py4j(dict_df)))
+
+def jvm_categorify_for_array(spark, df, col_name, dict_df):
+    gateway = spark.sparkContext._gateway
+    _jcls = gateway.jvm.org.apache.spark.sql.api.CategorifyForArray
+    return _j4py(spark, _jcls.categorify(_py4j(spark.sparkContext), _py4j(df), _py4j(col_name), _py4j(dict_df)))
+
+def jvm_categorify_for_multi_level_array(spark, df, col_name, dict_df, sep_list):
+    gateway = spark.sparkContext._gateway
+    _jcls = gateway.jvm.org.apache.spark.sql.api.CategorifyForMultiLevelArray
+    return _j4py(spark, _jcls.categorify(_py4j(spark.sparkContext), _py4j(df), _py4j(col_name), _py4j(dict_df), _py4j(sep_list, gateway)))
+
+def jvm_categorify_by_freq_for_array(spark, df, col_name, dict_df):
+    gateway = spark.sparkContext._gateway
+    _jcls = gateway.jvm.org.apache.spark.sql.api.CategorifyByFreqForArray
+    return _j4py(spark, _jcls.categorify(_py4j(spark.sparkContext), _py4j(df), _py4j(col_name), _py4j(dict_df)))
+
+def jvm_get_negative_sample(spark, df, col_name, dict_df, neg_cnt):
+    gateway = spark.sparkContext._gateway
+    _jcls = gateway.jvm.org.apache.spark.sql.api.NegativeSample
+    return _j4py(spark, _jcls.add(_py4j(spark.sparkContext), _py4j(df), _py4j(col_name), _py4j(dict_df), _py4j(neg_cnt)))
+
+def jvm_get_negative_feature(spark, df, tgt_name, src_name, dict_df, neg_cnt):
+    gateway = spark.sparkContext._gateway
+    _jcls = gateway.jvm.org.apache.spark.sql.api.NegativeFeature
+    return _j4py(spark, _jcls.add(_py4j(spark.sparkContext), _py4j(df), _py4j(tgt_name), _py4j(src_name), _py4j(dict_df), _py4j(neg_cnt)))
+
+def jvm_get_negative_feature_for_array(spark, df, tgt_name, src_name, dict_df, neg_cnt):
+    gateway = spark.sparkContext._gateway
+    _jcls = gateway.jvm.org.apache.spark.sql.api.NegativeFeatureForArray
+    return _j4py(spark, _jcls.add(_py4j(spark.sparkContext), _py4j(df), _py4j(tgt_name), _py4j(src_name), _py4j(dict_df), _py4j(neg_cnt)))
+
+def jvm_scala_df_test_int(spark, df, col_name):
+    gateway = spark.sparkContext._gateway
+    _jcls = gateway.jvm.org.apache.spark.sql.api.ScalaDFTest
+    return _j4py(spark, _jcls.scala_test_int(_py4j(spark.sparkContext), _py4j(df), _py4j(col_name)))
+
+def jvm_scala_df_test_str(spark, df, col_name):
+    gateway = spark.sparkContext._gateway
+    _jcls = gateway.jvm.org.apache.spark.sql.api.ScalaDFTest
+    return _j4py(spark, _jcls.scala_test_str(_py4j(spark.sparkContext), _py4j(df), _py4j(col_name)))
+
+def jvm_scala_df_test_broadcast(spark, df, col_name, dict_df):
+    gateway = spark.sparkContext._gateway
+    _jcls = gateway.jvm.org.apache.spark.sql.api.ScalaDFTest
+    return _j4py(spark, _jcls.scala_test_broadcast(_py4j(spark.sparkContext), _py4j(df), _py4j(col_name), _py4j(dict_df)))
diff --git a/RecDP/resources/recdp_overview.png b/RecDP/resources/recdp_overview.png
new file mode 100644
index 000000000..d02758dfd
Binary files /dev/null and b/RecDP/resources/recdp_overview.png differ
diff --git a/RecDP/run_docker b/RecDP/run_docker
new file mode 100755
index 000000000..a6e9497bc
--- /dev/null
+++ b/RecDP/run_docker
@@ -0,0 +1,3 @@
+#docker run --network host -v ${your_local_codes_data_dir}:/home/vmagent/app/recdp -w /home/vmagent/app/ -it xuechendi/recdp_spark3.1 /bin/bash
+docker run -it --privileged --network host -v /mnt/nvme2/chendi/BlueWhale/recdp:/home/vmagent/app/recdp -v /mnt/nvme2/chendi/intel-bigdata/OAP/native-sql-engine/:/home/vmagent/app/native-sql-engine -v /mnt/nvme2/chendi/BlueWhale/gazelle_lib:/home/vmagent/app/gazelle_lib -w /home/vmagent/app/ xuechendi/recdp_spark3.1:gazelle_devel /bin/bash
+#docker run -it --privileged --network host -v /mnt/nvme2/chendi/BlueWhale/recdp:/home/vmagent/app/recdp -w /home/vmagent/app/ xuechendi/recdp_spark3.1 /bin/bash
diff --git a/RecDP/setup.py b/RecDP/setup.py
new file mode 100644
index 000000000..81caa5360
--- /dev/null
+++ b/RecDP/setup.py
@@ -0,0 +1,64 @@
+import setuptools
+from setuptools.command.install import install
+
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+
+
+class post_install(install):
+    def run(self):
+        install.run(self)
+        import os
+        try:
+            import pyspark
+            vspark = str(pyspark.__version__)
+            print(f"Detect system pyspark, version is {vspark}")
+        except:
+            vspark = "3.2.0"
+            print(f"Didn't find system pyspark, use default version {vspark}, other version can be found {self.install_lib}/pyrecdp/ScalaProcessUtils/")
+        import shutil
+        scala_jar = "/30/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar" if vspark.startswith(
+            "3.0"
+        ) else "/31/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+        print(f"mkdir {self.install_lib}/pyrecdp/ScalaProcessUtils/target/")
+        os.makedirs(os.path.dirname(
+            f"{self.install_lib}/pyrecdp/ScalaProcessUtils/target/"),
+                    exist_ok=True)
+        print(
+            f"cp {self.build_lib}/ScalaProcessUtils/built/{scala_jar} to {self.install_lib}/pyrecdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+        )
+        shutil.copy(
+            f"{self.build_lib}/ScalaProcessUtils/built/{scala_jar}",
+            f"{self.install_lib}/pyrecdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+        )
+        print(f"cp {self.build_lib}/ScalaProcessUtils/built/ {self.install_lib}/pyrecdp/ScalaProcessUtils/")
+        shutil.copytree(f"{self.build_lib}/ScalaProcessUtils/built/",
+                        f"{self.install_lib}/pyrecdp/ScalaProcessUtils/built")
+
+
+setuptools.setup(
+    name="pyrecdp",
+    version="0.1.2",
+    author="INTEL AIA BDF",
+    author_email="chendi.xue@intel.com",
+    description=
+    "A data processing bundle for spark based recommender system operations",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/oap-project/recdp",
+    project_urls={
+        "Bug Tracker": "https://github.com/oap-project/recdp",
+    },
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+    ],
+    include_package_data=True,
+    package_dir={},
+    packages=["pyrecdp", "ScalaProcessUtils"],
+    package_data={"": ["*.jar"]},
+    python_requires=">=3.6",
+    cmdclass={'install': post_install},
+    zip_safe=False,
+    install_requires=[])


Test patch:
diff --git a/RecDP/examples/notebooks/operations/test_FeatureAdd_w_scala_udf.ipynb b/RecDP/examples/notebooks/operations/test_FeatureAdd_w_scala_udf.ipynb
new file mode 100644
index 000000000..5c2b389b2
--- /dev/null
+++ b/RecDP/examples/notebooks/operations/test_FeatureAdd_w_scala_udf.ipynb
@@ -0,0 +1,448 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/mnt/nvme2/chendi/BlueWhale/recdp\n"
+     ]
+    }
+   ],
+   "source": [
+    "#!/env/bin/python\n",
+    "\n",
+    "import init\n",
+    "\n",
+    "import findspark\n",
+    "findspark.init()\n",
+    "\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pyspark.sql import *\n",
+    "from pyspark import *\n",
+    "import pyspark.sql.functions as f\n",
+    "from timeit import default_timer as timer\n",
+    "import logging\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.utils import *\n",
+    "from pyspark.sql.functions import pandas_udf, PandasUDFType\n",
+    "\n",
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/recsys2021_0608_scalaudf_test/\"\n",
+    "original_folder = \"/recsys2021_0608/\"\n",
+    "\n",
+    "scala_udf_jars = \"/mnt/nvme2/chendi/BlueWhale/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar\"\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "t0 = timer()\n",
+    "spark = SparkSession.builder.master('yarn')\\\n",
+    "    .appName(\"scala_udf_column\")\\\n",
+    "    .config(\"spark.sql.broadcastTimeout\", \"7200\")\\\n",
+    "    .config(\"spark.cleaner.periodicGC.interval\", \"10min\")\\\n",
+    "    .config(\"spark.driver.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.executor.extraClassPath\", f\"{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.executor.memory\", \"20g\")\\\n",
+    "    .config(\"spark.executor.memoryOverhead\", \"6g\")\\\n",
+    "    .getOrCreate()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example 1.1: CategorifyByFreqForArray"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "recdp-scala-extension is enabled\n",
+      "per core memory size is 5.000 GB and shuffle_disk maximum capacity is 8589934592.000 GB\n",
+      "+--------+--------------------+\n",
+      "|hashtags|       hashtags_orig|\n",
+      "+--------+--------------------+\n",
+      "|    null|                null|\n",
+      "|      80|1A703973FF98425EE...|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|     119|A31D528F61979FC70...|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "|    null|                null|\n",
+      "+--------+--------------------+\n",
+      "only showing top 20 rows\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "dict_df = spark.read.parquet(\"/recsys2020_0608_categorify_example/recsys_dicts/hashtags\")\n",
+    "df = spark.read.parquet(\"/recsys2021_0608/\")\n",
+    "df = df.select(\"hashtags\").withColumn(\"hashtags_orig\", f.col(\"hashtags\"))\n",
+    "\n",
+    "proc = DataProcessor(spark, \"hdfs://\", \"/recdp_operations_example/\")\n",
+    "op_categorify = Categorify(\n",
+    "    ['hashtags'],\n",
+    "    dict_dfs=[{'col_name': 'hashtags', 'dict': dict_df}],\n",
+    "    hint = 'udf',\n",
+    "    doSplit = True,\n",
+    "    keepMostFrequent = True\n",
+    ")\n",
+    "proc.reset_ops([op_categorify])\n",
+    "proc.get_sample(df)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example 1.2: CategorifyForArray"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "recdp-scala-extension is enabled\n",
+      "per core memory size is 5.000 GB and shuffle_disk maximum capacity is 8589934592.000 GB\n",
+      "+--------------------+--------------------+\n",
+      "|            hashtags|       hashtags_orig|\n",
+      "+--------------------+--------------------+\n",
+      "|                  []|                null|\n",
+      "|                [80]|1A703973FF98425EE...|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|[5680, 6617, 120,...|A31D528F61979FC70...|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "|                  []|                null|\n",
+      "+--------------------+--------------------+\n",
+      "only showing top 20 rows\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "dict_df = spark.read.parquet(\"/recsys2020_0608_categorify_example/recsys_dicts/hashtags\")\n",
+    "df = spark.read.parquet(\"/recsys2021_0608/\")\n",
+    "df = df.select(\"hashtags\").withColumn(\"hashtags_orig\", f.col(\"hashtags\"))\n",
+    "\n",
+    "proc = DataProcessor(spark, \"hdfs://\", \"/recdp_operations_example/\")\n",
+    "op_categorify = Categorify(\n",
+    "    ['hashtags'],\n",
+    "    dict_dfs=[{'col_name': 'hashtags', 'dict': dict_df}],\n",
+    "    hint = 'udf',\n",
+    "    doSplit = True\n",
+    ")\n",
+    "proc.reset_ops([op_categorify])\n",
+    "proc.get_sample(df)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example 2: Categorify"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "recdp-scala-extension is enabled\n",
+      "per core memory size is 5.000 GB and shuffle_disk maximum capacity is 8589934592.000 GB\n",
+      "+-------+-------------+\n",
+      "|mention| mention_orig|\n",
+      "+-------+-------------+\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|      7|pedrojubierna|\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "|     22|             |\n",
+      "+-------+-------------+\n",
+      "only showing top 20 rows\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "dict_df = spark.read.parquet(\"/recsys2021_0608_example/recsys_dicts/mention\")\n",
+    "df = spark.read.parquet(\"/recsys2021_0608_example/train_with_categorified_features_test\")\n",
+    "df = df.select(\"mention\").withColumn(\"mention_orig\", f.col(\"mention\"))\n",
+    "\n",
+    "proc = DataProcessor(spark, \"hdfs://\", \"/recdp_operations_example/\")\n",
+    "op_categorify = Categorify(\n",
+    "    ['mention'],\n",
+    "    dict_dfs=[{'col_name': 'mention', 'dict': dict_df}],\n",
+    "    hint = 'udf'\n",
+    ")\n",
+    "proc.reset_ops([op_categorify])\n",
+    "proc.get_sample(df)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example 3: CodegenSeparator"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[('tweet_timestamp', 'bigint')]\n",
+      "+---------------+\n",
+      "|tweet_timestamp|\n",
+      "+---------------+\n",
+      "|     1613574302|\n",
+      "|     1612628499|\n",
+      "|     1613233141|\n",
+      "|     1613373619|\n",
+      "|     1614053869|\n",
+      "|     1613130630|\n",
+      "|     1613721703|\n",
+      "|     1612706347|\n",
+      "|     1613905759|\n",
+      "|     1613854589|\n",
+      "|     1613217462|\n",
+      "|     1612584698|\n",
+      "|     1613590692|\n",
+      "|     1613805413|\n",
+      "|     1612540790|\n",
+      "|     1613509274|\n",
+      "|     1613437436|\n",
+      "|     1614099587|\n",
+      "|     1612914478|\n",
+      "|     1612815075|\n",
+      "+---------------+\n",
+      "only showing top 20 rows\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "gateway = spark.sparkContext._gateway\n",
+    "spark._jsparkSession.udf().register(\"CodegenSeparator\", gateway.jvm.org.apache.spark.sql.api.CodegenSeparator())\n",
+    "spark._jsparkSession.udf().register(\"CodegenSeparator0\", gateway.jvm.org.apache.spark.sql.api.CodegenSeparator0())\n",
+    "spark._jsparkSession.udf().register(\"CodegenSeparator1\", gateway.jvm.org.apache.spark.sql.api.CodegenSeparator1())\n",
+    "spark._jsparkSession.udf().register(\"CodegenSeparator2\", gateway.jvm.org.apache.spark.sql.api.CodegenSeparator2())\n",
+    "df = spark.read.parquet(\"/recsys2021_0608_example/train_with_categorified_features_test\")\n",
+    "df = df.select(\"tweet_timestamp\")\n",
+    "df = df.withColumn('tweet_timestamp', f.expr(\"CodegenSeparator2(tweet_timestamp)\"))\n",
+    "print(df.dtypes)\n",
+    "df.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example 4: sortStringArrayByFrequency and sortIntegerArrayByFrequency"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "-RECORD 0--------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 000097973EC6EAD0F129E47A652711F8                                                                     \n",
+      " tweet_id             | [9A8B5B7DCF2211FB642C78DF8DF54699]                                                                   \n",
+      " sorted_tweet_id      | [9A8B5B7DCF2211FB642C78DF8DF54699]                                                                   \n",
+      "-RECORD 1--------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 00019AA271E5715BC857B49D97C162AB                                                                     \n",
+      " tweet_id             | [29C0142E44F15DCFE3EB7E03F961FF83]                                                                   \n",
+      " sorted_tweet_id      | [29C0142E44F15DCFE3EB7E03F961FF83]                                                                   \n",
+      "-RECORD 2--------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 0006DF11AA00736A5DEA555325BCFC53                                                                     \n",
+      " tweet_id             | [C60548620D1D5812600C7AAE6B8035B9, 3A4C792AD82431945039D07041566E06]                                 \n",
+      " sorted_tweet_id      | [C60548620D1D5812600C7AAE6B8035B9, 3A4C792AD82431945039D07041566E06]                                 \n",
+      "-RECORD 3--------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 000A44D2EB9F62FD43F1629396E79027                                                                     \n",
+      " tweet_id             | [20459E1E9AF78B609D19D93C2239181C]                                                                   \n",
+      " sorted_tweet_id      | [20459E1E9AF78B609D19D93C2239181C]                                                                   \n",
+      "-RECORD 4--------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 000A5181FB805AECDD8B3C9885E747E3                                                                     \n",
+      " tweet_id             | [29156C67DEB8F721CFC8AD41CC394614]                                                                   \n",
+      " sorted_tweet_id      | [29156C67DEB8F721CFC8AD41CC394614]                                                                   \n",
+      "-RECORD 5--------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 000C74CB320E603AFCBD627EA87B6F52                                                                     \n",
+      " tweet_id             | [BED406E79A7E89B2F6681A3F97EFA3B9]                                                                   \n",
+      " sorted_tweet_id      | [BED406E79A7E89B2F6681A3F97EFA3B9]                                                                   \n",
+      "-RECORD 6--------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 0011AD9F8685B8F9C78C567DE0D3A36D                                                                     \n",
+      " tweet_id             | [290819EE41D3D540EE497EFCE807B033, BA4FB7F72F14D69589232E91532E37B6]                                 \n",
+      " sorted_tweet_id      | [BA4FB7F72F14D69589232E91532E37B6, 290819EE41D3D540EE497EFCE807B033]                                 \n",
+      "-RECORD 7--------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 0012CE19CDFC95AD77327BD0987A6AD0                                                                     \n",
+      " tweet_id             | [8DF521BBDF4F3C08D482DA5A53DAE15B]                                                                   \n",
+      " sorted_tweet_id      | [8DF521BBDF4F3C08D482DA5A53DAE15B]                                                                   \n",
+      "-RECORD 8--------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 00144DB842898B44D023CDA3A2DE629C                                                                     \n",
+      " tweet_id             | [F3FD6F836B0000E7F36D7AF233147938]                                                                   \n",
+      " sorted_tweet_id      | [F3FD6F836B0000E7F36D7AF233147938]                                                                   \n",
+      "-RECORD 9--------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 0014BA51669944920AC6E33C5B95B0DB                                                                     \n",
+      " tweet_id             | [E64C8BC498F22BC8106862DA4460F7EB]                                                                   \n",
+      " sorted_tweet_id      | [E64C8BC498F22BC8106862DA4460F7EB]                                                                   \n",
+      "-RECORD 10-------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 00157C4D94462B388CCC77298FAF1A64                                                                     \n",
+      " tweet_id             | [F8764755FD1E2698108E45ADEE16C55A, B21D91AB251C41D161835C97BF837BF9]                                 \n",
+      " sorted_tweet_id      | [F8764755FD1E2698108E45ADEE16C55A, B21D91AB251C41D161835C97BF837BF9]                                 \n",
+      "-RECORD 11-------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 00183428F77726C5ADF6A915F92E2554                                                                     \n",
+      " tweet_id             | [E6512F790B1B625EC3577721BA508AEB, 3D3A5962A7421F7211DE3E110347B0BB, 456E6C67BC21F192858EADA71155... \n",
+      " sorted_tweet_id      | [4183774FF1CD612EA898EC86498C5129, 456E6C67BC21F192858EADA71155B2EC, 3D3A5962A7421F7211DE3E110347... \n",
+      "-RECORD 12-------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 0018CAD2EA842B5174480A3FE676B4E0                                                                     \n",
+      " tweet_id             | [90407B49F4579872D644685064A2B5B4]                                                                   \n",
+      " sorted_tweet_id      | [90407B49F4579872D644685064A2B5B4]                                                                   \n",
+      "-RECORD 13-------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 001ED7F203FDBF8B7852B05871681A2D                                                                     \n",
+      " tweet_id             | [0CF74BC01E04409D1EE0F11A26997D0A]                                                                   \n",
+      " sorted_tweet_id      | [0CF74BC01E04409D1EE0F11A26997D0A]                                                                   \n",
+      "-RECORD 14-------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 001FD1CBA364E80403ADB793434C7EC2                                                                     \n",
+      " tweet_id             | [5CEE8D0133C3D6E9C3E25DF02460004A]                                                                   \n",
+      " sorted_tweet_id      | [5CEE8D0133C3D6E9C3E25DF02460004A]                                                                   \n",
+      "-RECORD 15-------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 00265BB2E9944297E0D82D52C430275B                                                                     \n",
+      " tweet_id             | [AA62ABA22C6A07D95BAE5BB1AB88F3F0]                                                                   \n",
+      " sorted_tweet_id      | [AA62ABA22C6A07D95BAE5BB1AB88F3F0]                                                                   \n",
+      "-RECORD 16-------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 002892FA3EBA69066E64539680D02426                                                                     \n",
+      " tweet_id             | [4400D64CF418243BA6C326D3228AD4F4]                                                                   \n",
+      " sorted_tweet_id      | [4400D64CF418243BA6C326D3228AD4F4]                                                                   \n",
+      "-RECORD 17-------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 002909AF49CE54E02C744EFBA5BB0096                                                                     \n",
+      " tweet_id             | [AD2A8EEB6942DA572C4D0264003D2FB1]                                                                   \n",
+      " sorted_tweet_id      | [AD2A8EEB6942DA572C4D0264003D2FB1]                                                                   \n",
+      "-RECORD 18-------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 002A46EDB320995FBE36846F5EBA17D4                                                                     \n",
+      " tweet_id             | [1A58A439428D68F7302E07234428472F]                                                                   \n",
+      " sorted_tweet_id      | [1A58A439428D68F7302E07234428472F]                                                                   \n",
+      "-RECORD 19-------------------------------------------------------------------------------------------------------------------\n",
+      " engaged_with_user_id | 002B452AA089B5774CAEA44A26E8E382                                                                     \n",
+      " tweet_id             | [5F219297994025A56C3763A21ACC53D0]                                                                   \n",
+      " sorted_tweet_id      | [5F219297994025A56C3763A21ACC53D0]                                                                   \n",
+      "only showing top 20 rows\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 1.1 prepare dataFrames\n",
+    "# 1.2 create RecDP DataProcessor\n",
+    "spark.udf.registerJavaFunction(\"SortStringArrayByFrequency\",\"com.intel.recdp.SortStringArrayByFrequency\")\n",
+    "df = spark.read.parquet(\"/recsys2021_0608_example/train_with_categorified_features_test\")\n",
+    "df = df.select(\"engaged_with_user_id\", 'tweet_id').groupby('engaged_with_user_id').agg(f.collect_list('tweet_id').alias('tweet_id'))\n",
+    "df = df.withColumn('sorted_tweet_id', f.expr('SortStringArrayByFrequency(tweet_id)'))\n",
+    "\n",
+    "df.show(vertical=True, truncate=100)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/notebooks/operations/test_sep_FillNA.ipynb b/RecDP/examples/notebooks/operations/test_sep_FillNA.ipynb
new file mode 100644
index 000000000..a575e021d
--- /dev/null
+++ b/RecDP/examples/notebooks/operations/test_sep_FillNA.ipynb
@@ -0,0 +1,253 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "spark.stop()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## DLRM Categorify"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/mnt/nvme2/chendi/BlueWhale/frameworks.bigdata.bluewhale/RecDP\n"
+     ]
+    }
+   ],
+   "source": [
+    "###### Start spark job ######\n",
+    "import init\n",
+    "\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.utils import *\n",
+    "\n",
+    "import logging\n",
+    "from timeit import default_timer as timer\n",
+    "import os\n",
+    "from pyspark import *\n",
+    "from pyspark.sql import *\n",
+    "from pyspark.sql.functions import *\n",
+    "from pyspark.sql.types import *\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Define Schema\n",
+    "LABEL_COL = 0\n",
+    "INT_COLS = list(range(1, 14))\n",
+    "CAT_COLS = list(range(14, 40))\n",
+    "label_fields = [StructField('_c%d' % LABEL_COL, IntegerType())]\n",
+    "int_fields = [StructField('_c%d' % i, IntegerType()) for i in INT_COLS]\n",
+    "str_fields = [StructField('_c%d' % i, StringType()) for i in CAT_COLS]\n",
+    "schema = StructType(label_fields + int_fields + str_fields)\n",
+    "\n",
+    "to_be_categorified = [23, 35, 14, 33]\n",
+    "\n",
+    "def categorifyAllFeatures(df, proc, output_name=\"categorified\", gen_dict=False):\n",
+    "    dict_dfs = []\n",
+    "    to_categorify_cols = ['_c%d' % i for i in CAT_COLS]\n",
+    "    #to_categorify_cols = ['_c%d' % i for i in to_be_categorified]\n",
+    "    if gen_dict:\n",
+    "        # only call below function when target dicts were not pre-prepared        \n",
+    "        op_gen_dict = GenerateDictionary(to_categorify_cols, isParquet=False)\n",
+    "        proc.reset_ops([op_gen_dict])\n",
+    "        t1 = timer()\n",
+    "        dict_dfs = proc.generate_dicts(df)\n",
+    "        t2 = timer()\n",
+    "        print(\"Generate Dictionary took %.3f\" % (t2 - t1))\n",
+    "    else:\n",
+    "        # or we can simply load from pre-gened\n",
+    "        dict_dfs = [{'col_name': name, 'dict': proc.spark.read.parquet(\n",
+    "            \"%s/%s/%s/%s\" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in to_categorify_cols]    \n",
+    "\n",
+    "    # start to do categorify\n",
+    "    op_categorify = Categorify(to_categorify_cols, dict_dfs=dict_dfs)\n",
+    "    op_fillna_for_categorified = FillNA(to_categorify_cols, 0)\n",
+    "    proc.append_ops([op_categorify, op_fillna_for_categorified])\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name)\n",
+    "    t2 = timer()\n",
+    "    print(\"Categorify took %.3f\" % (t2 - t1))\n",
+    "    \n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "per core memory size is 5.000 GB and shuffle_disk maximum capacity is 1200.000 GB\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-2-4b70b73cff33>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mschema\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mschema\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moption\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'sep'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'\\t'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcsv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_names\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;31m#df = spark.read.parquet(\"/dlrm/categorified_stage1\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcategorifyAllFeatures\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"dlrm_categorified\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgen_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m<ipython-input-1-3e02c0249b07>\u001b[0m in \u001b[0;36mcategorifyAllFeatures\u001b[0;34m(df, proc, output_name, gen_dict)\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_ops\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mop_gen_dict\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0mt1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m         \u001b[0mdict_dfs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate_dicts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m         \u001b[0mt2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Generate Dictionary took %.3f\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mt2\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/mnt/nvme2/chendi/BlueWhale/frameworks.bigdata.bluewhale/RecDP/pyrecdp/data_processor.py\u001b[0m in \u001b[0;36mgenerate_dicts\u001b[0;34m(self, df)\u001b[0m\n\u001b[1;32m    964\u001b[0m                     \"We haven't support apply generate_dict to not GenerateDictionary operator yet.\")\n\u001b[1;32m    965\u001b[0m             \u001b[0msave_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_tmp_cache_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 966\u001b[0;31m             \u001b[0mdfs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dict_dfs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspark\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    967\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mdict_df\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdfs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    968\u001b[0m             materialized_dfs.append({'col_name': dict_df['col_name'], 'dict': self.materialize(\n",
+      "\u001b[0;32m/mnt/nvme2/chendi/BlueWhale/frameworks.bigdata.bluewhale/RecDP/pyrecdp/data_processor.py\u001b[0m in \u001b[0;36mto_dict_dfs\u001b[0;34m(self, df, spark, cache_path)\u001b[0m\n\u001b[1;32m    766\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    767\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mto_dict_dfs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspark\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 768\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate_dict_dfs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspark\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwithCount\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misParquet\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    769\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    770\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/mnt/nvme2/chendi/BlueWhale/frameworks.bigdata.bluewhale/RecDP/pyrecdp/data_processor.py\u001b[0m in \u001b[0;36mgenerate_dict_dfs\u001b[0;34m(self, df, spark, cache_path, withCount, isParquet)\u001b[0m\n\u001b[1;32m     99\u001b[0m             \u001b[0mwindowed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mWindow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartitionBy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'column_id'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0morderBy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspk_func\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdesc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'count'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m             \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwithColumn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'dict_col_id'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspk_func\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrow_number\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mover\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwindowed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m             \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"parquet\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"overwrite\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcache_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m             \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparquet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcache_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m             \u001b[0mi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/hadoop/spark/python/pyspark/sql/readwriter.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, path, format, mode, partitionBy, **options)\u001b[0m\n\u001b[1;32m   1107\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jwrite\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1108\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1109\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jwrite\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1110\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1111\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0msince\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1.4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/hadoop/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1301\u001b[0m             \u001b[0mproto\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEND_COMMAND_PART\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1302\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1303\u001b[0;31m         \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1304\u001b[0m         return_value = get_return_value(\n\u001b[1;32m   1305\u001b[0m             answer, self.gateway_client, self.target_id, self.name)\n",
+      "\u001b[0;32m/hadoop/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36msend_command\u001b[0;34m(self, command, retry, binary)\u001b[0m\n\u001b[1;32m   1031\u001b[0m         \u001b[0mconnection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_connection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1032\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1033\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconnection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1034\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbinary\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1035\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_connection_guard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/hadoop/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36msend_command\u001b[0;34m(self, command)\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1199\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1200\u001b[0;31m             \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msmart_decode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstream\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1201\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Answer received: {0}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0manswer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1202\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0manswer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproto\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRETURN_MESSAGE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/usr/lib64/python3.7/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    587\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 589\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    590\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    591\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/hadoop/spark/python/pyspark/context.py\u001b[0m in \u001b[0;36msignal_handler\u001b[0;34m(signal, frame)\u001b[0m\n\u001b[1;32m    283\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0msignal_handler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msignal\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    284\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcancelAllJobs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 285\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    286\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    287\u001b[0m         \u001b[0;31m# see http://stackoverflow.com/questions/23206787/\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/dlrm/\"\n",
+    "csv_folder = \"/dlrm/csv_raw_data\"\n",
+    "file = \"/dlrm/raw_data/day_0\"\n",
+    "#path = os.path.join(path_prefix, file)\n",
+    "\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "t0 = timer()\n",
+    "spark = SparkSession\\\n",
+    "    .builder\\\n",
+    "    .master('yarn')\\\n",
+    "    .appName(\"DLRM\")\\\n",
+    "    .config(\"spark.executor.memory\", \"20g\")\\\n",
+    "    .config(\"spark.executor.memoryOverhead\", \"3g\")\\\n",
+    "    .config(\"spark.cleaner.periodicGC.interval\", \"10min\")\\\n",
+    "    .getOrCreate()\n",
+    "\n",
+    "files = [\"day_%d\" % i for i in range(0, 24)]\n",
+    "file_names = [os.path.join(path_prefix, csv_folder, filename) for filename in files]\n",
+    "\n",
+    "proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity=\"1200GB\")\n",
+    "df = spark.read.schema(schema).option('sep', '\\t').csv(file_names)\n",
+    "#df = spark.read.parquet(\"/dlrm/categorified_stage1\")\n",
+    "df = categorifyAllFeatures(df, proc, output_name=\"dlrm_categorified\", gen_dict=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "History Log\n",
+    "\n",
+    "* per core memory size is 5.000 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+    "* Generate Dictionary took 2373.706\n",
+    "* categorify threshold is 50.000 M rows, flush_threshold is 960.000 GB\n",
+    "* ('_c19', DataFrame[dict_col: string, dict_col_id: int], 3)\n",
+    "* ('_c30', DataFrame[dict_col: string, dict_col_id: int], 3)\n",
+    "* ('_c26', DataFrame[dict_col: string, dict_col_id: int], 10)\n",
+    "* ('_c32', DataFrame[dict_col: string, dict_col_id: int], 14)\n",
+    "* ('_c39', DataFrame[dict_col: string, dict_col_id: int], 36)\n",
+    "* ('_c22', DataFrame[dict_col: string, dict_col_id: int], 63)\n",
+    "* ('_c38', DataFrame[dict_col: string, dict_col_id: int], 108)\n",
+    "* ('_c29', DataFrame[dict_col: string, dict_col_id: int], 154)\n",
+    "* ('_c31', DataFrame[dict_col: string, dict_col_id: int], 976)\n",
+    "* ('_c21', DataFrame[dict_col: string, dict_col_id: int], 1543)\n",
+    "* ('_c27', DataFrame[dict_col: string, dict_col_id: int], 2208)\n",
+    "* ('_c20', DataFrame[dict_col: string, dict_col_id: int], 7122)\n",
+    "* ('_c17', DataFrame[dict_col: string, dict_col_id: int], 7424)\n",
+    "* ('_c28', DataFrame[dict_col: string, dict_col_id: int], 11938)\n",
+    "* ('_c37', DataFrame[dict_col: string, dict_col_id: int], 12973)\n",
+    "* ('_c16', DataFrame[dict_col: string, dict_col_id: int], 17295)\n",
+    "* ('_c18', DataFrame[dict_col: string, dict_col_id: int], 20265)\n",
+    "* ('_c15', DataFrame[dict_col: string, dict_col_id: int], 39060)\n",
+    "* ('_c25', DataFrame[dict_col: string, dict_col_id: int], 405282)\n",
+    "* ('_c36', DataFrame[dict_col: string, dict_col_id: int], 590151)\n",
+    "* ('_c24', DataFrame[dict_col: string, dict_col_id: int], 3067955)\n",
+    "* ('_c34', DataFrame[dict_col: string, dict_col_id: int], 40790947)\n",
+    "* ('_c23', DataFrame[dict_col: string, dict_col_id: int], 130229466)\n",
+    "* ('_c35', DataFrame[dict_col: string, dict_col_id: int], 187188509)\n",
+    "* ('_c14', DataFrame[dict_col: string, dict_col_id: int], 227605431)\n",
+    "* ('_c33', DataFrame[dict_col: string, dict_col_id: int], 292775613)\n",
+    "* _c19 will do bhj\n",
+    "* _c30 will do bhj\n",
+    "* _c26 will do bhj\n",
+    "* _c32 will do bhj\n",
+    "* _c39 will do bhj\n",
+    "* _c22 will do bhj\n",
+    "* _c38 will do bhj\n",
+    "* _c29 will do bhj\n",
+    "* _c31 will do bhj\n",
+    "* _c21 will do bhj\n",
+    "* _c27 will do bhj\n",
+    "* _c20 will do bhj\n",
+    "* _c17 will do bhj\n",
+    "* _c28 will do bhj\n",
+    "* _c37 will do bhj\n",
+    "* _c16 will do bhj\n",
+    "* _c18 will do bhj\n",
+    "* _c15 will do bhj\n",
+    "* _c25 will do bhj\n",
+    "* _c36 will do bhj\n",
+    "* _c24 will do bhj\n",
+    "* _c34 will do bhj\n",
+    "* etstimated_to_shuffle_size for _c23 is 749.453 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for _c35 is 725.014 GB, will do smj and spill to disk\n",
+    "* etstimated_to_shuffle_size for _c14 is 700.576 GB, will do smj\n",
+    "* etstimated_to_shuffle_size for _c33 is 676.137 GB, will do smj and spill to disk\n",
+    "* do bhj to _c19\n",
+    "* do bhj to _c30\n",
+    "* do bhj to _c26\n",
+    "* do bhj to _c32\n",
+    "* do bhj to _c39\n",
+    "* do bhj to _c22\n",
+    "* do bhj to _c38\n",
+    "* do bhj to _c29\n",
+    "* do bhj to _c31\n",
+    "* do bhj to _c21\n",
+    "* do bhj to _c27\n",
+    "* do bhj to _c20\n",
+    "* do bhj to _c17\n",
+    "* do bhj to _c28\n",
+    "* do bhj to _c37\n",
+    "* do bhj to _c16\n",
+    "* do bhj to _c18\n",
+    "* do bhj to _c15\n",
+    "* do bhj to _c25\n",
+    "* do bhj to _c36\n",
+    "* do bhj to _c24\n",
+    "* do bhj to _c34\n",
+    "* do smj to _c23\n",
+    "* do smj to _c35\n",
+    "* do smj to _c14\n",
+    "* do smj to _c33\n",
+    "* Categorify took 12443.532"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/notebooks/operations/test_w_nativesql.ipynb b/RecDP/examples/notebooks/operations/test_w_nativesql.ipynb
new file mode 100644
index 000000000..d8dc22c3d
--- /dev/null
+++ b/RecDP/examples/notebooks/operations/test_w_nativesql.ipynb
@@ -0,0 +1,571 @@
+{
+ "cells": [
+  {
+   "attachments": {
+    "image.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzsAAAEeCAYAAACt0QuDAAAgAElEQVR4Ae2dPY7kONKG+zSfvzfoBBaYMyQW49UBBmvJWbf8BtJbuwtoY4y8wQ5QXtprDNZZVKPMPQM/hCRKZIhUSkpSKVFPAlkpUlQw4iEp6tVfffnvf/9r/vzzT74woA/QB+gD9AH6AH2APkAfoA/QB4rpA6JzvojQ4QMBCEAAAhCAAAQgAAEIQKAkAqJzEDsltSixQAACEIAABCAAAQhAAAI1AcQOHQECEIAABCAAAQhAAAIQKJIAYqfIZiUoCEAAAhCAAAQgAAEIQACxQx+AAAQgAAEIQAACEIAABIokgNgpslkJCgIQgAAEIAABCEAAAhBA7NAHIAABCEAAAhCAAAQgAIEiCSB2imxWgoIABCAAAQhAAAIQgAAEEDv0AQhAAAIQgAAEIAABCECgSAKInSKblaAgAAEIQAACEIAABCAAAcQOfQACEIAABCAAAQhAAAIQKJIAYqfIZiUoCEAAAhCAAAQgAAEIQACxQx+AAAQgAAEIQAACEIAABIokgNgpslkJCgIQgAAEIAABCEAAAhBA7NAHIAABCEAAAhCAAAQgAIEiCSB2imxWgoIABLZL4GYu57Oprp++i59XU53P5lx/L+bmryUVJPBprtXZnKurUTSDpdNlNvUO2jBdBY6lNsYLPcKBwiIEIACByQQQO5NRURACEIBACgIhscMB7TKyTxI7t4s5nyuj9eqyGCZstXZ9E1yiCAQgAIG9EEDs7KWl8BMCECiEQEjshPIKCTdVGBs64L9dzua86pUW+keqboQdCEDgeAQQO8drcyKGAASeSiBw4NrewrbObVFPDX555ZsRO4H2Wx7V5C1rgbX67XqT3aMgBCAAgc0SQOxstmlwDAIQ2CqB+sDzrJ+riRwE1wfp9lkcufXJL9fYsuvb38hVg89r1T6f0thonu8ZPv9TlxP/nOeAPCHl5FsbwSo93wNXM+6tN+1tZtFnke6tlx6gy7iMhrexdaJAxWjjc3l7TGxnuxdTvX54C9tj9eoYdd8ypmnTYb3WbX4hAAEIQCBMALET5kIuBCAAgSiB5oBZH5D6IqbeuD1wtgfaxjkA9w6023wvL1B7c8ArB/vOQW9bh7ttsJy1FyhvtJ9SNpAncXex6PWDGNoD+G6D1maXvrfeOtz+1vU5cdfZEbGjXvLgCpyueu1/KOZBTFZ06LYXXFaw9uv6vBFuVsx1jmlObvyOnTabHwhAAAIQGCeA2Bnnw1oIQAACAwLNQWx/UNsU0GIncDAvBQMH0MG8Qa32QHt4wFv749ziZMWOK4AacxGf6uNr961m8XJjdpp6LRfNQwd0b70qP0vsWB9aG62w8Xk09ffP3oRj9mOyokbZt/z01b5J9U7kEOo3ChFJCEAAAhAYEkDsDJmQAwEIQGCUwDSxEzuIDeRPPJDVB97WySa/v+qh07bcmKjytrnnT7vevRhR1+EJklY8hF6zXRe+t77zulnwbNt1rQ1H6AXbJuivEjfBMu1VFudKmhaW1pPF9dorO1FObQ332sQ6wi8EIAABCHgEEDseDhIQgAAE7hMIHtgaJWKiB6eqnFQXLev7kkrsDESKVOOKidiBv3WnXW+f9/F/e9HVmLW3d8lv7IqILTNcb6v0/Osy04sdPxbrVx9TerHTBNP0KVtfgMPEPtKhYQECEIAABGoCiB06AgQgAIGZBCaJHS1+ujpyiZ3+ANm7StPVOy6qvG3uHVjfE0NunXa5E0i9n3ZV/XtvvSvGug3Ti52gEOzqm3kbW5CTuqLk2B7l0Nryb8XTG5OGAAQgAAFNALGjiZCGAAQgcIdA8ArL4GA0clA7KDcuQlxXPEHirNBXG2LlujebBY7ofRuNIOufZ3EqqxcDgk0XCaWDgsUpOLY+uC6h2ImKU8e/uqmqkStUSsgtETtSXSjWOm/4vJbvHSkIQAACENAEEDuaCGkIQAAC9whowdKm5RYo98x7IzqcA9RIuXm3scmtTv1tVaG3psXFTvglBwM/u4N6x/f2IXwbX2ib2hf7/IwcnNvlmqcSJvfW6zbQzEM2Wx8Ht8u12/oabyhG78YkdYaEyCP1TuQw1qYaFWkIQAACEOgJIHZ6FixBAAIQmEzAHhg3z3jIGf3m4NmKAWtoUrnggby10P82tvz/nyP1+wfxVtA4gqg30SzVB+z2+RAlnpyyvu8T/s+OJ26sH049M9c7rtSLnj910EpAPSI6bGWajfI5JkyDtzbOFFnd80K6ThtXIN+6zS8EIAABCIQJIHbCXMiFAAQgsDkCndjZnGdHcmgosPJHv/C2wfyOUQMEIACBzRNA7Gy+iXAQAhCAQEMAsbORnhC5lS2bd2vXly0QDEMAAhBYnwBiZ33m1AgBCEBgEQHEziJsGTYK37KYoSJ5espcq8AthHkqwyoEIACB4gggdoprUgKCAAQgAAEIQAACEIAABIQAYod+AAEIQAACEIAABCAAAQgUSQCxU2SzEhQEIAABCEAAAhCAAAQggNihD0AAAhCAAAQgAAEIQAACRRJA7BTZrAQFAQhAAAIQgAAEIAABCCB26AMQgAAEIAABCEAAAhCAQJEEEDtFNitBQQACEIAABCAAAQhAAAKIHfoABCAAAQhAAAIQgAAEIFAkAcROkc1KUBCAAAQgAAEIQAACEIAAYoc+AAEIQAACEIAABCAAAQgUSQCxU2SzEhQEIAABCEAAAhCAAAQggNihD0AAAhCAAAQgAAEIQAACRRJA7BTZrAQFAQhAAAIQgAAEIAABCMwSO7fL2ZzPzbe6fnb0Pq9Vl3++3O7mdwVYgAAEIAABCEAAAhCAAAQgkInAdLHzeTUXK3A+r6Y6X0wta9xlczOXc2XqYrH8TIFgFgIQgAAEIAABCEAAAhCAgEtguthxtxIhU12NXNuRqzr6Ko+kY/muGZYhAAEIQAACEIAABCAAAQjkIjBL7PS3sbVXdRA7udoFuxCAAAQgAAEIQAACEIDAgwRmiZ2urvoWteZ2tdgVnFh+Z4MFCEAAAhCAAAQgAAEIQAACGQksEzvGGLnKI+8iiImaWP6cWH7+/GnEQb4woA/QB+gDfR/4UZ3M169f6++v3/6o95F/fPu1yxtbd6p+sE9lXqEP0AfoA/SBXfUB0QRLP3L88EX+3P3cLrW4acpNeBEBLyi4i5QCEIAABGYT+Hgzr28fzWYfb+bl9GreB0Y+zNvLi6mLeWXezeupzR9sQwYEIAABCECgPALTxU79prX+1dPOG6brqzyhV1L3z/icvZcYlIeRiCAAAQg8g0BEvIjAeXkzIok+3l7MixVHgfQzvKZOCEAAAhCAwFoEZoidtVyiHghAAAIQmETAETVu+ffXk3ltL/cgdlwyLEMAAhCAwNEIIHaO1uLECwEIFELAuVXNi0iu9vS3tiF2PDgkIAABCEDgYAQQOwdrcMKFAATKICBXb9zb02xU98SNXm+34xcCEIAABCBQIgHETomtSkwQgEDBBOSKTljoGBN4hocXFBTcFwgNAhCAAATuEUDs3CPEeghAAAJbIvD+OnjFdHeF5/3VnNoXE7guy1Ug+zrqrqxbgGUIQAACEIBAoQQQO4U2LGFBAAIQgAAEIAABCEDg6AQQO0fvAcQPAQhAAAIQgAAEIACBQgkgdgptWMKCAAQgAAEIQAACEIDA0Qkgdo7eA4gfAhCAAAQgAAEIQAAChRJA7BTasIQFAQhAAAIQgAAEIACBoxNA7By9BxA/BCAAAQhAAAIQgAAECiWA2Cm0YQkLAhCAAAQgAAEIQAACRyeA2Dl6DyB+CEAAAhCAAAQgAAEIFEoAsVNowxIWBCAAAQhAAAIQgAAEjk4AsXP0HkD8EIDA7gj89vsvJsV3d4HjMAQgAAEIQGAmAcTOTGAUhwAEIPBsAimEjtjgAwEIQAACECidAGKn9BYmPghAoDgCiJ3impSAIAABCEAgEwHETiawmIUABCCQiwBiJxdZ7EIAAhCAQGkEEDultSjxQAACxRNA7BTfxAQIAQhAAAKJCCB2EoHEDAT2QOD99WS+fv1af1/ePjqX5+Z3G7LwFAKInadgp1IIQAACENghAcTODhsNlyGwiMDHm3m1AufjzbycXs27GJqbv6hyNkpJALGTkia2IAABCECgZAKzxM7tcjbnc/Otrp8tl5u5tHn1uupq7JrPa9WVP19uJXMkNgjsjMC7eT29GKt9OudFBL28mf6aj10TKW9X87sqAcTOqripDAIQgAAEdkxgutj5vJqLFTifV1OdL6aRLzdzcQROx0KXOVfGbt6VYQECEHgOASVq7G1sJ3u1R3ulyuvVpNclgNhZlze1QQACEIDAfglMFztejHI1x4qXsNiRqzr91R9jdNozRwICEFiRwId5ewlc1REP6tvb9LqR8it6TVU9AcROz4IlCEAAAhCAwBiBZWJHrtp0V3P829iswNHiRqfHnGIdBCCQj4BcxXFfTqBrkvWv9cM8zZp75fX2pPMTQOzkZ0wNEIAABCBQBoEFYufTXCt7VUdD6K/4aHGj03pL0hCAQG4CcoUmIHTeXx1x4z6bEymf203s3yWA2LmLiAIQgAAEIACBmsBssSMvKbBXb0IMZb28i0CLG50Obavzfv78acRBvjCgDyToAz+q7rXT9vXTv377w/z55w9TnfpXUlc/2rqi5RP4wrh+aL+WSuwwrujL9AH6AH2APrCHPiCaYOlH4vsif+5/5IrOuNAx9UsJ2qs+vKDgPlJKQAACEFhAIJXYWVA1m0AAAhCAAAR2RWC62Lld+tdIO6+flis29nXU8uu+YTr8qupd8cFZCEAAApsjgNjZXJPgEAQgAAEIbJTAdLGz0QBwCwIQgMDRCCB2jtbixAsBCEAAAksJIHaWkmM7CEAAAk8igNh5EniqhQAEIACB3RFA7OyuyXAYAhA4OgHEztF7APFDAAIQgMBUAoidqaQoBwEIQGAjBBA7G2kI3IAABCAAgc0TQOxsvolwEAIQgIBPALHj8yAFAQhAAAIQiBFA7MTIkA8BCEBgowQQOxttGNyCAAQgAIHNEUDsbK5JcAgCEIDAOAHEzjgf1kIAAhCAAAQsAcSOJcEvBCAAgZ0QQOzspKFwEwIQgAAEnk4AsfP0JsABCKxH4K9/fzcpvut5TE0hAoidEBXyIAABCEAAAkMCiJ0hE3IgUCyBFEJHbPB5LgHEznP5UzsEIAABCOyHAGJnP22FpxB4mABi52GEmzCA2NlEM+AEBCAAAQjsgABiZweNhIsQSEUAsZOK5HPtIHaey5/aIQABCEBgPwQQO/tpKzyFwMMEEDsPI9yEAcTOJpoBJyAAAQhAYAcEEDs7aCRchEAqAoidVCSfawex81z+1A4BCEAAAvshgNjZT1vhKQQeJoDYeRjhJgwgdjbRDDgBAQhAAAI7IIDY2UEj4SIEUhFA7KQiOcXOh3l7OZnTy5v5cIq/v57M169f6+/LW7/m4+2lyz+9jr/xDrHjAGURAhCAAAQgMEIAsTMCh1UQKI1AXrFz/+DePfB3D/rd/DKYv5vX04t5e38zL67YeX91xE9bRvTOx5t5Ob2aRuI4+REYiJ0IGLIhAAEIQAACigBiRwEhCYGSCeQTO5GDezmI7w72GzFUX7SI5T8F/jyRZozE2lydOYmg6S/ODL334hRN82Lcqzki+ISHztdpbRixo4mQhgAEIAABCIQJIHbCXMiFQJEE8omdFpc6uK+vWHhipxUHXjkRG3dEQ7bWmCnSjCPYpvjkxdmIGvcWNRE7In60uNFpXRViRxMhDQEIQAACEAgTQOyEuZALgSIJrC526ju0+mdR3EdR5IDePrvi5j8FvBIl00TaBE+13VYs2bjllys7EzhSBAIQgAAEILCQAGJnITg2g8AeCTxD7MjVC7maUf92V3lMkw7kP4XrQJQ0V2GsKOnE2PtrJ9Bk3al7zibidcCuW/L9tbmipa/k6LS7jSxzZUcTIQ0BCEAAAhAIE5gldm6Xszmfm291/ewsfl6rLv98ud3N7wqwAAEIrEpgbbGjD9pt2v7a4HXa5q/2GxAlQZEmLxjolE8j2NxncAb+Bux2ZdyXFUi5TjjxgoKOEQsQgAAEIACBBwlMFzufV3OxAufzaqrzxdSyxl02N3M5V6YuFst/0GE2hwAElhN4htgJiQMRN6H85ZE9uKUSJVp8dWkldrr8QfX9Swy8q0O1qLEvN7BvX2s2FnFly44KKK7sDGiTAQEIQAACEIgRmC52PAu9qJGrOvoqj6Rj+Z4ZEhCAwKoE8omdyMG9ekalf8V086C/Pbjv81fF0VcWEDtBMeZdgZn5soK+toeXuI3tYYQYgAAEIACBgxBYJnbkqk11NXIjW0zUxPIPwpUwIbBJAvnEzibDneDUXJHmP8vjCqIJlSUrgthJhhJDEIAABCBQOIEFYufTXKv2VjXETuHdg/BKI4DYKaNFETtltCNRQAACEIBAfgKzxY68pCB025p11V7Rsb8636an/P78+dOIg3xhQB9I0wdSiR3aI017LOWYSuwsrZ/tntv+8Ic/fYA+cLQ+IJpg6UdYfZE/9z9yRccXOvU2sRcRxPLvV0QJCEAgE4FUYieTe5idSCCV2JlYHcUgAAEIQAACuyUwXezcLv3rpdXrp2OvpI7l75YWjkNg5wQQOztvwNZ9xE4Z7UgUEIAABCCQn8B0sZPfF2qAAAQyE0DsZAa8knnEzkqgqQYCEIAABHZPALGz+yYkAAhMJ4DYmc5qyyURO1tuHXyDAAQgAIEtEUDsbKk18AUCmQkgdjIDXsk8Ymcl0FQDAQhAAAK7J4DY2X0TEgAEphNA7Exn9WjJv/3z3ybFN+QHYidEhTwIQAACEIDAkABiZ8iEHAgUSyCn2PnfP/5iUnxLgZ9C6IiN0AexE6JCHgQgAAEIQGBIALEzZEIOBIolgNhRTfuvL8ak+CqzkkTsBKCQBQEIQAACEFiZAGJnZeBUB4FnEkDsKPophI7YCHwQOwEoZEEAAhCAAARWJoDYWRk41UHgmQQQO4o+YkcBIQkBCEAAAhAoiwBip6z2JBoIjBJA7Cg8iB0FhCQEIAABCECgLAKInbLak2ggMEoAsaPwIHYUEJIQgAAEIACBsgggdspqT6KBwCgBxI7Cg9hRQEhCAAIQgAAEyiKA2CmrPYkGAqMEEDsKD2JHASEJAQhAAAIQKIsAYqes9iQaCIwSQOwoPIgdBYQkBCAAAQhAoCwCiJ2y2pNoIDBKALGj8CB2FBCSEIAABCAAgbIIIHbKak+igcAoAcSOwoPYUUBIQgACEIAABMoigNgpqz2JBgKjBBA7Cg9iRwEhCQEIQAACECiLAGKnrPYkGgiMEkDsKDyIHQWEJAQgAAEIQKAsAoidstqTaCAwSgCxo/AgdhQQkhCAAAQgAIGyCCB2ympPooHAKAHEjsKD2FFASEIAAhCAQFoCH+bt5WROL2/mwzH8/noyX79+rb8vb/2aWL6zKYszCSB2ZgKjOAT2TACxo1oPsaOAkIQABCAAgXQE3s3r6cW8vb+ZF1fsfLyZVytwPt7My+nVvEulsfx0Dh3SEmLnkM1O0EclsF+xM+/M2MfbS3fG7PRaTyHhJkfshLmQCwEIQAAC6QiIoHHFjms5ti6W727L8iQCM8XOp7lWZ3OuruazM38zl/PZnO3XWfd5rfr8y63bggUIQOA5BPYpduafGevOkpl22/4OAR88YsfnQQoCEIAABNITCAgXe7vayV7VaWuN5ad36jgWZ4gdETWVud6upnIEjTE3c/HSLbzPq6nOF9NInHbbXiEdhzCRQmBDBPYpdlqAgcmiQ+usk6s67v3POt1tIwuIHQ8HCQhAAAIQyEDAmaMG1mWd3OqmT8rF8gcGyLhHYIbYaU2JiPHETVjsyFWd6tqrG52+5xjrIQCB9ARKEzuhM2Ba3Oi0RxWx4+EgAQEIQAACGQiMiR1jjMxloTuuY/kZPCzaZBqxY29hO587gaPFjU4XTZXgILBRAqWJnQ6zcwZMixud7raRBcSOh4MEBCBwFALznoO0J5bk7WHulfOj0Ho4Ti123l8dcePcbh3Lf9iBYxtIIHZcgP3talrc6LS7FcsQgMA6BIoVO86ZMS1udNojjdjxcJCAAASOQGD+c5DBN4cdAdXDMQrr/hXTIhabKzh+fn9VJ5b/sCOHNpBY7Bhzu5yNvItAixudnkL958+fRhzkCwP6QJo+kErshNrjf//4i0nxDdmu8/74Zn799Zv5w+4TflSm+mG5/DDV6Vfz7Y8/zZ9S7lSZH3U5J99u5/ymurIT8vlv//y3SfEN2f7t919Mim/INnm2T/FLXyi8D+h9qrNv/PPPyL5zdJvCeXl8iHXt/YNogqUf8fWL/Jn8GTyz42xZv5SgMvWjOrygwAHDIgS2QSCV2AlFk0LoiI3hxz/Tdf/MWHP/c+iftQ1sc2VngIQMCEDgIAT0rVVu2GqdvY1NvznM3YRlCGyVwAyxo14xfe6v4HSvnW7zbLBylceuc19WYNfzCwEIrEtgn2InIyPETka4mIYABDZNQAma3ld5nifwdjApINuE3hzWb8wSBDZHYIbY2ZzvOAQBCMwkgNhRwBA7CghJCEDgMAQiYkeu4oy9hIA3hB2mhxQTKGKnmKYkEAjcJ4DYUYwQOwoISQhA4DAEBmKneUPbQOjwhrDDdIlSA0XslNqyxAWBAAHEjoKC2FFASEIAAuUTiDwH+f5q7LOO9rcRPn75/s1h5ZMiwjIIIHbKaEeigMAkAogdhQmxo4CQhAAEIAABCJRFALFTVnsSDQRGCSB2FB7EjgJCEgIQgAAEIFAWAcROWe1JNBAYJYDYUXgQOwoISQhAAAIQgEBZBBA7ZbUn0UBglABiR+FB7CggJCEAAQhAAAJlEUDslNWeRAOBUQKIHYUHsaOAkIQABCAAgZQE/vbPf5sU35Q+Hc0WYudoLU68hyaA2FHNj9hRQEhCAAIQgEBKAimEjtjgs5wAYmc5O7aEwO4I7FXsfP/+3Tz6DTYWYieIhUwIQAACEEhDALGThuMjVhA7j9BjWwjsjABiRzUYYkcBIQkBCEAAAikJIHZS0lxmC7GzjBtbQWCXBBA7qtkQOwoISQhAAAIQSEkAsZOS5jJbiJ1l3NgKArskgNhRzYbYUUBIQgACENgHgffXk/n69Wv9fXn78J3+eDMvp5MZ5PulVkkhdlbBPFoJYmcUDyshUBYBxI5qT8SOAkISAhA4AoGcc8Eq/N5fzenlzTQS5928nl6Mq3feX1/M29srYmeVxth+JYid7bcRHkIgGYGcE9z//vEXk+IbCvbRlxPI9sEPYieIhUwIQKBsAjnngjXIfby9eEJGrvK8vrc1ixCSxDtiZ4222EMdiJ09tBI+QiARgZwTXAqhIzZCH8SOT+W3338xKb6+VVIQgMBRCOScC9ZgKGKnFjRtZSJ2mlvWPszby6updQ9iZ42m2EUdiJ1dNBNOQiANgZwTHGLHb6Oc92mnEDpigw8EIHBMAjnnApeo+1yNve1MhIp91sb+zn+2RkRN/8yO2JGLOd4VH8SO2xSHXkbsHLr5Cf5oBHJOcIgdvzchdnwepCAAge0QyDkXdFHKSwK652oacdLdatYXMm8v/vM23aoZC/UzOh9DASQiyL0CNMNksqI554JkThZuCLFTeAMTHgRcAjknOMSOS9qYnBMcV3Z81qQgAIF5BHLOBZ0nA7ETEDVemW7LeQveywqcTbmy48A49iJi59jtT/QHI5BzgkPs+J0JsePzIAUBCGyHQM65wI3SvWVteFVH3iHgvFjA3fDecvtq6frKzal9Rkdvg9jRRA6bRuwctukJ/IgEck5wiB2/RyF2fB6kIACB7RDIORe4UYqYkdvI6t/uljZbQl4ZHREqtkgBvznnggLwrBLCTLHzaa7V2Zyrq/l03Pu8VuZ8Pjffy61bE8vvCrAAAQisSiDnBIfY8Zsy5wTHbWw+a1IQgMA8AjnnAuuJ97IAo14eEEjb7Ur7zTkXlMYqVzwzxM7NXM6Vud6upnLFzufVVOeLaSROW0aUUCw/VyTYhQAE7hLIOcEhdnz8OSc4xI7PmhQEIDCPQM65wHoiYsd9OYBc3enfujb8R6B2u9J+c84FpbHKFc8MsdO6ICLGETty9aa69td5bNr+Wsd12ubzCwEIrEcg5wSH2PHbMecEh9jxWZOCAATmEcg5F/Se+G9Hs6+ertfHXirQb1zMUs65oBhImQNB7GQGjHkIbIlAzgkOseO3dM4JDrHjsyYFAQjMI5BzLpjnSfmlc84F5dNLEyFiJw1HrEBgFwRyTnCIHb8L5JzgEDs+a1IQgMA8AjnngnmelF8651xQPr00EW5a7Pz8+dOIg3xhQB9I0wdSTXCh9kgldkK2v3//bh79huyaf30xKb4h26kmuJDtVGInZJu8NGMNjnDcch/IORdsOe5n+JZzLnhGPM+qUzTB0o/4/EX+TP6oZ3aiLyLgBQWTkVIQAmsRSDXBhfxNJXZCth8VOrJ98JNI7IRsp5rgQrZTiZ2QbfIgAIHyCeScC8qnNy/CnHPBPE+OW3qG2JE3rbWvl25/7Vumb5c+331ZQSz/uLiJHALPJZBzgkPs+G2bc4JD7PisH0vJW6FOpvnnhO5/eI/lP1YbW0NgCwRyzgVbiG9LPuScC7YU55Z9mSF2thwGvkEAAlMI5JzgEDt+C+Sc4BA7PuvlqeZtUcP/7B7LX14TW0JgSwRyzgVbinMLvuScC7YQ3x58QOzsoZXwEQKJCOSc4BA7fiPlnOAQOz7rxamPN/My+K/u9X87DOcvrogNIbAtAjnngpyRZrulOaPTOeeCjG4XZRqxU1RzEgwExgnknOAQOz77nBMcYsdnvTj1/lrfvia3sDW3sb2adzEWy19cERtCYFsEcs4FOSNF7OSkW65txE65bUtkEBgQyDnBIXZ83Igdn8cmU/KPDZ172Lr/8B7L32QQOAWB+QRyzgXzvWrElasAACAASURBVJm+BWJnOitK9gQOKnb6B0/rs3nObQwy2XVn+Zz8HhlLENgvgZwTHGLH7xeIHZ/HJlNK1Hy8vZiXt4/6yo4rgrr8TQaBUxCYTyDnXDDfm+lbIHams6JkT+C4YickZLz7t3lAte8mLJVCIOcEh9jxewlix+exyZTs80/trWvG2efH8jcZBE5BYD4B5oL5zJZukXMuWOrT0bZD7LgtPhA77mtI3YIsQ2CfBJjgVLvxf3YUkOMl5apNdzXfuaUtln88QkRcIgHmgvVaFbGzHutYTccVO+3/VZBJrr5toSXkTnDOvBfjRz4EdkWACU41F2JHASEJAQgcgQBzwXqtjNhZj3WspoOKHReHPL/TX8GRZ3bkXu36N3Srm7spyxDYGQEmONVgiB0FhCQEIHAEAswF67UyYmc91rGaEDv1W0ZPRq7i6IdQdToGkXwI7IUAE5xqKcSOAkISAhA4AgHmgvVaGbGzHutYTYid+kHU5sqOiBv3DTzda0hj9MiHwM4IMMGpBkPsKCAkIQCBIxBgLlivlRE767GO1XRIseM+lyPP7PTP5jRv4+keVuU2tli/IX+nBJjgVMMhdhQQkhCAwBEIMBes18qInfVYx2o6pNiJwSAfAqUTYIJTLYzYUUBIQgACRyDAXLBeKyN21mMdqwmxEyNDPgQKJMAEpxoVsaOAkIQABI5AgLlgvVZG7KzHOlYTYidGhnwIFEiACU41KmJHASEJAQgcgQBzwXqtjNhZj3WsJsROjAz5ECiQABOcalTEjgJCEgIQOAIB5oL1Whmxsx7rWE2InRgZ8iFQIAEmONWoiB0FhCQEIHAEAswF67UyYmc91rGaEDsxMuRDoEACTHCqURE7CsgxkinGwTFIEWWpBFKMAbER+vzvH38xKb4h29+/fzePfkN2c+YhdnLSnWYbsTONE6UgUAQBJjjVjIgdBeQYyRTj4BikiLJUAinGAGJnWu9A7EzjlLMUYicnXWxDYGMEmOBUgyB2FJBjJFOMg2OQIspSCaQYA4idab0DsTONU85SiJ2cdLENgY0RYIJTDYLYUUCOkUwxDqaQen89GftPql/ePpxNmn9gfeIfVztMWFyTQIoxgNiZ1mKInWmccpZKIHZu5nI+m7P9Vlfz2Xr8ea36/MstZxyTbOcc3JMcoBAEnkwg5xhIcY+22Ah9Hr1HW7YPfhA7QSylZ6YYB3cZfbyZVytwPt7My+nVNE84vJvX04t5e38zL4iduxgpkIdAijEgNkKfXc4FoUAS5SF2EoF8wEwaseMInM6Xz6upzhfTSBwRRJW5WhXUFVp3IefgXjcSaoPAMgI5x8AuJzjEzrKOtPOtUoyDWQhE7GhhE8qbZZTCEFhOIMUYEBuhzy7nglAgifIQO4lAPmAmm9iRqzqVo250+gGfF2+ac3AvdooNIbAigZxjYJcTHGJnxd63napSjIMp0djb2E7dVR1nK8SOA4PFtQmkGANiI/TZ5VwQCiRRHmInEcgHzKQRO/YWtvO5Ezha3Oj0Az4v3jTn4HadshOc3Kvt36dtjKlvZzgN810DkeWwXbklor8vnHvAI/DIrgnkHAO7nOAQO4ccGSnGwSxw9X7/xdi72uptETuzEFI4LYEUY0BshD67nAtCgSTKQ+wkAvmAmQRix629v11Nixuddrdaaznn4O5iiN6n3ZR4f30xb2+v88VO1O67edW3R3TOsAABn0DOMbDLCQ6x43eQg6RSjIO5qORk1at7bPiA2Amf+DImlj/XV8qXTyDFGBAboc8u54JQIInyEDuJQD5gJrHYMeZ2ORt5F4EWNzo9xeefP38acTDVN9Xgnu7PD1OdfjXf/mhj+FGZU/XD/PmjMr9+++OBuFy7P0z16zfzR0JO0+NL1zbUuQ7LnGMg1QQX6gspXlAQsmsSiZ2Q7VQTXMj2b7//YlJ8Q7aPkJdiHNzl9KMy1Q87rt19dpv3xzfz65J99x/fTGXnD7FxqswP2f/H8pkbHphrbfuV95tiDIiN0DjY41wQiiNVXs65IJWP9+z8qPo7iPzj1z/Mt19P5rRkXzZz3ySaYOlH4vsif5J86pcStC8i4AUFzS1r3VUXedVo+zae9wVXdtwG8s4I+rexDW6bc7dj+fAEUk1wIZCpJriQ7RRiJ2Q3ldgJ2U41wYVspxA6YuOonxTj4D47f9/cX9Xx8+V2537dfat+CbGlbo+TAt4c4W9BCgJCIMUYEBuhzy7nglAgifJyzgWJXBw3M3Zn0U7eLPmw2JErNt1rp8/NVR1LTa7y2HXuywrs+rV/cw7uYSwibvpJ6OPtpb917SGx49v1641MfH4hUgcmkHMM7HKCS3RlJ9Slck5wiJ0Q8el5KcbB9NoyllSixt7GFnwhQkY3ML0/AinGgNgIfXY5F4QCSZSXcy5I5OIMM4HjTLUfmmFstaIPi53VPE1QUc7Brd2TSae/ytL8Azn7z+Xs72nB6Tzfrq61uWd7gdmhIXKKJJBzDOxygkPsFNnP7wWVYhzcqyP/+pETX3LwEbrik98patgJgRRjQGyEPrucC0KBJMorSuyEhE0oLxG7VGYQO39/n305dxx+I2x6oRMovejKzgS7THAB2GS5BJjgXBrGcBub4nGQZIpx8GxUnPh6dgvsu/4UYwCxM60PlCN2IidYEDvTOsJapXIO7i6G91djr9zY34HwWSJ2Inbl9jhbj/xyVadrCRYCBHKOgV2ezePKTqCXlJ+VYhw8j1LkxNf7q7P/D9xq8jyHnZob3/W/SBDhZuexwXzpbP2cxbDPxsTyn+Pl3FpTjAGxEfrsci4IBZIorxSxEz3BgthJ1FMSmck5uBO5iBkIZCWQcwzscoJD7GTtb1s1nmIcPC22yIkvY0Tg9KJheye+WgH2/mZeuhf3NC9TeLX/gKi+O6F9kc/TALsVR3yuWb+YNx2Lu+nGl1OMAbER+uxyLggFkihv/2IncoLF8kHsWBLb+M05uLcRYRov3LNs+gxcmhqw8iwCOcfALic4xM6zuuJT600xDp4awJ4rHz0w2ugVqZjPsfwdtE+KMSA2Qp9dzgWhQBLl7V7sTDzBsuW7i3hmJ/kzO4lGx7PMeDvvRs2nOkOob7mTgZH2loX+zOaJh3ODPYgJTmFB7Cggx0imGAfHIJUhSm+OUfbH1qmiqyZjfsXyV3VuWWUpxgBiZxr73YudaWFuuhRiB7Hjd1Bv5y1ip399tl/w0VRq22mF2aPRbXV7JjjVMogdBeQYyRTj4BikMkTpzTGu/dRzgmv7weWYz7H8B6tbY/MUYwCxM62lEDvTOOUshdhB7Az6l3sFJtVVnUAl/n3bgwIzM3Y86cyM9KHiTHAKH2JHATlGMsU4OAapDFFG9tXRh58zuDDbZMTnPf/z1hRjALEzrSchdqZxylkKsYPYGfQvmXTkfwDVv+6DpIOSyzPEdlIhpe4p5Z/qhduGCU5xQewoIMdIphgHIVI5n1UI1bfLvIFwaK7Kp72lOTGZgc+t/Vh+4upzmEsxBhA701oGsTONU85SiB3Ejte/5KqOO+notFd4cUKerUn8xp3311qgWZc2fZbQOvmEXyY4BR2xo4AcI5liHIRIIXZCVGxe/0ylfc10fcJLnahK/yynrX/Jb8Rn9ea7LT+YHYs6xRhA7MTo+vmIHZ/HM1KIHcSO1+9E3MhVHfvJIRqyCCgldrLUYaHs+JcJTjUeYkcBOUYyxTgIkULshKiQt0UCKcYAYmdayyJ2pnHKWQqxg9hR/au5pcCeeUv/6mk5U5bhpQdyO0F3tYiXFahG7ZJMcB2KZgGxo4AcI5liHIRIIXZCVMjbIoEUYwCxM61lETvTOOUshdhJIHZyTnDfv383Kb45O9Es23IFJtNzQHI1pxNpztWpWf4VXpgJTjUwYkcBOUYyxTgIkco5F4TqIw8CSwmkGAOInWn0ETvTOOUshdhB7OTsX9jeGAEmONUgiB0F5BjJFOMgRCqn2Elx0kts8IGAEEgxBhA70/oSYmcap5ylEDuInZz9C9sbI8AEpxoEsaOAHCOZYhyESCF2QlTI2yKBFGMAsTOtZfcodv7vr5VJ8Z1GKH8pxA5iJ38vo4bNEGCCU02B2FFAjpFMMQ5CpBA7ISrkbZFAijGA2JnWsoidaZxylkLsIHZy9i9sb4wAE5xqEMSOAnKMZIpxECKF2AlRIW+LBFKMAcTOtJZF7EzjlLMUYgexk7N/YXtjBJjgVIMgdhSQYyRTjIMQKcROiAp5WySQYgwgdqa1LGJnGqecpRA7iJ2c/QvbGyPABKcaBLGjgBwjmWIchEghdkJUyNsigRRjALEzrWURO9M45SyF2EHs5Oxf2N4YASY41SCIHQXkGMkU4yBECrETokLeFgmkGAOInWkti9iZxilnKcQOYidn/8L2xggwwakGQewoIMdIphgHIVKInSGVFAd6Q6t5c1K8hUpsbPmTYgwgdqa1cIoxIDbW/JQ2BrKKnc9rZc7nc/O93NZsp2BduQY3E1wQN5kbJJBrDEioWx8HweZA7ASxlJ6ZYhyEGG19DDzj/+ykONALsc6ZV9qBXohVijEgNkKfrY+DkM8581KMAcTOYy2UT+x8Xk11vphG4tzM5VyZ6+djzj66da7BvfWBHZ3gNn6g92h7s/2QQK4xIDVtfRwMaRhjNj4GYhPcb7//YlJ8g0wOkJliHIQwbX0MROeCUDCJ8lIc6CVyZbIZxM775H86GoK69XEQ8jlnXooxEJsLcvld2hjIJnbkqk7lqBudztVAY3ZTTHBiQ3+2PrCjE9zGD/Q0Z0mnOMATG0f95BoDwnPr4yDY5hsfA7EJjnEQbM3JmSnGQaiyrY+B6FwQCiZRXooDvUSuTDZT2oFeKPAUYyB0PCR1bX0chHjkzEsxBmJzQS6/SxsDiJ0jP7Oz8QO90CDmIC9EZXoeE5xitfExEJvgGAeqHWcmU4yDUJVbP8hD7IRabZhX2oHeMEIz+crNvbESsr31cRDyOWceYicn3Wm2ETuIncdv5Qn0tRSDO2CWKzshKDPy7k1cU9eHqtzlBIfYCTVl8XlT+/lYuRCkrY8BxE6o1YZ5iB1uYxv2iuU5KY6HYie+lns1vmVpY2DTYufnz59GHOQLA/oAfYA+QB+gD9AH6AP0AfrAMfuAaIKlH+kzX+RP8s8GX1CQPEYMQgACEIAABCAAAQhAAAKbJZBP7Bhjbpf2tdPns/eygs3SwDEIQAACEIAABCAAAQhAoBgCWcVOMZQIBAIQgAAEIAABCEAAAhDYHQHEzu6aDIchAAEIQAACEIAABCAAgSkEEDtTKFEGAhCAAAQgAAEIQAACENgdAcTO7poMhyEAAQhAAAIQgAAEIACBKQQQO1MoUQYCEIAABCAAAQhAAAIQ2B0BxM7umgyHIQABCEAAAhCAAAQgAIEpBBA7UyhRBgIQgAAEIAABCEAAAhDYHQHEzu6aDIchAAEIQAACEIAABCAAgSkEEDtTKO24zOe14h+67rj9cH05gS33ffmHy9X1c3lwbAmBAIFH+tX88XIzl/PF3AJ+jGZ9Xk1VXU3f+8WO/QfklemHRSB/sO1oTazcEYGy+m4PXuI6t/3b3efPzXcsOuPlbM7eWDLGyBg5h+aXY48nxE7fg4pcmj+BFYmBoA5I4Hl9/9NcK/eg7YDwCTkDgbz9av54WSZ2pJ5Lp5AkprOTtthi+cbcLowtS2k/v0fqu06rfF7Nxar3WoS0Jwfm5jsmjbmZixY4zvp6fFwv6mQa4wmx43SS/S42nf92rQZnEPoJTE9Mfdo9w3C+d6auPWtgz1Q0k1ZT/9U5g9FPZv2ZBruNe3Zjv8zxPCsB6WeXa30g1J25Un2v6UdT+35/du187g+WZHzYfnm2nTZYTzxaf/xUpqqcusRmIBZvXFZXMxw7zeTU+XYOHRDGfWLN/gms3a/u90kjSsMZL5f+yk5gzIj/dl9fj7N6fKmDXtkudOAWy6+nE+5W2HrvPnTfjTVOtE/LsVg/J/Wbj+SHxoxsKONTxtlNiZ1o3TI9HWM8IXb6nrXjJRkU/cRSK/928PQduRc3TaBt2g6OSdHrwWfT8uvU7w2stp7OvprsunwWIOAQkD7kTQCxfqT6npz1GvR9x64s2v5Z16Fvw4nVo2zYZHD8qD4+iMWdYJT/1jfXrrts6+W3bALBNs/br/y5IrQ/78eWwK8FTH1yLDZmbHl3vbusxFN9q087HiX+7tY2OXngjFM7RsruAfuN7uh9V7VcL/ycPuyWifXnWH49x/Un1OwJBWNk/9CPnz5/ZJyJH9F6XCf3v4zY2X8bSk/uz7C18cgAa04qW9Wuy9i0/MrACZ1ZUHACO7GmHmvLlnfSMpC8SasZpPVJPlucXwhoAnoHHO1HTl9rbQz7fmBnX12NXAn1JgTZPlqPdtCmQ+MncFCqzsT5B5buJNjG4441d9lWy2/hBNbvV7P6ZE2/7asjY6YRRCHh1Daf6tsydusxGcuXzfS+ofCesL/wDt53Yw1WjxN9nKXmim7bWH5XoF0Q1o3Nfvw28503tzGeDGJH951dpttJx/F9eMCny+i0DC51Bs2xVy+qASN5k8SOOtDTZklDYEBAH9DodLeB7se2TzZnnusdfj3JOIKitRUVO4v6qzt+1EQV8L2fmLT/bbr22Z690xNkFzwLxRNYr1/d7ZOD/b/TVyNjZq7Y6XxQdXX50t6B8VR8N9hlgAftuyNtZY/LbJFO3NuM9jeWr4rVycamZW3njPbXnlVmPCF2Qp1nf3ky6aizZ+1l/36S8A/AmknIOQBsg9aD0Wch9TgHXvUBmdhoJ72usJtW23RlWIDACIHBAU2sH0n+nb6vbNV9Xw7Ouv7r+hGrxy0TX+4nHjVO1MFgPy6lPnccNumr9xB3vD7WHIPAGv3qXp+8qfHSzyGRMdOVd9er/t6VkXZsDtjq47NYfl0s8pzPMbrC7qI8XN91W+h2cV6+4Y6Dpq97V1/q7WL5rlFnuR4nzlxjV4We2enmGWecSXmxoeYna6akX67sFNGazQRyCbwgoJ/A1K089cPfFyMHVd690fZMQIyLnCHobkuzg0xNYFr81APSPePgHtzFKiL/0ARCO+BgP5rW95sDM3u269K9zUYm4q4/274frCfcGp5dGRetjS5f0oFY+nEZGTveOAu8XjTsDrmFEOj6j93XrtCv7vbJ+riony+q661/RiAwZmQ+6odU1Y4NOdCy80bTWF6sdgNVlx1XskXvZyGNXVgYXnuutE/s+0Rkf6r6U96+Kz7YPi7L/RzTdW+9f7cn7GL59TFVY1Pz7Wy6/UiLHRX/EccTYsftILtd1gN8t4HgOARmEiiz7zdnQy2K4QGiXcMvBOYQ2EK/6g9M53jel+XV0z2LIy3tpu8GTm493E45bLZOHWU8IXYe7oVbMJDygM8/E9Gd9e4ugW4hXnyAgCWQsu9bm/r3CWNCnykPnr7TfpKGwB0CW+hXjxy4PbLtHTSs3jiBnfTdHOIhh826tQ80nhA7Gx/fuAcBCEAAAhCAAAQgAAEILCOA2FnGja0gAAEIQAACEIAABCAAgY0TQOxsvIFwDwIQgAAEIAABCEAAAhBYRgCxs4wbW0EAAhCAAAQgAAEIQAACGyeA2Nl4A+EeBCAAAQhAAAIQgAAEILCMAGJnGTe2ggAEIAABCEAAAhCAAAQ2TgCxs/EGwj0IQAACEIAABCAAAQhAYBkBxM4ybmwFAQhAAAIQgAAEIAABCGycAGJn4w2EexCAAAQgAAEIQAACEIDAMgKInWXc2AoCEIAABCAAAQhAAAIQ2DgBxM7GGwj3IAABCEAAAhCAAAQgAIFlBBA7y7ixFQQgAAEIQAACEIAABCCwcQKInY03EO5BAAIQgAAEIAABCEAAAssIIHaWcWMrCEAAAhCAAAQgAAEIQGDjBBA7G28g3IMABCAAAQhAAAIQgAAElhFA7CzjxlYQgAAEIAABCEAAAhCAwMYJIHY23kC4BwEIQAACEIAABCAAAQgsI4DYWcaNrQYEbuZyvpjbIP8ZGY4vt4s5n8/mvBnfHuHhxPWImVnbqjp3wPPzWpnq+jkrynIKq/ZaJTCnzh30D40kb39x2EjFO+QjbqdhpFjohpiT3inHOSGmKzuP++1yvrv/dPvDlPLpYlliKRZ/LF/qkHWVOew0sgTzxrdB7Gy8gVK499e/v5vQ9zHbn+ZauTuDsR3HYzXN37r3RXbEl4QK7H//+IsJfef7uGSLPq4lWy/bxq8zJc/v37+b2HeZr81W7kT8iJ19buu31zox9HWm7B/mX19M9JswsLz9pWcjLqfk87d//tvEvgnx1KbSMPJZDH3Uc8qwhM1JydHazPn72++/mNg3Z72N7Xvc53uQpj/Mr1dv8X9/rUzo65eLxR/L97cmVQYBxE4Z7TgaRUjoSJ7/uZlLdTW3a9VeCWnO7viTip2M5FeulrTfWk00O46rs30nMj6vprJlz1Z8NPVdL72drrzvmJPSOycn3Z3pO5vz5VJfZbo4ts/3jTv1xBdDQkfy/E+YZXO2qI/33J05GmERiKvRbhL70Ja0V9cuo1ezHHa18046UqdnOwHPmNCRfP2RybWLy9Yd7FfuWWgnJi9Gzftibo6t/qqQLuf03QB7IzYu12ZsVFcTv7YU6x/iZKhdY360lILtFbLTlPfaMdpHYuzidSYfbxPFTiieYX+J82gO3K7B8dRGG/gZ4RNsj0boDPpwwPLUrJjQkXz9eQ4j90pWv1++Bft4bE5x92fNyTUvlqoKjLlQW+sxFBvz9WUsb77q9wea6vR0TOhIvv548dXjM7a/CMXZ+u/th5y+WvfN8TsweiGjmdn9n7uPdZfj5et9o7PPTMFUuIWEjuT5nyZ+99ikqX+Yb6eWZj88zsmvg9TWCSB2tt5CCfybLHbO7uVr2RFU5nq9mE4oyMFcdxBnhY91UMo723dlWzvdkZ9Nx8pbe6Ff2cbdAdm0tdls0xzoNOVk4uh3YCGb8/Imix2XReySuMfoPjs3roHXYksmk8nBWnbWkk3HWUrJlDwni506Nrfda0/UbQa9395kHe0vfb9oDi5a+15dYjPULpZZ+2vbsd7WvdqpynVJZTfWP6R8bbuq27Y7QLD11fb6uJviIgo1K2vnaj7lYGdSHxG7rh03Ha8zZf+IXtUREWQ/oXjqdnB9t4WdX4dh31/C651cZ9HlIdk2HWdTl0q4P5osdp7KqB8P0f1X1xZ6TnFwy2JXztkP1W3d16G2aJL1ds0Ysl1/fMy7feeOT8EKh5mTxU6orVpx2I3/2P7C8hkwaftmne/GNvRTcvrxINuF9399mSnl7diw9aVhKtamix0njo5fPL5+PFuf+d07AcTO3ltwgv/TxY6/I2wOXPodlbuDM0bvsPpyjUttOrDz1na98qPxTK2jL5f04MuY4C1swSs73oGiMzkLD+cM17kWj72/HosBO1VuYKuZ0PsrRmMwlS17sHanzpQ8p4odv9+1MQ387Bn35SMxSqydaHcna7HtbuMuq3UD9lfzaQ82xrDX67Td3vdmte4j0q7u2HS2H3BQ64J9Tc6W3zlA9DgEYrdHjSqelP1jktip/fTj6du/dq7/E2oz9+Ausr434C45nOvsNj3WHolPFkwWO89iNMYiyFrPKTWwwP7SGS+xMTewXzkn6kbGfC0I3KtJzbLX3d1uMHF5stgJtJW/T2oq7MbZIM7Qfkj6psTh7kPijvfjJ9LH3THjLUfKZ2IqEUwXO37sDb+IvzUavS7OizX7IIDY2Uc7PeTlY2LHTix6ItJpvXNo04MJz9qLlB+NNLLNoI6+XDcpjNqdvnL6lZ3AzvUqV1+c/G6i7v1tPGnTI3E1twWEbIkFaZt7k9uCOhMfrO1W7NSTd4B91573+pNmb8dEfeQQ6CMLxE7Mx861e31E++ikR/pl0vE28Ta2JqQ+HrldpT8L3gY8wqM+uLtcAtzv3IrojmUrDkfYiCcp+UwXO7bRV2YUYxFtC/HPEeHRcg5HKeOcuKgjDW43Q+xoexbfA7/TxY6tpG+r5rY/Z39j+1FsPhkwacZufRvXBNWWRexkYCqkEDu2v/B7jwBi5x6hAtZPFzvOpV53wpBJS+6N9nZYamKyk33Hyx4cyW9oArPr7QY6bfPdX79O2SnXZ6tcX9szTfYsVsqDC/FkutgJsFSTUO1/zVTH3qZH4nJv6RC/els9r/HY57MUy+M2+7qnLE0VOwNhVxsXRqF+5Z61jcQofdXpy/3kXkfo3L410i5q+/oKnWrfOAOxG+gfsoGy0bTriNiJ9ZGgneHBe7w9Y+yGgqz2sT3wj9uL04iumSV2Git1/fogMMq14VG3v4idUJtGnYvwibVHaycln/liZ2VGMRbRvukzDY+Fps06jspWHaHK68aQat9eELvjXO1Xou0/b8V8seO01S2yvwjGGbuy04gl4dbHHY6hHg/1a8hcLlK2T/dl3P1tv771vt2XSr6zrw5Xuyh3uthx4hZu9f4q5q8f6yLH2GhzBBA7m2uS9A5NFzsX4z5k3J8Ekp2Cs7NoXawnke45kZEdh4il7nYau9MbKT+GwLVVP4TZ7MQ7X2o/b+Za9Tv3Po4xw9PWTRc7YZaun/WLFOoJOM7CLV9d+7jEW3ed2HJfAlHzvhf4TJZSZ3eQMQ3XaKnJYqett+tDNi7Xf2cyFS7dhO6W6frLg2InwL4WT+rgIx58097hsTZs14ucaAhdRWgrcPuB20fcfNvX3BeQ3O0jQXZNpa5tt86U/WPKbWyuH2484ofuL15ZeYlJe/Ar+dJfYuuj7Rjh49px2YidlHymih3Xn7UZuXW7LNx82zflsc4uvx3jXbqeZ/o26zhGxtxwO/9knazv9hHOQXzd1vXBsNN/vLEX7Q2jK6aKHc9vNbeG9hdeedunB0zc+UWW7Rwcdrln424nZft0X2aK2LEnSNIyFY8mi53qaoYvQ+rjaUi4KsZUyQAAAdRJREFUaXc5zIncfRFA7OyrvTJ6Oza4Zd34DjKjYzs0PcZyh+HgcmIC9I/EQDEHgYIJsL9Yv3Fhvj7zvDUidvLy3ZH1kcEtZzDt2fTsEYkf7hkgu+zfs5zdjYcqGGH5kN25G5fAcm7MWyk/xp7+sZVWGvdjrA3HtzzOWhjlb+vU+wvabLTN7JW91Y55Rr1hZSICiJ1EIDEDAQhAAAIQgAAEIAABCGyLAGJnW+2BNxCAAAQgAAEIQAACEIBAIgKInUQgMQMBCEAAAhCAAAQgAAEIbIsAYmdb7YE3EIAABCAAAQhAAAIQgEAiAoidRCAxAwEIQAACEIAABCAAAQhsiwBiZ1vtgTcQgAAEIAABCEAAAhCAQCICiJ1EIDEDAQhAAAIQgAAEIAABCGyLAGJnW+2BNxCAAAQgAAEIQAACEIBAIgKInUQgMQMBCEAAAhCAAAQgAAEIbIsAYmdb7YE3EIAABCAAAQhAAAIQgEAiAoidRCAxAwEIQAACEIAABCAAAQhsiwBiZ1vtgTcQgAAEIAABCEAAAhCAQCICiJ1EIDEDAQhAAAIQgAAEIAABCGyLQC12/vOf/xhZ4AsD+gB9gD5AH6AP0AfoA/QB+gB9oJQ+IDrn/wEqtRWCGAhLYwAAAABJRU5ErkJggg=="
+    }
+   },
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "![image.png](attachment:image.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "| \t                   | plus_one\t        | replace_string\t| stringIndex(66)\t |stringIndex(6,322,546)|\n",
+    "| :------------------- | ------------------ | ----------------- | ------------------ | -------------------- |\n",
+    "|python_udf\t           |[0158_0](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=0)\t            |[0158_10](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=10)\t        |[0158_24](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=24)\t         |[0158_36](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=36)               |\n",
+    "|pandas_udf\t           |[0158_2](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=2)\t            |[0158_12](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=12)\t        |[0158_27](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=27)\t         |[0158_39](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=39)               |\n",
+    "|columnar_pandas_udf   |[0159_0](http://sr602:18080/history/application_1626660109287_0159/SQL/execution/?id=0)\t            |[0159_2](http://sr602:18080/history/application_1626660109287_0159/SQL/execution/?id=2)\t            |[0159_5](http://sr602:18080/history/application_1626660109287_0159/SQL/execution/?id=5)\t             |[0159_8](http://sr602:18080/history/application_1626660109287_0159/SQL/execution/?id=8)                |\n",
+    "|scala_udf\t           |[0158_4](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=4)\t            |[0158_14](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=14)\t        | \t                 |                      |\n",
+    "|scala_dataframe\t   |[0158_6](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=6)\t            |[0158_16](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=16)\t        |[0158_30](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=30)\t         |[0158_42](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=42)               |\n",
+    "|spark_inline\t       |[0158_8](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=8)\t            |[0158_18](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=18)\t        | \t                 |                      |\n",
+    "|bhj\t \t \t         |  [0158_21](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=21)\t            |[0158_33](http://sr602:18080/history/application_1626660109287_0158/SQL/execution/?id=33)            |                    |                      |"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/mnt/nvme2/chendi/BlueWhale/recdp\n"
+     ]
+    }
+   ],
+   "source": [
+    "#!/env/bin/python\n",
+    "\n",
+    "import init\n",
+    "\n",
+    "import findspark\n",
+    "findspark.init()\n",
+    "\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pyspark.sql import *\n",
+    "from pyspark import *\n",
+    "import pyspark.sql.functions as f\n",
+    "from pyspark.sql.types import *\n",
+    "from timeit import default_timer as timer\n",
+    "import logging\n",
+    "from pyrecdp.data_processor import *\n",
+    "from pyrecdp.utils import *\n",
+    "from pyspark.sql.functions import pandas_udf, PandasUDFType\n",
+    "\n",
+    "\n",
+    "def intUDFTest(df, proc, output_name, method=\"pandas_udf\"):\n",
+    "    ops = []\n",
+    "    def python_plus_one(x):\n",
+    "        return x + 1\n",
+    "    \n",
+    "    if method == 'python_udf':\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': 'a'}, udfImpl=f.udf(python_plus_one)))\n",
+    "        \n",
+    "    elif method == 'pandas_udf':\n",
+    "        @pandas_udf('int')\n",
+    "        def pd_plus_one(v):\n",
+    "            v1s = []\n",
+    "            for index, token in v.items():\n",
+    "                v1s.append(python_plus_one(token))\n",
+    "            return pd.Series(v1s, dtype=int)\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': 'a'}, udfImpl=pd_plus_one))\n",
+    "        \n",
+    "    elif method == 'scala_udf':\n",
+    "        proc.spark.udf.registerJavaFunction(\"scala_plus_one\",\"com.intel.recdp.ScalaUDFTestInt\")\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': \"f.expr('scala_plus_one(a)')\"}, op='inline'))\n",
+    "        \n",
+    "    elif method == 'scala_df':\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': \"f.col('a')\"}, op='inline'))\n",
+    "        ops.append(ScalaDFTest(cols=['tweet'], method='int'))       \n",
+    "        \n",
+    "    elif method == 'spark_inline':\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': \"f.col('a') + f.lit(1)\"}, op='inline'))\n",
+    "    \n",
+    "    ops.append(DropFeature(['a']))\n",
+    "\n",
+    "    # execute\n",
+    "    proc.reset_ops(ops)\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name, df_cnt=626242930)\n",
+    "    t2 = timer()\n",
+    "    print(f\"intUDFTest with [{method}] took %.3f secs\" % (t2 - t1))\n",
+    "\n",
+    "    return df\n",
+    "\n",
+    "def stringUDFTest(df, proc, output_name, method=\"pandas_udf\"):\n",
+    "    ops = []\n",
+    "    def python_replace_str(x):\n",
+    "        return \"_\".join([n for n in x.split('\\t')])\n",
+    "    \n",
+    "    if method == 'python_udf':\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': 'a'}, udfImpl=f.udf(python_replace_str)))\n",
+    "        \n",
+    "    elif method == 'pandas_udf':\n",
+    "        @pandas_udf('string')\n",
+    "        def pd_replace_str(v):\n",
+    "            v1s = []\n",
+    "            for index, token in v.items():\n",
+    "                v1s.append(python_replace_str(token))\n",
+    "            return pd.Series(v1s, dtype=str)\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': 'a'}, udfImpl=pd_replace_str))\n",
+    "        \n",
+    "    elif method == 'scala_udf':\n",
+    "        proc.spark.udf.registerJavaFunction(\"scala_replace_str\",\"com.intel.recdp.ScalaUDFTestStr\")\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': \"f.expr('scala_replace_str(a)')\"}, op='inline'))\n",
+    "        \n",
+    "    elif method == 'scala_df':\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': \"f.col('a')\"}, op='inline'))\n",
+    "        ops.append(ScalaDFTest(cols=['tweet'], method='str'))       \n",
+    "        \n",
+    "    elif method == 'spark_inline':\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': \"f.concat_ws('_', f.split(f.col('a'), '\\t'))\"}, op='inline'))\n",
+    "    \n",
+    "    ops.append(DropFeature(['a']))\n",
+    "\n",
+    "    # execute\n",
+    "    proc.reset_ops(ops)\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name, df_cnt=626242930)\n",
+    "    t2 = timer()\n",
+    "    print(f\"stringUDFTest with [{method}] took %.3f secs\" % (t2 - t1))\n",
+    "\n",
+    "    return df\n",
+    "\n",
+    "def broadcastUDFTest(df, proc, output_name, dict_df, method=\"pandas_udf\"):\n",
+    "    ops = []\n",
+    "    if method == 'broadcast':\n",
+    "        ops.append(FeatureAdd(cols={'tweet': \"f.col('a')\"}, op='inline'))\n",
+    "        ops.append(Categorify(['tweet'], dict_dfs = [{'col_name': 'tweet', 'dict': dict_df}]))\n",
+    "\n",
+    "    elif method == 'python_udf':\n",
+    "        t1 = timer()\n",
+    "        broadcast_handler = proc.spark.sparkContext.broadcast(dict((row['dict_col'], row['dict_col_id']) for row in dict_df.collect()))\n",
+    "        t2 = timer()\n",
+    "        print(f\"broadcast for [{method}] took %.3f secs\" % (t2 - t1))\n",
+    "        def python_replace_str(x):\n",
+    "            m = broadcast_handler.value\n",
+    "            if x in m:\n",
+    "                return m[x]\n",
+    "            else:\n",
+    "                return -1\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': 'a'}, udfImpl=f.udf(python_replace_str)))\n",
+    "        \n",
+    "    elif method == 'pandas_udf':\n",
+    "        t1 = timer()\n",
+    "        broadcast_handler = proc.spark.sparkContext.broadcast(dict((row['dict_col'], row['dict_col_id']) for row in dict_df.collect()))\n",
+    "        t2 = timer()\n",
+    "        print(f\"broadcast for [{method}] took %.3f secs\" % (t2 - t1))\n",
+    "        def python_replace_str(x):\n",
+    "            m = broadcast_handler.value\n",
+    "            if x in m:\n",
+    "                return m[x]\n",
+    "            else:\n",
+    "                return -1\n",
+    "        @pandas_udf('int')\n",
+    "        def pd_replace_str(v):\n",
+    "            v1s = []\n",
+    "            for index, token in v.items():\n",
+    "                v1s.append(python_replace_str(token))\n",
+    "            return pd.Series(v1s, dtype=int)\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': 'a'}, udfImpl=pd_replace_str))\n",
+    "        \n",
+    "    elif method == 'scala_df':\n",
+    "        ops.append(FeatureAdd(\n",
+    "            cols={'tweet': \"f.col('a')\"}, op='inline'))\n",
+    "        ops.append(ScalaDFTest(cols=['tweet'], dicts=[{'col_name': 'a', 'dict': dict_df}], method='broadcast'))       \n",
+    "    \n",
+    "    ops.append(DropFeature(['a']))\n",
+    "\n",
+    "    # execute\n",
+    "    proc.reset_ops(ops)\n",
+    "    t1 = timer()\n",
+    "    df = proc.transform(df, name=output_name, df_cnt=626242930)\n",
+    "    t2 = timer()\n",
+    "    print(f\"broadcastUDFTest with [{method}] took %.3f secs\" % (t2 - t1))\n",
+    "\n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "recdp-scala-extension is enabled\n",
+      "per core memory size is 2.500 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+      "intUDFTest with [python_udf] took 58.395 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|  986|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "intUDFTest with [pandas_udf] took 24.093 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|32437|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "intUDFTest with [scala_udf] took 7.559 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|32437|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "intUDFTest with [scala_df] took 7.266 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|32437|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "intUDFTest with [spark_inline] took 6.221 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|32437|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "stringUDFTest with [python_udf] took 231.697 secs\n",
+      "+--------------------+\n",
+      "|               tweet|\n",
+      "+--------------------+\n",
+      "|101_56898_137_144...|\n",
+      "+--------------------+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "stringUDFTest with [pandas_udf] took 187.985 secs\n",
+      "+--------------------+\n",
+      "|               tweet|\n",
+      "+--------------------+\n",
+      "|101_56898_137_144...|\n",
+      "+--------------------+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "stringUDFTest with [scala_udf] took 186.053 secs\n",
+      "+--------------------+\n",
+      "|               tweet|\n",
+      "+--------------------+\n",
+      "|101_56898_137_144...|\n",
+      "+--------------------+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "stringUDFTest with [scala_df] took 189.695 secs\n",
+      "+--------------------+\n",
+      "|               tweet|\n",
+      "+--------------------+\n",
+      "|101_56898_137_144...|\n",
+      "+--------------------+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "stringUDFTest with [spark_inline] took 270.320 secs\n",
+      "+--------------------+\n",
+      "|               tweet|\n",
+      "+--------------------+\n",
+      "|101_56898_137_144...|\n",
+      "+--------------------+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "do bhj to tweet\n",
+      "Adding a CodegenSeparator to pure BHJ WSCG case\n",
+      "broadcastUDFTest with [broadcast] took 11.146 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|    4|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "broadcast for [python_udf] took 0.165 secs\n",
+      "broadcastUDFTest with [python_udf] took 66.387 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|    8|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "broadcast for [pandas_udf] took 0.128 secs\n",
+      "broadcastUDFTest with [pandas_udf] took 31.237 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|    4|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "broadcastUDFTest with [scala_df] took 11.561 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|    4|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "do bhj to tweet\n",
+      "Adding a CodegenSeparator to pure BHJ WSCG case\n",
+      "broadcastUDFTest with [broadcast] took 20.810 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|  992|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "broadcast for [python_udf] took 48.764 secs\n",
+      "broadcastUDFTest with [python_udf] took 74.020 secs\n",
+      "+-------+\n",
+      "|  tweet|\n",
+      "+-------+\n",
+      "|1436881|\n",
+      "+-------+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "broadcast for [pandas_udf] took 41.758 secs\n",
+      "broadcastUDFTest with [pandas_udf] took 44.911 secs\n",
+      "+-------+\n",
+      "|  tweet|\n",
+      "+-------+\n",
+      "|1436881|\n",
+      "+-------+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "broadcastUDFTest with [scala_df] took 133.202 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|  992|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/recsys2021_0608_udf_test/\"\n",
+    "original_folder = \"/recsys2021_0608/\"\n",
+    "\n",
+    "native_sql_path = \"/mnt/nvme2/chendi/intel-bigdata/OAP/native-sql-engine/native-sql-engine/core/target/spark-columnar-core-1.2.0-snapshot-jar-with-dependencies.jar\"\n",
+    "native_arrow_datasource_path = \"/mnt/nvme2/chendi/intel-bigdata/OAP/native-sql-engine/arrow-data-source/standard/target/spark-arrow-datasource-standard-1.2.0-snapshot-jar-with-dependencies.jar\"\n",
+    "scala_udf_jars = \"/mnt/nvme2/chendi/BlueWhale/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar\"\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "t0 = timer()\n",
+    "\n",
+    "#### vanilla spark ####\n",
+    "spark = SparkSession.builder.master('yarn')\\\n",
+    "    .appName(\"udf_vanilla\")\\\n",
+    "    .config(\"spark.sql.broadcastTimeout\", \"7200\")\\\n",
+    "    .config(\"spark.cleaner.periodicGC.interval\", \"10min\")\\\n",
+    "    .config(\"spark.driver.extraClassPath\", \n",
+    "            f\"{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.executor.extraClassPath\",\n",
+    "            f\"{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.executor.memory\", \"10g\")\\\n",
+    "    .config(\"spark.executor.memoryOverhead\", \"16g\")\\\n",
+    "    .getOrCreate()\n",
+    "proc = DataProcessor(spark, path_prefix,\n",
+    "                     current_path=current_path, shuffle_disk_capacity=\"1200GB\", enable_gazelle=False)\n",
+    "\n",
+    "##################################################################\n",
+    "#['python_udf', 'pandas_udf', 'scala_udf', 'scala_df', 'spark_inline']\n",
+    "for method in ['python_udf', 'pandas_udf', 'scala_udf', 'scala_df', 'spark_inline']:\n",
+    "    df = spark.read.format(\"parquet\").load(path_prefix + original_folder)\n",
+    "    df = df.selectExpr(\"engaged_with_user_follower_count as a\").withColumn(\"a\", f.col(\"a\").cast(IntegerType()))\n",
+    "    df = intUDFTest(df, proc, output_name=\"decoded_with_extracted_features\", method=method)\n",
+    "    df.filter(\"tweet is not null\").show(1)\n",
+    "for method in ['python_udf', 'pandas_udf', 'scala_udf', 'scala_df', 'spark_inline']:\n",
+    "    df = spark.read.format(\"parquet\").load(path_prefix + original_folder)\n",
+    "    df = df.selectExpr(\"text_tokens as a\")\n",
+    "    df = stringUDFTest(df, proc, output_name=\"decoded_with_extracted_features\", method=method)\n",
+    "    df.filter(\"tweet is not null\").show(1)\n",
+    "for method in ['broadcast', 'python_udf', 'pandas_udf', 'scala_df']:\n",
+    "    df = spark.read.format(\"parquet\").load(path_prefix + original_folder)\n",
+    "    dict_df = spark.read.format(\"parquet\").load(\"/recsys2021_0608_processed/recsys_dicts/language\")\n",
+    "    df = df.selectExpr(\"language as a\")\n",
+    "    df = broadcastUDFTest(df, proc, output_name=\"decoded_with_extracted_features\", dict_df = dict_df, method=method)\n",
+    "    df.filter(\"tweet is not null\").filter(\"tweet != -1\").show(1)\n",
+    "for method in ['broadcast', 'python_udf', 'pandas_udf', 'scala_df']:\n",
+    "    df = spark.read.format(\"parquet\").load(\"/recsys2021_0608_processed/first_hashtags\")\n",
+    "    dict_df = spark.read.format(\"parquet\").load(\"/recsys2021_0608_processed/recsys_dicts/hashtags\")\n",
+    "    df = broadcastUDFTest(df, proc, output_name=\"decoded_with_extracted_features\", dict_df = dict_df, method=method)\n",
+    "    df.filter(\"tweet is not null\").filter(\"tweet != -1\").show(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "spark.stop()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "recdp-scala-extension is enabled\n",
+      "per core memory size is 2.500 GB and shuffle_disk maximum capacity is 1200.000 GB\n",
+      "intUDFTest with [pandas_udf] took 24.533 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|32437|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "stringUDFTest with [pandas_udf] took 129.790 secs\n",
+      "+--------------------+\n",
+      "|               tweet|\n",
+      "+--------------------+\n",
+      "|101_56898_137_144...|\n",
+      "+--------------------+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "broadcast for [pandas_udf] took 0.553 secs\n",
+      "broadcastUDFTest with [pandas_udf] took 22.516 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|   18|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n",
+      "broadcast for [pandas_udf] took 44.521 secs\n",
+      "broadcastUDFTest with [pandas_udf] took 49.415 secs\n",
+      "+-----+\n",
+      "|tweet|\n",
+      "+-----+\n",
+      "|26760|\n",
+      "+-----+\n",
+      "only showing top 1 row\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "path_prefix = \"hdfs://\"\n",
+    "current_path = \"/recsys2021_0608_udf_test/\"\n",
+    "original_folder = \"/recsys2021_0608/\"\n",
+    "\n",
+    "native_sql_path = \"/mnt/nvme2/chendi/intel-bigdata/OAP/native-sql-engine/native-sql-engine/core/target/spark-columnar-core-1.2.0-snapshot-jar-with-dependencies.jar\"\n",
+    "native_arrow_datasource_path = \"/mnt/nvme2/chendi/intel-bigdata/OAP/native-sql-engine/arrow-data-source/standard/target/spark-arrow-datasource-standard-1.2.0-snapshot-jar-with-dependencies.jar\"\n",
+    "scala_udf_jars = \"/mnt/nvme2/chendi/BlueWhale/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar\"\n",
+    "\n",
+    "##### 1. Start spark and initialize data processor #####\n",
+    "t0 = timer()\n",
+    "#### nativesql ####\n",
+    "spark = SparkSession.builder.master('yarn')\\\n",
+    "    .appName(\"udf_native-sql\")\\\n",
+    "    .config(\"spark.sql.broadcastTimeout\", \"7200\")\\\n",
+    "    .config(\"spark.cleaner.periodicGC.interval\", \"10min\")\\\n",
+    "    .config(\"spark.executorEnv.LD_LIBRARY_PATH\", \"/usr/local/lib64/\")\\\n",
+    "    .config(\"spark.driver.extraClassPath\", \n",
+    "            f\"{native_sql_path}:{native_arrow_datasource_path}:{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.executor.extraClassPath\",\n",
+    "            f\"{native_sql_path}:{native_arrow_datasource_path}:{scala_udf_jars}\")\\\n",
+    "    .config(\"spark.sql.extensions\", \"com.intel.oap.ColumnarPlugin, com.intel.oap.spark.sql.ArrowWriteExtension\")\\\n",
+    "    .config(\"spark.shuffle.manager\", \"org.apache.spark.shuffle.sort.ColumnarShuffleManager\")\\\n",
+    "    .config(\"spark.executor.memory\", \"10g\")\\\n",
+    "    .config(\"spark.executor.memoryOverhead\", \"16g\")\\\n",
+    "    .config(\"spark.memory.offHeap.use\", \"true\")\\\n",
+    "    .config(\"spark.memory.offHeap.size\", \"12G\")\\\n",
+    "    .config(\"spark.executor.extraJavaOptions\", \"-XX:MaxDirectMemorySize=25G\")\\\n",
+    "    .config(\"spark.oap.sql.columnar.arrowudf\", \"true\")\\\n",
+    "    .getOrCreate()\n",
+    "proc = DataProcessor(spark, path_prefix,\n",
+    "                    current_path=current_path, shuffle_disk_capacity=\"1200GB\", enable_gazelle=True)\n",
+    "\n",
+    "##################################################################\n",
+    "#['python_udf', 'pandas_udf', 'scala_udf', 'scala_df', 'spark_inline']\n",
+    "for method in ['pandas_udf']:\n",
+    "    df = spark.read.format(\"arrow\").load(path_prefix + original_folder)\n",
+    "    df = df.selectExpr(\"engaged_with_user_follower_count as a\").withColumn(\"a\", f.col(\"a\").cast(IntegerType()))\n",
+    "    df = intUDFTest(df, proc, output_name=\"decoded_with_extracted_features\", method=method)\n",
+    "    df.filter(\"tweet is not null\").show(1)\n",
+    "for method in ['pandas_udf']:\n",
+    "    df = spark.read.format(\"arrow\").load(path_prefix + original_folder)\n",
+    "    df = df.selectExpr(\"text_tokens as a\")\n",
+    "    df = stringUDFTest(df, proc, output_name=\"decoded_with_extracted_features\", method=method)\n",
+    "    df.filter(\"tweet is not null\").show(1)\n",
+    "for method in ['pandas_udf']:\n",
+    "    df = spark.read.format(\"arrow\").load(path_prefix + original_folder)\n",
+    "    dict_df = spark.read.format(\"arrow\").load(\"/recsys2021_0608_processed/recsys_dicts/language\")\n",
+    "    df = df.selectExpr(\"language as a\")\n",
+    "    df = broadcastUDFTest(df, proc, output_name=\"decoded_with_extracted_features\", dict_df = dict_df, method=method)\n",
+    "    df.filter(\"tweet is not null\").filter(\"tweet != -1\").show(1)\n",
+    "for method in ['pandas_udf']:\n",
+    "    df = spark.read.format(\"arrow\").load(path_prefix + original_folder)\n",
+    "    dict_df = spark.read.format(\"arrow\").load(\"/recsys2021_0608_processed/recsys_dicts/hashtags\")\n",
+    "    df = df.withColumn(\"a\", f.split(f.col('hashtags'), '\\t').getItem(0)).select(\"a\")\n",
+    "    df = df.repartition(200)\n",
+    "    df = broadcastUDFTest(df, proc, output_name=\"decoded_with_extracted_features\", dict_df = dict_df, method=method)\n",
+    "    df.filter(\"tweet is not null\").filter(\"tweet != -1\").show(1)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RecDP/examples/python_tests/__init__.py b/RecDP/examples/python_tests/__init__.py
new file mode 100644
index 000000000..3bff0c48e
--- /dev/null
+++ b/RecDP/examples/python_tests/__init__.py
@@ -0,0 +1,1 @@
+__all__ = ["recsys", "dlrm", "wnd", "dien"]
diff --git a/RecDP/examples/python_tests/dien/download_dataset b/RecDP/examples/python_tests/dien/download_dataset
new file mode 100755
index 000000000..fbf2b208a
--- /dev/null
+++ b/RecDP/examples/python_tests/dien/download_dataset
@@ -0,0 +1,21 @@
+#!/usr/bin/bash
+
+mkdir -p raw_data
+mkdir -p output
+mkdir -p j2c_test
+
+if [ ! -f "data.tar.gz" ]; then
+        wget https://zenodo.org/record/3463683/files/data.tar.gz
+fi
+
+tar -jxvf data.tar.gz
+yes | cp data/local_test_splitByUser output/
+yes | cp data/local_test_splitByUser j2c_test/
+mv data/local_test_splitByUser raw_data/
+rm -r data
+
+wget http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/reviews_Books.json.gz
+wget http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/meta_Books.json.gz
+gunzip reviews_Books.json.gz
+gunzip meta_Books.json.gz
+mv *json raw_data
diff --git a/RecDP/examples/python_tests/dien/init.py b/RecDP/examples/python_tests/dien/init.py
new file mode 100644
index 000000000..a51233f58
--- /dev/null
+++ b/RecDP/examples/python_tests/dien/init.py
@@ -0,0 +1,6 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
diff --git a/RecDP/examples/python_tests/dien/preprocessing_for_downloaded_test.py b/RecDP/examples/python_tests/dien/preprocessing_for_downloaded_test.py
new file mode 100644
index 000000000..90ed0a9cd
--- /dev/null
+++ b/RecDP/examples/python_tests/dien/preprocessing_for_downloaded_test.py
@@ -0,0 +1,318 @@
+import init
+
+import os
+import pandas as pd
+import numpy as np
+from pyspark.sql import *
+from pyspark import *
+import pyspark.sql.functions as f
+from pyspark.sql.types import *
+from timeit import default_timer as timer
+import logging
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+
+NUM_INSTS = 1
+TARGET_PATH = "/home/xxx/dien"
+
+def load_csv(spark, path):
+    review_id_field = StructField('reviewer_id', StringType())
+    asin_field = StructField('asin', StringType())
+    overall_field = StructField('overall', FloatType())
+    unix_time_field = StructField('unix_review_time', IntegerType())
+    reviews_info_schema = StructType(
+        [review_id_field, asin_field, overall_field, unix_time_field])
+
+    category_field = StructField('category', StringType())
+    item_info_schema = StructType([asin_field, category_field])
+
+    reviews_info_df = spark.read.schema(reviews_info_schema).option('sep', '\t').csv(path + "/reviews-info")
+    item_info_df = spark.read.schema(item_info_schema).option('sep', '\t').csv(path + "/item-info")
+
+    return reviews_info_df, item_info_df
+
+def process_reviews(spark, review_file, proc, name):
+    records_df = spark.read.json(review_file)
+    records_df = records_df.select('reviewerID', 'asin', 'overall', 'unixReviewTime')
+    records_df = records_df.withColumnRenamed('reviewerID', 'reviewer_id')
+    records_df = records_df.withColumnRenamed('unixReviewTime', 'unix_review_time')
+    records_df = proc.transform(records_df, name)
+    return records_df
+
+
+def process_meta(spark, meta_file, proc, name):
+    def process_single_meta(row):
+        obj = eval(row)
+        cat = obj['categories'][0][-1]
+        return [obj['asin'], cat]
+
+    meta_df = spark.read.text(meta_file)
+    proc_meta_udf = f.udf(lambda x: process_single_meta(x), ArrayType(StringType()))
+    meta_df = meta_df.withColumn('asin', proc_meta_udf(f.col('value')).getItem(0))
+    meta_df = meta_df.withColumn('category', proc_meta_udf(f.col('value')).getItem(1))
+    meta_df = meta_df.drop('value')
+    meta_df = proc.transform(meta_df, name)
+    return meta_df
+
+def collapse_by_hist(df, item_info_df, proc, output_name, min_num_hist = 0, max_num_hist = 100):
+    op_model_merge = ModelMerge([{'col_name': 'asin', 'dict': item_info_df}])
+    op_collapse_by_hist = CollapseByHist(['asin', 'category'], by = ['reviewer_id'], orderBy = ['unix_review_time', 'asin'], minNumHist = min_num_hist, maxNumHist = 100)
+    proc.reset_ops([op_model_merge, op_collapse_by_hist])
+    
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()
+    print(f"Merge with category and collapse hist took {t2 - t1} secs")
+    return df
+
+
+def get_dict_for_asin(df, proc):
+    dict_dfs = []
+    dict_names = ['asin']
+    dict_dfs = [{'col_name': name, 'dict': df.select(spk_func.col(name).alias('dict_col'))} for name in dict_names]
+    return dict_dfs
+
+
+def add_negative_sample(df, item_info_df, dict_dfs, proc, output_name):
+    # add negative_sample as new row
+    op_negative_sample = NegativeSample(['asin'], dict_dfs)
+    # negative row category is not correct
+    op_drop_category = DropFeature(['category'])
+    op_add_category = ModelMerge([{'col_name': 'asin', 'dict': item_info_df}])
+    op_select = SelectFeature(['pos', 'reviewer_id', 'asin', 'category', 'hist_asin', 'hist_category'])
+    proc.reset_ops([op_negative_sample, op_drop_category, op_add_category, op_select])
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()    
+    print(f"add_negative_sample took {t2 - t1} secs")
+    return df
+
+
+def add_negative_hist_cols(df, item_info_df, dict_dfs, proc, output_name):
+    # add negative_sample as new row
+    op_columns_sample = NegativeFeature({'noclk_hist_asin': 'hist_asin'}, dict_dfs, doSplit=True, sep='\x02', negCnt = 5)
+    proc.reset_ops([op_columns_sample])
+    t1 = timer()
+
+    df = proc.transform(df, output_name)
+    t2 = timer()    
+    print(f"add_negative_hist_cols took {t2 - t1} secs")
+    return df
+
+
+def load_processed_csv(spark, data_dir):
+    label_field = StructField('pos', IntegerType())
+    review_id_field = StructField('reviewer_id', StringType())
+    asin_field = StructField('asin', StringType())
+    category_field = StructField('category', StringType())
+    hist_asin_field = StructField('hist_asin', StringType())
+    hist_category_field = StructField('hist_category', StringType())
+    csv_schema = StructType(
+        [label_field, review_id_field, asin_field, category_field, hist_asin_field, hist_category_field])
+
+    return spark.read.schema(csv_schema).option('sep', '\t').csv(data_dir)
+
+
+def categorify_dien_data(df, user_df, asin_df, cat_df, asin_cat_df, proc, output_name):
+    df = df.select('pos', 'reviewer_id', 'asin', 'category', 'hist_asin', 'hist_category',
+                   'noclk_hist_asin', f.expr('noclk_hist_asin as noclk_hist_category'))
+
+    dict_dfs = []
+    dict_dfs.append({'col_name': 'reviewer_id', 'dict': user_df})
+    dict_dfs.append({'col_name': 'asin', 'dict': asin_df})
+    dict_dfs.append({'col_name': 'category', 'dict': cat_df})
+    dict_dfs.append({'col_name': 'hist_asin', 'dict': asin_df})
+    dict_dfs.append({'col_name': 'hist_category', 'dict': cat_df})
+    dict_dfs.append({'col_name': 'noclk_hist_asin', 'dict': asin_df})
+    dict_dfs.append({'col_name': 'noclk_hist_category', 'dict': asin_cat_df})
+
+    op_categorify = Categorify(['reviewer_id', 'asin', 'category'], dict_dfs=dict_dfs)
+    op_categorify_2 = Categorify(['hist_asin', 'hist_category'], dict_dfs=dict_dfs, doSplit=True, sep='\x02')
+    op_categorify_3 = Categorify(['noclk_hist_asin', 'noclk_hist_category'], dict_dfs=dict_dfs, doSplit=True, multiLevelSplit=True, multiLevelSep=['|'])
+    op_fillna = FillNA(['asin', 'category'], 0)
+    proc.reset_ops([op_categorify, op_categorify_2, op_categorify_3, op_fillna])
+
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()
+    print(f"categorify took {t2 - t1} secs")
+    return df
+
+
+def load_voc(proc, output_name):
+    import pickle as pkl
+    with open("/home/xxx/dien/" + f'/{output_name}.pkl', "rb") as f:
+        voc = dict((key, value) for (key,value) in pkl.load(f).items())
+    dict_df = convert_to_spark_df(voc, proc.spark)
+    return dict_df
+
+
+def load_uid_voc(proc):
+    # saving (using python)
+    # build uid_dict, mid_dict and cat_dict
+    t1 = timer()
+    dict_df = load_voc(proc, 'test_uid_voc')
+    t2 = timer()
+    print(f"load_uid_voc took {t2 - t1} secs")
+    return dict_df
+    
+
+def load_mid_voc(proc):
+    t1 = timer()
+    dict_df = load_voc(proc, 'test_mid_voc')
+    t2 = timer()
+    print(f"load_mid_voc took {t2 - t1} secs")
+    return dict_df
+    
+    
+def load_cat_voc(proc):
+    t1 = timer()
+    dict_df = load_voc(proc, 'test_cat_voc')
+    t2 = timer()
+    print(f"load_cat_voc took {t2 - t1} secs")
+    return dict_df
+    
+def save_to_local_splitByUser(df, proc, output_name):
+    hint = 'byRename'
+    #hint = 'byRewrite'
+    t1 = timer()
+    if 'noclk_hist_asin' in df.columns:
+        dict_df = df.select('pos', 'reviewer_id', 'asin', 'category',
+                            f.expr("concat_ws('\x02', hist_asin)"),
+                            f.expr("concat_ws('\x02', hist_category)"),
+                            f.expr("concat_ws('\x02', noclk_hist_asin)"),
+                            f.expr("concat_ws('\x02', noclk_hist_category)"))
+        if hint == 'byRename':
+            dict_df.repartition(1).write.format("csv").option('sep', '\t').mode("overwrite").save(f"{proc.path_prefix}{proc.current_path}/{output_name}-spark")
+            result_rename_or_convert(proc.current_path, output_name)
+            t2 = timer()
+            print(f"save_to_local_splitByUser took {t2 - t1} secs")
+            return dict_df
+        else:
+            collected = [[c1, c2, c3, c4, c5, c6, c7, c8] for (c1, c2, c3, c4, c5, c6, c7, c8) in dict_df.collect()]
+    else:
+        dict_df = df.select('pos', 'reviewer_id', 'asin', 'category',
+                            f.expr("concat_ws('\x02', hist_asin)"),
+                            f.expr("concat_ws('\x02', hist_category)"))
+        if hint == 'byRename':
+            dict_df.repartition(1).write.format("csv").option('sep', '\t').mode("overwrite").save(f"{proc.path_prefix}{TARGET_PATH}/{output_name}-spark")
+            result_rename_or_convert(proc.current_path, output_name)
+            t2 = timer()
+            print(f"save_to_local_splitByUser took {t2 - t1} secs")
+            return dict_df
+        else:
+            collected = [[c1, c2, c3, c4, c5, c6] for (c1, c2, c3, c4, c5, c6) in dict_df.collect()]
+
+    user_map = {}
+    for items in collected:
+        if items[1] not in user_map:
+            user_map[items[1]] = []
+        user_map[items[1]].append(items)
+    with open(TARGET_PATH + f"/{output_name}", 'w') as fp:
+        for user, r in user_map.items():
+            positive_sorted = sorted(r, key=lambda x: x[0])
+            for items in positive_sorted:
+                print('\t'.join([str(x) for x in items]), file=fp)
+
+    t2 = timer()
+    print(f"save_to_local_splitByUser took {t2 - t1} secs")
+    return dict_df
+
+
+def result_rename_or_convert(fpath, output_name):
+    source_path_dict = list_dir(fpath, False)
+    fix = "-spark"
+    tgt_path = f"{TARGET_PATH}/{output_name}"
+    idx = 0
+    if len(source_path_dict[output_name + fix]) == 1:
+        file_name = source_path_dict[output_name + fix][0]
+        shutil.copy(file_name, f"{tgt_path}")
+    else:
+        for file_name in source_path_dict[output_name + fix]:
+            #print(f"result renamed from {file_name} to {tgt_path}_{idx}")
+            shutil.copy(file_name, f"{tgt_path}_{idx}")
+            shutil.rmtree(file_name, ignore_errors=True)
+            idx += 1
+
+def main(option = '--advanced'):
+    if option == '--basic':
+        fmt = 'pkl'
+    elif option == '--advanced':
+        fmt = 'adv_pkl'
+    else:
+        raise NotImplementedError(f'{option} is not recognized.')
+
+    path_prefix = "file://"
+    current_path = "/home/vmagent/app/recdp/examples/python_tests/dien/output/"
+    original_folder = "/home/vmagent/app/recdp/examples/python_tests/dien/"
+
+    scala_udf_jars = "/home/vmagent/app/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+
+    ##### 1. Start spark and initialize data processor #####
+    t0 = timer()
+    spark = SparkSession.builder.master('spark://dr8s30:7077')\
+        .appName("dien_data_process")\
+        .config("spark.driver.memory", "20G")\
+        .config("spark.driver.memoryOverhead", "10G")\
+        .config("spark.executor.instances", "4")\
+        .config("spark.executor.cores", "32")\
+        .config("spark.executor.memory", "100G")\
+        .config("spark.executor.memoryOverhead", "20G")\
+        .config("spark.driver.extraClassPath", f"{scala_udf_jars}")\
+        .config("spark.executor.extraClassPath", f"{scala_udf_jars}")\
+        .getOrCreate()
+    spark.sparkContext.setLogLevel("ERROR")
+
+    # 1.1 prepare dataFrames
+    # 1.2 create RecDP DataProcessor
+    proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity="1200GB", spark_mode='standalone')
+    t1 = timer()
+    print(f"start spark process took {(t1 - t0)} secs")
+
+    # ===============================================
+    # Data ingestion
+    t0 = timer()
+    reviews_info_df = process_reviews(spark, "%s/%s/raw_data/reviews_Books.json" % (path_prefix, original_folder), proc, "reviews-info")
+    #reviews_info_df.repartition(1).write.format("csv").option('sep', '\t').mode("overwrite").save("%s/%s/j2c_test/reviews-info-spark" % (path_prefix, original_folder))
+    t1 = timer()
+    print(f"parse reviews-info with spark took {(t1 - t0)} secs")
+
+    t0 = timer()
+    item_info_df = process_meta(spark, '%s/%s/raw_data/meta_Books.json' % (path_prefix, original_folder), proc, "item-info")
+    #item_info_df.repartition(1).write.format("csv").option('sep', '\t').mode("overwrite").save("%s/%s/j2c_test/item-info-spark" % (path_prefix, original_folder))
+    t1 = timer()
+    print(f"parse item-info with spark took {(t1 - t0)} secs")
+
+    # ===============================================
+    # 1. join records with its category and then collapse history 
+    df = reviews_info_df
+    dict_dfs = get_dict_for_asin(df, proc)
+
+    uid_dict_df = load_uid_voc(proc)
+    mid_dict_df = load_mid_voc(proc)
+    cat_dict_df = load_cat_voc(proc)
+    if fmt == 'adv_pkl':
+        asin_df = item_info_df.withColumnRenamed('asin', 'dict_col')
+        cat_dict_for_merge_df = cat_dict_df.withColumnRenamed('dict_col', 'category')
+        op_asin_to_cat_id = ModelMerge([{'col_name': 'category', 'dict': cat_dict_for_merge_df}])
+        op_select = SelectFeature(['dict_col', 'dict_col_id'])
+        proc.reset_ops([op_asin_to_cat_id, op_select])
+        asin_cat_df = proc.apply(asin_df)
+        # for create noclk_hist_category, we should create a dict to mapping from asin to its cat_id
+        test_df = load_processed_csv(spark, path_prefix + original_folder + "raw_data/local_test_splitByUser")
+        dict_dfs[0]['col_name'] = 'hist_asin'
+        test_df = add_negative_hist_cols(test_df, item_info_df, dict_dfs, proc, "test_records_with_negative_hists")
+        test_df = categorify_dien_data(test_df, uid_dict_df, mid_dict_df, cat_dict_df, asin_cat_df, proc, "local_test_splitByUser.parquet")
+        test_df = save_to_local_splitByUser(test_df, proc, 'local_test_splitByUser')
+    t1 = timer()
+
+    print(f"Total process time is {(t1 - t0)} secs")
+
+    ####################################
+    
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        main(sys.argv[1])
+    else:
+        main()
diff --git a/RecDP/examples/python_tests/dien/preprocessing_for_inference.py b/RecDP/examples/python_tests/dien/preprocessing_for_inference.py
new file mode 100644
index 000000000..35e06ad67
--- /dev/null
+++ b/RecDP/examples/python_tests/dien/preprocessing_for_inference.py
@@ -0,0 +1,372 @@
+import init
+
+import os
+import numpy as np
+from pyspark.sql import *
+from pyspark import *
+import pyspark.sql.functions as f
+from pyspark.sql.types import *
+from timeit import default_timer as timer
+import logging
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+
+NUM_INSTS = 64
+
+def load_csv(spark, path):
+    review_id_field = StructField('reviewer_id', StringType())
+    asin_field = StructField('asin', StringType())
+    overall_field = StructField('overall', FloatType())
+    unix_time_field = StructField('unix_review_time', IntegerType())
+    reviews_info_schema = StructType(
+        [review_id_field, asin_field, overall_field, unix_time_field])
+
+    category_field = StructField('category', StringType())
+    item_info_schema = StructType([asin_field, category_field])
+
+    reviews_info_df = spark.read.schema(reviews_info_schema).option('sep', '\t').csv(path + "/reviews-info")
+    item_info_df = spark.read.schema(item_info_schema).option('sep', '\t').csv(path + "/item-info")
+
+    return reviews_info_df, item_info_df
+
+def process_reviews_2(spark, review_file, proc, name):
+    def process_single_review(row):
+        obj = eval(row)
+        return [obj['reviewerID'], obj['asin'], obj['overall'], obj['unixReviewTime']]
+
+    records_df = spark.read.text(review_file)
+    proc_udf = f.udf(lambda x: process_single_review(x), ArrayType(StringType()))
+    records_df = records_df.withColumn('parsed', proc_udf(f.col('value')))
+    records_df = records_df.withColumn('reviewer_id', f.col('parsed').getItem(0))
+    records_df = records_df.withColumn('asin', f.col('parsed').getItem(1))
+    records_df = records_df.withColumn('overall', f.col('parsed').getItem(2))
+    records_df = records_df.withColumn('unix_review_time', f.col('parsed').getItem(3))
+    records_df = records_df.drop('value').drop('parsed')
+    records_df = proc.transform(records_df, name)
+    return records_df
+
+
+def process_reviews(spark, review_file, proc, name):
+    records_df = spark.read.json(review_file)
+    records_df = records_df.select('reviewerID', 'asin', 'overall', 'unixReviewTime')
+    records_df = records_df.withColumnRenamed('reviewerID', 'reviewer_id')
+    records_df = records_df.withColumnRenamed('unixReviewTime', 'unix_review_time')
+    records_df = proc.transform(records_df, name)
+    return records_df
+
+
+def process_meta(spark, meta_file, proc, name):
+    def process_single_meta(row):
+        obj = eval(row)
+        cat = obj['categories'][0][-1]
+        return [obj['asin'], cat]
+
+    meta_df = spark.read.text(meta_file)
+    proc_meta_udf = f.udf(lambda x: process_single_meta(x), ArrayType(StringType()))
+    meta_df = meta_df.withColumn('asin', proc_meta_udf(f.col('value')).getItem(0))
+    meta_df = meta_df.withColumn('category', proc_meta_udf(f.col('value')).getItem(1))
+    meta_df = meta_df.drop('value')
+    meta_df = proc.transform(meta_df, name)
+    return meta_df
+
+def merge_item_to_reviews(df, item_info_df, proc):
+    # merge with item-info and reviews-info firstly
+    op_asin_to_cat_id = ModelMerge([{'col_name': 'asin', 'dict': item_info_df}])
+    proc.reset_ops([op_asin_to_cat_id])
+    df = proc.apply(df)
+    return df
+
+
+def collapse_by_hist(df, proc, output_name, min_num_hist = 0, max_num_hist = 100):
+    op_collapse_by_hist = CollapseByHist(['asin', 'category'], by = ['reviewer_id'], orderBy = ['unix_review_time', 'asin'], minNumHist = min_num_hist, maxNumHist = 100)
+    proc.reset_ops([op_collapse_by_hist])
+    
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()
+    print(f"collapse hist took {t2 - t1} secs")
+    return df
+
+
+def get_dict_for_asin(df, proc):
+    dict_dfs = []
+    dict_names = ['asin']
+    dict_dfs = [{'col_name': name, 'dict': df.select(spk_func.col(name).alias('dict_col'))} for name in dict_names]
+    return dict_dfs
+
+
+def add_negative_sample(df, dict_dfs, mid_cat_df, proc, output_name):
+    # add negative_sample as new row
+    op_negative_sample = NegativeSample(['asin'], dict_dfs)
+    # negative row category is not correct
+    mid_cat_df = mid_cat_df.withColumnRenamed('dict_col', 'asin').withColumnRenamed('dict_col_id', 'category')
+    op_drop_category = DropFeature(['category'])
+    op_add_category = ModelMerge([{'col_name': 'asin', 'dict': mid_cat_df}])
+
+    op_select = SelectFeature(['pos', 'reviewer_id', 'asin', 'category', 'hist_asin', 'hist_category'])
+    proc.reset_ops([op_negative_sample, op_drop_category, op_add_category, op_select])
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()    
+    print(f"add_negative_sample took {t2 - t1} secs")
+    return df
+
+
+def add_negative_hist_cols(df, dict_dfs, mid_dict_df, mid_cat_df, proc, output_name):
+    # since we already categorified asin, we need to create a number_asin dict
+    new_dict_dfs = dict_dfs
+    new_dict_dfs[0]['col_name'] = 'hist_asin'
+    op_columns_sample = NegativeFeature({'noclk_hist_asin': 'hist_asin'}, new_dict_dfs, doSplit=True, sep='\x02', negCnt = 5)
+    proc.reset_ops([op_columns_sample])
+    df = proc.apply(df)
+
+    # add one col in df to create noclk_hist_category
+    df = df.withColumn('noclk_hist_category', f.col('noclk_hist_asin'))
+
+    new_dict_dfs = []
+    new_dict_dfs.append({'col_name': 'noclk_hist_category', 'dict': mid_cat_df})
+    op_categorify = Categorify(['noclk_hist_category'], dict_dfs=new_dict_dfs, doSplit=True, multiLevelSplit=True, multiLevelSep=['|'])
+    proc.reset_ops([op_categorify])
+    df = proc.apply(df)
+
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()    
+    print(f"add_negative_hist_cols took {t2 - t1} secs")
+    return df
+
+
+def load_processed_csv(spark, data_dir):
+    label_field = StructField('pos', IntegerType())
+    review_id_field = StructField('reviewer_id', StringType())
+    asin_field = StructField('asin', StringType())
+    category_field = StructField('category', StringType())
+    hist_asin_field = StructField('hist_asin', StringType())
+    hist_category_field = StructField('hist_category', StringType())
+    csv_schema = StructType(
+        [label_field, review_id_field, asin_field, category_field, hist_asin_field, hist_category_field])
+
+    return spark.read.schema(csv_schema).option('sep', '\t').csv(data_dir)
+
+
+def categorify_dien_data(df, user_df, asin_df, cat_df, proc, output_name):
+    dict_dfs = []
+    dict_dfs.append({'col_name': 'reviewer_id', 'dict': user_df})
+    dict_dfs.append({'col_name': 'asin', 'dict': asin_df})
+    dict_dfs.append({'col_name': 'category', 'dict': cat_df})
+
+    op_categorify = Categorify(['reviewer_id', 'asin', 'category'], dict_dfs=dict_dfs)
+    op_fillna = FillNA(['asin', 'category'], 0)
+    proc.reset_ops([op_categorify, op_fillna])
+
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()
+    print(f"categorify took {t2 - t1} secs")
+    return df
+
+
+def load_voc(proc, output_name):
+    import pickle as pkl
+    with open("/home/xxx/dien/" + f'/{output_name}.pkl', "rb") as f:
+        voc = dict((key, value) for (key,value) in pkl.load(f).items())
+    dict_df = convert_to_spark_df(voc, proc.spark)
+    return dict_df
+
+
+def load_uid_voc(proc):
+    # saving (using python)
+    # build uid_dict, mid_dict and cat_dict
+    t1 = timer()
+    dict_df = load_voc(proc, 'test_uid_voc')
+    t2 = timer()
+    print(f"load_uid_voc took {t2 - t1} secs")
+    return dict_df
+    
+
+def load_mid_voc(proc):
+    t1 = timer()
+    dict_df = load_voc(proc, 'test_mid_voc')
+    t2 = timer()
+    print(f"load_mid_voc took {t2 - t1} secs")
+    return dict_df
+    
+    
+def load_cat_voc(proc):
+    t1 = timer()
+    dict_df = load_voc(proc, 'test_cat_voc')
+    t2 = timer()
+    print(f"load_cat_voc took {t2 - t1} secs")
+    return dict_df
+    
+    
+def save_to_local_splitByUser(df, proc, output_name):
+    hint = 'byRename'
+    #hint = 'byRewrite'
+    t1 = timer()
+    if 'noclk_hist_asin' in df.columns:
+        dict_df = df.select('pos', 'reviewer_id', 'asin', 'category',
+                            f.expr("concat_ws('\x02', hist_asin)"),
+                            f.expr("concat_ws('\x02', hist_category)"),
+                            f.expr("concat_ws('\x02', noclk_hist_asin)"),
+                            f.expr("concat_ws('\x02', noclk_hist_category)"))
+        if hint == 'byRename':
+            dict_df.repartition(NUM_INSTS).write.format("csv").option('sep', '\t').mode("overwrite").save(f"file:///home/vmagent/app/recdp/examples/python_tests/{proc.current_path}/{output_name}-spark")
+            result_rename_or_convert("/home/vmagent/app/recdp/examples/python_tests/" + proc.current_path, output_name)
+            t2 = timer()
+            print(f"save_to_local_splitByUser took {t2 - t1} secs")
+            return dict_df
+        else:
+            collected = [[c1, c2, c3, c4, c5, c6, c7, c8] for (c1, c2, c3, c4, c5, c6, c7, c8) in dict_df.collect()]
+    else:
+        dict_df = df.select('pos', 'reviewer_id', 'asin', 'category',
+                            f.expr("concat_ws('\x02', hist_asin)"),
+                            f.expr("concat_ws('\x02', hist_category)"))
+        if hint == 'byRename':
+            dict_df.repartition(NUM_INSTS).write.format("csv").option('sep', '\t').mode("overwrite").save(f"file:///home/vmagent/app/recdp/examples/python_tests/{proc.current_path}/{output_name}-spark")
+            result_rename_or_convert("/home/vmagent/app/recdp/examples/python_tests/" + proc.current_path, output_name)
+            t2 = timer()
+            print(f"save_to_local_splitByUser took {t2 - t1} secs")
+            return dict_df
+        else:
+            collected = [[c1, c2, c3, c4, c5, c6] for (c1, c2, c3, c4, c5, c6) in dict_df.collect()]
+
+    user_map = {}
+    for items in collected:
+        if items[1] not in user_map:
+            user_map[items[1]] = []
+        user_map[items[1]].append(items)
+    with open("/home/vmagent/app/recdp/examples/python_tests/" + proc.current_path + f"/{output_name}", 'w') as fp:
+        for user, r in user_map.items():
+            positive_sorted = sorted(r, key=lambda x: x[0])
+            for items in positive_sorted:
+                print('\t'.join([str(x) for x in items]), file=fp)
+
+    t2 = timer()
+    print(f"save_to_local_splitByUser took {t2 - t1} secs")
+    return dict_df
+
+
+def result_rename_or_convert(fpath, output_name):
+    source_path_dict = list_dir(fpath, False)
+    fix = "-spark"
+    tgt_path = f"/home/xxx/dien/{output_name}"
+    idx = 0
+    if len(source_path_dict[output_name + fix]) == 1:
+        file_name = source_path_dict[output_name + fix][0]
+        shutil.copy(file_name, f"{tgt_path}")
+    else:
+        for file_name in source_path_dict[output_name + fix]:
+            #print(f"result renamed from {file_name} to {tgt_path}_{idx}")
+            shutil.copy(file_name, f"{tgt_path}_{idx}")
+            shutil.rmtree(file_name, ignore_errors=True)
+            idx += 1
+
+
+def main(option = '--advanced'):
+    fmt = 'adv_pkl'
+
+    path_prefix = "hdfs://dr8s30:9000"
+    current_path = "/dien/output/"
+    original_folder = "/dien/"
+
+    scala_udf_jars = "/home/vmagent/app/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+
+    ##### 1. Start spark and initialize data processor #####
+    t0 = timer()
+    spark = SparkSession.builder.master('spark://dr8s30:7077')\
+        .appName("dien_data_process")\
+        .config("spark.driver.memory", "20G")\
+        .config("spark.driver.memoryOverhead", "10G")\
+        .config("spark.executor.instances", "4")\
+        .config("spark.executor.cores", "32")\
+        .config("spark.executor.memory", "100G")\
+        .config("spark.executor.memoryOverhead", "20G")\
+        .config("spark.driver.extraClassPath", f"{scala_udf_jars}")\
+        .config("spark.executor.extraClassPath", f"{scala_udf_jars}")\
+        .getOrCreate()
+    spark.sparkContext.setLogLevel("ERROR")
+
+    # 1.1 prepare dataFrames
+    # 1.2 create RecDP DataProcessor
+    proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity="1200GB", spark_mode='standalone')
+    t1 = timer()
+    print(f"start spark process took {(t1 - t0)} secs")
+
+    # ===============================================
+    # Data ingestion
+    t0 = timer()
+    reviews_info_df = process_reviews(spark, "%s/%s/raw_data/reviews_Books.json" % (path_prefix, original_folder), proc, "reviews-info")
+    #reviews_info_df.repartition(1).write.format("csv").option('sep', '\t').mode("overwrite").save("%s/%s/j2c_test/reviews-info-spark" % (path_prefix, original_folder))
+    t1 = timer()
+    print(f"parse reviews-info with spark took {(t1 - t0)} secs")
+
+    t0 = timer()
+    item_info_df = process_meta(spark, '%s/%s/raw_data/meta_Books.json' % (path_prefix, original_folder), proc, "item-info")
+    #item_info_df.repartition(1).write.format("csv").option('sep', '\t').mode("overwrite").save("%s/%s/j2c_test/item-info-spark" % (path_prefix, original_folder))
+    t1 = timer()
+    print(f"parse item-info with spark took {(t1 - t0)} secs")
+
+    # ===============================================
+    # basic: Do categorify for all columns
+    # reviews_info_df, item_info_df = load_csv(spark, path_prefix + original_folder)
+
+    # 1. join records with its category and then collapse history 
+    df = reviews_info_df
+    # asin could come from reviews or item, so we do a union here
+    #asin_df = df.select('asin').union(item_info_df.select('asin'))
+    asin_df = df
+
+    dict_dfs = get_dict_for_asin(df, proc)
+    uid_dict_df = load_uid_voc(proc)
+    mid_dict_df = load_mid_voc(proc)
+    cat_dict_df = load_cat_voc(proc)
+
+    df = merge_item_to_reviews(df, item_info_df, proc)
+
+    if fmt == 'adv_pkl':
+        mid_dict_for_merge_df = mid_dict_df.withColumnRenamed('dict_col', 'asin').withColumnRenamed('dict_col_id', 'asin_id')
+        cat_dict_for_merge_df = cat_dict_df.withColumnRenamed('dict_col', 'category').withColumnRenamed('dict_col_id', 'cat_id')
+
+        # generate categorified dict_dfs
+        op_asin_to_id = ModelMerge([{'col_name': 'dict_col', 'dict': mid_dict_for_merge_df.withColumnRenamed('asin', 'dict_col')}])
+        proc.reset_ops([op_asin_to_id])
+        asin_id_dict_df = proc.apply(dict_dfs[0]['dict'])
+        asin_id_dict_df = asin_id_dict_df.select('asin_id').withColumnRenamed('asin_id', 'dict_col')
+        categorified_dict_dfs = []
+        categorified_dict_dfs.append({'col_name': 'asin', 'dict': asin_id_dict_df})
+
+        # generate item id to cat id map
+        op_asin_id_to_cat = ModelMerge([{'col_name': 'asin', 'dict': item_info_df}])
+        proc.reset_ops([op_asin_id_to_cat])
+        asin_cat_df = proc.apply(mid_dict_for_merge_df)
+        asin_cat_df.printSchema()
+
+        op_asin_id_to_cat_id = ModelMerge([{'col_name': 'category', 'dict': cat_dict_for_merge_df}])
+        proc.reset_ops([op_asin_id_to_cat_id])
+        asin_cat_df = proc.apply(asin_cat_df)
+        asin_cat_df.printSchema()
+        asin_cat_df = asin_cat_df.select('asin_id', 'cat_id')
+        asin_cat_df = asin_cat_df.withColumnRenamed('asin_id', 'dict_col').withColumnRenamed('cat_id', 'dict_col_id')
+
+        # for create noclk_hist_category, we should create a dict to mapping from asin to its cat_id
+        df = categorify_dien_data(df, uid_dict_df, mid_dict_df, cat_dict_df, proc, "local_test_splitByUser.parquet")
+        df = collapse_by_hist(df, proc, "collapsed", min_num_hist = 2, max_num_hist = 100)
+        df = add_negative_sample(df, categorified_dict_dfs, asin_cat_df, proc, "records_with_negative_sample")
+
+        df = add_negative_hist_cols(df, categorified_dict_dfs, mid_dict_df, asin_cat_df,  proc, "records_with_negative_hists")
+        df = save_to_local_splitByUser(df, proc, 'local_test_splitByUser')
+
+
+    t1 = timer()
+
+    print(f"Total process time is {(t1 - t0)} secs")
+
+    ####################################
+    
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        main(sys.argv[1])
+    else:
+        main()
diff --git a/RecDP/examples/python_tests/dien/preprocessing_for_training.py b/RecDP/examples/python_tests/dien/preprocessing_for_training.py
new file mode 100644
index 000000000..d924c2ca9
--- /dev/null
+++ b/RecDP/examples/python_tests/dien/preprocessing_for_training.py
@@ -0,0 +1,404 @@
+import init
+
+import os
+import numpy as np
+from pyspark.sql import *
+from pyspark import *
+import pyspark.sql.functions as f
+from pyspark.sql.types import *
+from timeit import default_timer as timer
+import logging
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+
+NUM_INSTS = 1
+
+def load_csv(spark, path):
+    review_id_field = StructField('reviewer_id', StringType())
+    asin_field = StructField('asin', StringType())
+    overall_field = StructField('overall', FloatType())
+    unix_time_field = StructField('unix_review_time', IntegerType())
+    reviews_info_schema = StructType(
+        [review_id_field, asin_field, overall_field, unix_time_field])
+
+    category_field = StructField('category', StringType())
+    item_info_schema = StructType([asin_field, category_field])
+
+    reviews_info_df = spark.read.schema(reviews_info_schema).option('sep', '\t').csv(path + "/reviews-info")
+    item_info_df = spark.read.schema(item_info_schema).option('sep', '\t').csv(path + "/item-info")
+
+    return reviews_info_df, item_info_df
+
+def process_reviews_2(spark, review_file, proc, name):
+    def process_single_review(row):
+        obj = eval(row)
+        return [obj['reviewerID'], obj['asin'], obj['overall'], obj['unixReviewTime']]
+
+    records_df = spark.read.text(review_file)
+    proc_udf = f.udf(lambda x: process_single_review(x), ArrayType(StringType()))
+    records_df = records_df.withColumn('parsed', proc_udf(f.col('value')))
+    records_df = records_df.withColumn('reviewer_id', f.col('parsed').getItem(0))
+    records_df = records_df.withColumn('asin', f.col('parsed').getItem(1))
+    records_df = records_df.withColumn('overall', f.col('parsed').getItem(2))
+    records_df = records_df.withColumn('unix_review_time', f.col('parsed').getItem(3))
+    records_df = records_df.drop('value').drop('parsed')
+    records_df = proc.transform(records_df, name)
+    return records_df
+
+
+def process_reviews(spark, review_file, proc, name):
+    records_df = spark.read.json(review_file)
+    records_df = records_df.select('reviewerID', 'asin', 'overall', 'unixReviewTime')
+    records_df = records_df.withColumnRenamed('reviewerID', 'reviewer_id')
+    records_df = records_df.withColumnRenamed('unixReviewTime', 'unix_review_time')
+    records_df = proc.transform(records_df, name)
+    return records_df
+
+
+def process_meta(spark, meta_file, proc, name):
+    def process_single_meta(row):
+        obj = eval(row)
+        cat = obj['categories'][0][-1]
+        return [obj['asin'], cat]
+
+    meta_df = spark.read.text(meta_file)
+    proc_meta_udf = f.udf(lambda x: process_single_meta(x), ArrayType(StringType()))
+    meta_df = meta_df.withColumn('asin', proc_meta_udf(f.col('value')).getItem(0))
+    meta_df = meta_df.withColumn('category', proc_meta_udf(f.col('value')).getItem(1))
+    meta_df = meta_df.drop('value')
+    meta_df = proc.transform(meta_df, name)
+    return meta_df
+
+def merge_item_to_reviews(df, item_info_df, proc):
+    # merge with item-info and reviews-info firstly
+    op_asin_to_cat_id = ModelMerge([{'col_name': 'asin', 'dict': item_info_df}])
+    proc.reset_ops([op_asin_to_cat_id])
+    df = proc.apply(df)
+    return df
+
+
+def collapse_by_hist(df, proc, output_name, min_num_hist = 0, max_num_hist = 100):
+    op_collapse_by_hist = CollapseByHist(['asin', 'category'], by = ['reviewer_id'], orderBy = ['unix_review_time', 'asin'], minNumHist = min_num_hist, maxNumHist = 100)
+    proc.reset_ops([op_collapse_by_hist])
+    
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()
+    print(f"collapse hist took {t2 - t1} secs")
+    return df
+
+
+def get_dict_for_asin(df, proc):
+    dict_dfs = []
+    dict_names = ['asin']
+    dict_dfs = [{'col_name': name, 'dict': df.select(spk_func.col(name).alias('dict_col'))} for name in dict_names]
+    return dict_dfs
+
+
+def add_negative_sample(df, dict_dfs, mid_cat_df, proc, output_name):
+    # add negative_sample as new row
+    op_negative_sample = NegativeSample(['asin'], dict_dfs)
+    # negative row category is not correct
+    mid_cat_df = mid_cat_df.withColumnRenamed('dict_col', 'asin').withColumnRenamed('dict_col_id', 'category')
+    op_drop_category = DropFeature(['category'])
+    op_add_category = ModelMerge([{'col_name': 'asin', 'dict': mid_cat_df}])
+
+    op_select = SelectFeature(['pos', 'reviewer_id', 'asin', 'category', 'hist_asin', 'hist_category'])
+    proc.reset_ops([op_negative_sample, op_drop_category, op_add_category, op_select])
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()    
+    print(f"add_negative_sample took {t2 - t1} secs")
+    return df
+
+
+def add_negative_hist_cols(df, dict_dfs, mid_dict_df, mid_cat_df, proc, output_name):
+    # since we already categorified asin, we need to create a number_asin dict
+    new_dict_dfs = dict_dfs
+    new_dict_dfs[0]['col_name'] = 'hist_asin'
+    op_columns_sample = NegativeFeature({'noclk_hist_asin': 'hist_asin'}, new_dict_dfs, doSplit=True, sep='\x02', negCnt = 5)
+    proc.reset_ops([op_columns_sample])
+    df = proc.apply(df)
+
+    # add one col in df to create noclk_hist_category
+    df = df.withColumn('noclk_hist_category', f.col('noclk_hist_asin'))
+
+    new_dict_dfs = []
+    new_dict_dfs.append({'col_name': 'noclk_hist_category', 'dict': mid_cat_df})
+    op_categorify = Categorify(['noclk_hist_category'], dict_dfs=new_dict_dfs, doSplit=True, multiLevelSplit=True, multiLevelSep=['|'])
+    proc.reset_ops([op_categorify])
+    df = proc.apply(df)
+
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()    
+    print(f"add_negative_hist_cols took {t2 - t1} secs")
+    return df
+
+
+def load_processed_csv(spark, data_dir):
+    label_field = StructField('pos', IntegerType())
+    review_id_field = StructField('reviewer_id', StringType())
+    asin_field = StructField('asin', StringType())
+    category_field = StructField('category', StringType())
+    hist_asin_field = StructField('hist_asin', StringType())
+    hist_category_field = StructField('hist_category', StringType())
+    csv_schema = StructType(
+        [label_field, review_id_field, asin_field, category_field, hist_asin_field, hist_category_field])
+
+    return spark.read.schema(csv_schema).option('sep', '\t').csv(data_dir)
+
+
+def categorify_dien_data(df, user_df, asin_df, cat_df, proc, output_name):
+    dict_dfs = []
+    dict_dfs.append({'col_name': 'reviewer_id', 'dict': user_df})
+    dict_dfs.append({'col_name': 'asin', 'dict': asin_df})
+    dict_dfs.append({'col_name': 'category', 'dict': cat_df})
+
+    op_categorify = Categorify(['reviewer_id', 'asin', 'category'], dict_dfs=dict_dfs)
+    op_fillna = FillNA(['asin', 'category'], 0)
+    proc.reset_ops([op_categorify, op_fillna])
+
+    t1 = timer()
+    df = proc.transform(df, output_name)
+    t2 = timer()
+    print(f"categorify took {t2 - t1} secs")
+    return df
+
+
+def save_to_voc(df, proc, cols, default_name, default_v, output_name):
+    import pickle
+    col_name = ''
+    dtypes_list = []
+    if isinstance(cols, list):
+        col_name = cols[0]
+        dtypes_list = df.select(*cols).dtypes
+    elif isinstance(cols, str):
+        col_name = cols
+        dtypes_list = df.select(cols).dtypes
+    else:
+        raise ValueError("save_to_voc expects cols as String or list of String")
+
+    to_select = []
+    if len(dtypes_list) == 1 and 'array' not in dtypes_list[0][1]:
+        to_select.append(f.col(dtypes_list[0][0]))
+        dict_df = df.select(*to_select)
+    else:
+        for name, dtype in dtypes_list:
+            if 'array' in dtype:
+                to_select.append(f.col(name))
+            else:
+                to_select.append(f.array(f.col(name)))
+
+        dict_df = df.withColumn(col_name, f.array_union(*to_select))
+        dict_df = dict_df.select(f.explode(f.col(col_name)).alias(col_name))
+
+    op_gen_dict = GenerateDictionary([f"{col_name}"])
+    proc.reset_ops([op_gen_dict])
+    dict_dfs = proc.generate_dicts(dict_df)
+    dict_df = dict_dfs[0]['dict']
+    voc = {}
+    voc.update(dict((row['dict_col'], row['dict_col_id']) for row in dict_df.collect()))
+    voc_count = dict_df.count()
+    pickle.dump(voc_count, open("/home/xxx/dien/" + f'/{output_name}.pkl', "wb"), protocol=0)
+    pickle.dump(voc, open("/home/xxx/dien/" + f'/test_{output_name}.pkl', "wb"), protocol=0)
+
+    return dict_df
+
+
+def save_to_uid_voc(df, proc):
+    # saving (using python)
+    # build uid_dict, mid_dict and cat_dict
+    t1 = timer()
+    dict_df = save_to_voc(df, proc, ['reviewer_id'], 'A1Y6U82N6TYZPI', 0, 'uid_voc')
+    t2 = timer()
+    print(f"save_to_uid_voc took {t2 - t1} secs")
+    return dict_df
+    
+
+def save_to_mid_voc(df, proc):
+    t1 = timer()
+    dict_df = save_to_voc(df, proc, ['asin'], 'default_mid', 0, 'mid_voc')
+    t2 = timer()
+    print(f"save_to_mid_voc took {t2 - t1} secs")
+    return dict_df
+    
+    
+def save_to_cat_voc(df, proc):
+    t1 = timer()
+    dict_df = save_to_voc(df, proc, ['category'], 'default_cat', 0, 'cat_voc')
+    t2 = timer()
+    print(f"save_to_cat_voc took {t2 - t1} secs")
+    return dict_df
+    
+    
+def save_to_local_splitByUser(df, proc, output_name):
+    hint = 'byRename'
+    #hint = 'byRewrite'
+    t1 = timer()
+    if 'noclk_hist_asin' in df.columns:
+        dict_df = df.select('pos', 'reviewer_id', 'asin', 'category',
+                            f.expr("concat_ws('\x02', hist_asin)"),
+                            f.expr("concat_ws('\x02', hist_category)"),
+                            f.expr("concat_ws('\x02', noclk_hist_asin)"),
+                            f.expr("concat_ws('\x02', noclk_hist_category)"))
+        if hint == 'byRename':
+            dict_df.repartition(NUM_INSTS).write.format("csv").option('sep', '\t').mode("overwrite").save(f"file:///home/vmagent/app/recdp/examples/python_tests/{proc.current_path}/{output_name}-spark")
+            result_rename_or_convert("/home/vmagent/app/recdp/examples/python_tests/" + proc.current_path, output_name)
+            t2 = timer()
+            print(f"save_to_local_splitByUser took {t2 - t1} secs")
+            return dict_df
+        else:
+            collected = [[c1, c2, c3, c4, c5, c6, c7, c8] for (c1, c2, c3, c4, c5, c6, c7, c8) in dict_df.collect()]
+    else:
+        dict_df = df.select('pos', 'reviewer_id', 'asin', 'category',
+                            f.expr("concat_ws('\x02', hist_asin)"),
+                            f.expr("concat_ws('\x02', hist_category)"))
+        if hint == 'byRename':
+            dict_df.repartition(NUM_INSTS).write.format("csv").option('sep', '\t').mode("overwrite").save(f"file:///home/vmagent/app/recdp/examples/python_tests/{proc.current_path}/{output_name}-spark")
+            result_rename_or_convert("/home/vmagent/app/recdp/examples/python_tests/" + proc.current_path, output_name)
+            t2 = timer()
+            print(f"save_to_local_splitByUser took {t2 - t1} secs")
+            return dict_df
+        else:
+            collected = [[c1, c2, c3, c4, c5, c6] for (c1, c2, c3, c4, c5, c6) in dict_df.collect()]
+
+    user_map = {}
+    for items in collected:
+        if items[1] not in user_map:
+            user_map[items[1]] = []
+        user_map[items[1]].append(items)
+    with open("/home/vmagent/app/recdp/examples/python_tests/" + proc.current_path + f"/{output_name}", 'w') as fp:
+        for user, r in user_map.items():
+            positive_sorted = sorted(r, key=lambda x: x[0])
+            for items in positive_sorted:
+                print('\t'.join([str(x) for x in items]), file=fp)
+
+    t2 = timer()
+    print(f"save_to_local_splitByUser took {t2 - t1} secs")
+    return dict_df
+
+
+def result_rename_or_convert(fpath, output_name):
+    source_path_dict = list_dir(fpath, False)
+    fix = "-spark"
+    tgt_path = f"/home/xxx/dien/{output_name}"
+    idx = 0
+    if len(source_path_dict[output_name + fix]) == 1:
+        file_name = source_path_dict[output_name + fix][0]
+        shutil.copy(file_name, f"{tgt_path}")
+    else:
+        for file_name in source_path_dict[output_name + fix]:
+            #print(f"result renamed from {file_name} to {tgt_path}_{idx}")
+            shutil.copy(file_name, f"{tgt_path}_{idx}")
+            shutil.rmtree(file_name, ignore_errors=True)
+            idx += 1
+
+
+def main(option = '--advanced'):
+    fmt = 'adv_pkl'
+
+    path_prefix = "file://"
+    current_path = "/home/vmagent/app/recdp/examples/python_tests/dien/output/"
+    original_folder = "/home/vmagent/app/recdp/examples/python_tests/dien/"
+
+    scala_udf_jars = "/home/vmagent/app/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+
+    ##### 1. Start spark and initialize data processor #####
+    t0 = timer()
+    spark = SparkSession.builder.master('spark://dr8s30:7077')\
+        .appName("dien_data_process")\
+        .config("spark.driver.memory", "20G")\
+        .config("spark.driver.memoryOverhead", "10G")\
+        .config("spark.executor.instances", "4")\
+        .config("spark.executor.cores", "32")\
+        .config("spark.executor.memory", "100G")\
+        .config("spark.executor.memoryOverhead", "20G")\
+        .config("spark.driver.extraClassPath", f"{scala_udf_jars}")\
+        .config("spark.executor.extraClassPath", f"{scala_udf_jars}")\
+        .getOrCreate()
+    spark.sparkContext.setLogLevel("ERROR")
+
+    # 1.1 prepare dataFrames
+    # 1.2 create RecDP DataProcessor
+    proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity="1200GB", spark_mode='standalone')
+    t1 = timer()
+    print(f"start spark process took {(t1 - t0)} secs")
+
+    # ===============================================
+    # Data ingestion
+    t0 = timer()
+    reviews_info_df = process_reviews(spark, "%s/%s/raw_data/reviews_Books.json" % (path_prefix, original_folder), proc, "reviews-info")
+    #reviews_info_df.repartition(1).write.format("csv").option('sep', '\t').mode("overwrite").save("%s/%s/j2c_test/reviews-info-spark" % (path_prefix, original_folder))
+    t1 = timer()
+    print(f"parse reviews-info with spark took {(t1 - t0)} secs")
+
+    t0 = timer()
+    item_info_df = process_meta(spark, '%s/%s/raw_data/meta_Books.json' % (path_prefix, original_folder), proc, "item-info")
+    #item_info_df.repartition(1).write.format("csv").option('sep', '\t').mode("overwrite").save("%s/%s/j2c_test/item-info-spark" % (path_prefix, original_folder))
+    t1 = timer()
+    print(f"parse item-info with spark took {(t1 - t0)} secs")
+
+    # ===============================================
+    # basic: Do categorify for all columns
+    # reviews_info_df, item_info_df = load_csv(spark, path_prefix + original_folder)
+
+    # 1. join records with its category and then collapse history 
+    df = reviews_info_df
+    # asin could come from reviews or item, so we do a union here
+    #asin_df = df.select('asin').union(item_info_df.select('asin'))
+    asin_df = df
+
+    dict_dfs = get_dict_for_asin(df, proc)
+    uid_dict_df = save_to_uid_voc(df, proc)
+    mid_dict_df = save_to_mid_voc(asin_df, proc)
+    cat_dict_df = save_to_cat_voc(item_info_df, proc)
+
+    df = merge_item_to_reviews(df, item_info_df, proc)
+
+    if fmt == 'adv_pkl':
+        mid_dict_for_merge_df = mid_dict_df.withColumnRenamed('dict_col', 'asin').withColumnRenamed('dict_col_id', 'asin_id')
+        cat_dict_for_merge_df = cat_dict_df.withColumnRenamed('dict_col', 'category').withColumnRenamed('dict_col_id', 'cat_id')
+
+        # generate categorified dict_dfs
+        op_asin_to_id = ModelMerge([{'col_name': 'dict_col', 'dict': mid_dict_for_merge_df.withColumnRenamed('asin', 'dict_col')}])
+        proc.reset_ops([op_asin_to_id])
+        asin_id_dict_df = proc.apply(dict_dfs[0]['dict'])
+        asin_id_dict_df = asin_id_dict_df.select('asin_id').withColumnRenamed('asin_id', 'dict_col')
+        categorified_dict_dfs = []
+        categorified_dict_dfs.append({'col_name': 'asin', 'dict': asin_id_dict_df})
+
+        # generate item id to cat id map
+        op_asin_id_to_cat = ModelMerge([{'col_name': 'asin', 'dict': item_info_df}])
+        proc.reset_ops([op_asin_id_to_cat])
+        asin_cat_df = proc.apply(mid_dict_for_merge_df)
+        asin_cat_df.printSchema()
+
+        op_asin_id_to_cat_id = ModelMerge([{'col_name': 'category', 'dict': cat_dict_for_merge_df}])
+        proc.reset_ops([op_asin_id_to_cat_id])
+        asin_cat_df = proc.apply(asin_cat_df)
+        asin_cat_df.printSchema()
+        asin_cat_df = asin_cat_df.select('asin_id', 'cat_id')
+        asin_cat_df = asin_cat_df.withColumnRenamed('asin_id', 'dict_col').withColumnRenamed('cat_id', 'dict_col_id')
+
+        # for create noclk_hist_category, we should create a dict to mapping from asin to its cat_id
+        df = categorify_dien_data(df, uid_dict_df, mid_dict_df, cat_dict_df, proc, "local_train_splitByUser.parquet")
+        df = collapse_by_hist(df, proc, "collapsed", min_num_hist = 2, max_num_hist = 100)
+        df = add_negative_sample(df, categorified_dict_dfs, asin_cat_df, proc, "records_with_negative_sample")
+
+        df = add_negative_hist_cols(df, categorified_dict_dfs, mid_dict_df, asin_cat_df,  proc, "records_with_negative_hists")
+        df = save_to_local_splitByUser(df, proc, 'local_train_splitByUser')
+
+
+    t1 = timer()
+
+    print(f"Total process time is {(t1 - t0)} secs")
+
+    ####################################
+    
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        main(sys.argv[1])
+    else:
+        main()
diff --git a/RecDP/examples/python_tests/dien/run b/RecDP/examples/python_tests/dien/run
new file mode 100755
index 000000000..ba589113e
--- /dev/null
+++ b/RecDP/examples/python_tests/dien/run
@@ -0,0 +1,4 @@
+#!/usr/bin/bash
+
+python j2c_spark.py
+python dien_data_process_v1.py
diff --git a/RecDP/examples/python_tests/dlrm/DLRM_Performance.py b/RecDP/examples/python_tests/dlrm/DLRM_Performance.py
new file mode 100644
index 000000000..b9b576f0d
--- /dev/null
+++ b/RecDP/examples/python_tests/dlrm/DLRM_Performance.py
@@ -0,0 +1,90 @@
+import init
+
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+
+import logging
+from timeit import default_timer as timer
+import os
+from pyspark import *
+from pyspark.sql import *
+from pyspark.sql.functions import *
+from pyspark.sql.types import *
+import numpy as np
+import pandas as pd
+
+# Define Schema
+LABEL_COL = 0
+INT_COLS = list(range(1, 14))
+CAT_COLS = list(range(14, 40))
+label_fields = [StructField('_c%d' % LABEL_COL, IntegerType())]
+int_fields = [StructField('_c%d' % i, IntegerType()) for i in INT_COLS]
+str_fields = [StructField('_c%d' % i, StringType()) for i in CAT_COLS]
+schema = StructType(label_fields + int_fields + str_fields)
+
+to_be_categorified = [23, 35, 14, 33]
+
+def categorifyAllFeatures(df, proc, output_name="categorified", gen_dict=False, enable_freqlimit=False):
+    dict_dfs = []
+    to_categorify_cols = ['_c%d' % i for i in CAT_COLS]
+    #to_categorify_cols = ['_c%d' % i for i in to_be_categorified]
+    if gen_dict:
+        # only call below function when target dicts were not pre-prepared        
+        op_gen_dict = GenerateDictionary(to_categorify_cols, isParquet=False)
+        proc.reset_ops([op_gen_dict])
+        t1 = timer()
+        dict_dfs = proc.generate_dicts(df)
+        t2 = timer()
+        print("Generate Dictionary took %.3f" % (t2 - t1))
+    else:
+        # or we can simply load from pre-gened
+        dict_dfs = [{'col_name': name, 'dict': proc.spark.read.parquet(
+            "%s/%s/%s/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in to_categorify_cols]    
+
+    if enable_freqlimit:
+        dict_dfs = [{'col_name': dict_df['col_name'], 'dict': dict_df['dict'].filter('count >= 15')} for dict_df in dict_dfs]
+
+    # start to do categorify
+    op_categorify = Categorify(to_categorify_cols, dict_dfs=dict_dfs)
+    op_fillna_for_categorified = FillNA(to_categorify_cols, 0)
+    proc.append_ops([op_categorify, op_fillna_for_categorified])
+    t1 = timer()
+    df = proc.transform(df, name=output_name)
+    t2 = timer()
+    print("Categorify took %.3f" % (t2 - t1))
+    
+    return df
+
+def main():
+	path_prefix = "hdfs://"
+	current_path = "/dlrm/"
+	csv_folder = "/dlrm/csv_raw_data"
+	#path = os.path.join(path_prefix, file)
+
+	scala_udf_jars = "/home/vmagent/app/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+
+	##### 1. Start spark and initialize data processor #####
+	t0 = timer()
+	spark = SparkSession.builder.master('local[80]')\
+		.appName("DLRM")\
+		.config("spark.driver.memory", "400G")\
+		.config("spark.driver.memoryOverhead", "80G")\
+		.config("spark.executor.cores", "80")\
+		.config("spark.driver.extraClassPath", f"{scala_udf_jars}")\
+		.getOrCreate()
+
+	#files = ["day_%d" % i for i in range(0, 24)]
+	files = ["day_0"]
+	file_names = [os.path.join(path_prefix, csv_folder, filename) for filename in files]
+
+	proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity="1200GB", spark_mode='local')
+	df = spark.read.schema(schema).option('sep', '\t').csv(file_names)
+	#df = spark.read.parquet("/dlrm/categorified_stage1")
+	df = categorifyAllFeatures(df, proc, output_name="dlrm_categorified", gen_dict=True, enable_freqlimit=True)
+	t1 = timer()
+
+	print(f"Total process time is {(t1 - t0)} secs")
+
+
+if __name__ == "__main__":
+	main()
\ No newline at end of file
diff --git a/RecDP/examples/python_tests/dlrm/DLRM_Performance_nativesql.py b/RecDP/examples/python_tests/dlrm/DLRM_Performance_nativesql.py
new file mode 100644
index 000000000..7a0fe67df
--- /dev/null
+++ b/RecDP/examples/python_tests/dlrm/DLRM_Performance_nativesql.py
@@ -0,0 +1,114 @@
+import init
+
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+
+import logging
+from timeit import default_timer as timer
+import os
+from pyspark import *
+from pyspark.sql import *
+from pyspark.sql.functions import *
+from pyspark.sql.types import *
+import numpy as np
+import pandas as pd
+
+# Define Schema
+LABEL_COL = 0
+INT_COLS = list(range(1, 14))
+CAT_COLS = list(range(14, 40))
+label_fields = [StructField('_c%d' % LABEL_COL, IntegerType())]
+int_fields = [StructField('_c%d' % i, IntegerType()) for i in INT_COLS]
+str_fields = [StructField('_c%d' % i, StringType()) for i in CAT_COLS]
+schema = StructType(label_fields + int_fields + str_fields)
+
+to_be_categorified = [23, 35, 14, 33]
+
+def categorifyAllFeatures(df, proc, output_name="categorified", gen_dict=False, enable_freqlimit=False):
+    dict_dfs = []
+    to_categorify_cols = ['_c%d' % i for i in CAT_COLS]
+    #to_categorify_cols = ['_c%d' % i for i in to_be_categorified]
+    if gen_dict:
+        # only call below function when target dicts were not pre-prepared        
+        op_gen_dict = GenerateDictionary(to_categorify_cols, isParquet=True)
+        proc.reset_ops([op_gen_dict])
+        t1 = timer()
+        dict_dfs = proc.generate_dicts(df)
+        t2 = timer()
+        print("Generate Dictionary took %.3f" % (t2 - t1))
+    else:
+        # or we can simply load from pre-gened
+        dict_dfs = [{'col_name': name, 'dict': proc.spark.read.format("arrow").load(
+            "%s/%s/%s/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in to_categorify_cols]    
+
+    if enable_freqlimit:
+        dict_dfs = [{'col_name': dict_df['col_name'], 'dict': dict_df['dict'].filter('count >= 15')} for dict_df in dict_dfs]
+    else:
+        dict_dfs = [{'col_name': dict_df['col_name'], 'dict': dict_df['dict'].filter('dict_col_id > 0')} for dict_df in dict_dfs]
+
+    # start to do categorify
+    op_categorify = Categorify(to_categorify_cols, dict_dfs=dict_dfs)
+    op_fillna_for_categorified = FillNA(to_categorify_cols, 0)
+    proc.append_ops([op_categorify, op_fillna_for_categorified])
+    t1 = timer()
+    df = proc.transform(df, name=output_name, df_cnt = 4373472329)
+    t2 = timer()
+    print("Categorify took %.3f" % (t2 - t1))
+    
+    return df
+
+
+def main():
+    path_prefix = "hdfs://"
+    current_path = "/dlrm_nativesql/"
+    csv_folder = "/dlrm/csv_raw_data"
+    parquet_folder = "/dlrm_nativesql/dlrm_parquet"
+    #path = os.path.join(path_prefix, file)
+
+    scala_udf_jars = "/home/vmagent/app/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+    native_sql_path = "/home/vmagent/app/native-sql-engine/native-sql-engine/core/target/spark-columnar-core-1.2.0-snapshot-jar-with-dependencies.jar"
+    native_arrow_datasource_path = "/home/vmagent/app/native-sql-engine/arrow-data-source/standard/target/spark-arrow-datasource-standard-1.2.0-snapshot-jar-with-dependencies.jar"
+
+    exec_scala_udf_jars = "/mnt/nvme2/chendi/BlueWhale/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+    exec_native_sql_path = "/mnt/nvme2/chendi/intel-bigdata/OAP/native-sql-engine/native-sql-engine/core/target/spark-columnar-core-1.2.0-snapshot-jar-with-dependencies.jar"
+    exec_native_arrow_datasource_path = "/mnt/nvme2/chendi/intel-bigdata/OAP/native-sql-engine/arrow-data-source/standard/target/spark-arrow-datasource-standard-1.2.0-snapshot-jar-with-dependencies.jar"
+
+    ##### 1. Start spark and initialize data processor #####
+    t0 = timer()
+    spark = SparkSession.builder.master('yarn')\
+        .appName("DLRM_nativesql")\
+        .config("spark.memory.offHeap.size", "600G")\
+        .config("spark.driver.memory", "30G")\
+        .config("spark.driver.memoryOverhead", "30G")\
+        .config("spark.executor.instances", "10")\
+        .config("spark.executor.cores", "6")\
+        .config("spark.executor.memory", "30G")\
+        .config("spark.executor.memoryOverhead", "20G")\
+        .config("spark.sql.broadcastTimeout", "7200")\
+        .config("spark.cleaner.periodicGC.interval", "60min")\
+        .config("spark.driver.extraClassPath", 
+                f"{exec_native_sql_path}:{exec_native_arrow_datasource_path}:{exec_scala_udf_jars}")\
+        .config("spark.executor.extraClassPath", 
+                f"{exec_native_sql_path}:{exec_native_arrow_datasource_path}:{exec_scala_udf_jars}")\
+        .config("spark.sql.extensions", "com.intel.oap.ColumnarPlugin, com.intel.oap.spark.sql.ArrowWriteExtension")\
+        .config("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager")\
+        .config("spark.sql.execution.sort.spillThreshold", "536870912")\
+        .config("spark.oap.sql.columnar.sortmergejoin.lazyread", "true")\
+        .config("spark.sql.adaptive.enabled", "true")\
+        .getOrCreate()
+
+    files = ["day_%d" % i for i in range(0, 24)]
+    #files = ["day_0"]
+    file_names = [os.path.join(path_prefix, parquet_folder, filename) for filename in files]
+
+    proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity="1200GB", enable_gazelle=True, spark_mode='standalone')
+    #df = spark.read.format("arrow").option("originalFormat", "csv").option('sep', '\t').load(file_names)
+    df = spark.read.format("arrow").load(file_names)
+    df = categorifyAllFeatures(df, proc, output_name="dlrm_categorified", gen_dict=False, enable_freqlimit=False)
+    t1 = timer()
+
+    print(f"Total process time is {(t1 - t0)} secs")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/RecDP/examples/python_tests/dlrm/init.py b/RecDP/examples/python_tests/dlrm/init.py
new file mode 100644
index 000000000..e0981db4f
--- /dev/null
+++ b/RecDP/examples/python_tests/dlrm/init.py
@@ -0,0 +1,8 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
+import findspark
+findspark.init()
diff --git a/RecDP/examples/python_tests/haystack_sod/README.md b/RecDP/examples/python_tests/haystack_sod/README.md
new file mode 100644
index 000000000..32fc50789
--- /dev/null
+++ b/RecDP/examples/python_tests/haystack_sod/README.md
@@ -0,0 +1,65 @@
+# How to run
+
+```
+# download kaggle dataset
+python
+>> import kaggle
+>>         kaggle.api.authenticate()
+>>         kaggle.api.dataset_download_files('stackoverflow/stacksample', path=SO_PATH, unzip=True, quiet=False)
+
+# or download from website https://www.kaggle.com/stackoverflow/stacksample
+        
+# cp stack-overflow folder to ${current_path}/recdp/examples/python_tests/haystack_sod/
+
+# run elastic search server on master node(if you are running on multiple node, choose one)
+docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.9.2
+
+# run below codes to all physical nodes
+# Fix the recdp path to align with your env
+docker run -it --net=host -v /mnt/nvme2/chendi/BlueWhale/recdp:/home/vmagent/app/recdp -v /mnt/nvme2/chendi/BlueWhale/applications.ai.appliedml.haystack/:/home/haystack -v /mnt/nvme2/chendi/BlueWhale/applications.ai.appliedml.haystack/haystack/:/home/user/haystack xuechendi/haystack-cpu-recdp /bin/bash
+
+# after enter docker, start hdfs, spark master and workers
+
+# for first time, format hdfs
+${HADOOP_HOME}/bin/hdfs namenode -format
+
+# start hdfs
+${HADOOP_HOME}/sbin/start-dfs.sh
+
+# start spark
+vim ${SPARK_HOME}/conf/workers
+# add all workers hostname
+# Example
+sr602
+sr603
+sr604
+...
+
+# start spark master on one of your node
+${SPARK_HOME}/sbin/start-master.sh
+
+# start spark workers on all nodes
+${SPARK_HOME}/sbin/start-worker.sh spark://${master_hostname}:7077
+
+# start spark history server
+${SPARK_HOME}/sbin/start-history-server.sh
+
+# run gen sod script
+cd /home/vmagent/app/recdp/examples/python_tests/haystack_sod/
+
+# doing spark configuration in gen_sod_to_documentstore.py
+EMBEDDING_PARALLELISM = 8     # num_nodes * 2
+NUMCORES_PER_SOCKET = 64
+EMBEDDING_BATCH_SIZE = 262144 # 256k
+ELASTIC_SERVER = ${master_hostname}
+
+python gen_sod_to_documentstore.py
+```
+
+![image](https://user-images.githubusercontent.com/4355494/133181494-b55a9366-b61b-4cad-a498-907949ec67ca.png)
+
+##### checkout inferencing progress in spark executor stderr log
+![image](https://user-images.githubusercontent.com/4355494/133181797-c1a2a1fe-230c-45ff-a536-89ed143a51c8.png)
+
+
+
diff --git a/RecDP/examples/python_tests/haystack_sod/gen_milvus_dpr.py b/RecDP/examples/python_tests/haystack_sod/gen_milvus_dpr.py
new file mode 100644
index 000000000..c75a03ee4
--- /dev/null
+++ b/RecDP/examples/python_tests/haystack_sod/gen_milvus_dpr.py
@@ -0,0 +1,148 @@
+import init
+import init_haystack        
+from haystack.retriever.dense import DensePassageRetriever
+from haystack.utils import print_answers
+
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+
+import logging
+from timeit import default_timer as timer
+import os
+from pyspark import *
+from pyspark.sql import *
+from pyspark.sql.functions import *
+from pyspark.sql.types import *
+import numpy as np
+import pandas as pd
+from bs4 import BeautifulSoup
+
+SO_PATH = './stack-overflow/'
+EMBEDDING_PARALLELISM = 2
+NUMCORES_PER_SOCKET = 52
+EMBEDDING_BATCH_SIZE = 32768
+
+def main():
+    path_prefix = "file://"
+    current_path = os.getcwd()
+    data_folder = "stack-overflow/"
+
+    if not os.path.isdir(f"{current_path}/stack-overflow/"):
+        import kaggle
+        kaggle.api.authenticate()
+        kaggle.api.dataset_download_files('stackoverflow/stacksample', path=SO_PATH, unzip=True, quiet=False)
+
+    ##### 1. Start spark and initialize data processor #####
+    spark = SparkSession.builder.master('spark://sr602:7077')\
+        .appName("HAYSTACK_SOD")\
+        .config("spark.driver.memory", "60G")\
+        .config("spark.executor.memory", "250G")\
+        .config("spark.executor.instances", EMBEDDING_PARALLELISM)\
+        .config("spark.executor.cores", NUMCORES_PER_SOCKET)\
+        .config("spark.driver.maxResultSize", "16G")\
+        .config("spark.sql.execution.arrow.maxRecordsPerBatch", EMBEDDING_BATCH_SIZE)\
+        .config("spark.cleaner.periodicGC.interval", "7200min")\
+        .getOrCreate()
+
+#    SO_FILE = {'question': 'Questions.csv', 'answer': 'Answers.csv'}
+#    file_names = dict((key, path_prefix + os.path.join(current_path, data_folder, filename)) for key, filename in SO_FILE.items())
+#
+#    proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity="1200GB", spark_mode='local')
+#
+#    ############## Process answer ###############
+#    t0 = timer()
+#    if not os.path.isdir(f"{current_path}/answer.parquet"):
+#        answers_pdf = pd.read_csv(file_names['answer'], encoding="ISO-8859-1")
+#        answers_pdf.drop(columns=['OwnerUserId', 'CreationDate', 'Id'], inplace=True)
+#        answer_df = spark.createDataFrame(answers_pdf)
+#        answer_df = proc.transform(answer_df, name="answer.parquet")        
+#    else:
+#        answer_df = spark.read.parquet(f"{path_prefix}{current_path}/answer.parquet")
+#    t1 = timer()
+#    print(f"Load Answer.csv took {(t1 - t0)} secs")
+#
+#
+#    t0 = timer()
+#    answer_df = answer_df.withColumnRenamed("ParentId", "Id").withColumnRenamed("Body", "TopAnswer-html")
+#    op_collapse = CollapseByHist(by="Id", orderBy="Score")
+#    proc.reset_ops([op_collapse])
+#    top_answers_df = proc.apply(answer_df)
+#    top_answers_df = top_answers_df.drop('Score')
+#
+#    top_answers_df = proc.transform(top_answers_df, name="top_answer_processed.parquet")
+#    t1 = timer()
+#    print(f"Process Answer.csv took {(t1 - t0)} secs")
+#
+#    ############## Process question ###############
+#    t0 = timer()
+#    if not os.path.isdir(f"{current_path}/question.parquet"):
+#        questions_pdf = pd.read_csv(file_names['question'], encoding="ISO-8859-1")
+#        questions_pdf.drop(columns=['OwnerUserId', 'CreationDate', 'ClosedDate', 'Score'], inplace=True)
+#        question_df = spark.createDataFrame(questions_pdf)
+#        question_df = proc.transform(question_df, name="question.parquet")        
+#    else:
+#        question_df = spark.read.parquet(f"{path_prefix}{current_path}/question.parquet")
+#    t1 = timer()
+#    print(f"Load Question.csv took {(t1 - t0)} secs")
+#
+#    t0 = timer()
+#    convert_udf = f.udf(lambda x: BeautifulSoup(x, "lxml").get_text(), StringType())
+#    question_df = question_df.withColumnRenamed("Title", "text").withColumnRenamed("Body", "Question-html")
+#    op_merge = ModelMerge([{'col_name': ['Id'], 'dict': top_answers_df}])
+#    proc.reset_ops([op_merge])
+#    question_df = proc.apply(question_df)
+#    question_df = question_df.filter(f.col("TopAnswer-html").isNotNull())
+#    question_df = question_df.drop('Id')
+#    question_df = question_df.dropDuplicates()
+#    question_df = question_df.na.fill({'Question-html': '', 'TopAnswer-html': ''})
+#    question_df = question_df.withColumn("question-body", convert_udf(f.col("Question-html")))
+#    question_df = question_df.withColumn("answer", convert_udf(f.col("TopAnswer-html")))
+#    question_df = question_df.drop("Question-html").drop("TopAnswer-html")
+#
+#    question_df = proc.transform(question_df, name="question_processed.parquet")
+#    question_df.show()
+#    t1 = timer()
+#    print(f"Process Question.csv took {(t1 - t0)} secs")
+
+    ############# Embedding and write to DocumentStore ###############
+    question_df = spark.read.parquet(f"{path_prefix}{current_path}/question_processed.parquet")
+    question_df = question_df.repartition(EMBEDDING_PARALLELISM)
+
+    import init_haystack
+    from haystack.document_store import MilvusDocumentStore
+    from haystack.utils import launch_milvus
+
+    from haystack.utils import print_answers
+    from haystack.utils import launch_milvus
+    print('launching milvus & retriever...')
+
+    # get embeddings for question
+    from typing import Iterator
+    @pandas_udf(ArrayType(DoubleType()))
+    def pandas_udf_embed_queries(iterator: Iterator[pd.DataFrame]) -> Iterator[pd.Series]:
+        print(f"start launcu_milvus")
+        launch_milvus()
+        document_store = MilvusDocumentStore(host="localhost", username="", password="",
+                                                index="flat")
+        print(f"start create retriever")
+        retriever = DensePassageRetriever(document_store=document_store,
+                                    query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
+                                    passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
+                                    use_gpu=False,batch_size=512,
+                                    embed_title=True)
+        for df in iterator:
+            print(df.head())
+            document_store.write_documents(df.to_dict(orient="records"))
+            document_store.update_embeddings(document_store)
+            yield df
+
+    t0 = timer()
+    print(f"Start to do EmbeddingRetriever embed_queries ...")
+    debug_df = pandas_udf_embed_queries(f.struct(f.col("text"), f.col("question-body"), f.col("answer")))
+    t1 = timer()
+    print(f'EmbeddingRetriever embed_queries took {(t1 - t0)} secs')
+
+
+if __name__ == "__main__":
+    main()
+
diff --git a/RecDP/examples/python_tests/haystack_sod/gen_sod.py b/RecDP/examples/python_tests/haystack_sod/gen_sod.py
new file mode 100644
index 000000000..1dee7904d
--- /dev/null
+++ b/RecDP/examples/python_tests/haystack_sod/gen_sod.py
@@ -0,0 +1,104 @@
+import init
+
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+
+import logging
+from timeit import default_timer as timer
+import os
+from pyspark import *
+from pyspark.sql import *
+from pyspark.sql.functions import *
+from pyspark.sql.types import *
+import numpy as np
+import pandas as pd
+from bs4 import BeautifulSoup
+
+MODE = 0  # 0: actual, 1: debug
+SO_PATH = './stack-overflow/'
+
+def main():
+    path_prefix = "file://"
+    current_path = "/home/vmagent/app/recdp/examples/python_tests/haystack_sod/"
+    data_folder = "stack-overflow/"
+    #data_folder = ""
+
+    if not os.path.isdir(f"{current_path}/stack-overflow/"):
+        import kaggle
+        kaggle.api.authenticate()
+        kaggle.api.dataset_download_files('stackoverflow/stacksample', path=SO_PATH, unzip=True, quiet=False)
+
+    ##### 1. Start spark and initialize data processor #####
+    spark = SparkSession.builder.master('local[*]')\
+        .appName("HAYSTACK_SOD")\
+        .config("spark.driver.memory", "400G")\
+        .config("spark.driver.memoryOverhead", "80G")\
+        .config("spark.executor.cores", "80")\
+        .config("spark.driver.maxResultSize", "16G")\
+        .config("spark.sql.execution.arrow.maxRecordsPerBatch", "32768")\
+        .getOrCreate()
+
+    #files = ["day_%d" % i for i in range(0, 24)]
+    SO_FILE = {'question': 'Questions.csv', 'answer': 'Answers.csv'}
+    file_names = dict((key, path_prefix + os.path.join(current_path, data_folder, filename)) for key, filename in SO_FILE.items())
+
+    proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity="1200GB", spark_mode='local')
+
+    ############## Process answer ###############
+    t0 = timer()
+    if not os.path.isdir(f"{current_path}/answer.parquet"):
+        answers_pdf = pd.read_csv(file_names['answer'], encoding="ISO-8859-1")
+        answers_pdf.drop(columns=['OwnerUserId', 'CreationDate', 'Id'], inplace=True)
+        answer_df = spark.createDataFrame(answers_pdf)
+        answer_df = proc.transform(answer_df, name="answer.parquet")        
+    else:
+        answer_df = spark.read.parquet(f"{path_prefix}{current_path}/answer.parquet")
+    t1 = timer()
+    print(f"Load Answer.csv took {(t1 - t0)} secs")
+
+
+    t0 = timer()
+    answer_df = answer_df.withColumnRenamed("ParentId", "Id").withColumnRenamed("Body", "TopAnswer-html")
+    op_collapse = CollapseByHist(by="Id", orderBy="Score")
+    proc.reset_ops([op_collapse])
+    top_answers_df = proc.apply(answer_df)
+    top_answers_df = top_answers_df.drop('Score')
+
+    top_answers_df = proc.transform(top_answers_df, name="top_answer_processed.parquet")
+    #top_answers_df.show()
+    t1 = timer()
+    print(f"Process Answer.csv took {(t1 - t0)} secs")
+
+    ############## Process question ###############
+    t0 = timer()
+    if not os.path.isdir(f"{current_path}/question.parquet"):
+        questions_pdf = pd.read_csv(file_names['question'], encoding="ISO-8859-1")
+        questions_pdf.drop(columns=['OwnerUserId', 'CreationDate', 'ClosedDate', 'Score'], inplace=True)
+        question_df = spark.createDataFrame(questions_pdf)
+        question_df = proc.transform(question_df, name="question.parquet")        
+    else:
+        question_df = spark.read.parquet(f"{path_prefix}{current_path}/question.parquet")
+    t1 = timer()
+    print(f"Load Question.csv took {(t1 - t0)} secs")
+
+    t0 = timer()
+    convert_udf = f.udf(lambda x: BeautifulSoup(x, "lxml").get_text(), StringType())
+    question_df = question_df.withColumnRenamed("Title", "text").withColumnRenamed("Body", "Question-html")
+    op_merge = ModelMerge([{'col_name': ['Id'], 'dict': top_answers_df}])
+    proc.reset_ops([op_merge])
+    question_df = proc.apply(question_df)
+    question_df = question_df.filter(f.col("TopAnswer-html").isNotNull())
+    question_df = question_df.drop('Id')
+    question_df = question_df.dropDuplicates()
+    question_df = question_df.na.fill({'Question-html': '', 'TopAnswer-html': ''})
+    question_df = question_df.withColumn("question-body", convert_udf(f.col("Question-html")))
+    question_df = question_df.withColumn("answer", convert_udf(f.col("TopAnswer-html")))
+    question_df = question_df.drop("Question-html").drop("TopAnswer-html")
+
+    question_df = proc.transform(question_df, name="question_processed.parquet")
+    question_df.show()
+    t1 = timer()
+    print(f"Process Question.csv took {(t1 - t0)} secs")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/RecDP/examples/python_tests/haystack_sod/gen_sod_to_documentstore.py b/RecDP/examples/python_tests/haystack_sod/gen_sod_to_documentstore.py
new file mode 100644
index 000000000..ca8ba4987
--- /dev/null
+++ b/RecDP/examples/python_tests/haystack_sod/gen_sod_to_documentstore.py
@@ -0,0 +1,154 @@
+import init
+
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+
+import logging
+from timeit import default_timer as timer
+import os
+from pyspark import *
+from pyspark.sql import *
+from pyspark.sql.functions import *
+from pyspark.sql.types import *
+import numpy as np
+import pandas as pd
+from bs4 import BeautifulSoup
+
+SO_PATH = './stack-overflow/'
+EMBEDDING_PARALLELISM = 8
+NUMCORES_PER_SOCKET = 64
+EMBEDDING_BATCH_SIZE = 262144
+ELASTIC_SERVER = "dr8s30"
+
+def main():
+    path_prefix = "hdfs://dr8s30:9000/"
+    current_local_path = "/home/vmagent/app/recdp/examples/python_tests/haystack_sod/"
+    current_path = "/haystack_sod/"
+    data_folder = "stack-overflow/"
+
+    #if not os.path.isdir(f"{current_path}/stack-overflow/"):
+    #    import kaggle
+    #    kaggle.api.authenticate()
+    #    kaggle.api.dataset_download_files('stackoverflow/stacksample', path=SO_PATH, unzip=True, quiet=False)
+
+    ##### 1. Start spark and initialize data processor #####
+    spark = SparkSession.builder.master(f'spark://{ELASTIC_SERVER}:7077')\
+        .appName("HAYSTACK_SOD")\
+        .config("spark.driver.memory", "60G")\
+        .config("spark.executor.memory", "250G")\
+        .config("spark.executor.instances", EMBEDDING_PARALLELISM)\
+        .config("spark.executor.cores", NUMCORES_PER_SOCKET)\
+        .config("spark.driver.maxResultSize", "16G")\
+        .config("spark.sql.execution.arrow.maxRecordsPerBatch", EMBEDDING_BATCH_SIZE)\
+        .config("spark.cleaner.periodicGC.interval", "7200min")\
+        .config("spark.cleaner.referenceTracking", "false")\
+        .config("spark.sql.files.minPartitionNum", "1")\
+        .config("spark.executor.heartbeatInterval", "10000000")\
+        .config("spark.network.timeout", "10000000")\
+        .getOrCreate()
+
+    #.config("spark.sql.files.maxPartitionBytes", "162MB")\
+    SO_FILE = {'question': 'Questions.csv', 'answer': 'Answers.csv'}
+    file_names = dict((key, os.path.join(current_local_path, data_folder, filename)) for key, filename in SO_FILE.items())
+
+    proc = DataProcessor(spark, path_prefix, current_path=current_path, shuffle_disk_capacity="1200GB", spark_mode='local')
+
+    ############## Process answer ###############
+    t0 = timer()
+    print("load ", file_names['answer'])
+    answers_pdf = pd.read_csv(file_names['answer'], encoding="ISO-8859-1")
+    answers_pdf.drop(columns=['OwnerUserId', 'CreationDate', 'Id'], inplace=True)
+    answer_df = spark.createDataFrame(answers_pdf)
+    answer_df = proc.transform(answer_df, name="answer.parquet")        
+    t1 = timer()
+    print(f"Load Answer.csv took {(t1 - t0)} secs")
+
+
+    t0 = timer()
+    answer_df = answer_df.withColumnRenamed("ParentId", "Id").withColumnRenamed("Body", "TopAnswer-html")
+    op_collapse = CollapseByHist(by="Id", orderBy="Score")
+    proc.reset_ops([op_collapse])
+    top_answers_df = proc.apply(answer_df)
+    top_answers_df = top_answers_df.drop('Score')
+
+    top_answers_df = proc.transform(top_answers_df, name="top_answer_processed.parquet")
+    t1 = timer()
+    print(f"Process Answer.csv took {(t1 - t0)} secs")
+
+    ############## Process question ###############
+    t0 = timer()
+    print("load ", file_names['question'])
+    questions_pdf = pd.read_csv(file_names['question'], encoding="ISO-8859-1")
+    questions_pdf.drop(columns=['OwnerUserId', 'CreationDate', 'ClosedDate', 'Score'], inplace=True)
+    question_df = spark.createDataFrame(questions_pdf)
+    question_df = proc.transform(question_df, name="question.parquet")        
+    t1 = timer()
+    print(f"Load Question.csv took {(t1 - t0)} secs")
+
+    t0 = timer()
+    convert_udf = f.udf(lambda x: BeautifulSoup(x, "lxml").get_text(), StringType())
+    question_df = question_df.withColumnRenamed("Title", "text").withColumnRenamed("Body", "Question-html")
+    op_merge = ModelMerge([{'col_name': ['Id'], 'dict': top_answers_df}])
+    proc.reset_ops([op_merge])
+    question_df = proc.apply(question_df)
+    question_df = question_df.filter(f.col("TopAnswer-html").isNotNull())
+    question_df = question_df.drop('Id')
+    question_df = question_df.dropDuplicates()
+    question_df = question_df.na.fill({'Question-html': '', 'TopAnswer-html': ''})
+    question_df = question_df.withColumn("question-body", convert_udf(f.col("Question-html")))
+    question_df = question_df.withColumn("answer", convert_udf(f.col("TopAnswer-html")))
+    question_df = question_df.drop("Question-html").drop("TopAnswer-html")
+
+    question_df = proc.transform(question_df, name="question_processed.parquet")
+    question_df.show()
+    t1 = timer()
+    print(f"Process Question.csv took {(t1 - t0)} secs")
+
+    ############# Embedding and write to DocumentStore ###############
+    question_df = question_df.repartition(EMBEDDING_PARALLELISM)
+
+    import init_haystack
+    from haystack.document_store.elasticsearch import ElasticsearchDocumentStore
+
+    from haystack.retriever.dense import EmbeddingRetriever
+    from haystack.utils import print_answers
+    from haystack.utils import launch_es
+    print('launching es & retriever...')
+
+    # get embeddings for question
+    from typing import Iterator
+    @pandas_udf(ArrayType(DoubleType()))
+    def pandas_udf_embed_queries(iterator: Iterator[pd.DataFrame]) -> Iterator[pd.Series]:
+        launch_es()
+        document_store = ElasticsearchDocumentStore(host=ELASTIC_SERVER, username="", password="",
+                                                index="document",
+                                                embedding_field="question_emb",
+                                                embedding_dim=768,
+                                                excluded_meta_data=["question_emb"])
+
+        retriever = EmbeddingRetriever(document_store=document_store, embedding_model="deepset/sentence_bert", use_gpu=False)
+        for df in iterator:
+            df["question_emb"] = retriever.embed_queries(texts=list(df['text'].values))
+            try:
+                document_store.write_documents(df.to_dict(orient="records"))
+            except:
+                pass
+            yield df["question_emb"]
+
+    total_len = question_df.count()
+    print('generating embeddings len = %d ...' % (total_len))
+    t0 = timer()
+    print(f"Start to do EmbeddingRetriever embed_queries ...")
+    question_df = question_df.withColumn("question_emb", pandas_udf_embed_queries(f.struct(f.col("text"), f.col("question-body"), f.col("answer"))))
+    question_df = proc.transform(question_df, name = "question_with_embed.parquet")
+    question_df.show()
+    t1 = timer()
+    print(f'EmbeddingRetriever embed_queries took {(t1 - t0)} secs')
+
+    print('!!!ALL DONE!!!')
+
+    question_df = question_df.withColumn("emb_size", f.size(f.col("question_emb")))
+    question_df.show()    
+
+if __name__ == "__main__":
+    main()
diff --git a/RecDP/examples/python_tests/haystack_sod/init.py b/RecDP/examples/python_tests/haystack_sod/init.py
new file mode 100644
index 000000000..a51233f58
--- /dev/null
+++ b/RecDP/examples/python_tests/haystack_sod/init.py
@@ -0,0 +1,6 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
diff --git a/RecDP/examples/python_tests/haystack_sod/init_haystack.py b/RecDP/examples/python_tests/haystack_sod/init_haystack.py
new file mode 100644
index 000000000..3742f3932
--- /dev/null
+++ b/RecDP/examples/python_tests/haystack_sod/init_haystack.py
@@ -0,0 +1,6 @@
+import os
+import sys
+import pathlib
+import_path = "/home/haystack/"
+sys.path.append(import_path)
+print(sys.path)
diff --git a/RecDP/examples/python_tests/haystack_sod/run_docker_haystack.sh b/RecDP/examples/python_tests/haystack_sod/run_docker_haystack.sh
new file mode 100644
index 000000000..c043045e2
--- /dev/null
+++ b/RecDP/examples/python_tests/haystack_sod/run_docker_haystack.sh
@@ -0,0 +1,1 @@
+docker run -it --net=host -v /mnt/nvme2/chendi/BlueWhale/recdp:/home/vmagent/app/recdp -v /mnt/nvme2/chendi/BlueWhale/applications.ai.appliedml.haystack/:/home/haystack -v /mnt/nvme2/chendi/BlueWhale/applications.ai.appliedml.haystack/haystack/:/home/user/haystack xuechendi/haystack-cpu-recdp /bin/bash
diff --git a/RecDP/examples/python_tests/haystack_sod/simple_test.py b/RecDP/examples/python_tests/haystack_sod/simple_test.py
new file mode 100644
index 000000000..00afc08b6
--- /dev/null
+++ b/RecDP/examples/python_tests/haystack_sod/simple_test.py
@@ -0,0 +1,25 @@
+import init_haystack
+from haystack.document_store.elasticsearch import ElasticsearchDocumentStore
+
+from haystack.retriever.dense import EmbeddingRetriever
+from haystack.utils import print_answers
+import time
+
+# give a simple test
+from haystack.pipeline import FAQPipeline
+document_store = ElasticsearchDocumentStore(host="localhost", username="", password="",
+					index="document",
+					embedding_field="question_emb",
+					embedding_dim=768,
+					excluded_meta_data=["question_emb"])
+
+retriever = EmbeddingRetriever(document_store=document_store, embedding_model="deepset/sentence_bert", use_gpu=False)
+pipe = FAQPipeline(retriever=retriever)
+
+start = time.time()
+prediction = pipe.run(query="Python - how to get current date?", top_k_retriever=3)
+print_answers(prediction, details="all")
+end = time.time()
+print('Took %.3f secs' % (end - start))
+
+print('!!!ALL DONE!!!')
\ No newline at end of file
diff --git a/RecDP/examples/python_tests/koalas/categorify.py b/RecDP/examples/python_tests/koalas/categorify.py
new file mode 100644
index 000000000..8b43287f0
--- /dev/null
+++ b/RecDP/examples/python_tests/koalas/categorify.py
@@ -0,0 +1,29 @@
+import findspark
+findspark.init()
+
+import os
+import sys
+
+import pandas as pd
+import numpy as np
+import databricks.koalas as ks
+from pyspark.sql import SparkSession
+
+import timeit
+
+
+spark = SparkSession.builder.master('yarn')\
+        .appName("Recsys2021_data_process")\
+        .getOrCreate()
+
+start = timeit.timeit()
+ks.set_option('compute.max_rows', 10000000)
+kdf = ks.read_parquet('/recsys2021_0608')
+kdf = ks.read_parquet("/recsys2021_0608_processed/first_hashtags")
+#kser = kdf['language'].fillna("").factorize()[0]
+kser = kdf['a'].fillna("").factorize()[0]
+
+ks.DataFrame(kser).to_parquet('/koalas_example/hashtags')
+
+end = timeit.timeit()
+print("koalas categorify took %.6f secs" % (end - start) * 1000000)
diff --git a/RecDP/examples/python_tests/recsys2020/RecsysSchema.py b/RecDP/examples/python_tests/recsys2020/RecsysSchema.py
new file mode 100644
index 000000000..b2225ec55
--- /dev/null
+++ b/RecDP/examples/python_tests/recsys2020/RecsysSchema.py
@@ -0,0 +1,105 @@
+from pyspark.sql.types import *
+
+
+class RecsysSchema:
+    def __init__(self):
+        self.string_cols1 = [
+            'text_tokens',
+            'hashtags',  # Tweet Features
+            'tweet_id',       #
+            'present_media',          #
+            'present_links',          #
+            'present_domains',        #
+            'tweet_type',     #
+            'language',       #
+        ]
+        self.int_cols1 = [
+            'tweet_timestamp',
+        ]
+        self.string_cols2 = [
+            'engaged_with_user_id',
+        ]
+        self.int_cols2 = [
+            'engaged_with_user_follower_count',  # Engaged With User Features
+            'engaged_with_user_following_count',      #
+        ]
+        self.bool_cols1 = [
+            'engaged_with_user_is_verified',          #
+        ]
+        self.int_cols3 = [
+            'engaged_with_user_account_creation',
+        ]
+        self.string_cols3 = [
+            'enaging_user_id',
+        ]
+        self.int_cols4 = [
+            'enaging_user_follower_count',  # Engaging User Features
+            'enaging_user_following_count',      #
+        ]
+        self.bool_cols2 = [
+            'enaging_user_is_verified',
+        ]
+        self.int_cols5 = [
+            'enaging_user_account_creation',
+        ]
+        self.bool_cols3 = [
+            'engagee_follows_engager',  # Engagement Features
+
+        ]
+        self.float_cols = [
+            'reply_timestamp',  # Target Reply
+            'retweet_timestamp',  # Target Retweet
+            'retweet_with_comment_timestamp',  # Target Retweet with comment
+            'like_timestamp',  # Target Like
+        ]
+
+        # After some conversion
+        self.int_cols6 = [
+            'tweet_timestamp',
+            'engaged_with_user_follower_count',  # Engaged With User Features
+            'engaged_with_user_following_count',      #
+            'engaged_with_user_account_creation',
+            'enaging_user_follower_count',  # Engaging User Features
+            'enaging_user_following_count',           #
+            'enaging_user_account_creation',
+        ]
+
+    def toStructType(self):
+        str_fields1 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols1]
+        int_fields1 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols1]
+        str_fields2 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols2]
+        int_fields2 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols2]
+        bool_fields1 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols1]
+        int_fields3 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols3]
+        str_fields3 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols3]
+        int_fields4 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols4]
+        bool_fields2 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols2]
+        int_fields5 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols5]
+        bool_fields3 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols3]
+        float_fields = [StructField('%s' % i, FloatType())
+                        for i in self.float_cols]
+        return StructType(
+            str_fields1
+            + int_fields1
+            + str_fields2
+            + int_fields2
+            + bool_fields1
+            + int_fields3
+            + str_fields3
+            + int_fields4
+            + bool_fields2
+            + int_fields5
+            + bool_fields3
+            + float_fields
+        )
diff --git a/RecDP/examples/python_tests/recsys2020/init.py b/RecDP/examples/python_tests/recsys2020/init.py
new file mode 100644
index 000000000..366ea2bfe
--- /dev/null
+++ b/RecDP/examples/python_tests/recsys2020/init.py
@@ -0,0 +1,10 @@
+import findspark
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(
+    __file__).parent.parent.parent.parent.absolute())
+# print(import_path)
+sys.path.append(import_path)
+
+findspark.init()
diff --git a/RecDP/examples/python_tests/recsys2020/recsys_data_process.py b/RecDP/examples/python_tests/recsys2020/recsys_data_process.py
new file mode 100644
index 000000000..a27dbfc89
--- /dev/null
+++ b/RecDP/examples/python_tests/recsys2020/recsys_data_process.py
@@ -0,0 +1,468 @@
+#!/env/bin/python
+
+import init
+
+import findspark
+findspark.init()
+
+import os
+import pandas as pd
+import numpy as np
+from pyspark.sql import *
+from pyspark import *
+import pyspark.sql.functions as f
+from timeit import default_timer as timer
+import logging
+from RecsysSchema import RecsysSchema
+from pyrecdp.data_processor import *
+from pyrecdp.encoder import *
+from pyrecdp.utils import *
+import hashlib
+
+
+def decodeBertTokenizerAndExtractFeatures(df, proc, output_name):
+    # modification on original feature
+    op_fillna_str = FillNA(
+        ['present_domains', 'present_links', 'hashtags', 'present_media', 'tweet_id'], "")
+    op_fillna_num = FillNA(['reply_timestamp', 'retweet_timestamp',
+                        'retweet_with_comment_timestamp', 'like_timestamp'], 0)
+    
+    op_feature_modification_type_convert = FeatureModification(cols=['tweet_timestamp',
+                                                                     'engaged_with_user_follower_count',
+                                                                     'engaged_with_user_following_count',
+                                                                     'engaged_with_user_account_creation',
+                                                                     'engaging_user_follower_count',
+                                                                     'engaging_user_following_count',
+                                                                     'engaging_user_account_creation',
+                                                                     'reply_timestamp',
+                                                                     'retweet_timestamp',
+                                                                     'retweet_with_comment_timestamp',
+                                                                     'like_timestamp'], op='toInt')
+    
+    op_feature_target_classify = FeatureModification(cols={
+        "reply_timestamp": "f.when(f.col('reply_timestamp') > 0, 1).otherwise(0)",
+        "retweet_timestamp": "f.when(f.col('retweet_timestamp') > 0, 1).otherwise(0)",
+        "retweet_with_comment_timestamp": "f.when(f.col('retweet_with_comment_timestamp') > 0, 1).otherwise(0)",
+        "like_timestamp": "f.when(f.col('like_timestamp') > 0, 1).otherwise(0)"}, op='inline')
+    
+    # adding new features
+    op_feature_to_be_categorified = FeatureAdd(
+        cols={"present_domains_indicator": "f.col('present_domains')",\
+              "present_links_indicator": "f.col('present_links')",\
+              "hashtags_indicator": "f.col('hashtags')",\
+              "language_indicator": "f.col('language')",\
+              "tweet_id_indicator": "f.col('tweet_id')",\
+              "present_media_indicator": "f.col('present_media')",\
+              "tweet_type_indicator": "f.col('tweet_type')",\
+              "engaged_with_user_id_indicator": "f.col('engaged_with_user_id')",\
+              "engaging_user_id_indicator": "f.col('engaging_user_id')"
+             }, 
+        op='inline')
+    
+    op_feature_from_original = FeatureAdd(
+        cols={"has_photo": "f.col('present_media').contains('Photo').cast(t.IntegerType())",
+              "has_video": "f.col('present_media').contains('Vedio').cast(t.IntegerType())",
+              "has_gif": "f.col('present_media').contains('GIF').cast(t.IntegerType())",             
+              "a_ff_rate": "f.col('engaged_with_user_following_count')/f.col('engaged_with_user_follower_count')",
+              "b_ff_rate": "f.col('engaging_user_following_count') /f.col('engaging_user_follower_count')",
+              "dt_dow": "f.dayofweek(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())",
+              "dt_hour": "f.hour(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())",  
+              "dt_minute": "f.minute(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())",
+              "dt_second": "f.second(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())",
+              'present_domains_indicator': "f.concat_ws('_', f.slice(f.split(f.col('present_domains_indicator'),'\t'), 1, 2))",
+              'len_hashtags': "f.when(f.col('hashtags') == '', f.lit(0)).otherwise(f.size(f.split(f.col('hashtags'), '\t')))",
+              'len_domains': "f.when(f.col('present_domains') == '', f.lit(0)).otherwise(f.size(f.split(f.col('present_domains'), '\t')))",
+              'len_links': "f.when(f.col('present_links') == '', f.lit(0)).otherwise(f.size(f.split(f.col('present_links'), '\t')))",
+              'engage_time': "f.least(f.col('reply_timestamp'), f.col('retweet_timestamp'), f.col('retweet_with_comment_timestamp'), f.col('like_timestamp'))"   
+        }, op='inline')
+    op_fillna_tweet_timestamp = FillNA(['tweet_timestamp'], -1)
+    ops = [op_fillna_str, op_fillna_num,
+           op_feature_modification_type_convert, op_feature_target_classify,
+           op_feature_to_be_categorified, op_feature_from_original, op_fillna_tweet_timestamp]
+
+    ########## Tweet token feature engineering ##########
+    from transformers import BertTokenizer
+    tokenizer = BertTokenizer.from_pretrained(
+        'bert-base-multilingual-cased', do_lower_case=False)
+
+    # define UDF
+    tokenizer_decode = f.udf(lambda x: tokenizer.decode(
+        [int(n) for n in x.split('\t')]))
+    format_url = f.udf(lambda x: x.replace(
+        'https : / / t. co / ', 'https://t.co/').replace('@ ', '@'))
+
+    def extract_hash(text, split_text='@', no=0):
+        text = text.lower()
+        uhash = ''
+        text_split = text.split('@')
+        if len(text_split) > (no+1):
+            text_split = text_split[no+1].split(' ')
+            cl_loop = True
+            uhash += clean_text(text_split[0])
+            while cl_loop:
+                if len(text_split) > 1:
+                    if text_split[1] in ['_']:
+                        uhash += clean_text(text_split[1]) + \
+                            clean_text(text_split[2])
+                        text_split = text_split[2:]
+                    else:
+                        cl_loop = False
+                else:
+                    cl_loop = False
+        hash_object = hashlib.md5(uhash.encode('utf-8'))
+        return hash_object.hexdigest()
+
+    def clean_text(text):
+        if len(text) > 1:
+            if text[-1] in ['!', '?', ':', ';', '.', ',']:
+                return(text[:-1])
+        return(text)
+    
+    # udf defines upon tweet
+    to_notsign = f.udf(lambda x: x.replace('\[CLS\] RT @', ''))
+    count_space = f.udf(lambda x: x.count(' '))
+    count_text_length = f.udf(lambda x: len(x))
+    user_defined_hash = f.udf(lambda x: extract_hash(x, split_text='RT @', no=0))
+    count_at = f.udf(lambda x: x.count('@'))
+    user_define_hash_1 = f.udf(lambda x: extract_hash(x))
+    user_define_hash_2 = f.udf(lambda x: extract_hash(x, no=1))
+
+    # decode
+    op_feature_modification_tokenizer_decode = FeatureAdd(cols={'tweet': 'text_tokens'}, udfImpl=tokenizer_decode)
+    op_feature_modification_format_url = FeatureModification(cols=['tweet'], udfImpl=format_url)
+
+    # adding new features
+    op_feature_add_tweet_indicator = FeatureAdd(cols={"tweet_indicator": "f.col('tweet')"}, op='inline')
+    op_feature_add_tweet_nortsign = FeatureAdd(cols={'tweet_nortsign': 'tweet'}, udfImpl=to_notsign)
+    op_feature_add_count_words = FeatureAdd(cols={'count_words': 'tweet'}, udfImpl=count_space)
+    op_feature_add_count_char = FeatureAdd(cols={'count_char': 'tweet'}, udfImpl=count_text_length)
+    op_feature_add_tw_uhash = FeatureAdd(cols={'tw_uhash': 'tweet'}, udfImpl=user_defined_hash)
+    op_feature_add_tw_hash = FeatureAdd(cols={'tw_hash': "f.hash(f.col('tweet'))%1000000000"}, op='inline')
+    # features upon tweet_nortsign
+    op_feature_add_count_at = FeatureAdd(cols={'count_ats': 'tweet_nortsign'}, udfImpl=count_at)
+    op_feature_add_tw_uhash0 = FeatureAdd(cols={'tw_hash0': 'tweet_nortsign'}, udfImpl=user_define_hash_1)
+    op_feature_add_tw_uhash1 = FeatureAdd(cols={'tw_hash1': 'tweet_nortsign'}, udfImpl=user_define_hash_2)
+
+    ops += [op_feature_modification_tokenizer_decode, op_feature_modification_format_url,
+            op_feature_add_tweet_indicator,
+            op_feature_add_tweet_nortsign, op_feature_add_count_words, op_feature_add_count_char,
+            op_feature_add_tw_uhash, op_feature_add_tw_hash, op_feature_add_count_at,
+            op_feature_add_tw_uhash0, op_feature_add_tw_uhash1]
+    proc.reset_ops(ops)
+
+    # execute
+    t1 = timer()
+    df = proc.transform(df, name=output_name)
+    t2 = timer()
+    print("BertTokenizer decode and feature extacting took %.3f" % (t2 - t1))
+
+    return df
+
+def splitByDate(df, proc, train_output, test_output, numFolds=5):
+    # 1.1 get timestamp range
+    import datetime
+    min_timestamp = df.select('tweet_timestamp').agg({'tweet_timestamp': 'min'}).collect()[0]['min(tweet_timestamp)']
+    max_timestamp = df.select('tweet_timestamp').agg({'tweet_timestamp': 'max'}).collect()[0]['max(tweet_timestamp)']
+    seconds_in_day = 3600 * 24
+
+    print(
+        "min_timestamp is %s, max_timestamp is %s, 20 days max is %s" % (
+            datetime.datetime.fromtimestamp(min_timestamp).strftime('%Y-%m-%d %H:%M:%S'),
+            datetime.datetime.fromtimestamp(max_timestamp).strftime('%Y-%m-%d %H:%M:%S'),
+            datetime.datetime.fromtimestamp(min_timestamp + 20 * seconds_in_day).strftime('%Y-%m-%d %H:%M:%S')
+        ))
+
+    time_range_split = {
+        'train': (min_timestamp, seconds_in_day * 18 + min_timestamp),
+        'test': (seconds_in_day * 18 + min_timestamp, max_timestamp)
+    }
+
+    print(time_range_split)
+
+    # 1.2 save ranged data for train
+    # filtering out train range data and save
+    train_start, train_end = time_range_split['train']
+    test_start, test_end = time_range_split['test']
+    t1 = timer()
+    train_df = df.filter(
+        (f.col('tweet_timestamp') >= f.lit(train_start)) & (f.col('tweet_timestamp') < f.lit(train_end)))
+    train_df = train_df.withColumn("fold", f.round(f.rand(seed=42)*numFolds))
+    train_df.write.format('parquet').mode('overwrite').save(proc.path_prefix + proc.current_path + train_output)
+    t2 = timer()
+    print("split to train took %.3f" % (t2 - t1))
+    
+    t1 = timer()
+    test_df = df.filter(
+        (f.col('tweet_timestamp') >= f.lit(test_start)) & (f.col('tweet_timestamp') < f.lit(test_end)))
+    test_df.write.format('parquet').mode('overwrite').save(proc.path_prefix + proc.current_path + test_output)
+    t2 = timer()
+    print("split to test took %.3f" % (t2 - t1))
+    
+    return (proc.spark.read.parquet(proc.path_prefix + proc.current_path + train_output),
+            proc.spark.read.parquet(proc.path_prefix + proc.current_path + test_output))
+
+
+def categorifyFeatures(df, proc, output_name="train_with_categorified_features", gen_dict=True, sampleRatio=1):
+    if gen_dict:
+        # only call below function when target dicts were not pre-prepared
+        op_multiItems = GenerateDictionary(
+            ['present_domains_indicator', 'present_links_indicator', 'hashtags_indicator'], doSplit=True)
+        op_singleItems = GenerateDictionary(
+            ['tweet_id_indicator', 'language_indicator', 
+             {'src_cols': ['engaged_with_user_id_indicator', 'engaging_user_id_indicator'], 'col_name': 'user_id'}])
+        op_tweet = GenerateDictionary(
+            ['tweet_indicator'], doSplit=True, withCount=True, sep=' ')
+        
+        proc.reset_ops([op_multiItems, op_singleItems, op_tweet])
+        t1 = timer()
+        dict_dfs = proc.generate_dicts(df)
+        t2 = timer()
+        print("Generate Dictionary took %.3f" % (t2 - t1))
+    else:
+        # or we can simply load from pre-gened
+        dict_names = ['hashtags_indicator', 'language_indicator', 'present_domains_indicator',
+                      'present_links_indicator', 'tweet_id_indicator', 'user_id_indicator', 'tweet_indicator']
+        dict_dfs = [{'col_name': name, 'dict': proc.spark.read.parquet(
+            "%s/%s/%s/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in dict_names]
+
+    # pre-defined dict
+    # pre-define
+    media = {
+        '': 0,
+        'GIF': 1,
+        'GIF_GIF': 2,
+        'GIF_Photo': 3,
+        'GIF_Video': 4,
+        'Photo': 5,
+        'Photo_GIF': 6,
+        'Photo_Photo': 7,
+        'Photo_Video': 8,
+        'Video': 9,
+        'Video_GIF': 10,
+        'Video_Photo': 11,
+        'Video_Video': 12
+    }
+
+    tweet_type = {'Quote': 0, 'Retweet': 1, 'TopLevel': 2}
+
+    media_df = proc.spark.createDataFrame(convert_to_spark_dict(media))
+    tweet_type_df = proc.spark.createDataFrame(
+        convert_to_spark_dict(tweet_type))
+
+    dict_dfs.append({'col_name': 'present_media_indicator', 'dict': media_df})
+    dict_dfs.append({'col_name': 'tweet_type_indicator', 'dict': tweet_type_df})
+
+    for i in dict_dfs:
+        dict_name = i['col_name']
+        dict_df = i['dict']
+        print("%s has numRows as %d" % (dict_name, dict_df.count()))
+
+    ###### 2. define operations and append them to data processor ######
+
+    # 1. define operations
+    # 1.1 filter on tweet dict
+    i = 0
+    for dict_df in dict_dfs:
+        if dict_df['col_name'] == 'tweet':
+            tweet_dict_df = dict_df['dict']
+            df_cnt = tweet_dict_df.count()
+            freqRange = [2, df_cnt * 0.9]
+            tweet_dict_df = tweet_dict_df.filter((f.col('count') <= f.lit(freqRange[1])) & (f.col('count') >= f.lit(freqRange[0])))
+            dict_dfs[i]['dict'] = tweet_dict_df
+        i += 1      
+
+    # 1.3 categorify
+    # since language dict is small, we may use udf to make partition more even
+    op_categorify_1 = Categorify(
+        ['present_domains_indicator', 'present_links_indicator', 'hashtags_indicator'], dict_dfs=dict_dfs, doSplit=True, keepMostFrequent=True)
+    op_categorify_2 = Categorify(['language_indicator', 'present_media_indicator', 'tweet_type_indicator'], dict_dfs=dict_dfs)
+    op_categorify_3 = Categorify([{'engaged_with_user_id_indicator': 'user_id'}, {'engaging_user_id_indicator': 'user_id'}], dict_dfs=dict_dfs)
+    op_categorify_4 = Categorify(['tweet_id_indicator'], dict_dfs=dict_dfs)
+    op_categorify_5 = Categorify(['tweet_indicator'], dict_dfs=dict_dfs, doSplit=True, sep=' ', doSortForArray=True)
+    #### below are features upon categorified tweet
+    op_feature_add_tw_word = FeatureAdd({
+        'tw_first_word': "f.col('tweet_indicator').getItem(0)",
+        'tw_second_word': "f.col('tweet_indicator').getItem(1)",
+        'tw_last_word': "f.col('tweet_indicator').getItem(f.size(f.col('tweet_indicator')) - 1)",
+        'tw_llast_word': "f.col('tweet_indicator').getItem(f.size(f.col('tweet_indicator')) - 1)",
+        'tw_len': "f.size(f.col('tweet_indicator'))"
+    }, op='inline')
+    ops = [op_categorify_1, op_categorify_2, op_categorify_3, op_categorify_4, op_categorify_5, op_feature_add_tw_word]
+    proc.reset_ops(ops)
+
+    t1 = timer()
+    if sampleRatio < 1 and sampleRatio > 0:
+        df = df.sample(sampleRatio)
+    df = proc.transform(df, name=output_name)
+    t2 = timer()
+    print("categorify took %.3f" % (t2 - t1))    
+    return df, dict_dfs
+
+
+def encodingFeatures(df, proc, output_name, gen_dict, sampleRatio=1):   
+    targets = ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp']
+    y_mean_all = []
+    
+    t1 = timer()
+    if gen_dict:
+        for tgt in targets:
+            tmp = df.groupBy().mean(tgt).collect()[0]
+            y_mean = tmp[f"avg({tgt})"]
+            y_mean_all.append(y_mean)
+        schema = t.StructType([t.StructField(tgt, t.FloatType(), True) for tgt in targets])
+        y_mean_all_df = proc.spark.createDataFrame([tuple(y_mean_all)], schema)
+        y_mean_all_df.write.format("parquet").mode("overwrite").save(
+            "%s/%s/%s/targets_mean" % (proc.path_prefix, proc.current_path, proc.dicts_path))
+    y_mean_all_df = proc.spark.read.parquet(
+        "%s/%s/%s/targets_mean" % (proc.path_prefix, proc.current_path, proc.dicts_path))
+
+    te_features = [
+        'present_media',
+        'tweet_type',
+        'language',
+        'engaged_with_user_id',
+        'engaging_user_id',
+        ['present_domains','language','engagee_follows_engager','tweet_type','present_media','engaged_with_user_is_verified'],
+        ['engaged_with_user_id','tweet_type','language'],
+        ['tw_first_word','tweet_type','language'],
+        ['tw_last_word','tweet_type','language'],
+        ['tw_hash0','tweet_type','language'],
+        ['tw_hash1','tweet_type','language'],
+        ['tw_uhash','tweet_type','language'],
+        ['tw_hash'],
+        ['present_media','tweet_type','language','engaged_with_user_is_verified','engaging_user_is_verified','engagee_follows_engager'],
+        ['present_domains','present_media','tweet_type','language'],
+        ['present_links','present_media','tweet_type','language'],
+        ['hashtags','present_media','tweet_type','language']
+        
+    ]
+    ce_features = ['present_media', 'tweet_type', 'language', 'engaged_with_user_id', 'engaging_user_id']
+    fe_features = ['present_media', 'tweet_type', 'language', 'engaged_with_user_id', 'engaging_user_id']
+    encoding_features = [("TE", te_features), ("CE", ce_features), ("FE", fe_features)]
+
+    te_train_dfs = []
+    te_test_dfs = []
+    ce_train_dfs = []
+    ce_test_dfs = []
+    fe_train_dfs = []
+    fe_test_dfs = []
+    for feature_type, features in encoding_features:
+        for c in features:
+            target_tmp = targets
+            out_name = ""
+            out_col_list = []
+            for tgt in target_tmp:
+                if isinstance(c, list):
+                    out_col_list.append(f'G{feature_type}_' + '_'.join(c) + f'_{tgt}')
+                    out_name = f'G{feature_type}_' + '_'.join(c)
+                else:
+                    out_col_list.append(f'{feature_type}_{c}_{tgt}')
+                    out_name = f'{feature_type}_{c}'
+            if gen_dict:
+                start = timer()
+                if feature_type == 'TE':
+                    encoder = TargetEncoder(proc, c, target_tmp, out_col_list, out_name, out_dtype=t.FloatType(), y_mean_list=y_mean_all)
+                    te_train_df, te_test_df = encoder.transform(df)
+                    te_train_dfs.append({'col_name': ['fold'] + (c if isinstance(c, list) else [c]), 'dict': te_train_df})
+                    te_test_dfs.append({'col_name': c, 'dict': te_test_df})
+                    print(f"generating target encoding for %s upon %s took %.1f seconds"%(str(c), str(target_tmp), timer()-start))
+
+                elif feature_type == 'CE':
+                    encoder = CountEncoder(proc, c, target_tmp, out_col_list, out_name)  
+                    ce_train_df, ce_test_df = encoder.transform(df)
+                    ce_train_dfs.append({'col_name': c if isinstance(c, list) else [c], 'dict': ce_train_df})
+                    ce_test_dfs.append({'col_name': c, 'dict': ce_test_df})  
+                    print(f"generating count encoding for %s upon %s took %.1f seconds"%(str(c), str(target_tmp), timer()-start))
+
+                elif feature_type == 'FE':
+                    # For frequency encoding, we don't need to merge with train data
+                    encoder = FrequencyEncoder(proc, c, target_tmp, out_col_list, out_name) 
+                    fe_train_df, fe_test_df = encoder.transform(df)
+                    fe_train_dfs.append({'col_name': c if isinstance(c, list) else [c], 'dict': fe_train_df})
+                    print(f"generating frequency encoding for %s upon %s took %.1f seconds"%(str(c), str(target_tmp), timer()-start))
+
+            else:
+                te_train_path = "%s/%s/%s/train/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name)
+                te_test_path = "%s/%s/%s/test/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name) 
+                if feature_type == 'TE':
+                    te_train_dfs.append({'col_name': ['fold'] + (c if isinstance(c, list) else [c]), 'dict': proc.spark.read.parquet(te_train_path)})
+                    te_test_dfs.append({'col_name': c, 'dict': proc.spark.read.parquet(te_test_path)})
+                if feature_type == 'CE':
+                    ce_train_dfs.append({'col_name': c if isinstance(c, list) else [c], 'dict': proc.spark.read.parquet(te_train_path)})
+                    ce_test_dfs.append({'col_name': c, 'dict': proc.spark.read.parquet(te_test_path)})
+                if feature_type == 'FE':
+                    # For frequency encoding, we don't need to merge with train data
+                    fe_train_dfs.append({'col_name': c if isinstance(c, list) else [c], 'dict': proc.spark.read.parquet(te_train_path)})
+
+    t2 = timer()
+    print("Generate encoding feature totally took %.3f" % (t2 - t1))
+
+    # merge dicts to original table
+    if sampleRatio < 1 and sampleRatio > 0:
+        df = df.sample(sampleRatio)
+    i = 3
+    for train_dfs in [te_train_dfs, ce_train_dfs, fe_train_dfs]:
+        op_merge_to_train = ModelMerge(train_dfs)
+        proc.reset_ops([op_merge_to_train])
+        i -= 1
+        _output_name = output_name if i == 0 else f"{output_name}_{i}"
+
+        t1 = timer()
+        df = proc.transform(df, name=_output_name)
+        t2 = timer()
+        print("encodingFeatures took %.3f" % (t2 - t1))
+    
+    return (df, te_train_dfs, te_test_dfs, y_mean_all_df)
+
+def main():
+    path_prefix = "hdfs://"
+    current_path = "/recsys2020_example/"
+    original_folder = "/recsys2021_0608/"
+    dicts_folder = "recsys_dicts/"
+    recsysSchema = RecsysSchema()
+
+    ##### 1. Start spark and initialize data processor #####
+    scala_udf_jars = "/home/vmagent/app/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+
+    t0 = timer()
+    spark = SparkSession.builder.master('local[80]')\
+        .appName("Recsys2020_data_process")\
+        .config("spark.driver.memory", "480G")\
+        .config("spark.executor.cores", "80")\
+        .config("spark.driver.extraClassPath", f"{scala_udf_jars}")\
+        .config("spark.sql.broadcastTimeout", "7200")\
+        .config("spark.cleaner.periodicGC.interval", "10min")\
+        .getOrCreate()
+
+    schema = recsysSchema.toStructType()
+
+    # 1.1 prepare dataFrames
+    # 1.2 create RecDP DataProcessor
+    proc = DataProcessor(spark, path_prefix,
+                        current_path=current_path, dicts_path=dicts_folder, shuffle_disk_capacity="1200GB")
+    df = spark.read.parquet(path_prefix + original_folder)
+    df = df.withColumnRenamed('enaging_user_following_count', 'engaging_user_following_count')
+    df = df.withColumnRenamed('enaging_user_is_verified', 'engaging_user_is_verified')
+
+    # fast test, comment for full dataset
+    # df.sample(0.01).write.format("parquet").mode("overwrite").save("%s/sample_0_0_1" % current_path)
+    # df = spark.read.parquet("%s/sample_0_0_1" % current_path)
+
+    # ===============================================
+    # decode tweet_tokens
+    df = decodeBertTokenizerAndExtractFeatures(df, proc, output_name="decoded_with_extracted_features")
+
+    # ===============================================
+    # splitting and sampling
+    df, test_df = splitByDate(df, proc, train_output="train", test_output="test", numFolds=5)
+
+    # ===============================================
+    # generate dictionary for categorify indexing
+    df, dict_dfs = categorifyFeatures(df, proc, output_name="train_with_categorified_features", gen_dict=True, sampleRatio=0.03)
+
+    # ===============================================
+    # encoding features
+    df, te_train_dfs, te_test_dfs, y_mean_all_df = encodingFeatures(df, proc, output_name="train_with_features_sample_0_0_3", gen_dict=True, sampleRatio=0.03)
+    
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/RecDP/examples/python_tests/recsys2020/recsys_for_friesian_integration.py b/RecDP/examples/python_tests/recsys2020/recsys_for_friesian_integration.py
new file mode 100644
index 000000000..03202db4a
--- /dev/null
+++ b/RecDP/examples/python_tests/recsys2020/recsys_for_friesian_integration.py
@@ -0,0 +1,79 @@
+import init
+
+import os
+import pandas as pd
+import numpy as np
+from pyspark.sql import *
+from pyspark import *
+import pyspark.sql.functions as f
+from timeit import default_timer as timer
+import logging
+from RecsysSchema import RecsysSchema
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+import hashlib
+
+def categorifyAllFeatures(df, proc, output_name="categorified", gen_dicts=False):
+    # 1. define operations
+    # 1.1 fill na and features
+    op_fillna_str = FillNA(
+        ['present_domains', 'present_links', 'hashtags'], "")
+
+    # 1.3 categorify
+    # since language dict is small, we may use udf to make partition more even
+    #'present_domains', 'present_links', 
+    op_categorify_multi = Categorify(
+        ['present_domains', 'present_links', 'hashtags'], gen_dicts=gen_dicts, doSplit=True, keepMostFrequent=True)
+    op_fillna_for_categorified = FillNA(['present_domains', 'present_links', 'hashtags'], -1)
+
+    # transform
+    proc.append_ops([op_fillna_str, op_categorify_multi, op_fillna_for_categorified])
+    t1 = timer()
+    df = proc.transform(df, name=output_name)
+    t2 = timer()
+    print("Data Process 1 and udf categorify took %.3f" % (t2 - t1))
+
+    return df
+
+def main():
+	path_prefix = "hdfs://"
+	current_path = "/recsys2020_0608_categorify_example_1/"
+	original_folder = "/recsys2021_0608/"
+	dicts_folder = "recsys_dicts/"
+	recsysSchema = RecsysSchema()
+
+	scala_udf_jars = "/home/vmagent/app/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+
+	##### 1. Start spark and initialize data processor #####
+	t0 = timer()
+	spark = SparkSession.builder.master('local[80]')\
+		.appName("Recsys2020_data_process_support_for_friesian")\
+		.config("spark.driver.memory", "400G")\
+		.config("spark.driver.memoryOverhead", "80G")\
+		.config("spark.executor.cores", "80")\
+		.config("spark.driver.extraClassPath", f"{scala_udf_jars}")\
+		.getOrCreate()
+
+	schema = recsysSchema.toStructType()
+
+	# 1.1 prepare dataFrames
+	# 1.2 create RecDP DataProcessor
+	proc = DataProcessor(spark, path_prefix,
+			current_path=current_path, dicts_path=dicts_folder, shuffle_disk_capacity="1200GB", spark_mode='local')
+
+	# ===============================================
+	# basic: Do categorify for all columns
+	df = spark.read.parquet(path_prefix + original_folder)
+
+	# rename firstly
+	df = df.withColumnRenamed('enaging_user_following_count', 'engaging_user_following_count')
+	df = df.withColumnRenamed('enaging_user_is_verified', 'engaging_user_is_verified')
+	df = categorifyAllFeatures(df, proc, gen_dicts=True)
+
+	t1 = timer()
+
+	print(f"Total process time is {(t1 - t0)} secs")
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/RecDP/examples/python_tests/recsys2020/test.py b/RecDP/examples/python_tests/recsys2020/test.py
new file mode 100644
index 000000000..1b5a2f6f3
--- /dev/null
+++ b/RecDP/examples/python_tests/recsys2020/test.py
@@ -0,0 +1,2 @@
+from init import *
+from pyrecdp.data_processor import *
diff --git a/RecDP/examples/python_tests/recsys2021/RecsysSchema.py b/RecDP/examples/python_tests/recsys2021/RecsysSchema.py
new file mode 100644
index 000000000..8447f50be
--- /dev/null
+++ b/RecDP/examples/python_tests/recsys2021/RecsysSchema.py
@@ -0,0 +1,105 @@
+from pyspark.sql.types import *
+
+
+class RecsysSchema:
+    def __init__(self):
+        self.string_cols1 = [
+            'text_tokens',
+            'hashtags',  # Tweet Features
+            'tweet_id',       #
+            'present_media',          #
+            'present_links',          #
+            'present_domains',        #
+            'tweet_type',     #
+            'language',       #
+        ]
+        self.int_cols1 = [
+            'tweet_timestamp',
+        ]
+        self.string_cols2 = [
+            'engaged_with_user_id',
+        ]
+        self.int_cols2 = [
+            'engaged_with_user_follower_count',  # Engaged With User Features
+            'engaged_with_user_following_count',      #
+        ]
+        self.bool_cols1 = [
+            'engaged_with_user_is_verified',          #
+        ]
+        self.int_cols3 = [
+            'engaged_with_user_account_creation',
+        ]
+        self.string_cols3 = [
+            'engaging_user_id',
+        ]
+        self.int_cols4 = [
+            'enaging_user_follower_count',  # Engaging User Features
+            'enaging_user_following_count',      #
+        ]
+        self.bool_cols2 = [
+            'enaging_user_is_verified',
+        ]
+        self.int_cols5 = [
+            'enaging_user_account_creation',
+        ]
+        self.bool_cols3 = [
+            'engagee_follows_engager',  # Engagement Features
+
+        ]
+        self.float_cols = [
+            'reply_timestamp',  # Target Reply
+            'retweet_timestamp',  # Target Retweet
+            'retweet_with_comment_timestamp',  # Target Retweet with comment
+            'like_timestamp',  # Target Like
+        ]
+
+        # After some conversion
+        self.int_cols6 = [
+            'tweet_timestamp',
+            'engaged_with_user_follower_count',  # Engaged With User Features
+            'engaged_with_user_following_count',      #
+            'engaged_with_user_account_creation',
+            'enaging_user_follower_count',  # Engaging User Features
+            'enaging_user_following_count',           #
+            'enaging_user_account_creation',
+        ]
+
+    def toStructType(self):
+        str_fields1 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols1]
+        int_fields1 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols1]
+        str_fields2 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols2]
+        int_fields2 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols2]
+        bool_fields1 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols1]
+        int_fields3 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols3]
+        str_fields3 = [StructField('%s' % i, StringType())
+                       for i in self.string_cols3]
+        int_fields4 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols4]
+        bool_fields2 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols2]
+        int_fields5 = [StructField('%s' % i, IntegerType())
+                       for i in self.int_cols5]
+        bool_fields3 = [StructField('%s' % i, BooleanType())
+                        for i in self.bool_cols3]
+        float_fields = [StructField('%s' % i, FloatType())
+                        for i in self.float_cols]
+        return StructType(
+            str_fields1
+            + int_fields1
+            + str_fields2
+            + int_fields2
+            + bool_fields1
+            + int_fields3
+            + str_fields3
+            + int_fields4
+            + bool_fields2
+            + int_fields5
+            + bool_fields3
+            + float_fields
+        )
diff --git a/RecDP/examples/python_tests/recsys2021/final_submission_feature_engineering.py b/RecDP/examples/python_tests/recsys2021/final_submission_feature_engineering.py
new file mode 100644
index 000000000..857f28dd5
--- /dev/null
+++ b/RecDP/examples/python_tests/recsys2021/final_submission_feature_engineering.py
@@ -0,0 +1,572 @@
+#!/env/bin/python
+
+import init
+
+import findspark
+findspark.init()
+
+import os
+import pandas as pd
+import numpy as np
+from pyspark.sql import *
+from pyspark import *
+import pyspark.sql.functions as f
+import pyspark.sql.types as t
+from timeit import default_timer as timer
+import logging
+from RecsysSchema import RecsysSchema
+from pyrecdp.data_processor import *
+from pyrecdp.encoder import *
+from pyrecdp.utils import *
+import hashlib
+
+target_list = [
+ 'reply_timestamp',
+ 'retweet_timestamp',
+ 'retweet_with_comment_timestamp',
+ 'like_timestamp'
+]
+
+final_feature_list = [
+ 'engaged_with_user_follower_count',
+ 'engaged_with_user_following_count',
+ 'engaged_with_user_is_verified',
+ 'engaging_user_follower_count',
+ 'engaging_user_following_count',
+ 'engaging_user_is_verified',
+ 'has_photo',
+ 'has_video',
+ 'has_gif',
+ 'a_ff_rate',
+ 'b_ff_rate', 
+ 'dt_hour',
+ 'dt_dow',
+ 'has_mention',  
+ 'mentioned_bucket_id',    
+ 'mentioned_count',    
+ 'most_used_word_bucket_id',
+ 'second_used_word_bucket_id',
+ 'TE_tweet_type_reply_timestamp',
+ 'TE_tweet_type_retweet_timestamp',
+ 'TE_dt_dow_retweet_timestamp',
+ 'TE_most_used_word_bucket_id_reply_timestamp',
+ 'TE_most_used_word_bucket_id_retweet_timestamp',
+ 'TE_most_used_word_bucket_id_retweet_with_comment_timestamp',
+ 'TE_most_used_word_bucket_id_like_timestamp',
+ 'TE_second_used_word_bucket_id_reply_timestamp',
+ 'TE_second_used_word_bucket_id_retweet_timestamp',
+ 'TE_second_used_word_bucket_id_retweet_with_comment_timestamp',
+ 'TE_second_used_word_bucket_id_like_timestamp',
+ 'TE_mentioned_bucket_id_retweet_timestamp',
+ 'TE_mentioned_bucket_id_retweet_with_comment_timestamp',
+ 'TE_mentioned_bucket_id_like_timestamp',
+ 'TE_mentioned_bucket_id_reply_timestamp',
+ 'TE_language_reply_timestamp',
+ 'TE_language_retweet_timestamp',
+ 'TE_language_retweet_with_comment_timestamp',
+ 'TE_language_like_timestamp',
+ 'TE_mentioned_count_reply_timestamp',
+ 'TE_mentioned_count_retweet_timestamp',
+ 'TE_mentioned_count_retweet_with_comment_timestamp',
+ 'TE_mentioned_count_like_timestamp',
+ 'TE_engaged_with_user_id_reply_timestamp',
+ 'TE_engaged_with_user_id_retweet_timestamp',
+ 'TE_engaged_with_user_id_retweet_with_comment_timestamp',
+ 'TE_engaged_with_user_id_like_timestamp',
+ 'GTE_language_engaged_with_user_id_reply_timestamp',
+ 'GTE_language_engaged_with_user_id_retweet_timestamp',
+ 'GTE_language_engaged_with_user_id_retweet_with_comment_timestamp',
+ 'GTE_language_engaged_with_user_id_like_timestamp',
+ 'GTE_tweet_type_engaged_with_user_id_reply_timestamp',
+ 'GTE_tweet_type_engaged_with_user_id_retweet_timestamp',
+ 'GTE_tweet_type_engaged_with_user_id_retweet_with_comment_timestamp',
+ 'GTE_tweet_type_engaged_with_user_id_like_timestamp',
+ 'GTE_has_mention_engaging_user_id_reply_timestamp',
+ 'GTE_has_mention_engaging_user_id_retweet_timestamp',
+ 'GTE_has_mention_engaging_user_id_retweet_with_comment_timestamp',
+ 'GTE_has_mention_engaging_user_id_like_timestamp',
+ 'GTE_mentioned_bucket_id_engaging_user_id_reply_timestamp',
+ 'GTE_mentioned_bucket_id_engaging_user_id_retweet_timestamp',
+ 'GTE_mentioned_bucket_id_engaging_user_id_retweet_with_comment_timestamp',
+ 'GTE_mentioned_bucket_id_engaging_user_id_like_timestamp',
+ 'GTE_language_engaging_user_id_reply_timestamp',
+ 'GTE_language_engaging_user_id_retweet_timestamp',
+ 'GTE_language_engaging_user_id_retweet_with_comment_timestamp',
+ 'GTE_language_engaging_user_id_like_timestamp',
+ 'GTE_tweet_type_engaging_user_id_reply_timestamp',
+ 'GTE_tweet_type_engaging_user_id_retweet_timestamp',
+ 'GTE_tweet_type_engaging_user_id_retweet_with_comment_timestamp',
+ 'GTE_tweet_type_engaging_user_id_like_timestamp',
+ 'GTE_dt_dow_engaged_with_user_id_reply_timestamp',
+ 'GTE_dt_dow_engaged_with_user_id_retweet_timestamp',
+ 'GTE_dt_dow_engaged_with_user_id_retweet_with_comment_timestamp',
+ 'GTE_dt_dow_engaged_with_user_id_like_timestamp',
+ 'GTE_mentioned_count_engaging_user_id_reply_timestamp',
+ 'GTE_mentioned_count_engaging_user_id_retweet_timestamp',
+ 'GTE_mentioned_count_engaging_user_id_retweet_with_comment_timestamp',
+ 'GTE_mentioned_count_engaging_user_id_like_timestamp',
+ 'GTE_dt_hour_engaged_with_user_id_reply_timestamp',
+ 'GTE_dt_hour_engaged_with_user_id_retweet_timestamp',
+ 'GTE_dt_hour_engaged_with_user_id_retweet_with_comment_timestamp',
+ 'GTE_dt_hour_engaged_with_user_id_like_timestamp',
+ 'GTE_dt_dow_engaging_user_id_reply_timestamp',
+ 'GTE_dt_dow_engaging_user_id_retweet_timestamp',
+ 'GTE_dt_dow_engaging_user_id_retweet_with_comment_timestamp',
+ 'GTE_dt_dow_engaging_user_id_like_timestamp',
+ 'GTE_dt_hour_engaging_user_id_reply_timestamp',
+ 'GTE_dt_hour_engaging_user_id_retweet_timestamp',
+ 'GTE_dt_hour_engaging_user_id_retweet_with_comment_timestamp',
+ 'GTE_dt_hour_engaging_user_id_like_timestamp'
+]
+
+def decodeBertTokenizerAndExtractFeatures(df, proc, output_name):
+    from transformers import BertTokenizer
+    tokenizer = BertTokenizer.from_pretrained(
+        'bert-base-multilingual-cased', do_lower_case=False)
+
+    # define UDF
+    tokenizer_decode = f.udf(lambda x: tokenizer.decode(
+        [int(n) for n in x.split('\t')]))
+    format_url = f.udf(lambda x: x.replace(
+        'https : / / t. co / ', 'https://t.co/').replace('@ ', '@'))
+
+    # define decode udf operations
+    op_feature_modification_tokenizer_decode = FeatureAdd(
+        cols={'tweet': 'text_tokens'}, udfImpl=tokenizer_decode)
+    op_feature_modification_format_url = FeatureModification(
+        cols=['tweet'], udfImpl=format_url)
+    
+    op_feature_target_classify = FeatureModification(cols={
+        "reply_timestamp": "f.when(f.col('reply_timestamp') > 0, 1).otherwise(0)",
+        "retweet_timestamp": "f.when(f.col('retweet_timestamp') > 0, 1).otherwise(0)",
+        "retweet_with_comment_timestamp": "f.when(f.col('retweet_with_comment_timestamp') > 0, 1).otherwise(0)",
+        "like_timestamp": "f.when(f.col('like_timestamp') > 0, 1).otherwise(0)"}, op='inline')
+    
+    # define new features
+    op_feature_from_original = FeatureAdd(
+        cols={"has_photo": "f.col('present_media').contains('Photo').cast(t.IntegerType())",
+              "has_video": "f.col('present_media').contains('Vedio').cast(t.IntegerType())",
+              "has_gif": "f.col('present_media').contains('GIF').cast(t.IntegerType())",             
+              "a_ff_rate": "f.col('engaged_with_user_following_count')/f.col('engaged_with_user_follower_count')",
+              "b_ff_rate": "f.col('engaging_user_following_count') /f.col('engaging_user_follower_count')",
+              "dt_dow": "f.dayofweek(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())",
+              "dt_hour": "f.hour(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType())",           
+              "mention": "f.regexp_extract(f.col('tweet'), r'[^RT]\s@(\S+)', 1)",
+              "has_mention": "(f.col('mention')!= '').cast(t.IntegerType())"
+        }, op='inline')
+
+    # execute
+    proc.reset_ops([op_feature_modification_tokenizer_decode,
+                    op_feature_modification_format_url,
+                    op_feature_target_classify,
+                    op_feature_from_original])
+    t1 = timer()
+    df = proc.transform(df, name=output_name)
+    t2 = timer()
+    print("BertTokenizer decode and feature extacting took %.3f" % (t2 - t1))
+
+    return df
+
+def categorifyFeatures(df, proc, output_name, gen_dict, sampleRatio=1):
+    # 1. prepare dictionary
+    dict_dfs = []
+    if gen_dict:
+        # only call below function when target dicts were not pre-prepared
+        op_gen_dict_multiItems = GenerateDictionary(['tweet'], doSplit=True, sep=' ', bucketSize=100)
+        op_gen_dict_singleItems = GenerateDictionary(['mention'], bucketSize=100)
+        proc.reset_ops([op_gen_dict_multiItems, op_gen_dict_singleItems])
+        t1 = timer()
+        dict_dfs = proc.generate_dicts(df)
+        t2 = timer()
+        print("Generate Dictionary took %.3f" % (t2 - t1))
+    else:
+        dict_names = ['tweet', 'mention']
+        dict_dfs = [{'col_name': name, 'dict': proc.spark.read.parquet(
+            "%s/%s/%s/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in dict_names]
+    # 2. since we need both mentioned_bucket_id and mentioned_count, add two mention id dict_dfs
+    for dict_df in dict_dfs:
+        if dict_df['col_name'] == 'mention':
+            dict_dfs.append({'col_name': 'mentioned_bucket_id', 'dict':dict_df['dict']})
+            dict_dfs.append({'col_name': 'mentioned_count', 'dict':dict_df['dict'].drop('dict_col_id').withColumnRenamed('count', 'dict_col_id')})
+    op_feature_add = FeatureAdd({"mentioned_bucket_id": "f.col('mention')", "mentioned_count": "f.col('mention')"}, op='inline')
+    
+    # 3. categorify
+    op_categorify_multiItems = Categorify([{'bucketized_tweet_word': 'tweet'}], dict_dfs=dict_dfs, doSplit=True, sep=' ')
+    op_categorify_singleItem = Categorify(['mentioned_bucket_id', 'mentioned_count'], dict_dfs=dict_dfs)
+    proc.reset_ops([op_feature_add, op_categorify_multiItems, op_categorify_singleItem])
+    
+    # 4. get most and second used bucketized_tweet_word
+    op_feature_add_sorted_bucketized_tweet_word = FeatureAdd(
+        cols={'sorted_bucketized_tweet_word': "f.expr('sortIntArrayByFrequency(bucketized_tweet_word)')"}, op='inline')
+    op_feature_add_convert = FeatureAdd(
+        cols={'most_used_word_bucket_id': "f.when(f.size(f.col('sorted_bucketized_tweet_word'))>0, f.col('sorted_bucketized_tweet_word').getItem(0)).otherwise(np.nan)",
+             'second_used_word_bucket_id': "f.when(f.size(f.col('sorted_bucketized_tweet_word'))>1, f.col('sorted_bucketized_tweet_word').getItem(1)).otherwise(np.nan)"}, op='inline')
+    proc.append_ops([op_feature_add_sorted_bucketized_tweet_word, op_feature_add_convert])
+
+    # 5. transform
+    t1 = timer()
+    if sampleRatio != 1:
+        df = df.sample(sampleRatio)
+    df = proc.transform(df, name=output_name)
+    t2 = timer()
+    print("categorify and getMostAndSecondUsedWordBucketId took %.3f" % (t2 - t1))
+    return (df, dict_dfs)
+
+
+def encodingFeatures(df, proc, output_name, gen_dict, sampleRatio=1):   
+    targets = ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp']
+    y_mean_all = []
+    
+    t1 = timer()
+    if gen_dict:
+        for tgt in targets:
+            tmp = df.groupBy().mean(tgt).collect()[0]
+            y_mean = tmp[f"avg({tgt})"]
+            y_mean_all.append(y_mean)
+        schema = t.StructType([t.StructField(tgt, t.FloatType(), True) for tgt in targets])
+        y_mean_all_df = proc.spark.createDataFrame([tuple(y_mean_all)], schema)
+        y_mean_all_df.write.format("parquet").mode("overwrite").save(
+            "%s/%s/%s/targets_mean" % (proc.path_prefix, proc.current_path, proc.dicts_path))
+    y_mean_all_df = proc.spark.read.parquet(
+        "%s/%s/%s/targets_mean" % (proc.path_prefix, proc.current_path, proc.dicts_path))
+
+    features = [
+            'engaged_with_user_id',
+            'language',
+            'dt_dow',
+            'tweet_type',
+            'most_used_word_bucket_id',
+            'second_used_word_bucket_id',
+            'mentioned_count',
+            'mentioned_bucket_id',
+            ['has_mention', 'engaging_user_id'],
+            ['mentioned_count', 'engaging_user_id'],
+            ['mentioned_bucket_id', 'engaging_user_id'],
+            ['language', 'engaged_with_user_id'],
+            ['language', 'engaging_user_id'],
+            ['dt_dow', 'engaged_with_user_id'],
+            ['dt_dow', 'engaging_user_id'],
+            ['dt_hour', 'engaged_with_user_id'],
+            ['dt_hour', 'engaging_user_id'],
+            ['tweet_type', 'engaged_with_user_id'],
+            ['tweet_type', 'engaging_user_id']
+    ]
+    excludes = {'dt_dow': ['reply_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'],
+               'tweet_type': ['like_timestamp', 'retweet_with_comment_timestamp']
+              }
+
+    te_train_dfs = []
+    te_test_dfs = []
+    for c in features:
+        target_tmp = targets
+        out_name = ""
+        if str(c) in excludes:
+            target_tmp = []
+            for tgt in targets:
+                if tgt not in excludes[c]:
+                    target_tmp.append(tgt)
+        out_col_list = []
+        for tgt in target_tmp:
+            if isinstance(c, list):
+                out_col_list.append('GTE_'+'_'.join(c)+'_'+tgt)
+                out_name = 'GTE_'+'_'.join(c)
+            else:
+                out_col_list.append(f'TE_{c}_{tgt}')
+                out_name = f'TE_{c}'
+        if gen_dict:
+            start = timer()
+            encoder = TargetEncoder(proc, c, target_tmp, out_col_list, out_name, out_dtype=t.FloatType(), y_mean_list=y_mean_all)
+            te_train_df, te_test_df = encoder.transform(df)
+            te_train_dfs.append({'col_name': ['fold'] + (c if isinstance(c, list) else [c]), 'dict': te_train_df})
+            te_test_dfs.append({'col_name': c, 'dict': te_test_df})
+            print(f"generating target encoding for %s upon %s took %.1f seconds"%(str(c), str(target_tmp), timer()-start))
+        else:
+            te_train_path = "%s/%s/%s/train/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name)
+            te_test_path = "%s/%s/%s/test/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name)               
+            te_train_dfs.append({'col_name': ['fold'] + (c if isinstance(c, list) else [c]), 'dict': proc.spark.read.parquet(te_train_path)})
+            te_test_dfs.append({'col_name': c, 'dict': proc.spark.read.parquet(te_test_path)})
+    t2 = timer()
+    print("Generate encoding feature totally took %.3f" % (t2 - t1))
+
+    # merge dicts to original table
+    op_merge_to_train = ModelMerge(te_train_dfs)
+    proc.reset_ops([op_merge_to_train])
+    
+    # select features
+    op_select = SelectFeature(target_list + final_feature_list)
+    proc.append_ops([op_select])
+
+    t1 = timer()
+    if sampleRatio != 1:
+        df = df.sample(sampleRatio)
+    df = proc.transform(df, name=output_name)
+    t2 = timer()
+    print("encodingFeatures took %.3f" % (t2 - t1))
+    
+    return (df, te_train_dfs, te_test_dfs, y_mean_all_df)
+
+
+def splitByDate(df, proc, train_output, test_output, numFolds=5):
+    # 1.1 get timestamp range
+    import datetime
+    min_timestamp = df.select('tweet_timestamp').agg({'tweet_timestamp': 'min'}).collect()[0]['min(tweet_timestamp)']
+    max_timestamp = df.select('tweet_timestamp').agg({'tweet_timestamp': 'max'}).collect()[0]['max(tweet_timestamp)']
+    seconds_in_day = 3600 * 24
+
+    print(
+        "min_timestamp is %s, max_timestamp is %s, 20 days max is %s" % (
+            datetime.datetime.fromtimestamp(min_timestamp).strftime('%Y-%m-%d %H:%M:%S'),
+            datetime.datetime.fromtimestamp(max_timestamp).strftime('%Y-%m-%d %H:%M:%S'),
+            datetime.datetime.fromtimestamp(min_timestamp + 20 * seconds_in_day).strftime('%Y-%m-%d %H:%M:%S')
+        ))
+
+    time_range_split = {
+        'train': (min_timestamp, seconds_in_day * 18 + min_timestamp),
+        'test': (seconds_in_day * 18 + min_timestamp, max_timestamp)
+    }
+
+    print(time_range_split)
+
+    # 1.2 save ranged data for train
+    # filtering out train range data and save
+    train_start, train_end = time_range_split['train']
+    test_start, test_end = time_range_split['test']
+    t1 = timer()
+    train_df = df.filter(
+        (f.col('tweet_timestamp') >= f.lit(train_start)) & (f.col('tweet_timestamp') < f.lit(train_end)))
+    train_df = train_df.withColumn("fold", f.round(f.rand(seed=42)*(numFolds-1)).cast("int"))
+    train_df.write.format('parquet').mode('overwrite').save(proc.path_prefix + proc.current_path + train_output)
+    t2 = timer()
+    print("split to train took %.3f" % (t2 - t1))
+    
+    t1 = timer()
+    test_df = df.filter(
+        (f.col('tweet_timestamp') >= f.lit(test_start)) & (f.col('tweet_timestamp') < f.lit(test_end)))
+    test_df.write.format('parquet').mode('overwrite').save(proc.path_prefix + proc.current_path + test_output)
+    t2 = timer()
+    print("split to test took %.3f" % (t2 - t1))
+    
+    return (proc.spark.read.parquet(proc.path_prefix + proc.current_path + train_output),
+            proc.spark.read.parquet(proc.path_prefix + proc.current_path + test_output))
+
+
+def mergeFeaturesToTest(df, dict_dfs, te_test_dfs, y_mean_all_df, proc, output_name):
+    # categorify test data with train generated dictionary
+    # 1. since we need both mentioned_bucket_id and mentioned_count, add two mention id dict_dfs
+    for dict_df in dict_dfs:
+        if dict_df['col_name'] == 'mention':
+            dict_dfs.append({'col_name': 'mentioned_bucket_id', 'dict':dict_df['dict']})
+            dict_dfs.append({'col_name': 'mentioned_count', 'dict':dict_df['dict'].drop('dict_col_id').withColumnRenamed('count', 'dict_col_id')})
+    op_feature_add = FeatureAdd({"mentioned_bucket_id": "f.col('mention')", "mentioned_count": "f.col('mention')"}, op='inline')
+    
+    # 2. categorify
+    op_categorify_multiItems = Categorify([{'bucketized_tweet_word': 'tweet'}], dict_dfs=dict_dfs, doSplit=True, sep=' ')
+    op_categorify_singleItem = Categorify(['mentioned_bucket_id', 'mentioned_count'], dict_dfs=dict_dfs)
+    proc.reset_ops([op_feature_add, op_categorify_multiItems, op_categorify_singleItem])
+    
+    # 3. get most and second used bucketized_tweet_word
+    op_feature_add_sorted_bucketized_tweet_word = FeatureAdd(
+        cols={'sorted_bucketized_tweet_word': "f.expr('sortIntArrayByFrequency(bucketized_tweet_word)')"}, op='inline')
+    op_feature_add_convert = FeatureAdd(
+        cols={'most_used_word_bucket_id': "f.when(f.size(f.col('sorted_bucketized_tweet_word'))>0, f.col('sorted_bucketized_tweet_word').getItem(0)).otherwise(np.nan)",
+             'second_used_word_bucket_id': "f.when(f.size(f.col('sorted_bucketized_tweet_word'))>1, f.col('sorted_bucketized_tweet_word').getItem(1)).otherwise(np.nan)"}, op='inline')
+    proc.append_ops([op_feature_add_sorted_bucketized_tweet_word, op_feature_add_convert])
+    
+    # 4. merge dicts to original table
+    op_merge_to_test = ModelMerge(te_test_dfs)
+    proc.append_ops([op_merge_to_test])
+        
+    # 5. set null in encoding features to y_mean
+    y_mean_all = y_mean_all_df.collect()[0]
+    for tgt in target_list:
+        to_fill_list = []
+        for feature in final_feature_list:
+            if 'TE_' in feature and tgt in feature:
+                to_fill_list.append(feature)
+        op_fill_na = FillNA(to_fill_list, y_mean_all[tgt])
+        proc.append_ops([op_fill_na])
+    
+    # select features
+    op_select = SelectFeature(target_list + final_feature_list)
+    proc.append_ops([op_select])
+
+    t1 = timer()
+    df = proc.transform(df, name=output_name)
+    t2 = timer()
+    print("mergeFeaturesToTest took %.3f" % (t2 - t1))
+
+
+def get_encoding_features_dicts(proc):
+    targets = ['reply_timestamp', 'retweet_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp']
+    y_mean_all = []
+    y_mean_all_df = proc.spark.read.parquet(
+        "%s/%s/%s/targets_mean" % (proc.path_prefix, proc.current_path, proc.dicts_path))
+    features = [
+            'engaged_with_user_id',
+            'language',
+            'dt_dow',
+            'tweet_type',
+            'most_used_word_bucket_id',
+            'second_used_word_bucket_id',
+            'mentioned_count',
+            'mentioned_bucket_id',
+            ['has_mention', 'engaging_user_id'],
+            ['mentioned_count', 'engaging_user_id'],
+            ['mentioned_bucket_id', 'engaging_user_id'],
+            ['language', 'engaged_with_user_id'],
+            ['language', 'engaging_user_id'],
+            ['dt_dow', 'engaged_with_user_id'],
+            ['dt_dow', 'engaging_user_id'],
+            ['dt_hour', 'engaged_with_user_id'],
+            ['dt_hour', 'engaging_user_id'],
+            ['tweet_type', 'engaged_with_user_id'],
+            ['tweet_type', 'engaging_user_id']
+    ]
+    excludes = {'dt_dow': ['reply_timestamp', 'retweet_with_comment_timestamp', 'like_timestamp'],
+               'tweet_type': ['like_timestamp', 'retweet_with_comment_timestamp']
+              }
+
+    te_train_dfs = []
+    te_test_dfs = []
+    for c in features:
+        target_tmp = targets
+        out_name = ""
+        if str(c) in excludes:
+            target_tmp = []
+            for tgt in targets:
+                if tgt not in excludes[c]:
+                    target_tmp.append(tgt)
+        out_col_list = []
+        for tgt in target_tmp:
+            if isinstance(c, list):
+                out_col_list.append('GTE_'+'_'.join(c)+'_'+tgt)
+                out_name = 'GTE_'+'_'.join(c)
+            else:
+                out_col_list.append(f'TE_{c}_{tgt}')
+                out_name = f'TE_{c}'
+        te_train_path = "%s/%s/%s/train/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name)
+        te_test_path = "%s/%s/%s/test/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, out_name)
+        te_train_dfs.append({'col_name': ['fold'] + (c if isinstance(c, list) else [c]), 'dict': proc.spark.read.parquet(te_train_path)})
+        te_test_dfs.append({'col_name': c, 'dict': proc.spark.read.parquet(te_test_path)})
+        
+    return (te_train_dfs, te_test_dfs, y_mean_all_df)
+
+def setup(path_prefix,current_path,dicts_folder):
+    recsysSchema = RecsysSchema()
+
+    ##### 1. Start spark and initialize data processor #####
+    scala_udf_jars = "/home/vmagent/app/script/test/recdp/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+
+    t0 = timer()
+    spark = SparkSession.builder.master('local[64]')\
+        .appName("Recsys2021_data_process")\
+        .config("spark.driver.memory", '450g')\
+        .config("spark.worker.memory", "450g")\
+        .config("spark.local.dir", "/home/vmagent/app/data")\
+        .config("spark.sql.broadcastTimeout", "7200")\
+        .config("spark.cleaner.periodicGC.interval", "15min")\
+        .config("spark.executorEnv.HF_DATASETS_OFFLINE", "1")\
+        .config("spark.executorEnv.TRANSFORMERS_OFFLINE", "1")\
+        .config("spark.executor.memory", "20g")\
+        .config("spark.executor.memoryOverhead", "10g")\
+        .config("spark.executor.cores", "64")\
+        .config("spark.driver.extraClassPath", f"{scala_udf_jars}")\
+        .config("spark.executor.extraClassPath", f"{scala_udf_jars}")\
+        .getOrCreate()
+
+    schema = recsysSchema.toStructType()
+
+    proc = DataProcessor(spark, path_prefix,
+                        current_path=current_path, dicts_path=dicts_folder, shuffle_disk_capacity="1500GB",spark_mode='local')
+    
+    return spark, schema,proc
+
+def train():
+    path_prefix = "hdfs://localhost:9000/"
+    current_path = "/recsys2021/feateng_test/"
+    original_folder = "/recsys2021/oridata/train/"
+    dicts_folder = "recsys_dicts/"
+
+    spark, schema,proc = setup(path_prefix,current_path,dicts_folder)
+
+    # 1.1 prepare dataFrames
+    df = spark.read.parquet(path_prefix + original_folder)
+    df = df.withColumnRenamed('enaging_user_following_count', 'engaging_user_following_count')
+    df = df.withColumnRenamed('enaging_user_is_verified', 'engaging_user_is_verified')
+    print("data loaded!")
+
+    # # fast test, comment for full dataset
+    # df.sample(0.01).write.format("parquet").mode("overwrite").save(path_prefix+"%s/sample_0_0_1" % current_path)
+    # df = spark.read.parquet(path_prefix+"%s/sample_0_0_1" % current_path)
+    # print("data sampled!")
+
+    # ===============================================
+    # decode tweet_tokens
+    df = decodeBertTokenizerAndExtractFeatures(df, proc, output_name="decoded_with_extracted_features")
+    print("data decoded!")
+
+    # ===============================================
+    # splitting and sampling
+    df, test_df = splitByDate(df, proc, train_output="train", test_output="test", numFolds=5)
+    print("data splited!")
+
+    # ===============================================
+    # generate dictionary for categorify indexing
+    df, dict_dfs = categorifyFeatures(df, proc, output_name="train_with_categorified_features", gen_dict=True, sampleRatio=1)
+    print("data categorified!")
+
+    # ===============================================
+    # encoding features
+    df, te_train_dfs, te_test_dfs, y_mean_all_df = encodingFeatures(df, proc, output_name="train_with_features_sample_0_0_3", gen_dict=True, sampleRatio=0.03)
+    print("data encoded!")
+
+    # ===============================================
+    # adding features to test
+    ### Below codes is used to prepare for mergeFeaturesToTest for test separately ###
+    # test_df = spark.read.parquet(path_prefix+current_path+"test")
+    # dict_names = ['tweet', 'mention']
+    # dict_dfs = [{'col_name': name, 'dict': spark.read.parquet(
+    #     "%s/%s/%s/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in dict_names]
+    # te_train_dfs, te_test_dfs, y_mean_all_df = get_encoding_features_dicts(proc)
+    ##################################################################################
+    test_df = mergeFeaturesToTest(test_df, dict_dfs, te_test_dfs, y_mean_all_df, proc, output_name="test_with_features")
+    print("test data merged!")
+
+def valid():
+    path_prefix = "hdfs://localhost:9000/"
+    current_path = "/recsys2021/feateng/"
+    original_folder = "/recsys2021/oridata/valid/"
+    dicts_folder = "recsys_dicts/"
+
+    spark, schema,proc = setup(path_prefix,current_path,dicts_folder)
+    
+    df = spark.read.schema(schema).option('sep', '\x01').csv(path_prefix + original_folder)
+    df = df.withColumnRenamed('enaging_user_follower_count', 'engaging_user_follower_count')
+    df = df.withColumnRenamed('enaging_user_is_verified', 'engaging_user_is_verified')
+    df = df.withColumnRenamed('enaging_user_following_count', 'engaging_user_following_count')
+    df = df.withColumnRenamed('enaging_user_account_creation', 'engaging_user_account_creation')
+    print("data loaded!")
+
+    # ===============================================
+    # decode tweet_tokens
+    df = decodeBertTokenizerAndExtractFeatures(df, proc, output_name="decoded_with_extracted_features_val")
+    print("data decoded!")
+
+    # ===============================================
+    # adding features to valid
+    ### Below codes is used to prepare for mergeFeaturesToTest for test separately ###
+    dict_names = ['tweet', 'mention']
+    dict_dfs = [{'col_name': name, 'dict': spark.read.parquet(
+        "%s/%s/%s/%s" % (proc.path_prefix, proc.current_path, proc.dicts_path, name))} for name in dict_names]
+    te_train_dfs, te_val_dfs, y_mean_all_df = get_encoding_features_dicts(proc)
+    ##################################################################################
+    val_df = mergeFeaturesToTest(df, dict_dfs, te_val_dfs, y_mean_all_df, proc, output_name="val_with_features")
+    print("val data merged!")
+
+if __name__ == "__main__":
+    mode = "train"
+    if mode == "train":
+        train()
+    elif mode == "valid":
+        valid()
+    
\ No newline at end of file
diff --git a/RecDP/examples/python_tests/recsys2021/init.py b/RecDP/examples/python_tests/recsys2021/init.py
new file mode 100644
index 000000000..e0981db4f
--- /dev/null
+++ b/RecDP/examples/python_tests/recsys2021/init.py
@@ -0,0 +1,8 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
+import findspark
+findspark.init()
diff --git a/RecDP/examples/scala_tests/pom.xml b/RecDP/examples/scala_tests/pom.xml
new file mode 100644
index 000000000..6ad841064
--- /dev/null
+++ b/RecDP/examples/scala_tests/pom.xml
@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.intel.broaddp</groupId>
+  <artifactId>test</artifactId>
+  <version>0.1.0</version>
+  <packaging>jar</packaging>
+
+  <licenses>
+    <license>
+      <name>The Apache Software License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+    </license>
+  </licenses>
+
+  <properties>
+    <spark.version>3.0.0</spark.version>
+    <scala.binary.version>2.12</scala.binary.version>
+    <scala.version>2.12.8</scala.version>
+    <jars.target.dir>${project.build.directory}/scala-${scala.binary.version}/jars</jars.target.dir>
+  </properties>
+  <dependencies>
+    <!-- Prevent our dummy JAR from being included in Spark distributions or uploaded to YARN -->
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-sql_${scala.binary.version}</artifactId>
+      <version>${spark.version}</version>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>net.alchim31.maven</groupId>
+        <artifactId>scala-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>scala-compile-first</id>
+            <phase>process-resources</phase>
+            <goals>
+              <goal>add-source</goal>
+              <goal>compile</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>scala-test-compile</id>
+            <phase>process-test-resources</phase>
+            <goals>
+              <goal>testCompile</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.0</version>
+        <configuration>
+          <source>1.8</source>
+          <target>1.8</target>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>compile</phase>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>3.0.0</version>
+        <configuration>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
diff --git a/RecDP/examples/scala_tests/run_test.sh b/RecDP/examples/scala_tests/run_test.sh
new file mode 100755
index 000000000..3cbed7b37
--- /dev/null
+++ b/RecDP/examples/scala_tests/run_test.sh
@@ -0,0 +1,1 @@
+spark-submit --master yarn --name DLRM_JOIN_FILLNA_SCALA --class "org.apache.spark.broaddp.test.dlrm.JoinFillNAWithUDF" /mnt/nvme2/chendi/BlueWhale/frameworks.bigdata.bluewhale/BroadDP/examples/scala_tests/target/test-0.1.0-jar-with-dependencies.jar
diff --git a/RecDP/examples/scala_tests/src/main/scala/org/apache/spark/intel/broaddp/test/dlrm/JoinFillNA.scala b/RecDP/examples/scala_tests/src/main/scala/org/apache/spark/intel/broaddp/test/dlrm/JoinFillNA.scala
new file mode 100644
index 000000000..3bc5bbfb6
--- /dev/null
+++ b/RecDP/examples/scala_tests/src/main/scala/org/apache/spark/intel/broaddp/test/dlrm/JoinFillNA.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.broaddp.test.dlrm
+
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.expressions._
+import org.apache.spark.sql.SparkSession
+import java.util.UUID.randomUUID
+
+object JoinFillNA {
+  def main(args: Array[String]): Unit = {
+    val spark = SparkSession.builder.master("yarn").appName("DLRM_JOIN_FILLNA_SCALA").getOrCreate()
+    val catCols = (14 until 40 toArray).map(i => "_c" + i)
+    val intCols = (1 until 14 toArray).map(i => "_c" + i)
+
+    val path_prefix = "hdfs://"
+    val parquet_folder = "/dlrm/parquet_raw_data/"
+    val parquet_path = s"$path_prefix/$parquet_folder"
+    val files = (0 until 24 toList).map(i => s"$parquet_folder/day_$i")
+    val models_folder = "/dlrm/models/"
+
+    val start = System.nanoTime
+    var df = spark.read.parquet(files:_*)
+
+    for( i <- 14 to 39) {
+      val colName = s"_c$i"
+      val model = spark.read.parquet(s"$models_folder/_c$i").withColumn("id", row_number().over(
+        Window.orderBy(desc("model_count")))).withColumn("id", col("id") - 1).select(col("data").alias(colName), col("id"))
+      val broadcastModel = broadcast(model)
+      df = df.join(broadcastModel, df(colName) === broadcastModel(colName), joinType="left")
+        .drop(colName)
+        .withColumnRenamed("id", colName)
+    }
+
+    df.printSchema
+    df = df.na.fill(0, cols = intCols ++ catCols)
+    val saved_path = s"/dlrm/tmp/${randomUUID().toString}/"
+    df.write.format("parquet").mode("overwrite").save(saved_path)
+    
+    val end = System.nanoTime
+    println("Total time: " + (end - start) / 1e9d)
+    println("Saved to: " + saved_path)
+    spark.stop()
+  }
+} 
diff --git a/RecDP/examples/scala_tests/src/main/scala/org/apache/spark/intel/broaddp/test/dlrm/JoinFillNASplit.scala b/RecDP/examples/scala_tests/src/main/scala/org/apache/spark/intel/broaddp/test/dlrm/JoinFillNASplit.scala
new file mode 100644
index 000000000..49f4e0e7d
--- /dev/null
+++ b/RecDP/examples/scala_tests/src/main/scala/org/apache/spark/intel/broaddp/test/dlrm/JoinFillNASplit.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package  org.apache.spark.broaddp.test.dlrm
+
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.expressions._
+import org.apache.spark.sql.SparkSession
+import java.util.UUID.randomUUID
+
+object JoinFillNASplit {
+  def main(args: Array[String]): Unit = {
+    val spark = SparkSession.builder.master("yarn").appName("DLRM_JOIN_FILLNA_SCALA").getOrCreate()
+    val catCols = (14 until 40 toArray).map(i => "_c" + i)
+    val intCols = (1 until 14 toArray).map(i => "_c" + i)
+
+    val path_prefix = "hdfs://"
+    val parquet_folder = "/dlrm/parquet_raw_data/"
+    val parquet_path = s"$path_prefix/$parquet_folder"
+    val files = (0 until 24 toList).map(i => s"$parquet_folder/day_$i")
+    val models_folder = "/dlrm/models/"
+
+    val start = System.nanoTime
+    var df = spark.read.parquet(files:_*)
+
+    for( i <- 14 to 39) {
+      val colName = s"_c$i"
+      val model = spark.read.parquet(s"$models_folder/_c$i").withColumn("id", row_number().over(
+        Window.orderBy(desc("model_count")))).withColumn("id", col("id") - 1).select(col("data").alias(colName), col("id"))
+      val broadcastModel = broadcast(model)
+      df = df.join(broadcastModel, df(colName) === broadcastModel(colName), joinType="left")
+        .drop(colName)
+        .withColumnRenamed("id", colName)
+    }
+
+    df.printSchema
+    df = df.repartition(2000)
+    df = df.na.fill(0, cols = intCols ++ catCols)
+    val saved_path = s"/dlrm/tmp/${randomUUID().toString}/"
+    df.write.format("parquet").mode("overwrite").save(saved_path)
+    
+    val end = System.nanoTime
+    println("Total time: " + (end - start) / 1e9d)
+    println("Saved to: " + saved_path)
+    spark.stop()
+  }
+} 
diff --git a/RecDP/examples/scala_tests/src/main/scala/org/apache/spark/intel/broaddp/test/dlrm/JoinFillNAWithUDF.scala b/RecDP/examples/scala_tests/src/main/scala/org/apache/spark/intel/broaddp/test/dlrm/JoinFillNAWithUDF.scala
new file mode 100644
index 000000000..6a98358fd
--- /dev/null
+++ b/RecDP/examples/scala_tests/src/main/scala/org/apache/spark/intel/broaddp/test/dlrm/JoinFillNAWithUDF.scala
@@ -0,0 +1,94 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.broaddp.test.dlrm
+
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql._
+import org.apache.spark.sql.expressions._
+import java.util.UUID.randomUUID
+import org.apache.spark.sql.functions.udf
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.util.collection.OpenHashMap
+
+
+object JoinFillNAWithUDF {
+  var spark: SparkSession = _
+
+  def categorify_udf(df: DataFrame, dict_df: DataFrame, col_name: String): DataFrame = {
+    val sorted_data = dict_df.orderBy(desc("model_count")).collect()
+    // option 1, use OpenHashMap
+    val broadcast_data = new OpenHashMap[String, Int](sorted_data.size)
+    sorted_data.zipWithIndex.foreach { case (row, idx) =>
+      broadcast_data.update(row(0).asInstanceOf[String], idx)
+    }
+    // option 2, use Map
+    // val broadcast_data: Array[(String, Int)] = sorted_data.zipWithIndex.map{case (row, idx) => (row(0).asInstanceOf[String], idx)}.toMap
+
+    val broadcast_handler = spark.sparkContext.broadcast(broadcast_data)
+
+    def mapToIndex(x: String): Int = {
+      val broadcasted = broadcast_handler.value
+      if (broadcasted.contains(x)) {
+          return broadcasted(x)
+      } else {
+          return -1
+      }
+    }
+
+    val udf_mapToIndex = udf(mapToIndex(_))
+    spark.udf.register("udf_mapToIndex", udf_mapToIndex.asNondeterministic())
+    df.withColumn(col_name, udf_mapToIndex(col(col_name)))
+  }
+
+  def categorify(df: DataFrame, dict_dfs: Array[(String, DataFrame)]): DataFrame = {
+    var new_df = df
+    dict_dfs.foreach{case(col, dict_df) => 
+      new_df = categorify_udf(new_df, dict_df, col)
+    }
+    new_df
+  }
+
+  def main(args: Array[String]): Unit = {
+    spark = SparkSession.builder.master("yarn").appName("DLRM_JOIN_FILLNA_WITH_UDF_SCALA").getOrCreate()
+    val catCols = (14 until 40 toArray).map(i => "_c" + i)
+    val intCols = (1 until 14 toArray).map(i => "_c" + i)
+
+    val path_prefix = "hdfs://"
+    val parquet_folder = "/dlrm/parquet_raw_data/"
+    val parquet_path = s"$path_prefix/$parquet_folder"
+    val files = (0 until 24 toList).map(i => s"$parquet_folder/day_$i")
+    val models_folder = "/dlrm/models/"
+
+    val start = System.nanoTime
+    var df = spark.read.parquet(files:_*)
+    val dict_dfs = catCols.map(col => (col, spark.read.parquet(s"$models_folder/$col")))
+
+    // categorify
+    // option 1, using UDF
+    df = categorify(df, dict_dfs)
+
+    // FillNA
+    df = df.na.fill(0, cols = intCols ++ catCols)
+
+    // Save result to HDFS
+    val saved_path = s"/dlrm/tmp/${randomUUID().toString}/"
+    df.write.format("parquet").mode("overwrite").save(saved_path)
+    // save to /dlrm/tmp/a5a0319c-e149-41db-ab20-29c8c3e88b47
+    val end = System.nanoTime
+    println("Total time: " + (end - start) / 1e9d)
+    println("Saved to: " + saved_path)
+    spark.stop()
+  }
+}
diff --git a/RecDP/tests/data/part-00008-3f7afb26-5b6e-4f4a-8a44-f439ddc4319f-c000.snappy.parquet b/RecDP/tests/data/part-00008-3f7afb26-5b6e-4f4a-8a44-f439ddc4319f-c000.snappy.parquet
new file mode 100644
index 000000000..842392d3e
Binary files /dev/null and b/RecDP/tests/data/part-00008-3f7afb26-5b6e-4f4a-8a44-f439ddc4319f-c000.snappy.parquet differ
diff --git a/RecDP/tests/init.py b/RecDP/tests/init.py
new file mode 100644
index 000000000..ed9fa0375
--- /dev/null
+++ b/RecDP/tests/init.py
@@ -0,0 +1,6 @@
+import os
+import sys
+import pathlib
+import_path = str(pathlib.Path(__file__).parent.parent.absolute())
+print(import_path)
+sys.path.append(import_path)
diff --git a/RecDP/tests/test_categorify.py b/RecDP/tests/test_categorify.py
new file mode 100644
index 000000000..38284c816
--- /dev/null
+++ b/RecDP/tests/test_categorify.py
@@ -0,0 +1,52 @@
+#!/env/bin/python
+
+import os
+import pathlib
+import sys
+
+import numpy as np
+import pandas as pd
+import pyrecdp
+import pyspark.sql.functions as f
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+from pyspark import *
+from pyspark.sql import *
+
+
+def main():
+    path_prefix = "file://"
+    cur_folder = str(pathlib.Path(__file__).parent.absolute())
+    folder = cur_folder + "/data"
+    path = path_prefix + folder
+    recdp_path = pyrecdp.__path__[0]
+    scala_udf_jars = recdp_path + "/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+    print(scala_udf_jars)
+
+    ##### 1. Start spark and initialize data processor #####
+    spark = SparkSession.builder.master("local[1]")\
+        .config('spark.eventLog.enabled', False)\
+        .config('spark.driver.maxResultSize', '16G')\
+        .config('spark.driver.memory', '10g')\
+        .config('spark.worker.memory', '10g')\
+        .config('spark.executor.memory', '10g')\
+        .config("spark.driver.extraClassPath", f"{scala_udf_jars}")\
+        .config("spark.executor.extraClassPath", f"{scala_udf_jars}")\
+        .appName("test_categorify")\
+        .getOrCreate()
+    
+    proc = DataProcessor(spark, path_prefix, cur_folder)
+    print(f"DataSource path is {path}")
+    df = spark.read.parquet(f"{path}")
+    df = df.select("tweet", "language")
+    df.printSchema()
+
+    proc.reset_ops([Categorify(['language'])])
+    df = proc.apply(df)
+    df = df.fillna('None')
+    df.show(vertical = True)
+    df.explain()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/RecDP/tests/test_codegenseperate.py b/RecDP/tests/test_codegenseperate.py
new file mode 100644
index 000000000..2fb3d95d6
--- /dev/null
+++ b/RecDP/tests/test_codegenseperate.py
@@ -0,0 +1,54 @@
+#!/env/bin/python
+
+import os
+import pathlib
+import sys
+
+import numpy as np
+import pandas as pd
+import pyrecdp
+import pyspark.sql.functions as f
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+from pyspark import *
+from pyspark.sql import *
+
+def main():
+    path_prefix = "file://"
+    cur_folder = str(pathlib.Path(__file__).parent.absolute())
+    folder = cur_folder + "/data"
+    path = path_prefix + folder
+    recdp_path = pyrecdp.__path__[0]
+    scala_udf_jars = recdp_path + "/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+    print(scala_udf_jars)
+
+    ##### 1. Start spark and initialize data processor #####
+    spark = SparkSession.builder.master("local[1]")\
+        .config('spark.eventLog.enabled', False)\
+        .config('spark.driver.maxResultSize', '16G')\
+        .config('spark.driver.memory', '10g')\
+        .config('spark.worker.memory', '10g')\
+        .config('spark.executor.memory', '10g')\
+        .config("spark.driver.extraClassPath", f"{scala_udf_jars}")\
+        .config("spark.executor.extraClassPath", f"{scala_udf_jars}")\
+        .appName("test_codegenseperator")\
+        .getOrCreate()
+    
+    proc = DataProcessor(spark)
+    print(f"DataSource path is {path}")
+    df = spark.read.parquet(f"{path}")
+    df.printSchema()
+
+    dict_df = df.filter("like_timestamp is not null").groupBy("engaged_with_user_id").agg({"like_timestamp": "count"})
+
+    df.join(dict_df, "engaged_with_user_id")
+    vspark = str(spark.version)
+    if vspark.startswith("3.1") or vspark.startswith("3.0"):
+        df = df.withColumn("engaged_with_user_id", f.expr(f"CodegenSeparator1(engaged_with_user_id)"))
+    df = df.fillna('None')
+    df.show(1, vertical = True)
+    df.explain()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/RecDP/tests/test_sortArrayByFrequency.py b/RecDP/tests/test_sortArrayByFrequency.py
new file mode 100644
index 000000000..d3d893b4f
--- /dev/null
+++ b/RecDP/tests/test_sortArrayByFrequency.py
@@ -0,0 +1,51 @@
+#!/env/bin/python
+
+import os
+import pathlib
+import sys
+
+import numpy as np
+import pandas as pd
+import pyrecdp
+import pyspark.sql.functions as f
+import pyspark.sql.types as t
+from pyrecdp.data_processor import *
+from pyrecdp.utils import *
+from pyspark import *
+from pyspark.sql import *
+
+def main():
+    path_prefix = "file://"
+    cur_folder = str(pathlib.Path(__file__).parent.absolute())
+    folder = cur_folder + "/data"
+    path = path_prefix + folder
+    recdp_path = pyrecdp.__path__[0]
+    scala_udf_jars = recdp_path + "/ScalaProcessUtils/target/recdp-scala-extensions-0.1.0-jar-with-dependencies.jar"
+    print(scala_udf_jars)
+
+    ##### 1. Start spark and initialize data processor #####
+    spark = SparkSession.builder.master("local[1]")\
+        .config('spark.eventLog.enabled', False)\
+        .config('spark.driver.maxResultSize', '16G')\
+        .config('spark.driver.memory', '10g')\
+        .config('spark.worker.memory', '10g')\
+        .config('spark.executor.memory', '10g')\
+        .config("spark.driver.extraClassPath", f"{scala_udf_jars}")\
+        .config("spark.executor.extraClassPath", f"{scala_udf_jars}")\
+        .appName("test_sortArrayByFrequency")\
+        .getOrCreate()
+    
+    proc = DataProcessor(spark)
+    print(f"DataSource path is {path}")
+    df = spark.read.parquet(f"{path}")
+    df = df.select("language", "tweet_timestamp")
+    df = df.withColumn("dt_hour", f.dayofweek(f.from_unixtime(f.col('tweet_timestamp'))).cast(t.IntegerType()))
+    df = df.groupby('dt_hour').agg(f.collect_list("language").alias("language_list"))
+    df = df.filter("size(language_list) > 3")
+    df = df.withColumn("sorted_langugage", f.expr(f"sortStringArrayByFrequency(language_list)"))
+    df.printSchema()
+    df.show(24, vertical = True, truncate = 100)
+
+
+if __name__ == "__main__":
+    main()