[Nexus 5X Original Android Development] osm0sis:
Ah yes.. install_keyring will be the real snag. Everything else seems to work so far.

diff -Nur a/init.recovery.bullhead.rc b/init.recovery.bullhead.rc
--- a/init.recovery.bullhead.rc	2017-08-27 13:53:45.398067800 -0300
+++ b/init.recovery.bullhead.rc	2017-08-27 13:52:28.256967100 -0300
@@ -1,7 +1,10 @@
# separate copy needed to use /sbin/linker64 instead of /system/bin/linker64
-service sbinqseecomd /sbin/qseecomd
- oneshot
+#service sbinqseecomd /sbin/qseecomd
+# oneshot
+service sbinqseecomd /vendor/bin/qseecomd
disabled
+ # add path for vendor libs
+ setenv LD_LIBRARY_PATH /vendor/lib64
user root
group root
seclabel u:r:recovery:s0
@@ -13,6 +16,10 @@
group root
seclabel u:r:recovery:s0

+on boot
+# start pulldecryptfiles
+# start sbinqseecomd
+
on fs
# needed to make qseecomd work in recovery
symlink /dev/block/platform/soc.0/f9824900.sdhci /dev/block/bootdevice
@@ -21,6 +28,14 @@
chmod 0664 /dev/ion
chown system system /dev/ion
start pulldecryptfiles
+ install_keyring

-on property:ro.crypto.state=encrypted
+#on property:ro.crypto.state=encrypted
+# mount ext4 /dev/block/bootdevice/by-name/vendor /vendor
+# start sbinqseecomd
+
+on property:pulldecryptfiles.finished=1
start sbinqseecomd
+
+on property:init.svc.sbinqseecomd=running
+ start recovery
diff -Nur a/sbin/pulldecryptfiles.sh b/sbin/pulldecryptfiles.sh
--- a/sbin/pulldecryptfiles.sh	2017-08-27 13:53:48.367661600 -0300
+++ b/sbin/pulldecryptfiles.sh	2017-08-27 13:49:40.852704500 -0300
@@ -4,19 +4,42 @@
# This allows us to decrypt the device in recovery and still be
# able to unmount /vendor when we are done.

+mount -o rw,remount rootfs /
+mount -t ext4 -o ro /dev/block/bootdevice/by-name/system /system
+mkdir /tmp/system
+cp -r /system/lib64 /tmp/system/
+cp -r /system/bin /tmp/system/
+umount /system
+mkdir /system/lib64
+mkdir /system/bin
+mv /tmp/system/lib64/* /system/lib64/
+mv /tmp/system/bin/* /system/bin/
+rmdir /tmp/system/lib64 /tmp/system/bin /tmp/system
+
mkdir -p /vendor
-mount -t ext4 -o ro /dev/block/platform/soc.0/f9824900.sdhci/by-name/vendor /vendor
+mkdir -p /tmp/vendor
+#mount -t ext4 -o ro /dev/block/platform/soc.0/f9824900.sdhci/by-name/vendor /vendor
+mount -t ext4 -o ro /dev/block/bootdevice/by-name/vendor /vendor
+
+#cp /vendor/lib64/libQSEEComAPI.so /sbin/libQSEEComAPI.so
+#cp /vendor/lib64/hw/keystore.msm8992.so /sbin/keystore.msm8992.so
+#cp /vendor/lib64/libdrmfs.so /sbin/libdrmfs.so
+#cp /vendor/lib64/libdrmtime.so /sbin/libdrmtime.so
+#cp /vendor/lib64/librpmb.so /sbin/librpmb.so
+#cp /vendor/lib64/libssd.so /sbin/libssd.so
+#cp /vendor/lib64/libdiag.so /sbin/libdiag.so
+#cp /vendor/lib64/libkmcrypto.so /sbin/libkmcrypto.so

-cp /vendor/lib64/libQSEEComAPI.so /sbin/libQSEEComAPI.so
-cp /vendor/lib64/hw/keystore.msm8992.so /sbin/keystore.msm8992.so
-cp /vendor/lib64/libdrmfs.so /sbin/libdrmfs.so
-cp /vendor/lib64/libdrmtime.so /sbin/libdrmtime.so
-cp /vendor/lib64/librpmb.so /sbin/librpmb.so
-cp /vendor/lib64/libssd.so /sbin/libssd.so
-cp /vendor/lib64/libdiag.so /sbin/libdiag.so
-cp /vendor/lib64/libkmcrypto.so /sbin/libkmcrypto.so
+cp -r /vendor/lib64 /tmp/vendor/
+cp -r /vendor/bin /tmp/vendor/

umount /vendor

-mkdir -p /vendor/lib64/hw
-cp /sbin/keystore.msm8992.so /vendor/lib64/hw/keystore.msm8992.so
+mv /tmp/vendor/* /vendor/
+rmdir /tmp/vendor
+
+#mkdir -p /vendor/lib64/hw
+#cp /sbin/keystore.msm8992.so /vendor/lib64/hw/keystore.msm8992.so
+
+setprop pulldecryptfiles.finished 1
+exit 0

Edit: Updated with a proper patch