Repository: nfrechette/acl (cpp)

Issue:
Look into ARM NEON fused multiply-add: vfmaq_f32
`vfmaq_f32(...)`


Discussion:
Decompression is about 2% faster on my Pixel 3.

Patch:
diff --git a/includes/acl/compression/stream/write_decompression_stats.h b/includes/acl/compression/stream/write_decompression_stats.h
--- a/includes/acl/compression/stream/write_decompression_stats.h
+++ b/includes/acl/compression/stream/write_decompression_stats.h
@@ -130,7 +130,9 @@ namespace acl
 							cache_flusher->flush_buffer(compressed_clips[clip_index], compressed_clips[clip_index]->get_size());
 						}
 						cache_flusher->end_flushing();
-
+					}
+					else
+					{
 						// If we want the cache warm, decompress everything once to prime it
 						DecompressionContextType* context = contexts[0];
 						context->seek(sample_time, SampleRoundingPolicy::None);
diff --git a/includes/acl/math/quat_32.h b/includes/acl/math/quat_32.h
--- a/includes/acl/math/quat_32.h
+++ b/includes/acl/math/quat_32.h
@@ -220,14 +220,29 @@ namespace acl
 		float32x4_t l_yxwz = vrev64q_f32(lhs);
 		float32x4_t l_wzyx = vcombine_f32(vget_high_f32(l_yxwz), vget_low_f32(l_yxwz));
 		float32x4_t lwrx_lzrx_lyrx_lxrx = vmulq_f32(r_xxxx, l_wzyx);
+
+#if defined(ACL_NEON64_INTRINSICS)
+		float32x4_t result0 = vfmaq_f32(lxrw_lyrw_lzrw_lwrw, lwrx_lzrx_lyrx_lxrx, control_wzyx);
+#else
 		float32x4_t result0 = vmlaq_f32(lxrw_lyrw_lzrw_lwrw, lwrx_lzrx_lyrx_lxrx, control_wzyx);
+#endif
 
 		float32x4_t l_zwxy = vrev64q_u32(l_wzyx);
 		float32x4_t lzry_lwry_lxry_lyry = vmulq_f32(r_yyyy, l_zwxy);
+
+#if defined(ACL_NEON64_INTRINSICS)
+		float32x4_t result1 = vfmaq_f32(result0, lzry_lwry_lxry_lyry, control_zwxy);
+#else
 		float32x4_t result1 = vmlaq_f32(result0, lzry_lwry_lxry_lyry, control_zwxy);
+#endif
 
 		float32x4_t lyrz_lxrz_lwrz_lzrz = vmulq_f32(r_zzzz, l_yxwz);
+
+#if defined(ACL_NEON64_INTRINSICS)
+		return vfmaq_f32(result1, lyrz_lxrz_lwrz_lzrz, control_yxwz);
+#else
 		return vmlaq_f32(result1, lyrz_lxrz_lwrz_lzrz, control_yxwz);
+#endif
 #else
 		float lhs_x = quat_get_x(lhs);
 		float lhs_y = quat_get_y(lhs);
@@ -390,7 +405,7 @@ namespace acl
 		// using a AND/XOR with the bias (same number of instructions)
 		float dot = vector_dot(start, end);
 		float bias = dot >= 0.0f ? 1.0f : -1.0f;
-		Vector4_32 interpolated_rotation = vector_mul_add(vector_sub(vector_mul(end, bias), start), alpha, start);
+		Vector4_32 interpolated_rotation = vector_neg_mul_sub(vector_neg_mul_sub(end, bias, start), alpha, start);
 		// Use sqrt/div/mul to normalize because the sqrt/div are faster than rsqrt
 		float inv_len = 1.0f / sqrt(vector_length_squared(interpolated_rotation));
 		return vector_mul(interpolated_rotation, inv_len);
@@ -430,7 +445,7 @@ namespace acl
 		Vector4_32 end_vector = quat_to_vector(end);
 		float dot = vector_dot(start_vector, end_vector);
 		float bias = dot >= 0.0f ? 1.0f : -1.0f;
-		Vector4_32 interpolated_rotation = vector_mul_add(vector_sub(vector_mul(end_vector, bias), start_vector), alpha, start_vector);
+		Vector4_32 interpolated_rotation = vector_neg_mul_sub(vector_neg_mul_sub(end_vector, bias, start_vector), alpha, start_vector);
 		// TODO: Test with this instead: Rotation = (B * Alpha) + (A * (Bias * (1.f - Alpha)));
 		//Vector4_32 value = vector_add(vector_mul(end_vector, alpha), vector_mul(start_vector, bias * (1.0f - alpha)));
 		return quat_normalize(vector_to_quat(interpolated_rotation));
diff --git a/includes/acl/math/vector4_32.h b/includes/acl/math/vector4_32.h
--- a/includes/acl/math/vector4_32.h
+++ b/includes/acl/math/vector4_32.h
@@ -547,7 +547,9 @@ namespace acl
 	// output = (input * scale) + offset
 	inline Vector4_32 ACL_SIMD_CALL vector_mul_add(Vector4_32Arg0 input, Vector4_32Arg1 scale, Vector4_32Arg2 offset)
 	{
-#if defined(ACL_NEON_INTRINSICS)
+#if defined(ACL_NEON64_INTRINSICS)
+		return vfmaq_f32(offset, input, scale);
+#elif defined(ACL_NEON_INTRINSICS)
 		return vmlaq_f32(offset, input, scale);
 #else
 		return vector_add(vector_mul(input, scale), offset);
@@ -556,7 +558,9 @@ namespace acl
 
 	inline Vector4_32 ACL_SIMD_CALL vector_mul_add(Vector4_32Arg0 input, float scale, Vector4_32Arg2 offset)
 	{
-#if defined(ACL_NEON_INTRINSICS)
+#if defined(ACL_NEON64_INTRINSICS)
+		return vfmaq_n_f32(offset, input, scale);
+#elif defined(ACL_NEON_INTRINSICS)
 		return vmlaq_n_f32(offset, input, scale);
 #else
 		return vector_add(vector_mul(input, scale), offset);
@@ -566,13 +570,26 @@ namespace acl
 	// output = offset - (input * scale)
 	inline Vector4_32 ACL_SIMD_CALL vector_neg_mul_sub(Vector4_32Arg0 input, Vector4_32Arg1 scale, Vector4_32Arg2 offset)
 	{
-#if defined(ACL_NEON_INTRINSICS)
+#if defined(ACL_NEON64_INTRINSICS)
+		return vfmsq_f32(offset, input, scale);
+#elif defined(ACL_NEON_INTRINSICS)
 		return vmlsq_f32(offset, input, scale);
 #else
 		return vector_sub(offset, vector_mul(input, scale));
 #endif
 	}
 
+	inline Vector4_32 ACL_SIMD_CALL vector_neg_mul_sub(Vector4_32Arg0 input, float scale, Vector4_32Arg2 offset)
+	{
+#if defined(ACL_NEON64_INTRINSICS)
+		return vfmsq_n_f32(offset, input, scale);
+#elif defined(ACL_NEON_INTRINSICS)
+		return vmlsq_n_f32(offset, input, scale);
+#else
+		return vector_sub(offset, vector_mul(input, scale));
+#endif
+	}
+
 	inline Vector4_32 ACL_SIMD_CALL vector_lerp(Vector4_32Arg0 start, Vector4_32Arg1 end, float alpha)
 	{
 		return vector_mul_add(vector_sub(end, start), alpha, start);
diff --git a/includes/acl/math/vector4_64.h b/includes/acl/math/vector4_64.h
--- a/includes/acl/math/vector4_64.h
+++ b/includes/acl/math/vector4_64.h
@@ -407,6 +407,11 @@ namespace acl
 		return vector_sub(offset, vector_mul(input, scale));
 	}
 
+	inline Vector4_64 vector_neg_mul_sub(const Vector4_64& input, double scale, const Vector4_64& offset)
+	{
+		return vector_sub(offset, vector_mul(input, scale));
+	}
+
 	inline Vector4_64 vector_lerp(const Vector4_64& start, const Vector4_64& end, double alpha)
 	{
 		return vector_mul_add(vector_sub(end, start), alpha, start);
diff --git a/tools/graph_generation/gen_decomp_delta_stats.py b/tools/graph_generation/gen_decomp_delta_stats.py
--- a/tools/graph_generation/gen_decomp_delta_stats.py
+++ b/tools/graph_generation/gen_decomp_delta_stats.py
@@ -39,8 +39,8 @@ def bytes_to_mb(num_bytes):
 	return num_bytes / (1024 * 1024)
 
 if __name__ == "__main__":
-	if len(sys.argv) != 2:
-		print('Usage: python gen_decomp_delta_stats.py <path/to/input_file.sjson>')
+	if len(sys.argv) != 2 and len(sys.argv) != 3:
+		print('Usage: python gen_decomp_delta_stats.py <path/to/input_file.sjson> [-warm]')
 		sys.exit(1)
 
 	input_sjson_file = sys.argv[1]
@@ -56,9 +56,13 @@ def bytes_to_mb(num_bytes):
 		input_sjson_data = sjson.loads(file.read())
 
 	clip_names = []
+	if len(sys.argv) == 3 and sys.argv[2] == '-warm':
+		label = 'warm'
+	else:
+		label = 'cold'
 
-	decomp_delta_cold_us_csv_file = open('decomp_delta_cold_forward_stats_us.csv', 'w')
-	decomp_delta_cold_mbsec_csv_file = open('decomp_delta_cold_forward_stats_mbsec.csv', 'w')
+	decomp_delta_us_csv_file = open('decomp_delta_{}_forward_stats_us.csv'.format(label), 'w')
+	decomp_delta_mbsec_csv_file = open('decomp_delta_{}_forward_stats_mbsec.csv'.format(label), 'w')
 
 	pose_size_per_clip = {}
 	per_entry_data = []
@@ -68,11 +72,11 @@ def bytes_to_mb(num_bytes):
 
 		if len(clip_names) == 0:
 			clip_names = get_clip_names(entry['stats_dir'])
-			print('Variants,Config,Version,{}'.format(','.join(clip_names)), file = decomp_delta_cold_us_csv_file)
-			print('Variants,Config,Version,{}'.format(','.join(clip_names)), file = decomp_delta_cold_mbsec_csv_file)
+			print('Variants,Config,Version,{}'.format(','.join(clip_names)), file = decomp_delta_us_csv_file)
+			print('Variants,Config,Version,{}'.format(','.join(clip_names)), file = decomp_delta_mbsec_csv_file)
 
-		pose_cold_medians_ms = {}
-		bone_cold_medians_ms = {}
+		pose_medians_ms = {}
+		bone_medians_ms = {}
 		clip_names = []
 
 		stat_files = get_clip_stat_files(entry['stats_dir'])
@@ -86,11 +90,11 @@ def bytes_to_mb(num_bytes):
 
 			run_data = clip_sjson_data['runs'][0]['decompression_time_per_sample']
 
-			forward_data_pose_cold = run_data['forward_pose_cold']['data']
-			forward_data_bone_cold = run_data['forward_bone_cold']['data']
+			forward_data_pose = run_data['forward_pose_{}'.format(label)]['data']
+			forward_data_bone = run_data['forward_bone_{}'.format(label)]['data']
 
-			pose_cold_medians_ms[clip_name] = numpy.median(forward_data_pose_cold)
-			bone_cold_medians_ms[clip_name] = numpy.median(forward_data_bone_cold)
+			pose_medians_ms[clip_name] = numpy.median(forward_data_pose)
+			bone_medians_ms[clip_name] = numpy.median(forward_data_bone)
 
 			if 'pose_size' in clip_sjson_data['runs'][0]:
 				pose_size = clip_sjson_data['runs'][0]['pose_size']
@@ -99,39 +103,39 @@ def bytes_to_mb(num_bytes):
 		data = {}
 		data['name'] = entry['name']
 		data['version'] = entry['version']
-		data['pose_cold_medians_ms'] = pose_cold_medians_ms
-		data['bone_cold_medians_ms'] = bone_cold_medians_ms
+		data['pose_medians_ms'] = pose_medians_ms
+		data['bone_medians_ms'] = bone_medians_ms
 		data['clip_names'] = clip_names
 		per_entry_data.append(data)
 
 	for data in per_entry_data:
-		pose_cold_medians_ms = data['pose_cold_medians_ms']
-		bone_cold_medians_ms = data['bone_cold_medians_ms']
+		pose_medians_ms = data['pose_medians_ms']
+		bone_medians_ms = data['bone_medians_ms']
 		clip_names = data['clip_names']
 
-		pose_cold_medians_us = []
-		bone_cold_medians_us = []
-		pose_cold_medians_mbsec = []
-		bone_cold_medians_mbsec = []
+		pose_medians_us = []
+		bone_medians_us = []
+		pose_medians_mbsec = []
+		bone_medians_mbsec = []
 
 		for clip_name in clip_names:
 			pose_size = pose_size_per_clip[clip_name]
-			pose_cold_median_ms = pose_cold_medians_ms[clip_name]
-			bone_cold_median_ms = bone_cold_medians_ms[clip_name]
+			pose_cold_median_ms = pose_medians_ms[clip_name]
+			bone_cold_median_ms = bone_medians_ms[clip_name]
 
 			# Convert the elapsed time from milliseconds into microseconds
-			pose_cold_medians_us.append(str(ms_to_us(pose_cold_median_ms)))
-			bone_cold_medians_us.append(str(ms_to_us(bone_cold_median_ms)))
+			pose_medians_us.append(str(ms_to_us(pose_cold_median_ms)))
+			bone_medians_us.append(str(ms_to_us(bone_cold_median_ms)))
 
 			# Convert the speed into MB/sec
-			pose_cold_medians_mbsec.append(str(bytes_to_mb(pose_size) / ms_to_s(pose_cold_median_ms)))
-			bone_cold_medians_mbsec.append(str(bytes_to_mb(pose_size) / ms_to_s(bone_cold_median_ms)))
+			pose_medians_mbsec.append(str(bytes_to_mb(pose_size) / ms_to_s(pose_cold_median_ms)))
+			bone_medians_mbsec.append(str(bytes_to_mb(pose_size) / ms_to_s(bone_cold_median_ms)))
 
-		print('decompress_pose,{},{},{}'.format(data['name'], data['version'], ','.join(pose_cold_medians_us)), file = decomp_delta_cold_us_csv_file)
-		print('decompress_bone,{},{},{}'.format(data['name'], data['version'], ','.join(bone_cold_medians_us)), file = decomp_delta_cold_us_csv_file)
+		print('decompress_pose,{},{},{}'.format(data['name'], data['version'], ','.join(pose_medians_us)), file = decomp_delta_us_csv_file)
+		print('decompress_bone,{},{},{}'.format(data['name'], data['version'], ','.join(bone_medians_us)), file = decomp_delta_us_csv_file)
 
-		print('decompress_pose,{},{},{}'.format(data['name'], data['version'], ','.join(pose_cold_medians_mbsec)), file = decomp_delta_cold_mbsec_csv_file)
-		print('decompress_bone,{},{},{}'.format(data['name'], data['version'], ','.join(bone_cold_medians_mbsec)), file = decomp_delta_cold_mbsec_csv_file)
+		print('decompress_pose,{},{},{}'.format(data['name'], data['version'], ','.join(pose_medians_mbsec)), file = decomp_delta_mbsec_csv_file)
+		print('decompress_bone,{},{},{}'.format(data['name'], data['version'], ','.join(bone_medians_mbsec)), file = decomp_delta_mbsec_csv_file)
 
-	decomp_delta_cold_us_csv_file.close()
-	decomp_delta_cold_mbsec_csv_file.close()
+	decomp_delta_us_csv_file.close()
+	decomp_delta_mbsec_csv_file.close()


Test patch:
diff --git a/tests/sources/math/test_vector4_impl.h b/tests/sources/math/test_vector4_impl.h
--- a/tests/sources/math/test_vector4_impl.h
+++ b/tests/sources/math/test_vector4_impl.h
@@ -350,15 +350,20 @@ void test_vector4_impl(const Vector4Type& zero, const QuatType& identity, const
 	REQUIRE(scalar_near_equal(vector_get_w(vector_mul_add(test_value10, test_value11, test_value2)), (test_value10_flt[3] * test_value11_flt[3]) + test_value2_flt[3], threshold));
 
 	REQUIRE(scalar_near_equal(vector_get_x(vector_mul_add(test_value10, test_value11_flt[0], test_value2)), (test_value10_flt[0] * test_value11_flt[0]) + test_value2_flt[0], threshold));
-	REQUIRE(scalar_near_equal(vector_get_y(vector_mul_add(test_value10, test_value11_flt[1], test_value2)), (test_value10_flt[1] * test_value11_flt[1]) + test_value2_flt[1], threshold));
-	REQUIRE(scalar_near_equal(vector_get_z(vector_mul_add(test_value10, test_value11_flt[2], test_value2)), (test_value10_flt[2] * test_value11_flt[2]) + test_value2_flt[2], threshold));
-	REQUIRE(scalar_near_equal(vector_get_w(vector_mul_add(test_value10, test_value11_flt[3], test_value2)), (test_value10_flt[3] * test_value11_flt[3]) + test_value2_flt[3], threshold));
+	REQUIRE(scalar_near_equal(vector_get_y(vector_mul_add(test_value10, test_value11_flt[0], test_value2)), (test_value10_flt[1] * test_value11_flt[0]) + test_value2_flt[1], threshold));
+	REQUIRE(scalar_near_equal(vector_get_z(vector_mul_add(test_value10, test_value11_flt[0], test_value2)), (test_value10_flt[2] * test_value11_flt[0]) + test_value2_flt[2], threshold));
+	REQUIRE(scalar_near_equal(vector_get_w(vector_mul_add(test_value10, test_value11_flt[0], test_value2)), (test_value10_flt[3] * test_value11_flt[0]) + test_value2_flt[3], threshold));
 
 	REQUIRE(scalar_near_equal(vector_get_x(vector_neg_mul_sub(test_value10, test_value11, test_value2)), (test_value10_flt[0] * -test_value11_flt[0]) + test_value2_flt[0], threshold));
 	REQUIRE(scalar_near_equal(vector_get_y(vector_neg_mul_sub(test_value10, test_value11, test_value2)), (test_value10_flt[1] * -test_value11_flt[1]) + test_value2_flt[1], threshold));
 	REQUIRE(scalar_near_equal(vector_get_z(vector_neg_mul_sub(test_value10, test_value11, test_value2)), (test_value10_flt[2] * -test_value11_flt[2]) + test_value2_flt[2], threshold));
 	REQUIRE(scalar_near_equal(vector_get_w(vector_neg_mul_sub(test_value10, test_value11, test_value2)), (test_value10_flt[3] * -test_value11_flt[3]) + test_value2_flt[3], threshold));
 
+	REQUIRE(scalar_near_equal(vector_get_x(vector_neg_mul_sub(test_value10, test_value11_flt[0], test_value2)), (test_value10_flt[0] * -test_value11_flt[0]) + test_value2_flt[0], threshold));
+	REQUIRE(scalar_near_equal(vector_get_y(vector_neg_mul_sub(test_value10, test_value11_flt[0], test_value2)), (test_value10_flt[1] * -test_value11_flt[0]) + test_value2_flt[1], threshold));
+	REQUIRE(scalar_near_equal(vector_get_z(vector_neg_mul_sub(test_value10, test_value11_flt[0], test_value2)), (test_value10_flt[2] * -test_value11_flt[0]) + test_value2_flt[2], threshold));
+	REQUIRE(scalar_near_equal(vector_get_w(vector_neg_mul_sub(test_value10, test_value11_flt[0], test_value2)), (test_value10_flt[3] * -test_value11_flt[0]) + test_value2_flt[3], threshold));
+
 	//////////////////////////////////////////////////////////////////////////
 	// Comparisons and masking