From 1ad153e2c0c9622b30b065f9a801266494b6e8f5 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@collabora.com>
Date: Fri, 19 Aug 2022 15:41:29 -0400
Subject: [PATCH] pan/bi: Optimize bitwise arithmetic of booleans
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We can rewrite from the SFU to the higher throughput CVT.

total instructions in shared programs: 2674788 -> 2674280 (-0.02%)
instructions in affected programs: 118823 -> 118315 (-0.43%)
helped: 178
HURT: 16
helped stats (abs) min: 1.0 max: 8.0 x̄: 3.01 x̃: 3
helped stats (rel) min: 0.11% max: 2.63% x̄: 0.56% x̃: 0.45%
HURT stats (abs)   min: 1.0 max: 5.0 x̄: 1.75 x̃: 1
HURT stats (rel)   min: 0.25% max: 4.35% x̄: 2.76% x̃: 2.92%
95% mean confidence interval for instructions value: -2.88 -2.36
95% mean confidence interval for instructions %-change: -0.44% -0.13%
Instructions are helped.

total cycles in shared programs: 140627.36 -> 140640.17 (<.01%)
cycles in affected programs: 564.06 -> 576.88 (2.27%)
helped: 4
HURT: 166
helped stats (abs) min: 0.0625 max: 0.125 x̄: 0.09 x̃: 0
helped stats (rel) min: 5.56% max: 14.29% x̄: 8.97% x̃: 8.01%
HURT stats (abs)   min: 0.015625 max: 0.765625 x̄: 0.08 x̃: 0
HURT stats (rel)   min: 0.31% max: 13.64% x̄: 3.31% x̃: 2.29%
95% mean confidence interval for cycles value: 0.06 0.09
95% mean confidence interval for cycles %-change: 2.52% 3.51%
Cycles are HURT.

total cvt in shared programs: 14128.09 -> 14199.27 (0.50%)
cvt in affected programs: 4766.98 -> 4838.16 (1.49%)
helped: 4
HURT: 1875
helped stats (abs) min: 0.03125 max: 0.046875 x̄: 0.04 x̃: 0
helped stats (rel) min: 1.19% max: 1.49% x̄: 1.37% x̃: 1.39%
HURT stats (abs)   min: 0.015625 max: 0.765625 x̄: 0.04 x̃: 0
HURT stats (rel)   min: 0.10% max: 25.00% x̄: 2.82% x̃: 1.35%
95% mean confidence interval for cvt value: 0.04 0.04
95% mean confidence interval for cvt %-change: 2.63% 2.98%
Cvt are HURT.

total sfu in shared programs: 7549.31 -> 7232.88 (-4.19%)
sfu in affected programs: 2181 -> 1864.56 (-14.51%)
helped: 1889
HURT: 0
helped stats (abs) min: 0.0625 max: 3.0625 x̄: 0.17 x̃: 0
helped stats (rel) min: 1.36% max: 100.00% x̄: 22.93% x̃: 10.53%
95% mean confidence interval for sfu value: -0.18 -0.16
95% mean confidence interval for sfu %-change: -24.14% -21.71%
Sfu are helped.

total quadwords in shared programs: 1449848 -> 1449568 (-0.02%)
quadwords in affected programs: 13864 -> 13584 (-2.02%)
helped: 37
HURT: 2
helped stats (abs) min: 8.0 max: 8.0 x̄: 8.00 x̃: 8
helped stats (rel) min: 1.45% max: 25.00% x̄: 3.43% x̃: 2.04%
HURT stats (abs)   min: 8.0 max: 8.0 x̄: 8.00 x̃: 8
HURT stats (rel)   min: 6.25% max: 7.69% x̄: 6.97% x̃: 6.97%
95% mean confidence interval for quadwords value: -8.34 -6.02
95% mean confidence interval for quadwords %-change: -4.72% -1.07%
Quadwords are helped.

total threads in shared programs: 53556 -> 53557 (<.01%)
threads in affected programs: 1 -> 2 (100.00%)
helped: 1
HURT: 0

Signed-off-by: Alyssa Rosenzweig <alyssa@collabora.com>
---
 src/panfrost/bifrost/bifrost_compile.c        |  1 +
 src/panfrost/bifrost/bifrost_nir.h            |  1 +
 src/panfrost/bifrost/bifrost_nir_algebraic.py | 16 ++++++++++++++++
 3 files changed, 18 insertions(+)

diff --git a/src/panfrost/bifrost/bifrost_compile.c b/src/panfrost/bifrost/bifrost_compile.c
index c0f466b7e4b..64ca394c3e4 100644
--- a/src/panfrost/bifrost/bifrost_compile.c
+++ b/src/panfrost/bifrost/bifrost_compile.c
@@ -4571,6 +4571,7 @@ bi_optimize_nir(nir_shader *nir, unsigned gpu_id, bool is_blend)
                 NIR_PASS(progress, nir, nir_opt_cse);
         }
 
+        NIR_PASS(progress, nir, bifrost_nir_opt_boolean_bitwise);
         NIR_PASS(progress, nir, nir_lower_alu_to_scalar, bi_scalarize_filter, NULL);
         NIR_PASS(progress, nir, nir_lower_phis_to_scalar, true);
         NIR_PASS(progress, nir, nir_opt_vectorize, bi_vectorize_filter, NULL);
diff --git a/src/panfrost/bifrost/bifrost_nir.h b/src/panfrost/bifrost/bifrost_nir.h
index b94ba6eaf38..ba0208012b8 100644
--- a/src/panfrost/bifrost/bifrost_nir.h
+++ b/src/panfrost/bifrost/bifrost_nir.h
@@ -27,3 +27,4 @@
 
 bool bifrost_nir_lower_algebraic_late(nir_shader *shader);
 bool bifrost_nir_lower_xfb(nir_shader *shader);
+bool bifrost_nir_opt_boolean_bitwise(nir_shader *shader);
diff --git a/src/panfrost/bifrost/bifrost_nir_algebraic.py b/src/panfrost/bifrost/bifrost_nir_algebraic.py
index 9a6085815fd..77fad35ff31 100644
--- a/src/panfrost/bifrost/bifrost_nir_algebraic.py
+++ b/src/panfrost/bifrost/bifrost_nir_algebraic.py
@@ -28,6 +28,20 @@ a = 'a'
 b = 'b'
 c = 'c'
 
+# In general, bcsel is cheaper than bitwise arithmetic on Mali. On
+# Bifrost, we can implement bcsel as either CSEL or MUX to schedule to either
+# execution unit. On Valhall, bitwise arithmetic may be on the SFU whereas MUX
+# is on the higher throughput CVT unit. We get a zero argument for free relative
+# to the bitwise op, which would be LSHIFT_* internally taking a zero anyway.
+#
+# As such, it's beneficial to reexpress bitwise arithmetic of booleans as bcsel.
+opt_bool_bitwise = [
+    (('iand', 'a@1', 'b@1'), ('bcsel', a, b, False)),
+    (('ior', 'a@1', 'b@1'), ('bcsel', a, a, b)),
+    (('iand', 'a@1', ('inot', 'b@1')), ('bcsel', b, 0, a)),
+    (('ior', 'a@1', ('inot', 'b@1')), ('bcsel', b, a, True)),
+]
+
 algebraic_late = [
     # Canonical form. The scheduler will convert back if it makes sense.
     (('fmul', a, 2.0), ('fadd', a, a)),
@@ -69,6 +83,8 @@ def run():
 
     print('#include "bifrost_nir.h"')
 
+    print(nir_algebraic.AlgebraicPass("bifrost_nir_opt_boolean_bitwise",
+                                      opt_bool_bitwise).render())
     print(nir_algebraic.AlgebraicPass("bifrost_nir_lower_algebraic_late",
                                       algebraic_late).render())
 
-- 
2.35.1

