From dd11fc9629ea8d2e014dd345b35cda71df76f7a9 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 9 Jun 2023 11:03:09 -0400
Subject: [PATCH] gallivm: Switch to reg intrinsics

This is pretty straightforward, since we don't try to "coalesce" register access
the way a GPU backend would. In the old path, we generated register load/store
instructions internally when hitting register sources/destinations. In the new
path, we just translate the register load/store intrinsics to the LLVM
loads/stores and we're back where we started. It's a bit more code, but it's
more straightforward.

Notably, although this continues to use registers, this does NOT use any
nir_legacy functionality :-)

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 src/gallium/auxiliary/gallivm/lp_bld_nir.c    | 178 ++++++++++--------
 src/gallium/auxiliary/gallivm/lp_bld_nir.h    |   6 +-
 .../auxiliary/gallivm/lp_bld_nir_aos.c        |   6 +-
 .../auxiliary/gallivm/lp_bld_nir_soa.c        |  41 ++--
 4 files changed, 127 insertions(+), 104 deletions(-)

diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir.c b/src/gallium/auxiliary/gallivm/lp_bld_nir.c
index 9e6e42e0871..8b714fbeba4 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_nir.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_nir.c
@@ -150,36 +150,10 @@ glsl_sampler_to_pipe(int sampler_dim, bool is_array)
 }
 
 
-static LLVMValueRef get_ssa_src(struct lp_build_nir_context *bld_base, nir_ssa_def *ssa)
-{
-   return bld_base->ssa_defs[ssa->index];
-}
-
-
-static LLVMValueRef
-get_src(struct lp_build_nir_context *bld_base, nir_src src);
-
-
-static LLVMValueRef
-get_reg_src(struct lp_build_nir_context *bld_base, nir_reg_src src)
-{
-   struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, src.reg);
-   LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
-   struct lp_build_context *reg_bld = get_int_bld(bld_base, true, src.reg->bit_size);
-   LLVMValueRef indir_src = NULL;
-   if (src.indirect)
-      indir_src = get_src(bld_base, *src.indirect);
-   return bld_base->load_reg(bld_base, reg_bld, &src, indir_src, reg_storage);
-}
-
-
 static LLVMValueRef
 get_src(struct lp_build_nir_context *bld_base, nir_src src)
 {
-   if (src.is_ssa)
-      return get_ssa_src(bld_base, src.ssa);
-   else
-      return get_reg_src(bld_base, src.reg);
+   return bld_base->ssa_defs[src.ssa->index];
 }
 
 
@@ -204,46 +178,15 @@ assign_ssa_dest(struct lp_build_nir_context *bld_base, const nir_ssa_def *ssa,
 }
 
 
-static void
-assign_reg(struct lp_build_nir_context *bld_base, const nir_reg_dest *reg,
-           unsigned write_mask,
-           LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
-{
-   assert(write_mask != 0x0);
-   struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, reg->reg);
-   LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
-   struct lp_build_context *reg_bld = get_int_bld(bld_base, true, reg->reg->bit_size);
-   LLVMValueRef indir_src = NULL;
-   if (reg->indirect)
-      indir_src = get_src(bld_base, *reg->indirect);
-   bld_base->store_reg(bld_base, reg_bld, reg, write_mask,
-                       indir_src, reg_storage, vals);
-}
-
-
 static void
 assign_dest(struct lp_build_nir_context *bld_base,
             const nir_dest *dest,
             LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
 {
-   if (dest->is_ssa)
-      assign_ssa_dest(bld_base, &dest->ssa, vals);
-   else
-      assign_reg(bld_base, &dest->reg, 0xf, vals);
+   assign_ssa_dest(bld_base, &dest->ssa, vals);
 }
 
 
-static void
-assign_alu_dest(struct lp_build_nir_context *bld_base,
-                const nir_alu_dest *dest,
-                LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
-{
-   if (dest->dest.is_ssa)
-      assign_ssa_dest(bld_base, &dest->dest.ssa, vals);
-   else
-      assign_reg(bld_base, &dest->dest.reg, dest->write_mask, vals);
-}
-
 static LLVMValueRef
 fcmp32(struct lp_build_nir_context *bld_base,
        enum pipe_compare_func compare,
@@ -1219,17 +1162,6 @@ visit_alu(struct lp_build_nir_context *bld_base,
       src_bit_size[i] = nir_src_bit_size(instr->src[i].src);
    }
 
-   if (instr->op == nir_op_mov &&
-       is_aos(bld_base) &&
-       !instr->dest.dest.is_ssa) {
-      for (unsigned i = 0; i < 4; i++) {
-         if (instr->dest.write_mask & (1 << i)) {
-            assign_reg(bld_base, &instr->dest.dest.reg, (1 << i), src);
-         }
-      }
-      return;
-   }
-
    LLVMValueRef result[NIR_MAX_VEC_COMPONENTS];
    if (instr->op == nir_op_vec4 ||
        instr->op == nir_op_vec3 ||
@@ -1279,7 +1211,7 @@ visit_alu(struct lp_build_nir_context *bld_base,
                                nir_dest_bit_size(instr->dest.dest));
       }
    }
-   assign_alu_dest(bld_base, &instr->dest, result);
+   assign_ssa_dest(bld_base, &instr->dest.dest.ssa, result);
 }
 
 
@@ -1387,7 +1319,6 @@ visit_load_input(struct lp_build_nir_context *bld_base,
    bld_base->load_var(bld_base, nir_var_shader_in, nc, bit_size, &var, 0, NULL, 0, indir_index, result);
 }
 
-
 static void
 visit_store_output(struct lp_build_nir_context *bld_base,
                    nir_intrinsic_instr *instr)
@@ -1417,6 +1348,74 @@ visit_store_output(struct lp_build_nir_context *bld_base,
                        bit_size, &var, mask, NULL, 0, indir_index, src);
 }
 
+
+static void
+visit_load_reg(struct lp_build_nir_context *bld_base,
+               nir_intrinsic_instr *instr,
+               LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
+{
+   struct gallivm_state *gallivm = bld_base->base.gallivm;
+   LLVMBuilderRef builder = gallivm->builder;
+
+   nir_ssa_def *handle = instr->src[0].ssa;
+   nir_intrinsic_instr *decl = nir_instr_as_intrinsic(handle->parent_instr);
+   unsigned base = nir_intrinsic_base(instr);
+
+   struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, decl);
+   LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
+   unsigned bit_size = nir_intrinsic_bit_size(decl);
+   struct lp_build_context *reg_bld = get_int_bld(bld_base, true, bit_size);
+   LLVMValueRef indir_src = NULL;
+   if (instr->intrinsic == nir_intrinsic_load_reg_indirect)
+      indir_src = get_src(bld_base, instr->src[1]);
+
+   LLVMValueRef val = bld_base->load_reg(bld_base, reg_bld, decl, base, indir_src, reg_storage);
+
+   if (nir_dest_num_components(instr->dest) > 1) {
+      for (unsigned i = 0; i < nir_dest_num_components(instr->dest); i++)
+         result[i] = LLVMBuildExtractValue(builder, val, i, "");
+   } else {
+      result[0] = val;
+   }
+}
+
+
+static void
+visit_store_reg(struct lp_build_nir_context *bld_base,
+                nir_intrinsic_instr *instr)
+{
+   struct gallivm_state *gallivm = bld_base->base.gallivm;
+   LLVMBuilderRef builder = gallivm->builder;
+
+   unsigned base = nir_intrinsic_base(instr);
+   unsigned write_mask = nir_intrinsic_write_mask(instr);
+   assert(write_mask != 0x0);
+
+   LLVMValueRef val = get_src(bld_base, instr->src[0]);
+   LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS] = { NULL };
+   if (nir_src_num_components(instr->src[0]) > 1) {
+      for (unsigned i = 0; i < nir_src_num_components(instr->src[0]); i++)
+         vals[i] = LLVMBuildExtractValue(builder, val, i, "");
+   } else {
+      vals[0] = val;
+   }
+
+   nir_ssa_def *handle = instr->src[1].ssa;
+   nir_intrinsic_instr *decl = nir_instr_as_intrinsic(handle->parent_instr);
+
+   struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, decl);
+   LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
+   unsigned bit_size = nir_intrinsic_bit_size(decl);
+   struct lp_build_context *reg_bld = get_int_bld(bld_base, true, bit_size);
+   LLVMValueRef indir_src = NULL;
+   if (instr->intrinsic == nir_intrinsic_store_reg_indirect)
+      indir_src = get_src(bld_base, instr->src[2]);
+
+   bld_base->store_reg(bld_base, reg_bld, decl, write_mask, base,
+                       indir_src, reg_storage, vals);
+}
+
+
 static bool
 compact_array_index_oob(struct lp_build_nir_context *bld_base, nir_variable *var, const uint32_t index)
 {
@@ -2080,6 +2079,17 @@ visit_intrinsic(struct lp_build_nir_context *bld_base,
 {
    LLVMValueRef result[NIR_MAX_VEC_COMPONENTS] = {0};
    switch (instr->intrinsic) {
+   case nir_intrinsic_decl_reg:
+      /* already handled */
+      break;
+   case nir_intrinsic_load_reg:
+   case nir_intrinsic_load_reg_indirect:
+      visit_load_reg(bld_base, instr, result);
+      break;
+   case nir_intrinsic_store_reg:
+   case nir_intrinsic_store_reg_indirect:
+      visit_store_reg(bld_base, instr);
+      break;
    case nir_intrinsic_load_input:
       visit_load_input(bld_base, instr, result);
       break;
@@ -2771,19 +2781,23 @@ handle_shader_output_decl(struct lp_build_nir_context *bld_base,
 */
 static LLVMTypeRef
 get_register_type(struct lp_build_nir_context *bld_base,
-                  nir_register *reg)
+                  nir_intrinsic_instr *reg)
 {
    if (is_aos(bld_base))
       return bld_base->base.int_vec_type;
 
+   unsigned num_array_elems = nir_intrinsic_num_array_elems(reg);
+   unsigned bit_size = nir_intrinsic_bit_size(reg);
+   unsigned num_components = nir_intrinsic_num_components(reg);
+
    struct lp_build_context *int_bld =
-      get_int_bld(bld_base, true, reg->bit_size == 1 ? 32 : reg->bit_size);
+      get_int_bld(bld_base, true, bit_size == 1 ? 32 : bit_size);
 
    LLVMTypeRef type = int_bld->vec_type;
-   if (reg->num_components > 1)
-      type = LLVMArrayType(type, reg->num_components);
-   if (reg->num_array_elems)
-      type = LLVMArrayType(type, reg->num_array_elems);
+   if (num_components > 1)
+      type = LLVMArrayType(type, num_components);
+   if (num_array_elems)
+      type = LLVMArrayType(type, num_array_elems);
 
    return type;
 }
@@ -2794,14 +2808,14 @@ bool lp_build_nir_llvm(struct lp_build_nir_context *bld_base,
 {
    struct nir_function *func;
 
-   nir_convert_from_ssa(nir, true, false);
-   nir_lower_locals_to_regs(nir);
+   nir_convert_from_ssa(nir, true, true);
+   nir_lower_locals_to_reg_intrinsics(nir);
    nir_remove_dead_derefs(nir);
    nir_remove_dead_variables(nir, nir_var_function_temp, NULL);
 
    if (is_aos(bld_base)) {
       nir_move_vec_src_uses_to_dest(nir);
-      nir_lower_vec_to_movs(nir, NULL, NULL);
+      nir_lower_vec_to_regs(nir, NULL, NULL);
    }
 
    nir_foreach_shader_out_variable(variable, nir)
@@ -2831,7 +2845,7 @@ bool lp_build_nir_llvm(struct lp_build_nir_context *bld_base,
 
    func = (struct nir_function *)exec_list_get_head(&nir->functions);
 
-   nir_foreach_register(reg, &func->impl->registers) {
+   nir_foreach_register_new(reg, func->impl) {
       LLVMTypeRef type = get_register_type(bld_base, reg);
       LLVMValueRef reg_alloc = lp_build_alloca(bld_base->base.gallivm,
                                                type, "reg");
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir.h b/src/gallium/auxiliary/gallivm/lp_bld_nir.h
index 8884c9e6bd5..e78b94017ae 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_nir.h
+++ b/src/gallium/auxiliary/gallivm/lp_bld_nir.h
@@ -161,13 +161,15 @@ struct lp_build_nir_context
 
    LLVMValueRef (*load_reg)(struct lp_build_nir_context *bld_base,
                             struct lp_build_context *reg_bld,
-                            const nir_reg_src *reg,
+                            const nir_intrinsic_instr *decl,
+                            unsigned base,
                             LLVMValueRef indir_src,
                             LLVMValueRef reg_storage);
    void (*store_reg)(struct lp_build_nir_context *bld_base,
                      struct lp_build_context *reg_bld,
-                     const nir_reg_dest *reg,
+                     const nir_intrinsic_instr *decl,
                      unsigned writemask,
+                     unsigned base,
                      LLVMValueRef indir_src,
                      LLVMValueRef reg_storage,
                      LLVMValueRef dst[NIR_MAX_VEC_COMPONENTS]);
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir_aos.c b/src/gallium/auxiliary/gallivm/lp_bld_nir_aos.c
index 36a2304a04d..40cc473bb43 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_nir_aos.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_nir_aos.c
@@ -162,7 +162,8 @@ emit_store_var(struct lp_build_nir_context *bld_base,
 static LLVMValueRef
 emit_load_reg(struct lp_build_nir_context *bld_base,
               struct lp_build_context *reg_bld,
-              const nir_reg_src *reg,
+              const nir_intrinsic_instr *decl,
+              unsigned base,
               LLVMValueRef indir_src,
               LLVMValueRef reg_storage)
 {
@@ -205,7 +206,8 @@ swizzle_writemask(struct lp_build_nir_aos_context *bld,
 static void
 emit_store_reg(struct lp_build_nir_context *bld_base,
                struct lp_build_context *reg_bld,
-               const nir_reg_dest *reg,
+               const nir_intrinsic_instr *decl,
+               unsigned base,
                unsigned writemask,
                LLVMValueRef indir_src,
                LLVMValueRef reg_storage,
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c b/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c
index a93668159f7..d968f227c1f 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c
@@ -827,19 +827,20 @@ static void emit_store_var(struct lp_build_nir_context *bld_base,
  */
 static LLVMValueRef reg_chan_pointer(struct lp_build_nir_context *bld_base,
                                            struct lp_build_context *reg_bld,
-                                           const nir_register *reg,
+                                           const nir_intrinsic_instr *decl,
                                            LLVMValueRef reg_storage,
                                            int array_index, int chan)
 {
    struct gallivm_state *gallivm = bld_base->base.gallivm;
-   int nc = reg->num_components;
+   int nc = nir_intrinsic_num_components(decl);
+   int nae = nir_intrinsic_num_array_elems(decl);
 
    LLVMTypeRef chan_type = reg_bld->vec_type;
    if (nc > 1)
       chan_type = LLVMArrayType(chan_type, nc);
 
-   if (reg->num_array_elems > 0) {
-      LLVMTypeRef array_type = LLVMArrayType(chan_type, reg->num_array_elems);
+   if (nae > 0) {
+      LLVMTypeRef array_type = LLVMArrayType(chan_type, nae);
       reg_storage = lp_build_array_get_ptr2(gallivm, array_type, reg_storage,
                                             lp_build_const_int32(gallivm, array_index));
    }
@@ -853,18 +854,20 @@ static LLVMValueRef reg_chan_pointer(struct lp_build_nir_context *bld_base,
 
 static LLVMValueRef emit_load_reg(struct lp_build_nir_context *bld_base,
                                   struct lp_build_context *reg_bld,
-                                  const nir_reg_src *reg,
+                                  const nir_intrinsic_instr *decl,
+                                  unsigned base,
                                   LLVMValueRef indir_src,
                                   LLVMValueRef reg_storage)
 {
    struct gallivm_state *gallivm = bld_base->base.gallivm;
    LLVMBuilderRef builder = gallivm->builder;
-   int nc = reg->reg->num_components;
+   int nc = nir_intrinsic_num_components(decl);
+   int num_array_elems = nir_intrinsic_num_array_elems(decl);
    LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS] = { NULL };
    struct lp_build_context *uint_bld = &bld_base->uint_bld;
-   if (reg->indirect) {
-      LLVMValueRef indirect_val = lp_build_const_int_vec(gallivm, uint_bld->type, reg->base_offset);
-      LLVMValueRef max_index = lp_build_const_int_vec(gallivm, uint_bld->type, reg->reg->num_array_elems - 1);
+   if (indir_src != NULL) {
+      LLVMValueRef indirect_val = lp_build_const_int_vec(gallivm, uint_bld->type, base);
+      LLVMValueRef max_index = lp_build_const_int_vec(gallivm, uint_bld->type, num_array_elems - 1);
       indirect_val = LLVMBuildAdd(builder, indirect_val, indir_src, "");
       indirect_val = lp_build_min(uint_bld, indirect_val, max_index);
       reg_storage = LLVMBuildBitCast(builder, reg_storage, LLVMPointerType(reg_bld->elem_type, 0), "");
@@ -875,8 +878,8 @@ static LLVMValueRef emit_load_reg(struct lp_build_nir_context *bld_base,
    } else {
       for (unsigned i = 0; i < nc; i++) {
          vals[i] = LLVMBuildLoad2(builder, reg_bld->vec_type,
-                                  reg_chan_pointer(bld_base, reg_bld, reg->reg, reg_storage,
-                                                   reg->base_offset, i), "");
+                                  reg_chan_pointer(bld_base, reg_bld, decl, reg_storage,
+                                                   base, i), "");
       }
    }
    return nc == 1 ? vals[0] : lp_nir_array_build_gather_values(builder, vals, nc);
@@ -884,8 +887,9 @@ static LLVMValueRef emit_load_reg(struct lp_build_nir_context *bld_base,
 
 static void emit_store_reg(struct lp_build_nir_context *bld_base,
                            struct lp_build_context *reg_bld,
-                           const nir_reg_dest *reg,
+                           const nir_intrinsic_instr *decl,
                            unsigned writemask,
+                           unsigned base,
                            LLVMValueRef indir_src,
                            LLVMValueRef reg_storage,
                            LLVMValueRef dst[NIR_MAX_VEC_COMPONENTS])
@@ -894,10 +898,11 @@ static void emit_store_reg(struct lp_build_nir_context *bld_base,
    struct gallivm_state *gallivm = bld_base->base.gallivm;
    LLVMBuilderRef builder = gallivm->builder;
    struct lp_build_context *uint_bld = &bld_base->uint_bld;
-   int nc = reg->reg->num_components;
-   if (reg->indirect) {
-      LLVMValueRef indirect_val = lp_build_const_int_vec(gallivm, uint_bld->type, reg->base_offset);
-      LLVMValueRef max_index = lp_build_const_int_vec(gallivm, uint_bld->type, reg->reg->num_array_elems - 1);
+   int nc = nir_intrinsic_num_components(decl);
+   int num_array_elems = nir_intrinsic_num_array_elems(decl);
+   if (indir_src != NULL) {
+      LLVMValueRef indirect_val = lp_build_const_int_vec(gallivm, uint_bld->type, base);
+      LLVMValueRef max_index = lp_build_const_int_vec(gallivm, uint_bld->type, num_array_elems - 1);
       indirect_val = LLVMBuildAdd(builder, indirect_val, indir_src, "");
       indirect_val = lp_build_min(uint_bld, indirect_val, max_index);
       reg_storage = LLVMBuildBitCast(builder, reg_storage, LLVMPointerType(reg_bld->elem_type, 0), "");
@@ -916,8 +921,8 @@ static void emit_store_reg(struct lp_build_nir_context *bld_base,
          continue;
       dst[i] = LLVMBuildBitCast(builder, dst[i], reg_bld->vec_type, "");
       lp_exec_mask_store(&bld->exec_mask, reg_bld, dst[i],
-                         reg_chan_pointer(bld_base, reg_bld, reg->reg, reg_storage,
-                                          reg->base_offset, i));
+                         reg_chan_pointer(bld_base, reg_bld, decl, reg_storage,
+                                          base, i));
    }
 }
 
-- 
2.39.2

