From 93be983db0a2ddea354ed0cf63c32ca4e204a584 Mon Sep 17 00:00:00 2001
From: duke <duke@openjdk.org>
Date: Wed, 12 Jun 2024 13:21:24 +0000
Subject: [PATCH] Backport ce5727df4436425b24b89f24c6e6b708575ec7c6

---
 src/hotspot/cpu/riscv/assembler_riscv.hpp |  2 +
 src/hotspot/cpu/riscv/riscv_v.ad          | 98 +++++++++++++++++++----
 2 files changed, 84 insertions(+), 16 deletions(-)

diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp
index 9385d63fd1a..1518eeecc84 100644
--- a/src/hotspot/cpu/riscv/assembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp
@@ -1793,9 +1793,11 @@ enum Nf {
 
   // Vector unordered indexed load instructions
   INSN(vluxei32_v, 0b0000111, 0b110, 0b01, 0b0);
+  INSN(vluxei64_v, 0b0000111, 0b111, 0b01, 0b0);
 
   // Vector unordered indexed store instructions
   INSN(vsuxei32_v, 0b0100111, 0b110, 0b01, 0b0);
+  INSN(vsuxei64_v, 0b0100111, 0b111, 0b01, 0b0);
 
 #undef INSN
 
diff --git a/src/hotspot/cpu/riscv/riscv_v.ad b/src/hotspot/cpu/riscv/riscv_v.ad
index b0c73440d88..2bba18d2c19 100644
--- a/src/hotspot/cpu/riscv/riscv_v.ad
+++ b/src/hotspot/cpu/riscv/riscv_v.ad
@@ -3670,12 +3670,11 @@ instruct vsignum_reg(vReg dst, vReg zero, vReg one, vRegMask_V0 v0) %{
 
 // ------------------------------ Vector Load Gather ---------------------------
 
-instruct gather_load(vReg dst, indirect mem, vReg idx) %{
-  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 4 ||
-            type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
+instruct gather_loadS(vReg dst, indirect mem, vReg idx) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
   match(Set dst (LoadVectorGather mem idx));
   effect(TEMP_DEF dst);
-  format %{ "gather_load $dst, $mem, $idx" %}
+  format %{ "gather_loadS $dst, $mem, $idx" %}
   ins_encode %{
     __ vmv1r_v(as_VectorRegister($dst$$reg), as_VectorRegister($idx$$reg));
     BasicType bt = Matcher::vector_element_basic_type(this);
@@ -3688,12 +3687,28 @@ instruct gather_load(vReg dst, indirect mem, vReg idx) %{
   ins_pipe(pipe_slow);
 %}
 
-instruct gather_load_masked(vReg dst, indirect mem, vReg idx, vRegMask_V0 v0, vReg tmp) %{
-  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 4 ||
-            type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
+instruct gather_loadD(vReg dst, indirect mem, vReg idx) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
+  match(Set dst (LoadVectorGather mem idx));
+  effect(TEMP_DEF dst);
+  format %{ "gather_loadD $dst, $mem, $idx" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vzext_vf2(as_VectorRegister($dst$$reg), as_VectorRegister($idx$$reg));
+    __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), (int)sew);
+    __ vluxei64_v(as_VectorRegister($dst$$reg), as_Register($mem$$base),
+                  as_VectorRegister($dst$$reg));
+ %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct gather_loadS_masked(vReg dst, indirect mem, vReg idx, vRegMask_V0 v0, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
   match(Set dst (LoadVectorGatherMasked mem (Binary idx v0)));
   effect(TEMP_DEF dst, TEMP tmp);
-  format %{ "gather_load_masked $dst, $mem, $idx, $v0\t# KILL $tmp" %}
+  format %{ "gather_loadS_masked $dst, $mem, $idx, $v0\t# KILL $tmp" %}
   ins_encode %{
     __ vmv1r_v(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
     BasicType bt = Matcher::vector_element_basic_type(this);
@@ -3708,14 +3723,32 @@ instruct gather_load_masked(vReg dst, indirect mem, vReg idx, vRegMask_V0 v0, vR
   ins_pipe(pipe_slow);
 %}
 
+instruct gather_loadD_masked(vReg dst, indirect mem, vReg idx, vRegMask_V0 v0, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
+  match(Set dst (LoadVectorGatherMasked mem (Binary idx v0)));
+  effect(TEMP_DEF dst, TEMP tmp);
+  format %{ "gather_loadD_masked $dst, $mem, $idx, $v0\t# KILL $tmp" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vzext_vf2(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
+    __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), (int)sew);
+    __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg),
+               as_VectorRegister($dst$$reg));
+    __ vluxei64_v(as_VectorRegister($dst$$reg), as_Register($mem$$base),
+                  as_VectorRegister($tmp$$reg), Assembler::v0_t);
+ %}
+  ins_pipe(pipe_slow);
+%}
+
 // ------------------------------ Vector Store Scatter -------------------------
 
-instruct scatter_store(indirect mem, vReg src, vReg idx, vReg tmp) %{
-  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4 ||
-            type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
+instruct scatter_storeS(indirect mem, vReg src, vReg idx, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
   match(Set mem (StoreVectorScatter mem (Binary src idx)));
   effect(TEMP tmp);
-  format %{ "scatter_store $mem, $idx, $src\t# KILL $tmp" %}
+  format %{ "scatter_storeS $mem, $idx, $src\t# KILL $tmp" %}
   ins_encode %{
     __ vmv1r_v(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
     BasicType bt = Matcher::vector_element_basic_type(this, $src);
@@ -3728,12 +3761,28 @@ instruct scatter_store(indirect mem, vReg src, vReg idx, vReg tmp) %{
   ins_pipe(pipe_slow);
 %}
 
-instruct scatter_store_masked(indirect mem, vReg src, vReg idx, vRegMask_V0 v0, vReg tmp) %{
-  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4 ||
-            type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
+instruct scatter_storeD(indirect mem, vReg src, vReg idx, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
+  match(Set mem (StoreVectorScatter mem (Binary src idx)));
+  effect(TEMP tmp);
+  format %{ "scatter_storeD $mem, $idx, $src\t# KILL $tmp" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this, $src);
+    Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
+    __ vsetvli_helper(bt, Matcher::vector_length(this, $src));
+    __ vzext_vf2(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
+    __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), (int)sew);
+    __ vsuxei64_v(as_VectorRegister($src$$reg), as_Register($mem$$base),
+                  as_VectorRegister($tmp$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct scatter_storeS_masked(indirect mem, vReg src, vReg idx, vRegMask_V0 v0, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx v0))));
   effect(TEMP tmp);
-  format %{ "scatter_store_masked $mem, $idx, $src, $v0\t# KILL $tmp" %}
+  format %{ "scatter_storeS_masked $mem, $idx, $src, $v0\t# KILL $tmp" %}
   ins_encode %{
     __ vmv1r_v(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
     BasicType bt = Matcher::vector_element_basic_type(this, $src);
@@ -3746,6 +3795,23 @@ instruct scatter_store_masked(indirect mem, vReg src, vReg idx, vRegMask_V0 v0,
   ins_pipe(pipe_slow);
 %}
 
+instruct scatter_storeD_masked(indirect mem, vReg src, vReg idx, vRegMask_V0 v0, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
+  match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx v0))));
+  effect(TEMP tmp);
+  format %{ "scatter_storeD_masked $mem, $idx, $src, $v0\t# KILL $tmp" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this, $src);
+    Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
+    __ vsetvli_helper(bt, Matcher::vector_length(this, $src));
+    __ vzext_vf2(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
+    __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), (int)sew);
+    __ vsuxei64_v(as_VectorRegister($src$$reg), as_Register($mem$$base),
+                  as_VectorRegister($tmp$$reg), Assembler::v0_t);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 // ------------------------------ Populate Index to a Vector -------------------
 
 instruct populateindex(vReg dst, iRegIorL2I src1, iRegIorL2I src2, vReg tmp) %{