[PATCH 3/4] bufhelp: avoid byte-wise load/store on RISC-V with Zicclsm

* cipher/bufhelp.h (bufhelp_u32_aligned_t, bufhelp_u64_aligned_t)
(buf_load32, buf_store32, buf_load64, buf_store64): New.
(buf_get_be32, buf_get_le32, buf_put_be32, buf_put_le32)
(buf_get_be64, buf_get_le64, buf_put_be64, buf_put_le64): Use
'buf_load32'/'buf_load64' and 'buf_store32'/'buf_store64'.
--

GCC expands accesses through 'aligned(1)' type to byte load/store
sequences when tuning model marks unaligned access slow, which is case
with default -mtune. This happens with aligned buffers too and neither
Zicclsm in -march nor -mno-strict-align changes it. Use inline assembly
with memory operand to get single instruction access, and naturally
aligned type when compiler can prove pointer alignment.

Benchmark on SpacemiT K1 (1600 Mhz), CAMELLIA128 cycles/byte:

             |   before    after   speedup
 ECB enc     |    33.62    31.39     1.07x
 CBC enc     |    39.91    32.28     1.24x
 CBC dec     |    38.50    32.12     1.20x
 CFB enc     |    40.06    31.99     1.25x
 CFB dec     |    37.20    32.14     1.16x
 OFB enc     |    38.78    32.11     1.21x
 CTR enc     |    37.18    32.08     1.16x
 XTS enc     |    40.09    32.45     1.24x

Signed-off-by: Jussi Kivilinna <[email protected]>
---
 cipher/bufhelp.h | 134 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 117 insertions(+), 17 deletions(-)

diff --git a/cipher/bufhelp.h b/cipher/bufhelp.h
index 6dcc622a..53516508 100644
--- a/cipher/bufhelp.h
+++ b/cipher/bufhelp.h
@@ -126,58 +126,158 @@ typedef struct bufhelp_u32_s
   u32 a;
 } __attribute__((packed, aligned(1), may_alias)) bufhelp_u32_t;
 
+typedef struct bufhelp_u32_aligned_s
+{
+  u32 a;
+} __attribute__(( aligned(4), may_alias)) bufhelp_u32_aligned_t;
+
+typedef struct bufhelp_u64_s
+{
+  u64 a;
+} __attribute__((packed, aligned(1), may_alias)) bufhelp_u64_t;
+
+typedef struct bufhelp_u64_aligned_s
+{
+  u64 a;
+} __attribute__((aligned(8), may_alias)) bufhelp_u64_aligned_t;
+
+#if defined(__riscv) && \
+    (defined(__riscv_zicclsm) && (__riscv_zicclsm >= 1000000)) && \
+    defined(HAVE_GCC_INLINE_ASM_RISCV)
+
+static inline u32 buf_load32(const void *_buf)
+{
+  if (CONSTANT_P(((uintptr_t)_buf & 3) == 0) && ((uintptr_t)_buf & 3) == 0)
+    {
+      /* Compiler could determinate that this pointer is always aligned. */
+      return ((const bufhelp_u32_aligned_t *)_buf)->a;
+    }
+  else
+    {
+      u64 val;
+      asm ("lw %0, %1"
+	   : "=r" (val)
+	   : "m" (*((const bufhelp_u32_t *)_buf)));
+      return val;
+    }
+}
+
+static inline void buf_store32(void *_buf, u32 val)
+{
+  if (CONSTANT_P(((uintptr_t)_buf & 3) == 0) && ((uintptr_t)_buf & 3) == 0)
+    {
+      /* Compiler could determinate that this pointer is always aligned. */
+      ((bufhelp_u32_aligned_t *)_buf)->a = val;
+    }
+  else
+    {
+      asm ("sw %1, %0"
+	   : "=m" (*((bufhelp_u32_t *)_buf))
+	   : "r" (val));
+    }
+}
+
+static inline u64 buf_load64(const void *_buf)
+{
+  if (CONSTANT_P(((uintptr_t)_buf & 7) == 0) && ((uintptr_t)_buf & 7) == 0)
+    {
+      /* Compiler could determinate that this pointer is always aligned. */
+      return ((const bufhelp_u64_aligned_t *)_buf)->a;
+    }
+  else
+    {
+      u64 val;
+      asm ("ld %0, %1"
+	   : "=r" (val)
+	   : "m" (*((const bufhelp_u64_t *)_buf)));
+      return val;
+    }
+}
+
+static inline void buf_store64(void *_buf, u64 val)
+{
+  if (CONSTANT_P(((uintptr_t)_buf & 7) == 0) && ((uintptr_t)_buf & 7) == 0)
+    {
+      /* Compiler could determinate that this pointer is always aligned. */
+      ((bufhelp_u64_aligned_t *)_buf)->a = val;
+    }
+  else
+    {
+      asm ("sd %1, %0"
+	  : "=m" (*((bufhelp_u64_t *)_buf))
+	  : "r" (val));
+    }
+}
+
+#else
+
+static inline u32 buf_load32(const void *_buf)
+{
+  return ((const bufhelp_u32_t *)_buf)->a;
+}
+
+static inline void buf_store32(void *_buf, u32 val)
+{
+  bufhelp_u32_t *out = _buf;
+  out->a = val;
+}
+
+static inline u64 buf_load64(const void *_buf)
+{
+  return ((const bufhelp_u64_t *)_buf)->a;
+}
+
+static inline void buf_store64(void *_buf, u64 val)
+{
+  bufhelp_u64_t *out = _buf;
+  out->a = val;
+}
+
+#endif
+
 /* Functions for loading and storing unaligned u32 values of different
    endianness.  */
 static inline u32 buf_get_be32(const void *_buf)
 {
-  return be_bswap32(((const bufhelp_u32_t *)_buf)->a);
+  return be_bswap32(buf_load32(_buf));
 }
 
 static inline u32 buf_get_le32(const void *_buf)
 {
-  return le_bswap32(((const bufhelp_u32_t *)_buf)->a);
+  return le_bswap32(buf_load32(_buf));
 }
 
 static inline void buf_put_be32(void *_buf, u32 val)
 {
-  bufhelp_u32_t *out = _buf;
-  out->a = be_bswap32(val);
+  buf_store32(_buf, be_bswap32(val));
 }
 
 static inline void buf_put_le32(void *_buf, u32 val)
 {
-  bufhelp_u32_t *out = _buf;
-  out->a = le_bswap32(val);
+  buf_store32(_buf, le_bswap32(val));
 }
 
 
-typedef struct bufhelp_u64_s
-{
-  u64 a;
-} __attribute__((packed, aligned(1), may_alias)) bufhelp_u64_t;
-
 /* Functions for loading and storing unaligned u64 values of different
    endianness.  */
 static inline u64 buf_get_be64(const void *_buf)
 {
-  return be_bswap64(((const bufhelp_u64_t *)_buf)->a);
+  return be_bswap64(buf_load64(_buf));
 }
 
 static inline u64 buf_get_le64(const void *_buf)
 {
-  return le_bswap64(((const bufhelp_u64_t *)_buf)->a);
+  return le_bswap64(buf_load64(_buf));
 }
 
 static inline void buf_put_be64(void *_buf, u64 val)
 {
-  bufhelp_u64_t *out = _buf;
-  out->a = be_bswap64(val);
+  buf_store64(_buf, be_bswap64(val));
 }
 
 static inline void buf_put_le64(void *_buf, u64 val)
 {
-  bufhelp_u64_t *out = _buf;
-  out->a = le_bswap64(val);
+  buf_store64(_buf, le_bswap64(val));
 }
 
 #endif /*BUFHELP_UNALIGNED_ACCESS*/
-- 
2.53.0