#define D2MODE 20
#pragma GCC optimize("O3")
#include <immintrin.h>
#include <xmmintrin.h>
#include <stdint.h>
#include <string.h>
#include <stdio.h>
#define AVX2M __attribute__((target("avx2,bmi2,lzcnt")))
#define AVX2 __attribute__((target("avx2")))
#define NMAX 4096
#define PFA 2
#define PFB 0
#ifndef PADA
#define PADA 0
#endif
#define LDA ((int)(2*NMAX + PADA))
#ifndef PADB
#define PADB 0
#endif
#define LDB ((int)(NMAX + PADB))
static int8_t A0b_raw[(size_t)NMAX*LDA + 8192] __attribute__((aligned(64)));
static int8_t *A0b = A0b_raw + 2048;
static int GLDA = LDA, GLDB = LDB;
static uint64_t A1b[NMAX*NMAX/8] __attribute__((aligned(64)));
static uint64_t Bpb[NMAX*NMAX/8] __attribute__((aligned(64)));
static uint64_t Pbb[NMAX*NMAX/8] __attribute__((aligned(64)));
static uint8_t PBb[(size_t)NMAX*LDB] __attribute__((aligned(64)));
static void generic_mm(int n, const int8_t*A, const int8_t*B, int8_t*C){
for (int i=0;i<n;i++) for (int j=0;j<n;j++){
int s=0; for(int k=0;k<n;k++) s += (int)A[(size_t)i*n+k]*(int)B[(size_t)k*n+j];
C[(size_t)i*n+j]=(int8_t)s;
}
}
AVX2 static inline __m256i bcast16(const int8_t*p){
__m256i v; __asm__("vpbroadcastw %1, %0" : "=x"(v) : "m"(*(const short*)p)); return v;
}
AVX2 static inline __m256i packlo8(__m256i a, __m256i b){
const __m256i m=_mm256_set1_epi16(0x00FF);
return _mm256_permute4x64_epi64(
_mm256_packus_epi16(_mm256_and_si256(a,m), _mm256_and_si256(b,m)), 0xD8);
}
#define AVX2BMI __attribute__((target("avx2,bmi")))
static const uint8_t EXP_CTL[32] __attribute__((aligned(32))) = {
0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3 };
static const uint8_t EXP_BIT[32] __attribute__((aligned(32))) = {
1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128,
1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128 };
static const uint8_t EXP_MSB[32] __attribute__((aligned(32))) = {
0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80, 0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,
0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80, 0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80 };
__attribute__((optimize("O3","unroll-loops"))) AVX2 static void expand_bits(int n){
const __m256i ctl=_mm256_load_si256((const __m256i*)EXP_CTL);
const __m256i bit=_mm256_load_si256((const __m256i*)EXP_BIT);
const __m256i msb=_mm256_load_si256((const __m256i*)EXP_MSB);
int W=n/64;
for(int i=0;i<n;i++){
const uint32_t*p=(const uint32_t*)(Pbb+(size_t)i*W); uint8_t*o=PBb+(size_t)i*GLDB;
for(int q=0;q<2*W;q++){
__m256i b=_mm256_set1_epi32((int)p[q]);
__m256i s=_mm256_shuffle_epi8(b,ctl);
__m256i e=_mm256_cmpeq_epi8(_mm256_and_si256(s,bit),bit);
_mm256_storeu_si256((__m256i*)(o+q*32), _mm256_and_si256(e,msb));
}
}
}
#define NIB 2
#define NJ 4
static const int16_t W0FF[16] __attribute__((aligned(32))) =
{0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,
0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF};
static int8_t BI3b[NMAX*NMAX] __attribute__((aligned(64)));
AVX2M static void mikro_u4(int n,const int8_t*A0,const int8_t*BI,int8_t*C,const uint8_t*PB,int jb,int lda,int ldb){
const int8_t *x0=A0;
long nn=(long)n;
const int8_t *x1=A0+lda;
const int8_t *bib=BI+(long)(n>>3)*512; long bidx=-((long)(n>>3)*512);
int8_t *C0=C+jb, *C1=C0+n;
const uint8_t *PB0=PB+jb, *PB1=PB0+ldb;
asm volatile( "vpxor %%xmm8,%%xmm8,%%xmm8\n\t"
"vpxor %%xmm9,%%xmm9,%%xmm9\n\t"
"vpxor %%xmm10,%%xmm10,%%xmm10\n\t"
"vpxor %%xmm11,%%xmm11,%%xmm11\n\t"
"vpxor %%xmm12,%%xmm12,%%xmm12\n\t"
"vpxor %%xmm13,%%xmm13,%%xmm13\n\t"
"vpxor %%xmm14,%%xmm14,%%xmm14\n\t"
"vpxor %%xmm15,%%xmm15,%%xmm15\n\t"
".p2align 5\n\t"
"1:\n\t"
"vpbroadcastd 0(%[x0]), %%ymm0\n\t"
"vpbroadcastd 0(%[x0],%[xd],1), %%ymm1\n\t"
"vmovdqa 0(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vmovdqa 32(%[bib],%[bidx]), %%ymm6\n\t"
"vpmaddubsw %%ymm6, %%ymm0, %%ymm7\n\t"
"vpmaddubsw %%ymm6, %%ymm1, %%ymm6\n\t"
"vmovdqa 64(%[bib],%[bidx]), %%ymm2\n\t"
"vpmaddubsw %%ymm2, %%ymm0, %%ymm3\n\t"
"vpmaddubsw %%ymm2, %%ymm1, %%ymm2\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm4, %%ymm12, %%ymm12\n\t"
"vmovdqa 96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vpaddw %%ymm7, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vpaddw %%ymm3, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm4, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[x0]), %%ymm0\n\t"
"vpbroadcastd 4(%[x0],%[xd],1), %%ymm1\n\t"
"vmovdqa 128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vmovdqa 160(%[bib],%[bidx]), %%ymm6\n\t"
"vpmaddubsw %%ymm6, %%ymm0, %%ymm7\n\t"
"vpmaddubsw %%ymm6, %%ymm1, %%ymm6\n\t"
"vmovdqa 192(%[bib],%[bidx]), %%ymm2\n\t"
"vpmaddubsw %%ymm2, %%ymm0, %%ymm3\n\t"
"vpmaddubsw %%ymm2, %%ymm1, %%ymm2\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm4, %%ymm12, %%ymm12\n\t"
"vmovdqa 224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vpaddw %%ymm7, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vpaddw %%ymm3, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm4, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 8(%[x0]), %%ymm0\n\t"
"vpbroadcastd 8(%[x0],%[xd],1), %%ymm1\n\t"
"vmovdqa 256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vmovdqa 288(%[bib],%[bidx]), %%ymm6\n\t"
"vpmaddubsw %%ymm6, %%ymm0, %%ymm7\n\t"
"vpmaddubsw %%ymm6, %%ymm1, %%ymm6\n\t"
"vmovdqa 320(%[bib],%[bidx]), %%ymm2\n\t"
"vpmaddubsw %%ymm2, %%ymm0, %%ymm3\n\t"
"vpmaddubsw %%ymm2, %%ymm1, %%ymm2\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm4, %%ymm12, %%ymm12\n\t"
"vmovdqa 352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vpaddw %%ymm7, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vpaddw %%ymm3, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm4, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[x0]), %%ymm0\n\t"
"vpbroadcastd 12(%[x0],%[xd],1), %%ymm1\n\t"
"vmovdqa 384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vmovdqa 416(%[bib],%[bidx]), %%ymm6\n\t"
"vpmaddubsw %%ymm6, %%ymm0, %%ymm7\n\t"
"vpmaddubsw %%ymm6, %%ymm1, %%ymm6\n\t"
"vmovdqa 448(%[bib],%[bidx]), %%ymm2\n\t"
"vpmaddubsw %%ymm2, %%ymm0, %%ymm3\n\t"
"vpmaddubsw %%ymm2, %%ymm1, %%ymm2\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm4, %%ymm12, %%ymm12\n\t"
"vmovdqa 480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vpaddw %%ymm7, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vpaddw %%ymm3, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm4, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 16(%[x0]), %%ymm0\n\t"
"vpbroadcastd 16(%[x0],%[xd],1), %%ymm1\n\t"
"vmovdqa 512(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vmovdqa 544(%[bib],%[bidx]), %%ymm6\n\t"
"vpmaddubsw %%ymm6, %%ymm0, %%ymm7\n\t"
"vpmaddubsw %%ymm6, %%ymm1, %%ymm6\n\t"
"vmovdqa 576(%[bib],%[bidx]), %%ymm2\n\t"
"vpmaddubsw %%ymm2, %%ymm0, %%ymm3\n\t"
"vpmaddubsw %%ymm2, %%ymm1, %%ymm2\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm4, %%ymm12, %%ymm12\n\t"
"vmovdqa 608(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vpaddw %%ymm7, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vpaddw %%ymm3, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm4, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 20(%[x0]), %%ymm0\n\t"
"vpbroadcastd 20(%[x0],%[xd],1), %%ymm1\n\t"
"vmovdqa 640(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vmovdqa 672(%[bib],%[bidx]), %%ymm6\n\t"
"vpmaddubsw %%ymm6, %%ymm0, %%ymm7\n\t"
"vpmaddubsw %%ymm6, %%ymm1, %%ymm6\n\t"
"vmovdqa 704(%[bib],%[bidx]), %%ymm2\n\t"
"vpmaddubsw %%ymm2, %%ymm0, %%ymm3\n\t"
"vpmaddubsw %%ymm2, %%ymm1, %%ymm2\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm4, %%ymm12, %%ymm12\n\t"
"vmovdqa 736(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vpaddw %%ymm7, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vpaddw %%ymm3, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm4, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 24(%[x0]), %%ymm0\n\t"
"vpbroadcastd 24(%[x0],%[xd],1), %%ymm1\n\t"
"vmovdqa 768(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vmovdqa 800(%[bib],%[bidx]), %%ymm6\n\t"
"vpmaddubsw %%ymm6, %%ymm0, %%ymm7\n\t"
"vpmaddubsw %%ymm6, %%ymm1, %%ymm6\n\t"
"vmovdqa 832(%[bib],%[bidx]), %%ymm2\n\t"
"vpmaddubsw %%ymm2, %%ymm0, %%ymm3\n\t"
"vpmaddubsw %%ymm2, %%ymm1, %%ymm2\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm4, %%ymm12, %%ymm12\n\t"
"vmovdqa 864(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vpaddw %%ymm7, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vpaddw %%ymm3, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm4, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 28(%[x0]), %%ymm0\n\t"
"vpbroadcastd 28(%[x0],%[xd],1), %%ymm1\n\t"
"vmovdqa 896(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vmovdqa 928(%[bib],%[bidx]), %%ymm6\n\t"
"vpmaddubsw %%ymm6, %%ymm0, %%ymm7\n\t"
"vpmaddubsw %%ymm6, %%ymm1, %%ymm6\n\t"
"vmovdqa 960(%[bib],%[bidx]), %%ymm2\n\t"
"vpmaddubsw %%ymm2, %%ymm0, %%ymm3\n\t"
"vpmaddubsw %%ymm2, %%ymm1, %%ymm2\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm4, %%ymm12, %%ymm12\n\t"
"vmovdqa 992(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm4\n\t"
"vpaddw %%ymm7, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vpaddw %%ymm3, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm4, %%ymm15, %%ymm15\n\t"
"addq $32, %[x0]\n\t"
"addq $1024, %[bidx]\n\t"
"jnz 1b\n\t"
"vpand %[mask], %%ymm8, %%ymm2\n\t"
"vpand %[mask], %%ymm9, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpxor 0(%[PB0]), %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 0(%[C0])\n\t"
"vpand %[mask], %%ymm10, %%ymm2\n\t"
"vpand %[mask], %%ymm11, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpxor 32(%[PB0]), %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 32(%[C0])\n\t"
"vpand %[mask], %%ymm12, %%ymm2\n\t"
"vpand %[mask], %%ymm13, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpxor 0(%[PB1]), %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 0(%[C1])\n\t"
"vpand %[mask], %%ymm14, %%ymm2\n\t"
"vpand %[mask], %%ymm15, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpxor 32(%[PB1]), %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 32(%[C1])\n\t"
: [x0]"+r"(x0), [bidx]"+r"(bidx)
: [xd]"r"((long)lda), [C0]"r"(C0), [C1]"r"(C1), [PB0]"r"(PB0), [PB1]"r"(PB1), [nn]"r"(nn), [mask]"m"(W0FF), [bib]"r"(bib)
: "ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15","cc","memory");
}
#define SM 2048
AVX2BMI static void corr1024(void);
AVX2BMI static void corr512(void);
AVX2BMI static void corr256(void);
AVX2BMI static void corr512(void){
const int n=512, W=8;
memset(Pbb,0,(size_t)n*W*8);
const int KB=64;
for (int kb=0; kb<n; kb+=KB){
int w0=kb>>6;
const char *bp0=(const char*)Bpb+(size_t)w0*4096;
for (int i=0;i<n;i++){
const uint64_t*row=A1b+(size_t)i*W+w0;
uint64_t b0=row[0], b1=0;
uint64_t *o=Pbb+(size_t)i*W;
asm volatile(
"vpxor %%xmm0,%%xmm0,%%xmm0\n\t"
"vpxor %%xmm1,%%xmm1,%%xmm1\n\t"
"testq %[b0], %[b0]\n\t"
"jz 3%=f\n\t"
"1%=:\n\t"
"tzcntq %[b0], %%r11\n\t"
"shlq $6, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"blsrq %[b0], %[b0]\n\t"
"jnz 1%=b\n\t"
"3%=:\n\t"
"testq %[b1], %[b1]\n\t"
"jz 4%=f\n\t"
"2%=:\n\t"
"tzcntq %[b1], %%r11\n\t"
"shlq $6, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"blsrq %[b1], %[b1]\n\t"
"jnz 2%=b\n\t"
"4%=:\n\t"
"vpxor 0(%[o]), %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[o])\n\t"
"vpxor 32(%[o]), %%ymm1, %%ymm1\n\t"
"vmovdqu %%ymm1, 32(%[o])\n\t"
: [b0]"+r"(b0), [b1]"+r"(b1)
: [bp0]"r"(bp0), [o]"r"(o)
: "r11","cc","memory","ymm0","ymm1");
}
}
}
AVX2BMI static void corr256(void){
const int n=256, W=4;
memset(Pbb,0,(size_t)n*W*8);
const int KB=64;
for (int kb=0; kb<n; kb+=KB){
int w0=kb>>6;
const char *bp0=(const char*)Bpb+(size_t)w0*2048;
for (int i=0;i<n;i++){
const uint64_t*row=A1b+(size_t)i*W+w0;
uint64_t b0=row[0], b1=0;
uint64_t *o=Pbb+(size_t)i*W;
asm volatile(
"vpxor %%xmm0,%%xmm0,%%xmm0\n\t"
"testq %[b0], %[b0]\n\t"
"jz 3%=f\n\t"
"1%=:\n\t"
"tzcntq %[b0], %%r11\n\t"
"shlq $5, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"blsrq %[b0], %[b0]\n\t"
"jnz 1%=b\n\t"
"3%=:\n\t"
"testq %[b1], %[b1]\n\t"
"jz 4%=f\n\t"
"2%=:\n\t"
"tzcntq %[b1], %%r11\n\t"
"shlq $5, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"blsrq %[b1], %[b1]\n\t"
"jnz 2%=b\n\t"
"4%=:\n\t"
"vpxor 0(%[o]), %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[o])\n\t"
: [b0]"+r"(b0), [b1]"+r"(b1)
: [bp0]"r"(bp0), [o]"r"(o)
: "r11","cc","memory","ymm0");
}
}
}
AVX2BMI static void corr1024(void){
const int n=1024, W=16;
memset(Pbb,0,(size_t)n*W*8);
const int KB=64;
for (int kb=0; kb<n; kb+=KB){
int w0=kb>>6;
const char *bp0=(const char*)Bpb+(size_t)w0*8192;
for (int i=0;i<n;i++){
const uint64_t*row=A1b+(size_t)i*W+w0;
uint64_t b0=row[0], b1=0;
uint64_t *o=Pbb+(size_t)i*W;
asm volatile(
"vpxor %%xmm0,%%xmm0,%%xmm0\n\t"
"vpxor %%xmm1,%%xmm1,%%xmm1\n\t"
"vpxor %%xmm2,%%xmm2,%%xmm2\n\t"
"vpxor %%xmm3,%%xmm3,%%xmm3\n\t"
"testq %[b0], %[b0]\n\t"
"jz 3%=f\n\t"
"1%=:\n\t"
"tzcntq %[b0], %%r11\n\t"
"shlq $7, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"vpxor 64(%%r11), %%ymm2, %%ymm2\n\t"
"vpxor 96(%%r11), %%ymm3, %%ymm3\n\t"
"blsrq %[b0], %[b0]\n\t"
"jnz 1%=b\n\t"
"3%=:\n\t"
"testq %[b1], %[b1]\n\t"
"jz 4%=f\n\t"
"2%=:\n\t"
"tzcntq %[b1], %%r11\n\t"
"shlq $7, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"vpxor 64(%%r11), %%ymm2, %%ymm2\n\t"
"vpxor 96(%%r11), %%ymm3, %%ymm3\n\t"
"blsrq %[b1], %[b1]\n\t"
"jnz 2%=b\n\t"
"4%=:\n\t"
"vpxor 0(%[o]), %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[o])\n\t"
"vpxor 32(%[o]), %%ymm1, %%ymm1\n\t"
"vmovdqu %%ymm1, 32(%[o])\n\t"
"vpxor 64(%[o]), %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 64(%[o])\n\t"
"vpxor 96(%[o]), %%ymm3, %%ymm3\n\t"
"vmovdqu %%ymm3, 96(%[o])\n\t"
: [b0]"+r"(b0), [b1]"+r"(b1)
: [bp0]"r"(bp0), [o]"r"(o)
: "r11","cc","memory","ymm0","ymm1","ymm2","ymm3");
}
}
}
static uint64_t TB[((size_t)NMAX/64)*256] __attribute__((aligned(64)));
AVX2BMI static void nibtab_build(const uint64_t *__restrict r, uint64_t *__restrict T, int W){
const int U = W>>2;
const __m256i *rp = (const __m256i*)r;
__m256i *tp = (__m256i*)T;
for (int q=0;q<U;q++){
const __m256i a1=rp[q], a2=rp[U+q], a4=rp[2*U+q], a8=rp[3*U+q];
const __m256i x12=_mm256_xor_si256(a1,a2), x14=_mm256_xor_si256(a1,a4),
x18=_mm256_xor_si256(a1,a8);
__m256i *o = tp+q;
o[ 0*U]=_mm256_setzero_si256();
o[ 1*U]=a1; o[ 2*U]=a2; o[ 4*U]=a4; o[ 8*U]=a8;
o[ 3*U]=x12; o[ 5*U]=x14;
o[ 6*U]=_mm256_xor_si256(a2,a4); o[ 7*U]=_mm256_xor_si256(x12,a4);
o[ 9*U]=x18; o[10*U]=_mm256_xor_si256(a2,a8);
o[12*U]=_mm256_xor_si256(a4,a8);
o[11*U]=_mm256_xor_si256(a2,x18); o[13*U]=_mm256_xor_si256(a4,x18);
o[14*U]=_mm256_xor_si256(_mm256_xor_si256(a2,a4),a8);
o[15*U]=_mm256_xor_si256(_mm256_xor_si256(x12,a4),a8);
}
}
#define DEF_NIBCORR(TAG, M) \
AVX2BMI static void nibcorr##TAG(void){ \
const int n=M, W=n/64, G=n>>2, GB=16, NY=W>>2; \
memset(Pbb,0,(size_t)n*W*8); \
for (int g0=0; g0<G; g0+=GB){ \
for (int t=0;t<GB;t++){ \
const int g=g0+t; \
uint64_t *T=TB+(size_t)t*16*W; \
const uint64_t *r=Bpb+(size_t)(g*4)*W; \
nibtab_build(r,T,W); \
} \
for (int i=0;i<n;i++){ \
const uint64_t *arow=A1b+(size_t)i*W+(g0>>4); \
__m256i acc[NY]; \
for (int q=0;q<NY;q++) acc[q]=_mm256_setzero_si256(); \
const uint64_t *T=TB; \
for (int wb=0; wb<(GB>>4); wb++){ \
const uint64_t word=arow[wb]; \
_Pragma("GCC unroll 16") \
for (int b=0;b<16;b++, T+=(size_t)16*W){ \
const uint64_t *e=T+(size_t)(((word>>(4*b))&15u)*W); \
for (int q=0;q<NY;q++) \
acc[q]=_mm256_xor_si256(acc[q],_mm256_load_si256((const __m256i*)(e+4*q))); \
} \
} \
uint64_t *o=Pbb+(size_t)i*W; \
for (int q=0;q<NY;q++){ \
__m256i *po=(__m256i*)(o+4*q); \
_mm256_store_si256(po,_mm256_xor_si256(_mm256_load_si256((const __m256i*)po),acc[q])); \
} \
} \
} \
}
DEF_NIBCORR(256,256)
AVX2 static void submm_tail(int m,int8_t*Cp){
GLDA = 2*(m) + PADA; GLDB = m + PADB;
nibcorr256(); expand_bits(m);
const int GB=64, IB=256;
for (int ib=0; ib<m; ib+=IB){
const int ihi = (ib+IB<m)? ib+IB : m;
for (int gb=0; gb<m; gb+=GB){
const int ghi = (gb+GB<m)? gb+GB : m;
for (int g=gb; g<ghi; g+=64){
const int8_t *bi = BI3b+(size_t)(g>>6)*(m/2)*128;
{ const int rev=0;
for (int i=ib; i<ihi; i+=2){ (void)rev;
const int8_t *ap=A0b+(size_t)i*GLDA;
mikro_u4 (m, ap, bi,
Cp+(size_t)i*m+g, PBb+(size_t)i*GLDB+g, 0, GLDA, GLDB);
}
}
}
}
}
}
static const signed char P1A[7][4] = {
{+1,+0,+0,+1},
{+0,+0,+1,+1},
{+1,+0,+0,+0},
{+0,+0,+0,+1},
{+1,+1,+0,+0},
{-1,+0,+1,+0},
{+0,+1,+0,-1},
};
static const signed char P1B[7][4] = {
{+1,+0,+0,+1},
{+1,+0,+0,+0},
{+0,+1,+0,-1},
{-1,+0,+1,+0},
{+0,+0,+0,+1},
{+1,+1,+0,+0},
{+0,+0,+1,+1},
};
static const signed char M2AN[7][7] = {
{4,4,2,2,4,4,4},
{4,4,2,2,4,4,4},
{2,2,1,1,2,2,2},
{2,2,1,1,2,2,2},
{4,4,2,2,4,4,4},
{4,4,2,2,4,4,4},
{4,4,2,2,4,4,4},
};
static const signed char M2A[7][7][4][3] = {
{
{{0,0,+1},{1,1,+1},{2,2,+1},{3,3,+1}},
{{1,0,+1},{1,1,+1},{3,2,+1},{3,3,+1}},
{{0,0,+1},{2,2,+1},{0,0,0},{0,0,0}},
{{1,1,+1},{3,3,+1},{0,0,0},{0,0,0}},
{{0,0,+1},{0,1,+1},{2,2,+1},{2,3,+1}},
{{0,0,-1},{1,0,+1},{2,2,-1},{3,2,+1}},
{{0,1,+1},{1,1,-1},{2,3,+1},{3,3,-1}},
},
{
{{2,0,+1},{2,2,+1},{3,1,+1},{3,3,+1}},
{{3,0,+1},{3,1,+1},{3,2,+1},{3,3,+1}},
{{2,0,+1},{2,2,+1},{0,0,0},{0,0,0}},
{{3,1,+1},{3,3,+1},{0,0,0},{0,0,0}},
{{2,0,+1},{2,1,+1},{2,2,+1},{2,3,+1}},
{{2,0,-1},{2,2,-1},{3,0,+1},{3,2,+1}},
{{2,1,+1},{2,3,+1},{3,1,-1},{3,3,-1}},
},
{
{{0,0,+1},{1,1,+1},{0,0,0},{0,0,0}},
{{1,0,+1},{1,1,+1},{0,0,0},{0,0,0}},
{{0,0,+1},{0,0,0},{0,0,0},{0,0,0}},
{{1,1,+1},{0,0,0},{0,0,0},{0,0,0}},
{{0,0,+1},{0,1,+1},{0,0,0},{0,0,0}},
{{0,0,-1},{1,0,+1},{0,0,0},{0,0,0}},
{{0,1,+1},{1,1,-1},{0,0,0},{0,0,0}},
},
{
{{2,2,+1},{3,3,+1},{0,0,0},{0,0,0}},
{{3,2,+1},{3,3,+1},{0,0,0},{0,0,0}},
{{2,2,+1},{0,0,0},{0,0,0},{0,0,0}},
{{3,3,+1},{0,0,0},{0,0,0},{0,0,0}},
{{2,2,+1},{2,3,+1},{0,0,0},{0,0,0}},
{{2,2,-1},{3,2,+1},{0,0,0},{0,0,0}},
{{2,3,+1},{3,3,-1},{0,0,0},{0,0,0}},
},
{
{{0,0,+1},{0,2,+1},{1,1,+1},{1,3,+1}},
{{1,0,+1},{1,1,+1},{1,2,+1},{1,3,+1}},
{{0,0,+1},{0,2,+1},{0,0,0},{0,0,0}},
{{1,1,+1},{1,3,+1},{0,0,0},{0,0,0}},
{{0,0,+1},{0,1,+1},{0,2,+1},{0,3,+1}},
{{0,0,-1},{0,2,-1},{1,0,+1},{1,2,+1}},
{{0,1,+1},{0,3,+1},{1,1,-1},{1,3,-1}},
},
{
{{0,0,-1},{1,1,-1},{2,0,+1},{3,1,+1}},
{{1,0,-1},{1,1,-1},{3,0,+1},{3,1,+1}},
{{0,0,-1},{2,0,+1},{0,0,0},{0,0,0}},
{{1,1,-1},{3,1,+1},{0,0,0},{0,0,0}},
{{0,0,-1},{0,1,-1},{2,0,+1},{2,1,+1}},
{{0,0,+1},{1,0,-1},{2,0,-1},{3,0,+1}},
{{0,1,-1},{1,1,+1},{2,1,+1},{3,1,-1}},
},
{
{{0,2,+1},{1,3,+1},{2,2,-1},{3,3,-1}},
{{1,2,+1},{1,3,+1},{3,2,-1},{3,3,-1}},
{{0,2,+1},{2,2,-1},{0,0,0},{0,0,0}},
{{1,3,+1},{3,3,-1},{0,0,0},{0,0,0}},
{{0,2,+1},{0,3,+1},{2,2,-1},{2,3,-1}},
{{0,2,-1},{1,2,+1},{2,2,+1},{3,2,-1}},
{{0,3,+1},{1,3,-1},{2,3,-1},{3,3,+1}},
},
};
static const signed char M2BN[7][7] = {
{4,2,4,4,2,4,4},
{2,1,2,2,1,2,2},
{4,2,4,4,2,4,4},
{4,2,4,4,2,4,4},
{2,1,2,2,1,2,2},
{4,2,4,4,2,4,4},
{4,2,4,4,2,4,4},
};
#define AS 131584
#define AW 1032
#define BS 66048
#define BW 1032
#define RIG_USE 1
static const signed char M2B[7][7][4][3] = {
{
{{0,0,+1},{1,1,+1},{2,2,+1},{3,3,+1}},
{{0,0,+1},{2,2,+1},{0,0,0},{0,0,0}},
{{0,1,+1},{1,1,-1},{2,3,+1},{3,3,-1}},
{{0,0,-1},{1,0,+1},{2,2,-1},{3,2,+1}},
{{1,1,+1},{3,3,+1},{0,0,0},{0,0,0}},
{{0,0,+1},{0,1,+1},{2,2,+1},{2,3,+1}},
{{1,0,+1},{1,1,+1},{3,2,+1},{3,3,+1}},
},
{
{{0,0,+1},{1,1,+1},{0,0,0},{0,0,0}},
{{0,0,+1},{0,0,0},{0,0,0},{0,0,0}},
{{0,1,+1},{1,1,-1},{0,0,0},{0,0,0}},
{{0,0,-1},{1,0,+1},{0,0,0},{0,0,0}},
{{1,1,+1},{0,0,0},{0,0,0},{0,0,0}},
{{0,0,+1},{0,1,+1},{0,0,0},{0,0,0}},
{{1,0,+1},{1,1,+1},{0,0,0},{0,0,0}},
},
{
{{0,2,+1},{1,3,+1},{2,2,-1},{3,3,-1}},
{{0,2,+1},{2,2,-1},{0,0,0},{0,0,0}},
{{0,3,+1},{1,3,-1},{2,3,-1},{3,3,+1}},
{{0,2,-1},{1,2,+1},{2,2,+1},{3,2,-1}},
{{1,3,+1},{3,3,-1},{0,0,0},{0,0,0}},
{{0,2,+1},{0,3,+1},{2,2,-1},{2,3,-1}},
{{1,2,+1},{1,3,+1},{3,2,-1},{3,3,-1}},
},
{
{{0,0,-1},{1,1,-1},{2,0,+1},{3,1,+1}},
{{0,0,-1},{2,0,+1},{0,0,0},{0,0,0}},
{{0,1,-1},{1,1,+1},{2,1,+1},{3,1,-1}},
{{0,0,+1},{1,0,-1},{2,0,-1},{3,0,+1}},
{{1,1,-1},{3,1,+1},{0,0,0},{0,0,0}},
{{0,0,-1},{0,1,-1},{2,0,+1},{2,1,+1}},
{{1,0,-1},{1,1,-1},{3,0,+1},{3,1,+1}},
},
{
{{2,2,+1},{3,3,+1},{0,0,0},{0,0,0}},
{{2,2,+1},{0,0,0},{0,0,0},{0,0,0}},
{{2,3,+1},{3,3,-1},{0,0,0},{0,0,0}},
{{2,2,-1},{3,2,+1},{0,0,0},{0,0,0}},
{{3,3,+1},{0,0,0},{0,0,0},{0,0,0}},
{{2,2,+1},{2,3,+1},{0,0,0},{0,0,0}},
{{3,2,+1},{3,3,+1},{0,0,0},{0,0,0}},
},
{
{{0,0,+1},{0,2,+1},{1,1,+1},{1,3,+1}},
{{0,0,+1},{0,2,+1},{0,0,0},{0,0,0}},
{{0,1,+1},{0,3,+1},{1,1,-1},{1,3,-1}},
{{0,0,-1},{0,2,-1},{1,0,+1},{1,2,+1}},
{{1,1,+1},{1,3,+1},{0,0,0},{0,0,0}},
{{0,0,+1},{0,1,+1},{0,2,+1},{0,3,+1}},
{{1,0,+1},{1,1,+1},{1,2,+1},{1,3,+1}},
},
{
{{2,0,+1},{2,2,+1},{3,1,+1},{3,3,+1}},
{{2,0,+1},{2,2,+1},{0,0,0},{0,0,0}},
{{2,1,+1},{2,3,+1},{3,1,-1},{3,3,-1}},
{{2,0,-1},{2,2,-1},{3,0,+1},{3,2,+1}},
{{3,1,+1},{3,3,+1},{0,0,0},{0,0,0}},
{{2,0,+1},{2,1,+1},{2,2,+1},{2,3,+1}},
{{3,0,+1},{3,1,+1},{3,2,+1},{3,3,+1}},
},
};
#ifndef RB
#define RB 16
#endif
#define D2H 256
static int8_t MMS2[7][D2H*D2H+64] __attribute__((aligned(64)));
#define A2_STORE(v0,v1) \
{ __m256i _w0=_mm256_and_si256(v0,mx), _w1=_mm256_and_si256(v1,mx); \
__m256i _l0=_mm256_unpacklo_epi16(_w0,_w0), _h0=_mm256_unpackhi_epi16(_w0,_w0); \
__m256i _l1=_mm256_unpacklo_epi16(_w1,_w1), _h1=_mm256_unpackhi_epi16(_w1,_w1); \
_mm256_store_si256((__m256i*)(dst+2*k), _mm256_permute2x128_si256(_l0,_h0,0x20)); \
_mm256_store_si256((__m256i*)(dst+2*k+32), _mm256_permute2x128_si256(_l0,_h0,0x31)); \
_mm256_store_si256((__m256i*)(dst+2*k+64), _mm256_permute2x128_si256(_l1,_h1,0x20)); \
_mm256_store_si256((__m256i*)(dst+2*k+96), _mm256_permute2x128_si256(_l1,_h1,0x31)); } \
w1[k>>6]=(uint64_t)(uint32_t)_mm256_movemask_epi8(v0) \
|((uint64_t)(uint32_t)_mm256_movemask_epi8(v1)<<32);
#define A2_TERM(TN) _mm256_sign_epi8(_mm256_load_si256((const __m256i*)(p##TN+k)),sv[TN])
#define A2_TERM32(TN) _mm256_sign_epi8(_mm256_load_si256((const __m256i*)(p##TN+k+32)),sv[TN])
#define A2_ROWS \
const int8_t* p0=X+(size_t)((size_t)T[0][0]*h+i)*n+(size_t)T[0][1]*h; \
const int8_t* p1=X+(size_t)((size_t)T[1][0]*h+i)*n+(size_t)T[1][1]*h; \
const int8_t* p2=X+(size_t)((size_t)T[2][0]*h+i)*n+(size_t)T[2][1]*h; \
const int8_t* p3=X+(size_t)((size_t)T[3][0]*h+i)*n+(size_t)T[3][1]*h;
#define A2_BODY(NT) \
for (int i=0;i<h;i++){ \
A2_ROWS \
int8_t*dst=A0b+(size_t)i*GLDA; uint64_t*w1=A1b+(size_t)i*W; \
for (int k=0;k<h;k+=64){ \
__m256i v0=A2_TERM(0), v1=A2_TERM32(0); \
if(NT>1){ v0=_mm256_add_epi8(v0,A2_TERM(1)); v1=_mm256_add_epi8(v1,A2_TERM32(1)); } \
if(NT>2){ v0=_mm256_add_epi8(v0,A2_TERM(2)); v1=_mm256_add_epi8(v1,A2_TERM32(2)); } \
if(NT>3){ v0=_mm256_add_epi8(v0,A2_TERM(3)); v1=_mm256_add_epi8(v1,A2_TERM32(3)); } \
A2_STORE(v0,v1) \
} \
}
#define B2_TERM(TN) _mm256_sign_epi8(_mm256_load_si256((const __m256i*)(pa##TN+64*g+t)),sv[TN])
#define B2_TERMB(TN) _mm256_sign_epi8(_mm256_load_si256((const __m256i*)(pb##TN+64*g+t)),sv[TN])
#define B2_BODY(NT) \
for (int p=0;p<h/2;p++){ \
const int8_t *pa0=X+(size_t)((size_t)T[0][0]*h+2*p)*n+(size_t)T[0][1]*h; \
const int8_t *pb0=pa0+n; \
const int8_t *pa1=X+(size_t)((size_t)T[1][0]*h+2*p)*n+(size_t)T[1][1]*h; \
const int8_t *pb1=pa1+n; \
const int8_t *pa2=X+(size_t)((size_t)T[2][0]*h+2*p)*n+(size_t)T[2][1]*h; \
const int8_t *pb2=pa2+n; \
const int8_t *pa3=X+(size_t)((size_t)T[3][0]*h+2*p)*n+(size_t)T[3][1]*h; \
const int8_t *pb3=pa3+n; \
uint64_t *w0p=Bpb+(size_t)(2*p)*W, *w1p=Bpb+(size_t)(2*p+1)*W; \
for (int g=0;g<ng;g++){ \
for (int t=0;t<64;t+=32){ \
__m256i va=B2_TERM(0), vb=B2_TERMB(0); \
if(NT>1){ va=_mm256_add_epi8(va,B2_TERM(1)); vb=_mm256_add_epi8(vb,B2_TERMB(1)); } \
if(NT>2){ va=_mm256_add_epi8(va,B2_TERM(2)); vb=_mm256_add_epi8(vb,B2_TERMB(2)); } \
if(NT>3){ va=_mm256_add_epi8(va,B2_TERM(3)); vb=_mm256_add_epi8(vb,B2_TERMB(3)); } \
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb); \
int8_t *o=BI3b+((size_t)g*(h/2)+p)*128; \
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20)); \
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31)); \
uint32_t mm0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(va,one),7)); \
uint32_t mm1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(vb,one),7)); \
if(t==0){ w0p[g]=(uint64_t)mm0; w1p[g]=(uint64_t)mm1; } \
else { w0p[g]|=(uint64_t)mm0<<32; w1p[g]|=(uint64_t)mm1<<32; } \
} \
} \
}
#define C7_BODY(STORE) \
for (int i=__RB0;i<__RB1;i++){ \
const int8_t*a0=M[0]+(size_t)i*m, *a1=M[1]+(size_t)i*m, *a2=M[2]+(size_t)i*m; \
const int8_t*a3=M[3]+(size_t)i*m, *a4=M[4]+(size_t)i*m, *a5=M[5]+(size_t)i*m; \
const int8_t*a6=M[6]+(size_t)i*m; \
int8_t*d0=D+(size_t)(r0+i)*n+c0, *d1=d0+m; \
int8_t*d2=d0+(size_t)m*n, *d3=d1+(size_t)m*n; \
for (int j=0;j<m;j+=32){ \
__m256i m0=_mm256_loadu_si256((const __m256i*)(a0+j)); \
__m256i m1=_mm256_loadu_si256((const __m256i*)(a1+j)); \
__m256i m2=_mm256_loadu_si256((const __m256i*)(a2+j)); \
__m256i m3=_mm256_loadu_si256((const __m256i*)(a3+j)); \
__m256i m4=_mm256_loadu_si256((const __m256i*)(a4+j)); \
__m256i m5=_mm256_loadu_si256((const __m256i*)(a5+j)); \
__m256i m6=_mm256_loadu_si256((const __m256i*)(a6+j)); \
STORE(_mm256_sub_epi8(_mm256_add_epi8(_mm256_add_epi8(m0,m3),m6),m4),(__m256i*)(d0+j)); \
STORE(_mm256_add_epi8(m2,m4),(__m256i*)(d1+j)); \
STORE(_mm256_add_epi8(m1,m3),(__m256i*)(d2+j)); \
STORE(_mm256_sub_epi8(_mm256_add_epi8(_mm256_add_epi8(m0,m2),m5),m1),(__m256i*)(d3+j)); \
} \
}
#define C7_NS(v,pp) _mm256_storeu_si256(pp,v)
#define C7_NT(v,pp) _mm256_stream_si256(pp,v)
AVX2 static void cmb7ns_r(int8_t*D,int n,int r0,int c0,int m,const int8_t*const*M,int lo,int hi){
const int __RB0=lo, __RB1=hi; C7_BODY(C7_NS) }
#if D2MODE == 17 || D2MODE == 18
#endif
#if D2MODE == 19
#endif
#if D2MODE == 20
#define P2_TERM(acc,TN) { __m256i _t=_mm256_load_si256((const __m256i*)(p##TN+k)); \
acc = neg[TN] ? _mm256_sub_epi8(acc,_t) : _mm256_add_epi8(acc,_t); }
#define P2_TERM32(acc,TN) { __m256i _t=_mm256_load_si256((const __m256i*)(p##TN+k+32)); \
acc = neg[TN] ? _mm256_sub_epi8(acc,_t) : _mm256_add_epi8(acc,_t); }
#define P2_ABODY(NT) \
{ const int8_t* pp[4]; \
for (int t=0;t<4;t++) pp[t]=X+(size_t)((size_t)T[t][0]*h)*n+(size_t)T[t][1]*h; \
for (int i=0;i<h;i++){ \
const int8_t*p0=pp[0],*p1=pp[1],*p2=pp[2],*p3=pp[3]; \
int8_t*dst=A0b+(size_t)i*GLDA; uint64_t*w1=A1b+(size_t)i*W; \
if(PFA){ const long off=(long)PFA*n; \
_mm_prefetch((const char*)(p0+off), _MM_HINT_T0); \
if(NT>1) _mm_prefetch((const char*)(p1+off), _MM_HINT_T0); \
if(NT>2) _mm_prefetch((const char*)(p2+off), _MM_HINT_T0); \
if(NT>3) _mm_prefetch((const char*)(p3+off), _MM_HINT_T0); } \
for (int k=0;k<h;k+=64){ \
__m256i v0=_mm256_load_si256((const __m256i*)(p0+k)); \
__m256i v1=_mm256_load_si256((const __m256i*)(p0+k+32)); \
if(neg[0]){ v0=_mm256_sub_epi8(_mm256_setzero_si256(),v0); \
v1=_mm256_sub_epi8(_mm256_setzero_si256(),v1); } \
if(NT>1){ P2_TERM(v0,1) P2_TERM32(v1,1) } \
if(NT>2){ P2_TERM(v0,2) P2_TERM32(v1,2) } \
if(NT>3){ P2_TERM(v0,3) P2_TERM32(v1,3) } \
A2_STORE(v0,v1) \
} \
for (int t=0;t<4;t++) pp[t]+=n; \
} \
}
__attribute__((optimize("O3","unroll-loops"))) AVX2 static void buildA_q3(int h,int n,const int8_t*X,
const signed char (*T)[3], int nt){
const __m256i mx=_mm256_set1_epi8(0x7F);
const int W=h/64;
int neg[4];
for (int t=0;t<4;t++) neg[t] = (nt>t && T[t][2]<0);
if (nt==4){ P2_ABODY(4) } else if (nt==3){ P2_ABODY(3) }
else if (nt==2){ P2_ABODY(2) } else { P2_ABODY(1) }
}
#define P2_BTERM(acc,TN) { __m256i _t=_mm256_load_si256((const __m256i*)(pa##TN+64*g+t)); \
acc = neg[TN] ? _mm256_sub_epi8(acc,_t) : _mm256_add_epi8(acc,_t); }
#define P2_BTERMB(acc,TN) { __m256i _t=_mm256_load_si256((const __m256i*)(pb##TN+64*g+t)); \
acc = neg[TN] ? _mm256_sub_epi8(acc,_t) : _mm256_add_epi8(acc,_t); }
#define P2_BBODY(NT) \
for (int p=0;p<h/2;p++){ \
const int8_t *pa0=X+(size_t)((size_t)T[0][0]*h+2*p)*n+(size_t)T[0][1]*h; \
const int8_t *pb0=pa0+n; \
const int8_t *pa1=X+(size_t)((size_t)T[1][0]*h+2*p)*n+(size_t)T[1][1]*h; \
const int8_t *pb1=pa1+n; \
const int8_t *pa2=X+(size_t)((size_t)T[2][0]*h+2*p)*n+(size_t)T[2][1]*h; \
const int8_t *pb2=pa2+n; \
const int8_t *pa3=X+(size_t)((size_t)T[3][0]*h+2*p)*n+(size_t)T[3][1]*h; \
const int8_t *pb3=pa3+n; \
if(PFB){ const long off=(long)PFB*2*n; \
for (int q=0;q<ng;q++){ \
_mm_prefetch((const char*)(pa0+off+64*q), _MM_HINT_T0); \
if(NT>1) _mm_prefetch((const char*)(pa1+off+64*q), _MM_HINT_T0); \
if(NT>2) _mm_prefetch((const char*)(pa2+off+64*q), _MM_HINT_T0); \
if(NT>3) _mm_prefetch((const char*)(pa3+off+64*q), _MM_HINT_T0); } }\
uint64_t *w0p=Bpb+(size_t)(2*p)*W, *w1p=Bpb+(size_t)(2*p+1)*W; \
for (int g=0;g<ng;g++){ \
for (int t=0;t<64;t+=32){ \
__m256i va=_mm256_load_si256((const __m256i*)(pa0+64*g+t)); \
__m256i vb=_mm256_load_si256((const __m256i*)(pb0+64*g+t)); \
if(neg[0]){ va=_mm256_sub_epi8(_mm256_setzero_si256(),va); \
vb=_mm256_sub_epi8(_mm256_setzero_si256(),vb); } \
if(NT>1){ P2_BTERM(va,1) P2_BTERMB(vb,1) } \
if(NT>2){ P2_BTERM(va,2) P2_BTERMB(vb,2) } \
if(NT>3){ P2_BTERM(va,3) P2_BTERMB(vb,3) } \
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb); \
int8_t *o=BI3b+((size_t)g*(h/2)+p)*128; \
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20)); \
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31)); \
{ uint32_t mm0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(va,one),7)); \
uint32_t mm1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(vb,one),7)); \
if(t==0){ w0p[g]=(uint64_t)mm0; w1p[g]=(uint64_t)mm1; } \
else { w0p[g]|=(uint64_t)mm0<<32; w1p[g]|=(uint64_t)mm1<<32; } }\
} \
} \
}
__attribute__((optimize("O3","unroll-loops"))) AVX2 static void buildB_q3(int h,int n,const int8_t*X,
const signed char (*T)[3], int nt){
const __m256i one=_mm256_set1_epi8(1);
const int ng=h/64, W=h/64;
int neg[4];
for (int t=0;t<4;t++) neg[t] = (nt>t && T[t][2]<0);
if (nt==4){ P2_BBODY(4) } else if (nt==3){ P2_BBODY(3) }
else if (nt==2){ P2_BBODY(2) } else { P2_BBODY(1) }
}
#define P2_TERM(acc,TN) { __m256i _t=_mm256_load_si256((const __m256i*)(p##TN+k)); \
acc = neg[TN] ? _mm256_sub_epi8(acc,_t) : _mm256_add_epi8(acc,_t); }
#define P2_TERM32(acc,TN) { __m256i _t=_mm256_load_si256((const __m256i*)(p##TN+k+32)); \
acc = neg[TN] ? _mm256_sub_epi8(acc,_t) : _mm256_add_epi8(acc,_t); }
#define P2_ABODY_B(NT) \
{ const int8_t* pp[4]; \
for (int t=0;t<4;t++) pp[t]=X+(size_t)((size_t)T[t][0]*h)*n+(size_t)T[t][1]*h; \
for (int i=0;i<h;i++){ \
const int8_t*p0=pp[0],*p1=pp[1],*p2=pp[2],*p3=pp[3]; \
int8_t*dst=A0base+(size_t)i*GLDA; uint64_t*w1=A1base+(size_t)i*W; \
if(PFA){ const long off=(long)PFA*n; \
_mm_prefetch((const char*)(p0+off), _MM_HINT_T0); \
if(NT>1) _mm_prefetch((const char*)(p1+off), _MM_HINT_T0); \
if(NT>2) _mm_prefetch((const char*)(p2+off), _MM_HINT_T0); \
if(NT>3) _mm_prefetch((const char*)(p3+off), _MM_HINT_T0); } \
for (int k=0;k<h;k+=64){ \
__m256i v0=_mm256_load_si256((const __m256i*)(p0+k)); \
__m256i v1=_mm256_load_si256((const __m256i*)(p0+k+32)); \
if(neg[0]){ v0=_mm256_sub_epi8(_mm256_setzero_si256(),v0); \
v1=_mm256_sub_epi8(_mm256_setzero_si256(),v1); } \
if(NT>1){ P2_TERM(v0,1) P2_TERM32(v1,1) } \
if(NT>2){ P2_TERM(v0,2) P2_TERM32(v1,2) } \
if(NT>3){ P2_TERM(v0,3) P2_TERM32(v1,3) } \
A2_STORE(v0,v1) \
} \
for (int t=0;t<4;t++) pp[t]+=n; \
} \
}
__attribute__((optimize("O3","unroll-loops"))) AVX2 static void buildA_q3_b(int h,int n,const int8_t*X,
const signed char (*T)[3], int nt,int8_t* A0base,uint64_t* A1base){
const __m256i mx=_mm256_set1_epi8(0x7F);
const int W=h/64;
int neg[4];
for (int t=0;t<4;t++) neg[t] = (nt>t && T[t][2]<0);
if (nt==4){ P2_ABODY_B(4) } else if (nt==3){ P2_ABODY_B(3) }
else if (nt==2){ P2_ABODY_B(2) } else { P2_ABODY_B(1) }
}
#define P2_BTERM(acc,TN) { __m256i _t=_mm256_load_si256((const __m256i*)(pa##TN+64*g+t)); \
acc = neg[TN] ? _mm256_sub_epi8(acc,_t) : _mm256_add_epi8(acc,_t); }
#define P2_BTERMB(acc,TN) { __m256i _t=_mm256_load_si256((const __m256i*)(pb##TN+64*g+t)); \
acc = neg[TN] ? _mm256_sub_epi8(acc,_t) : _mm256_add_epi8(acc,_t); }
#define P2_BBODY_B(NT) \
for (int p=0;p<h/2;p++){ \
const int8_t *pa0=X+(size_t)((size_t)T[0][0]*h+2*p)*n+(size_t)T[0][1]*h; \
const int8_t *pb0=pa0+n; \
const int8_t *pa1=X+(size_t)((size_t)T[1][0]*h+2*p)*n+(size_t)T[1][1]*h; \
const int8_t *pb1=pa1+n; \
const int8_t *pa2=X+(size_t)((size_t)T[2][0]*h+2*p)*n+(size_t)T[2][1]*h; \
const int8_t *pb2=pa2+n; \
const int8_t *pa3=X+(size_t)((size_t)T[3][0]*h+2*p)*n+(size_t)T[3][1]*h; \
const int8_t *pb3=pa3+n; \
if(PFB){ const long off=(long)PFB*2*n; \
for (int q=0;q<ng;q++){ \
_mm_prefetch((const char*)(pa0+off+64*q), _MM_HINT_T0); \
if(NT>1) _mm_prefetch((const char*)(pa1+off+64*q), _MM_HINT_T0); \
if(NT>2) _mm_prefetch((const char*)(pa2+off+64*q), _MM_HINT_T0); \
if(NT>3) _mm_prefetch((const char*)(pa3+off+64*q), _MM_HINT_T0); } }\
uint64_t *w0p=BPbase+(size_t)(2*p)*W, *w1p=BPbase+(size_t)(2*p+1)*W; \
for (int g=0;g<ng;g++){ \
for (int t=0;t<64;t+=32){ \
__m256i va=_mm256_load_si256((const __m256i*)(pa0+64*g+t)); \
__m256i vb=_mm256_load_si256((const __m256i*)(pb0+64*g+t)); \
if(neg[0]){ va=_mm256_sub_epi8(_mm256_setzero_si256(),va); \
vb=_mm256_sub_epi8(_mm256_setzero_si256(),vb); } \
if(NT>1){ P2_BTERM(va,1) P2_BTERMB(vb,1) } \
if(NT>2){ P2_BTERM(va,2) P2_BTERMB(vb,2) } \
if(NT>3){ P2_BTERM(va,3) P2_BTERMB(vb,3) } \
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb); \
int8_t *o=BIbase+((size_t)g*(h/2)+p)*128; \
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20)); \
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31)); \
{ uint32_t mm0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(va,one),7)); \
uint32_t mm1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(vb,one),7)); \
if(t==0){ w0p[g]=(uint64_t)mm0; w1p[g]=(uint64_t)mm1; } \
else { w0p[g]|=(uint64_t)mm0<<32; w1p[g]|=(uint64_t)mm1<<32; } }\
} \
} \
}
__attribute__((optimize("O3","unroll-loops"))) AVX2 static void buildB_q3_b(int h,int n,const int8_t*X,
const signed char (*T)[3], int nt,int8_t* BIbase,uint64_t* BPbase){
const __m256i one=_mm256_set1_epi8(1);
const int ng=h/64, W=h/64;
int neg[4];
for (int t=0;t<4;t++) neg[t] = (nt>t && T[t][2]<0);
if (nt==4){ P2_BBODY_B(4) } else if (nt==3){ P2_BBODY_B(3) }
else if (nt==2){ P2_BBODY_B(2) } else { P2_BBODY_B(1) }
}
#define DEF_NIBCORR_B(TAG, M) \
AVX2BMI static void nibcorr##TAG##_c(const uint64_t*A1base,const uint64_t*BPbase){ \
const int n=M, W=n/64, G=n>>2, GB=16, NY=W>>2; \
memset(Pbb,0,(size_t)n*W*8); \
for (int g0=0; g0<G; g0+=GB){ \
for (int t=0;t<GB;t++){ \
const int g=g0+t; \
uint64_t *T=TB+(size_t)t*16*W; \
const uint64_t *r=BPbase+(size_t)(g*4)*W; \
nibtab_build(r,T,W); \
} \
for (int i=0;i<n;i++){ \
const uint64_t *arow=A1base+(size_t)i*W+(g0>>4); \
__m256i acc[NY]; \
for (int q=0;q<NY;q++) acc[q]=_mm256_setzero_si256(); \
const uint64_t *T=TB; \
for (int wb=0; wb<(GB>>4); wb++){ \
const uint64_t word=arow[wb]; \
_Pragma("GCC unroll 16") \
for (int b=0;b<16;b++, T+=(size_t)16*W){ \
const uint64_t *e=T+(size_t)(((word>>(4*b))&15u)*W); \
for (int q=0;q<NY;q++) \
acc[q]=_mm256_xor_si256(acc[q],_mm256_load_si256((const __m256i*)(e+4*q))); \
} \
} \
uint64_t *o=Pbb+(size_t)i*W; \
for (int q=0;q<NY;q++){ \
__m256i *po=(__m256i*)(o+4*q); \
_mm256_store_si256(po,_mm256_xor_si256(_mm256_load_si256((const __m256i*)po),acc[q])); \
} \
} \
} \
}
DEF_NIBCORR_B(256,256)
AVX2 static void submm_tail_c(int m,int8_t*Cp,const int8_t* A0base,const uint64_t* A1base,
const int8_t* BIbase,const uint64_t* BPbase){
GLDA = 2*(m) + PADA; GLDB = m + PADB;
nibcorr256_c(A1base,BPbase);
expand_bits(m);
const int GB=64, IB=256;
for (int ib=0; ib<m; ib+=IB){
const int ihi = (ib+IB<m)? ib+IB : m;
for (int gb=0; gb<m; gb+=GB){
const int ghi = (gb+GB<m)? gb+GB : m;
for (int g=gb; g<ghi; g+=64){
const int8_t *bi = BIbase+(size_t)(g>>6)*(m/2)*128;
for (int i=ib; i<ihi; i+=2){
const int8_t *ap=A0base+(size_t)i*GLDA;
mikro_u4(m, ap, bi, Cp+(size_t)i*m+g, PBb+(size_t)i*GLDB+g, 0, GLDA, GLDB);
}
}
}
}
}
AVX2 static void fold_top_r(int8_t*C,int n,int m,int k1,const int8_t*M,int lo,int hi);
AVX2 static void fold_top_r(int8_t*C,int n,int m,int k1,const int8_t*M,int lo,int hi){
static const signed char SGN[7][4] = {
{+1, 0, 0,+1}, { 0, 0,+1,-1}, { 0,+1, 0,+1},
{+1, 0,+1, 0}, {-1,+1, 0, 0}, { 0, 0, 0,+1},
{+1, 0, 0, 0} };
static const int FIRST[4] = {0,2,1,0};
const int qr[4]={0,0,1,1}, qc[4]={0,1,0,1};
for (int q=0;q<4;q++){
const int s=SGN[k1][q]; if(!s) continue;
int8_t*D=C+(size_t)(qr[q]*m)*n+(size_t)qc[q]*m;
if (k1==FIRST[q]){
for (int i=lo;i<hi;i++){
const int8_t*src=M+(size_t)i*m; int8_t*dst=D+(size_t)i*n;
for (int j=0;j<m;j+=32)
_mm256_storeu_si256((__m256i*)(dst+j),_mm256_loadu_si256((const __m256i*)(src+j)));
}
} else {
for (int i=lo;i<hi;i++){
const int8_t*src=M+(size_t)i*m; int8_t*dst=D+(size_t)i*n;
for (int j=0;j<m;j+=32){
__m256i t=_mm256_loadu_si256((const __m256i*)(src+j));
__m256i d=_mm256_loadu_si256((const __m256i*)(dst+j));
_mm256_storeu_si256((__m256i*)(dst+j),
s>0? _mm256_add_epi8(d,t) : _mm256_sub_epi8(d,t));
}
}
}
}
}
static int8_t MMSFc[2*RB*512] __attribute__((aligned(64)));
AVX2 static void cmb7c(int h,const int8_t*const*M,int lo,int hi){
for (int i=lo;i<hi;i++){
const int8_t*a0=M[0]+(size_t)i*h, *a1=M[1]+(size_t)i*h, *a2=M[2]+(size_t)i*h;
const int8_t*a3=M[3]+(size_t)i*h, *a4=M[4]+(size_t)i*h, *a5=M[5]+(size_t)i*h;
const int8_t*a6=M[6]+(size_t)i*h;
int8_t*d0=MMSFc+(size_t)(i-lo)*512;
int8_t*d1=d0+h;
int8_t*d2=d0+(size_t)RB*512;
int8_t*d3=d2+h;
for (int j=0;j<h;j+=32){
__m256i m0=_mm256_loadu_si256((const __m256i*)(a0+j));
__m256i m1=_mm256_loadu_si256((const __m256i*)(a1+j));
__m256i m2=_mm256_loadu_si256((const __m256i*)(a2+j));
__m256i m3=_mm256_loadu_si256((const __m256i*)(a3+j));
__m256i m4=_mm256_loadu_si256((const __m256i*)(a4+j));
__m256i m5=_mm256_loadu_si256((const __m256i*)(a5+j));
__m256i m6=_mm256_loadu_si256((const __m256i*)(a6+j));
_mm256_storeu_si256((__m256i*)(d0+j),_mm256_sub_epi8(_mm256_add_epi8(_mm256_add_epi8(m0,m3),m6),m4));
_mm256_storeu_si256((__m256i*)(d1+j),_mm256_add_epi8(m2,m4));
_mm256_storeu_si256((__m256i*)(d2+j),_mm256_add_epi8(m1,m3));
_mm256_storeu_si256((__m256i*)(d3+j),_mm256_sub_epi8(_mm256_add_epi8(_mm256_add_epi8(m0,m2),m5),m1));
}
}
}
#define LD(P,O) _mm256_load_si256((const __m256i*)((P)+(O)))
#define A8(a,b) _mm256_add_epi8(a,b)
#define S8(a,b) _mm256_sub_epi8(a,b)
#define ASTORE_A(V0,K2) \
{ __m256i _w=_mm256_and_si256(V0,mx); \
__m256i _l=_mm256_unpacklo_epi16(_w,_w), _h=_mm256_unpackhi_epi16(_w,_w); \
_mm256_store_si256((__m256i*)(dst+2*k), _mm256_permute2x128_si256(_l,_h,0x20)); \
_mm256_store_si256((__m256i*)(dst+2*k+32), _mm256_permute2x128_si256(_l,_h,0x31)); \
msk[K2]=(uint32_t)_mm256_movemask_epi8(V0); }
#define ASTORE_B(V1,K2) \
{ __m256i _w=_mm256_and_si256(V1,mx); \
__m256i _l=_mm256_unpacklo_epi16(_w,_w), _h=_mm256_unpackhi_epi16(_w,_w); \
_mm256_store_si256((__m256i*)(dst+2*k+64), _mm256_permute2x128_si256(_l,_h,0x20)); \
_mm256_store_si256((__m256i*)(dst+2*k+96), _mm256_permute2x128_si256(_l,_h,0x31)); \
w1[k>>6]=(uint64_t)msk[K2]|((uint64_t)(uint32_t)_mm256_movemask_epi8(V1)<<32); }
AVX2 static void fuseA(int h,int n,const int8_t*X,int k1){
const __m256i mx=_mm256_set1_epi8(0x7F);
const __m256i zero=_mm256_setzero_si256();
const int W=h/64;
switch(k1){
case 0: {
const int8_t* u0=X+((size_t)(0*h))*n+(size_t)0*h;
const int8_t* u1=X+((size_t)(1*h))*n+(size_t)1*h;
const int8_t* u2=X+((size_t)(2*h))*n+(size_t)2*h;
const int8_t* u3=X+((size_t)(3*h))*n+(size_t)3*h;
const int8_t* u4=X+((size_t)(1*h))*n+(size_t)0*h;
const int8_t* u5=X+((size_t)(3*h))*n+(size_t)2*h;
const int8_t* u6=X+((size_t)(0*h))*n+(size_t)1*h;
const int8_t* u7=X+((size_t)(2*h))*n+(size_t)3*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int i=0;i<h;i++){ uint32_t msk[7];
for (int k=0;k<h;k+=64){
__m256i a0=LD(u0,k);
__m256i a1=LD(u1,k);
__m256i a2=LD(u2,k);
__m256i a3=LD(u3,k);
__m256i a4=LD(u4,k);
__m256i a5=LD(u5,k);
__m256i a6=LD(u6,k);
__m256i a7=LD(u7,k);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(A8(a0,a1),a2),a3); ASTORE_A(v0,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(A8(a4,a1),a5),a3); ASTORE_A(v0,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
__m256i v0=A8(a0,a2); ASTORE_A(v0,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
__m256i v0=A8(a1,a3); ASTORE_A(v0,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(A8(a0,a6),a2),a7); ASTORE_A(v0,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
__m256i v0=A8(S8(A8(S8(zero,a0),a4),a2),a5); ASTORE_A(v0,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
__m256i v0=S8(A8(S8(a6,a1),a7),a3); ASTORE_A(v0,6) }
a0=LD(u0,k+32);
a1=LD(u1,k+32);
a2=LD(u2,k+32);
a3=LD(u3,k+32);
a4=LD(u4,k+32);
a5=LD(u5,k+32);
a6=LD(u6,k+32);
a7=LD(u7,k+32);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)0*AW+(size_t)i*W;
__m256i v1=A8(A8(A8(a0,a1),a2),a3); ASTORE_B(v1,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)1*AW+(size_t)i*W;
__m256i v1=A8(A8(A8(a4,a1),a5),a3); ASTORE_B(v1,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)2*AW+(size_t)i*W;
__m256i v1=A8(a0,a2); ASTORE_B(v1,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)3*AW+(size_t)i*W;
__m256i v1=A8(a1,a3); ASTORE_B(v1,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)4*AW+(size_t)i*W;
__m256i v1=A8(A8(A8(a0,a6),a2),a7); ASTORE_B(v1,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)5*AW+(size_t)i*W;
__m256i v1=A8(S8(A8(S8(zero,a0),a4),a2),a5); ASTORE_B(v1,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)6*AW+(size_t)i*W;
__m256i v1=S8(A8(S8(a6,a1),a7),a3); ASTORE_B(v1,6) }
}
u0+=n;
u1+=n;
u2+=n;
u3+=n;
u4+=n;
u5+=n;
u6+=n;
u7+=n;
}
return; }
case 1: {
const int8_t* u0=X+((size_t)(2*h))*n+(size_t)0*h;
const int8_t* u1=X+((size_t)(2*h))*n+(size_t)2*h;
const int8_t* u2=X+((size_t)(3*h))*n+(size_t)1*h;
const int8_t* u3=X+((size_t)(3*h))*n+(size_t)3*h;
const int8_t* u4=X+((size_t)(3*h))*n+(size_t)0*h;
const int8_t* u5=X+((size_t)(3*h))*n+(size_t)2*h;
const int8_t* u6=X+((size_t)(2*h))*n+(size_t)1*h;
const int8_t* u7=X+((size_t)(2*h))*n+(size_t)3*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int i=0;i<h;i++){ uint32_t msk[7];
for (int k=0;k<h;k+=64){
__m256i a0=LD(u0,k);
__m256i a1=LD(u1,k);
__m256i a2=LD(u2,k);
__m256i a3=LD(u3,k);
__m256i a4=LD(u4,k);
__m256i a5=LD(u5,k);
__m256i a6=LD(u6,k);
__m256i a7=LD(u7,k);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(A8(a0,a1),a2),a3); ASTORE_A(v0,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(A8(a4,a2),a5),a3); ASTORE_A(v0,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
__m256i v0=A8(a0,a1); ASTORE_A(v0,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
__m256i v0=A8(a2,a3); ASTORE_A(v0,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(A8(a0,a6),a1),a7); ASTORE_A(v0,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(S8(S8(zero,a0),a1),a4),a5); ASTORE_A(v0,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
__m256i v0=S8(S8(A8(a6,a7),a2),a3); ASTORE_A(v0,6) }
a0=LD(u0,k+32);
a1=LD(u1,k+32);
a2=LD(u2,k+32);
a3=LD(u3,k+32);
a4=LD(u4,k+32);
a5=LD(u5,k+32);
a6=LD(u6,k+32);
a7=LD(u7,k+32);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)0*AW+(size_t)i*W;
__m256i v1=A8(A8(A8(a0,a1),a2),a3); ASTORE_B(v1,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)1*AW+(size_t)i*W;
__m256i v1=A8(A8(A8(a4,a2),a5),a3); ASTORE_B(v1,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)2*AW+(size_t)i*W;
__m256i v1=A8(a0,a1); ASTORE_B(v1,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)3*AW+(size_t)i*W;
__m256i v1=A8(a2,a3); ASTORE_B(v1,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)4*AW+(size_t)i*W;
__m256i v1=A8(A8(A8(a0,a6),a1),a7); ASTORE_B(v1,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)5*AW+(size_t)i*W;
__m256i v1=A8(A8(S8(S8(zero,a0),a1),a4),a5); ASTORE_B(v1,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)6*AW+(size_t)i*W;
__m256i v1=S8(S8(A8(a6,a7),a2),a3); ASTORE_B(v1,6) }
}
u0+=n;
u1+=n;
u2+=n;
u3+=n;
u4+=n;
u5+=n;
u6+=n;
u7+=n;
}
return; }
case 2: {
const int8_t* u0=X+((size_t)(0*h))*n+(size_t)0*h;
const int8_t* u1=X+((size_t)(1*h))*n+(size_t)1*h;
const int8_t* u2=X+((size_t)(1*h))*n+(size_t)0*h;
const int8_t* u3=X+((size_t)(0*h))*n+(size_t)1*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int i=0;i<h;i++){ uint32_t msk[7];
for (int k=0;k<h;k+=64){
__m256i a0=LD(u0,k);
__m256i a1=LD(u1,k);
__m256i a2=LD(u2,k);
__m256i a3=LD(u3,k);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
__m256i v0=A8(a0,a1); ASTORE_A(v0,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
__m256i v0=A8(a2,a1); ASTORE_A(v0,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
__m256i v0=a0; ASTORE_A(v0,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
__m256i v0=a1; ASTORE_A(v0,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
__m256i v0=A8(a0,a3); ASTORE_A(v0,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
__m256i v0=A8(S8(zero,a0),a2); ASTORE_A(v0,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
__m256i v0=S8(a3,a1); ASTORE_A(v0,6) }
a0=LD(u0,k+32);
a1=LD(u1,k+32);
a2=LD(u2,k+32);
a3=LD(u3,k+32);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)0*AW+(size_t)i*W;
__m256i v1=A8(a0,a1); ASTORE_B(v1,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)1*AW+(size_t)i*W;
__m256i v1=A8(a2,a1); ASTORE_B(v1,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)2*AW+(size_t)i*W;
__m256i v1=a0; ASTORE_B(v1,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)3*AW+(size_t)i*W;
__m256i v1=a1; ASTORE_B(v1,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)4*AW+(size_t)i*W;
__m256i v1=A8(a0,a3); ASTORE_B(v1,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)5*AW+(size_t)i*W;
__m256i v1=A8(S8(zero,a0),a2); ASTORE_B(v1,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)6*AW+(size_t)i*W;
__m256i v1=S8(a3,a1); ASTORE_B(v1,6) }
}
u0+=n;
u1+=n;
u2+=n;
u3+=n;
}
return; }
case 3: {
const int8_t* u0=X+((size_t)(2*h))*n+(size_t)2*h;
const int8_t* u1=X+((size_t)(3*h))*n+(size_t)3*h;
const int8_t* u2=X+((size_t)(3*h))*n+(size_t)2*h;
const int8_t* u3=X+((size_t)(2*h))*n+(size_t)3*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int i=0;i<h;i++){ uint32_t msk[7];
for (int k=0;k<h;k+=64){
__m256i a0=LD(u0,k);
__m256i a1=LD(u1,k);
__m256i a2=LD(u2,k);
__m256i a3=LD(u3,k);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
__m256i v0=A8(a0,a1); ASTORE_A(v0,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
__m256i v0=A8(a2,a1); ASTORE_A(v0,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
__m256i v0=a0; ASTORE_A(v0,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
__m256i v0=a1; ASTORE_A(v0,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
__m256i v0=A8(a0,a3); ASTORE_A(v0,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
__m256i v0=A8(S8(zero,a0),a2); ASTORE_A(v0,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
__m256i v0=S8(a3,a1); ASTORE_A(v0,6) }
a0=LD(u0,k+32);
a1=LD(u1,k+32);
a2=LD(u2,k+32);
a3=LD(u3,k+32);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)0*AW+(size_t)i*W;
__m256i v1=A8(a0,a1); ASTORE_B(v1,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)1*AW+(size_t)i*W;
__m256i v1=A8(a2,a1); ASTORE_B(v1,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)2*AW+(size_t)i*W;
__m256i v1=a0; ASTORE_B(v1,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)3*AW+(size_t)i*W;
__m256i v1=a1; ASTORE_B(v1,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)4*AW+(size_t)i*W;
__m256i v1=A8(a0,a3); ASTORE_B(v1,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)5*AW+(size_t)i*W;
__m256i v1=A8(S8(zero,a0),a2); ASTORE_B(v1,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)6*AW+(size_t)i*W;
__m256i v1=S8(a3,a1); ASTORE_B(v1,6) }
}
u0+=n;
u1+=n;
u2+=n;
u3+=n;
}
return; }
case 4: {
const int8_t* u0=X+((size_t)(0*h))*n+(size_t)0*h;
const int8_t* u1=X+((size_t)(0*h))*n+(size_t)2*h;
const int8_t* u2=X+((size_t)(1*h))*n+(size_t)1*h;
const int8_t* u3=X+((size_t)(1*h))*n+(size_t)3*h;
const int8_t* u4=X+((size_t)(1*h))*n+(size_t)0*h;
const int8_t* u5=X+((size_t)(1*h))*n+(size_t)2*h;
const int8_t* u6=X+((size_t)(0*h))*n+(size_t)1*h;
const int8_t* u7=X+((size_t)(0*h))*n+(size_t)3*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int i=0;i<h;i++){ uint32_t msk[7];
for (int k=0;k<h;k+=64){
__m256i a0=LD(u0,k);
__m256i a1=LD(u1,k);
__m256i a2=LD(u2,k);
__m256i a3=LD(u3,k);
__m256i a4=LD(u4,k);
__m256i a5=LD(u5,k);
__m256i a6=LD(u6,k);
__m256i a7=LD(u7,k);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(A8(a0,a1),a2),a3); ASTORE_A(v0,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(A8(a4,a2),a5),a3); ASTORE_A(v0,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
__m256i v0=A8(a0,a1); ASTORE_A(v0,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
__m256i v0=A8(a2,a3); ASTORE_A(v0,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(A8(a0,a6),a1),a7); ASTORE_A(v0,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(S8(S8(zero,a0),a1),a4),a5); ASTORE_A(v0,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
__m256i v0=S8(S8(A8(a6,a7),a2),a3); ASTORE_A(v0,6) }
a0=LD(u0,k+32);
a1=LD(u1,k+32);
a2=LD(u2,k+32);
a3=LD(u3,k+32);
a4=LD(u4,k+32);
a5=LD(u5,k+32);
a6=LD(u6,k+32);
a7=LD(u7,k+32);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)0*AW+(size_t)i*W;
__m256i v1=A8(A8(A8(a0,a1),a2),a3); ASTORE_B(v1,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)1*AW+(size_t)i*W;
__m256i v1=A8(A8(A8(a4,a2),a5),a3); ASTORE_B(v1,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)2*AW+(size_t)i*W;
__m256i v1=A8(a0,a1); ASTORE_B(v1,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)3*AW+(size_t)i*W;
__m256i v1=A8(a2,a3); ASTORE_B(v1,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)4*AW+(size_t)i*W;
__m256i v1=A8(A8(A8(a0,a6),a1),a7); ASTORE_B(v1,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)5*AW+(size_t)i*W;
__m256i v1=A8(A8(S8(S8(zero,a0),a1),a4),a5); ASTORE_B(v1,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)6*AW+(size_t)i*W;
__m256i v1=S8(S8(A8(a6,a7),a2),a3); ASTORE_B(v1,6) }
}
u0+=n;
u1+=n;
u2+=n;
u3+=n;
u4+=n;
u5+=n;
u6+=n;
u7+=n;
}
return; }
case 5: {
const int8_t* u0=X+((size_t)(0*h))*n+(size_t)0*h;
const int8_t* u1=X+((size_t)(1*h))*n+(size_t)1*h;
const int8_t* u2=X+((size_t)(2*h))*n+(size_t)0*h;
const int8_t* u3=X+((size_t)(3*h))*n+(size_t)1*h;
const int8_t* u4=X+((size_t)(1*h))*n+(size_t)0*h;
const int8_t* u5=X+((size_t)(3*h))*n+(size_t)0*h;
const int8_t* u6=X+((size_t)(0*h))*n+(size_t)1*h;
const int8_t* u7=X+((size_t)(2*h))*n+(size_t)1*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int i=0;i<h;i++){ uint32_t msk[7];
for (int k=0;k<h;k+=64){
__m256i a0=LD(u0,k);
__m256i a1=LD(u1,k);
__m256i a2=LD(u2,k);
__m256i a3=LD(u3,k);
__m256i a4=LD(u4,k);
__m256i a5=LD(u5,k);
__m256i a6=LD(u6,k);
__m256i a7=LD(u7,k);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(S8(S8(zero,a0),a1),a2),a3); ASTORE_A(v0,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(S8(S8(zero,a4),a1),a5),a3); ASTORE_A(v0,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
__m256i v0=A8(S8(zero,a0),a2); ASTORE_A(v0,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
__m256i v0=A8(S8(zero,a1),a3); ASTORE_A(v0,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
__m256i v0=A8(A8(S8(S8(zero,a0),a6),a2),a7); ASTORE_A(v0,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
__m256i v0=A8(S8(S8(a0,a4),a2),a5); ASTORE_A(v0,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
__m256i v0=S8(A8(A8(S8(zero,a6),a1),a7),a3); ASTORE_A(v0,6) }
a0=LD(u0,k+32);
a1=LD(u1,k+32);
a2=LD(u2,k+32);
a3=LD(u3,k+32);
a4=LD(u4,k+32);
a5=LD(u5,k+32);
a6=LD(u6,k+32);
a7=LD(u7,k+32);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)0*AW+(size_t)i*W;
__m256i v1=A8(A8(S8(S8(zero,a0),a1),a2),a3); ASTORE_B(v1,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)1*AW+(size_t)i*W;
__m256i v1=A8(A8(S8(S8(zero,a4),a1),a5),a3); ASTORE_B(v1,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)2*AW+(size_t)i*W;
__m256i v1=A8(S8(zero,a0),a2); ASTORE_B(v1,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)3*AW+(size_t)i*W;
__m256i v1=A8(S8(zero,a1),a3); ASTORE_B(v1,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)4*AW+(size_t)i*W;
__m256i v1=A8(A8(S8(S8(zero,a0),a6),a2),a7); ASTORE_B(v1,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)5*AW+(size_t)i*W;
__m256i v1=A8(S8(S8(a0,a4),a2),a5); ASTORE_B(v1,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)6*AW+(size_t)i*W;
__m256i v1=S8(A8(A8(S8(zero,a6),a1),a7),a3); ASTORE_B(v1,6) }
}
u0+=n;
u1+=n;
u2+=n;
u3+=n;
u4+=n;
u5+=n;
u6+=n;
u7+=n;
}
return; }
case 6: {
const int8_t* u0=X+((size_t)(0*h))*n+(size_t)2*h;
const int8_t* u1=X+((size_t)(1*h))*n+(size_t)3*h;
const int8_t* u2=X+((size_t)(2*h))*n+(size_t)2*h;
const int8_t* u3=X+((size_t)(3*h))*n+(size_t)3*h;
const int8_t* u4=X+((size_t)(1*h))*n+(size_t)2*h;
const int8_t* u5=X+((size_t)(3*h))*n+(size_t)2*h;
const int8_t* u6=X+((size_t)(0*h))*n+(size_t)3*h;
const int8_t* u7=X+((size_t)(2*h))*n+(size_t)3*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int i=0;i<h;i++){ uint32_t msk[7];
for (int k=0;k<h;k+=64){
__m256i a0=LD(u0,k);
__m256i a1=LD(u1,k);
__m256i a2=LD(u2,k);
__m256i a3=LD(u3,k);
__m256i a4=LD(u4,k);
__m256i a5=LD(u5,k);
__m256i a6=LD(u6,k);
__m256i a7=LD(u7,k);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
__m256i v0=S8(S8(A8(a0,a1),a2),a3); ASTORE_A(v0,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
__m256i v0=S8(S8(A8(a4,a1),a5),a3); ASTORE_A(v0,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
__m256i v0=S8(a0,a2); ASTORE_A(v0,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
__m256i v0=S8(a1,a3); ASTORE_A(v0,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
__m256i v0=S8(S8(A8(a0,a6),a2),a7); ASTORE_A(v0,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
__m256i v0=S8(A8(A8(S8(zero,a0),a4),a2),a5); ASTORE_A(v0,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
__m256i v0=A8(S8(S8(a6,a1),a7),a3); ASTORE_A(v0,6) }
a0=LD(u0,k+32);
a1=LD(u1,k+32);
a2=LD(u2,k+32);
a3=LD(u3,k+32);
a4=LD(u4,k+32);
a5=LD(u5,k+32);
a6=LD(u6,k+32);
a7=LD(u7,k+32);
{ int8_t* dst=A0b+(size_t)0*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)0*AW+(size_t)i*W;
__m256i v1=S8(S8(A8(a0,a1),a2),a3); ASTORE_B(v1,0) }
{ int8_t* dst=A0b+(size_t)1*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)1*AW+(size_t)i*W;
__m256i v1=S8(S8(A8(a4,a1),a5),a3); ASTORE_B(v1,1) }
{ int8_t* dst=A0b+(size_t)2*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)2*AW+(size_t)i*W;
__m256i v1=S8(a0,a2); ASTORE_B(v1,2) }
{ int8_t* dst=A0b+(size_t)3*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)3*AW+(size_t)i*W;
__m256i v1=S8(a1,a3); ASTORE_B(v1,3) }
{ int8_t* dst=A0b+(size_t)4*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)4*AW+(size_t)i*W;
__m256i v1=S8(S8(A8(a0,a6),a2),a7); ASTORE_B(v1,4) }
{ int8_t* dst=A0b+(size_t)5*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)5*AW+(size_t)i*W;
__m256i v1=S8(A8(A8(S8(zero,a0),a4),a2),a5); ASTORE_B(v1,5) }
{ int8_t* dst=A0b+(size_t)6*AS+(size_t)i*GLDA;
uint64_t* w1=A1b+(size_t)6*AW+(size_t)i*W;
__m256i v1=A8(S8(S8(a6,a1),a7),a3); ASTORE_B(v1,6) }
}
u0+=n;
u1+=n;
u2+=n;
u3+=n;
u4+=n;
u5+=n;
u6+=n;
u7+=n;
}
return; }
}
}
#define BSTORE(VX,VY,KK) \
{ __m256i va=VX, vb=VY; \
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb); \
int8_t *o=BI3b+(size_t)(KK)*BS+((size_t)g*(h/2)+p)*128; \
_mm256_store_si256((__m256i*)(o+2*tt), _mm256_permute2x128_si256(lo,hi,0x20)); \
_mm256_store_si256((__m256i*)(o+2*tt+32), _mm256_permute2x128_si256(lo,hi,0x31)); \
{ uint32_t mm0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(va,one),7)); \
uint32_t mm1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(vb,one),7)); \
uint64_t *w0p=Bpb+(size_t)(KK)*BW+(size_t)(2*p)*W; \
uint64_t *w1p=Bpb+(size_t)(KK)*BW+(size_t)(2*p+1)*W; \
if(tt==0){ w0p[g]=(uint64_t)mm0; w1p[g]=(uint64_t)mm1; } \
else { w0p[g]|=(uint64_t)mm0<<32; w1p[g]|=(uint64_t)mm1<<32; } } }
AVX2 static void fuseB(int h,int n,const int8_t*X,int k1){
const __m256i one=_mm256_set1_epi8(1), zero=_mm256_setzero_si256();
const int ng=h/64, W=h/64;
switch(k1){
case 0: {
const int8_t* u0=X+(size_t)(0*h)*n+(size_t)0*h;
const int8_t* u1=X+(size_t)(1*h)*n+(size_t)1*h;
const int8_t* u2=X+(size_t)(2*h)*n+(size_t)2*h;
const int8_t* u3=X+(size_t)(3*h)*n+(size_t)3*h;
const int8_t* u4=X+(size_t)(0*h)*n+(size_t)1*h;
const int8_t* u5=X+(size_t)(2*h)*n+(size_t)3*h;
const int8_t* u6=X+(size_t)(1*h)*n+(size_t)0*h;
const int8_t* u7=X+(size_t)(3*h)*n+(size_t)2*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int p=0;p<h/2;p++){
const int8_t* v0=u0+n;
const int8_t* v1=u1+n;
const int8_t* v2=u2+n;
const int8_t* v3=u3+n;
const int8_t* v4=u4+n;
const int8_t* v5=u5+n;
const int8_t* v6=u6+n;
const int8_t* v7=u7+n;
for (int g=0;g<ng;g++){
for (int tt=0;tt<64;tt+=32){
__m256i VA[7];
__m256i a0=LD(u0,64*g+tt);
__m256i a1=LD(u1,64*g+tt);
__m256i a2=LD(u2,64*g+tt);
__m256i a3=LD(u3,64*g+tt);
__m256i a4=LD(u4,64*g+tt);
__m256i a5=LD(u5,64*g+tt);
__m256i a6=LD(u6,64*g+tt);
__m256i a7=LD(u7,64*g+tt);
VA[0]=A8(A8(A8(a0,a1),a2),a3);
VA[1]=A8(a0,a2);
VA[2]=S8(A8(S8(a4,a1),a5),a3);
VA[3]=A8(S8(A8(S8(zero,a0),a6),a2),a7);
VA[4]=A8(a1,a3);
VA[5]=A8(A8(A8(a0,a4),a2),a5);
VA[6]=A8(A8(A8(a6,a1),a7),a3);
a0=LD(v0,64*g+tt);
a1=LD(v1,64*g+tt);
a2=LD(v2,64*g+tt);
a3=LD(v3,64*g+tt);
a4=LD(v4,64*g+tt);
a5=LD(v5,64*g+tt);
a6=LD(v6,64*g+tt);
a7=LD(v7,64*g+tt);
{ __m256i zvb=A8(A8(A8(a0,a1),a2),a3); BSTORE(VA[0],zvb,0) }
{ __m256i zvb=A8(a0,a2); BSTORE(VA[1],zvb,1) }
{ __m256i zvb=S8(A8(S8(a4,a1),a5),a3); BSTORE(VA[2],zvb,2) }
{ __m256i zvb=A8(S8(A8(S8(zero,a0),a6),a2),a7); BSTORE(VA[3],zvb,3) }
{ __m256i zvb=A8(a1,a3); BSTORE(VA[4],zvb,4) }
{ __m256i zvb=A8(A8(A8(a0,a4),a2),a5); BSTORE(VA[5],zvb,5) }
{ __m256i zvb=A8(A8(A8(a6,a1),a7),a3); BSTORE(VA[6],zvb,6) }
}
}
u0+=2*n;
u1+=2*n;
u2+=2*n;
u3+=2*n;
u4+=2*n;
u5+=2*n;
u6+=2*n;
u7+=2*n;
}
return; }
case 1: {
const int8_t* u0=X+(size_t)(0*h)*n+(size_t)0*h;
const int8_t* u1=X+(size_t)(1*h)*n+(size_t)1*h;
const int8_t* u2=X+(size_t)(0*h)*n+(size_t)1*h;
const int8_t* u3=X+(size_t)(1*h)*n+(size_t)0*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int p=0;p<h/2;p++){
const int8_t* v0=u0+n;
const int8_t* v1=u1+n;
const int8_t* v2=u2+n;
const int8_t* v3=u3+n;
for (int g=0;g<ng;g++){
for (int tt=0;tt<64;tt+=32){
__m256i VA[7];
__m256i a0=LD(u0,64*g+tt);
__m256i a1=LD(u1,64*g+tt);
__m256i a2=LD(u2,64*g+tt);
__m256i a3=LD(u3,64*g+tt);
VA[0]=A8(a0,a1);
VA[1]=a0;
VA[2]=S8(a2,a1);
VA[3]=A8(S8(zero,a0),a3);
VA[4]=a1;
VA[5]=A8(a0,a2);
VA[6]=A8(a3,a1);
a0=LD(v0,64*g+tt);
a1=LD(v1,64*g+tt);
a2=LD(v2,64*g+tt);
a3=LD(v3,64*g+tt);
{ __m256i zvb=A8(a0,a1); BSTORE(VA[0],zvb,0) }
{ __m256i zvb=a0; BSTORE(VA[1],zvb,1) }
{ __m256i zvb=S8(a2,a1); BSTORE(VA[2],zvb,2) }
{ __m256i zvb=A8(S8(zero,a0),a3); BSTORE(VA[3],zvb,3) }
{ __m256i zvb=a1; BSTORE(VA[4],zvb,4) }
{ __m256i zvb=A8(a0,a2); BSTORE(VA[5],zvb,5) }
{ __m256i zvb=A8(a3,a1); BSTORE(VA[6],zvb,6) }
}
}
u0+=2*n;
u1+=2*n;
u2+=2*n;
u3+=2*n;
}
return; }
case 2: {
const int8_t* u0=X+(size_t)(0*h)*n+(size_t)2*h;
const int8_t* u1=X+(size_t)(1*h)*n+(size_t)3*h;
const int8_t* u2=X+(size_t)(2*h)*n+(size_t)2*h;
const int8_t* u3=X+(size_t)(3*h)*n+(size_t)3*h;
const int8_t* u4=X+(size_t)(0*h)*n+(size_t)3*h;
const int8_t* u5=X+(size_t)(2*h)*n+(size_t)3*h;
const int8_t* u6=X+(size_t)(1*h)*n+(size_t)2*h;
const int8_t* u7=X+(size_t)(3*h)*n+(size_t)2*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int p=0;p<h/2;p++){
const int8_t* v0=u0+n;
const int8_t* v1=u1+n;
const int8_t* v2=u2+n;
const int8_t* v3=u3+n;
const int8_t* v4=u4+n;
const int8_t* v5=u5+n;
const int8_t* v6=u6+n;
const int8_t* v7=u7+n;
for (int g=0;g<ng;g++){
for (int tt=0;tt<64;tt+=32){
__m256i VA[7];
__m256i a0=LD(u0,64*g+tt);
__m256i a1=LD(u1,64*g+tt);
__m256i a2=LD(u2,64*g+tt);
__m256i a3=LD(u3,64*g+tt);
__m256i a4=LD(u4,64*g+tt);
__m256i a5=LD(u5,64*g+tt);
__m256i a6=LD(u6,64*g+tt);
__m256i a7=LD(u7,64*g+tt);
VA[0]=S8(S8(A8(a0,a1),a2),a3);
VA[1]=S8(a0,a2);
VA[2]=A8(S8(S8(a4,a1),a5),a3);
VA[3]=S8(A8(A8(S8(zero,a0),a6),a2),a7);
VA[4]=S8(a1,a3);
VA[5]=S8(S8(A8(a0,a4),a2),a5);
VA[6]=S8(S8(A8(a6,a1),a7),a3);
a0=LD(v0,64*g+tt);
a1=LD(v1,64*g+tt);
a2=LD(v2,64*g+tt);
a3=LD(v3,64*g+tt);
a4=LD(v4,64*g+tt);
a5=LD(v5,64*g+tt);
a6=LD(v6,64*g+tt);
a7=LD(v7,64*g+tt);
{ __m256i zvb=S8(S8(A8(a0,a1),a2),a3); BSTORE(VA[0],zvb,0) }
{ __m256i zvb=S8(a0,a2); BSTORE(VA[1],zvb,1) }
{ __m256i zvb=A8(S8(S8(a4,a1),a5),a3); BSTORE(VA[2],zvb,2) }
{ __m256i zvb=S8(A8(A8(S8(zero,a0),a6),a2),a7); BSTORE(VA[3],zvb,3) }
{ __m256i zvb=S8(a1,a3); BSTORE(VA[4],zvb,4) }
{ __m256i zvb=S8(S8(A8(a0,a4),a2),a5); BSTORE(VA[5],zvb,5) }
{ __m256i zvb=S8(S8(A8(a6,a1),a7),a3); BSTORE(VA[6],zvb,6) }
}
}
u0+=2*n;
u1+=2*n;
u2+=2*n;
u3+=2*n;
u4+=2*n;
u5+=2*n;
u6+=2*n;
u7+=2*n;
}
return; }
case 3: {
const int8_t* u0=X+(size_t)(0*h)*n+(size_t)0*h;
const int8_t* u1=X+(size_t)(1*h)*n+(size_t)1*h;
const int8_t* u2=X+(size_t)(2*h)*n+(size_t)0*h;
const int8_t* u3=X+(size_t)(3*h)*n+(size_t)1*h;
const int8_t* u4=X+(size_t)(0*h)*n+(size_t)1*h;
const int8_t* u5=X+(size_t)(2*h)*n+(size_t)1*h;
const int8_t* u6=X+(size_t)(1*h)*n+(size_t)0*h;
const int8_t* u7=X+(size_t)(3*h)*n+(size_t)0*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int p=0;p<h/2;p++){
const int8_t* v0=u0+n;
const int8_t* v1=u1+n;
const int8_t* v2=u2+n;
const int8_t* v3=u3+n;
const int8_t* v4=u4+n;
const int8_t* v5=u5+n;
const int8_t* v6=u6+n;
const int8_t* v7=u7+n;
for (int g=0;g<ng;g++){
for (int tt=0;tt<64;tt+=32){
__m256i VA[7];
__m256i a0=LD(u0,64*g+tt);
__m256i a1=LD(u1,64*g+tt);
__m256i a2=LD(u2,64*g+tt);
__m256i a3=LD(u3,64*g+tt);
__m256i a4=LD(u4,64*g+tt);
__m256i a5=LD(u5,64*g+tt);
__m256i a6=LD(u6,64*g+tt);
__m256i a7=LD(u7,64*g+tt);
VA[0]=A8(A8(S8(S8(zero,a0),a1),a2),a3);
VA[1]=A8(S8(zero,a0),a2);
VA[2]=S8(A8(A8(S8(zero,a4),a1),a5),a3);
VA[3]=A8(S8(S8(a0,a6),a2),a7);
VA[4]=A8(S8(zero,a1),a3);
VA[5]=A8(A8(S8(S8(zero,a0),a4),a2),a5);
VA[6]=A8(A8(S8(S8(zero,a6),a1),a7),a3);
a0=LD(v0,64*g+tt);
a1=LD(v1,64*g+tt);
a2=LD(v2,64*g+tt);
a3=LD(v3,64*g+tt);
a4=LD(v4,64*g+tt);
a5=LD(v5,64*g+tt);
a6=LD(v6,64*g+tt);
a7=LD(v7,64*g+tt);
{ __m256i zvb=A8(A8(S8(S8(zero,a0),a1),a2),a3); BSTORE(VA[0],zvb,0) }
{ __m256i zvb=A8(S8(zero,a0),a2); BSTORE(VA[1],zvb,1) }
{ __m256i zvb=S8(A8(A8(S8(zero,a4),a1),a5),a3); BSTORE(VA[2],zvb,2) }
{ __m256i zvb=A8(S8(S8(a0,a6),a2),a7); BSTORE(VA[3],zvb,3) }
{ __m256i zvb=A8(S8(zero,a1),a3); BSTORE(VA[4],zvb,4) }
{ __m256i zvb=A8(A8(S8(S8(zero,a0),a4),a2),a5); BSTORE(VA[5],zvb,5) }
{ __m256i zvb=A8(A8(S8(S8(zero,a6),a1),a7),a3); BSTORE(VA[6],zvb,6) }
}
}
u0+=2*n;
u1+=2*n;
u2+=2*n;
u3+=2*n;
u4+=2*n;
u5+=2*n;
u6+=2*n;
u7+=2*n;
}
return; }
case 4: {
const int8_t* u0=X+(size_t)(2*h)*n+(size_t)2*h;
const int8_t* u1=X+(size_t)(3*h)*n+(size_t)3*h;
const int8_t* u2=X+(size_t)(2*h)*n+(size_t)3*h;
const int8_t* u3=X+(size_t)(3*h)*n+(size_t)2*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int p=0;p<h/2;p++){
const int8_t* v0=u0+n;
const int8_t* v1=u1+n;
const int8_t* v2=u2+n;
const int8_t* v3=u3+n;
for (int g=0;g<ng;g++){
for (int tt=0;tt<64;tt+=32){
__m256i VA[7];
__m256i a0=LD(u0,64*g+tt);
__m256i a1=LD(u1,64*g+tt);
__m256i a2=LD(u2,64*g+tt);
__m256i a3=LD(u3,64*g+tt);
VA[0]=A8(a0,a1);
VA[1]=a0;
VA[2]=S8(a2,a1);
VA[3]=A8(S8(zero,a0),a3);
VA[4]=a1;
VA[5]=A8(a0,a2);
VA[6]=A8(a3,a1);
a0=LD(v0,64*g+tt);
a1=LD(v1,64*g+tt);
a2=LD(v2,64*g+tt);
a3=LD(v3,64*g+tt);
{ __m256i zvb=A8(a0,a1); BSTORE(VA[0],zvb,0) }
{ __m256i zvb=a0; BSTORE(VA[1],zvb,1) }
{ __m256i zvb=S8(a2,a1); BSTORE(VA[2],zvb,2) }
{ __m256i zvb=A8(S8(zero,a0),a3); BSTORE(VA[3],zvb,3) }
{ __m256i zvb=a1; BSTORE(VA[4],zvb,4) }
{ __m256i zvb=A8(a0,a2); BSTORE(VA[5],zvb,5) }
{ __m256i zvb=A8(a3,a1); BSTORE(VA[6],zvb,6) }
}
}
u0+=2*n;
u1+=2*n;
u2+=2*n;
u3+=2*n;
}
return; }
case 5: {
const int8_t* u0=X+(size_t)(0*h)*n+(size_t)0*h;
const int8_t* u1=X+(size_t)(0*h)*n+(size_t)2*h;
const int8_t* u2=X+(size_t)(1*h)*n+(size_t)1*h;
const int8_t* u3=X+(size_t)(1*h)*n+(size_t)3*h;
const int8_t* u4=X+(size_t)(0*h)*n+(size_t)1*h;
const int8_t* u5=X+(size_t)(0*h)*n+(size_t)3*h;
const int8_t* u6=X+(size_t)(1*h)*n+(size_t)0*h;
const int8_t* u7=X+(size_t)(1*h)*n+(size_t)2*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int p=0;p<h/2;p++){
const int8_t* v0=u0+n;
const int8_t* v1=u1+n;
const int8_t* v2=u2+n;
const int8_t* v3=u3+n;
const int8_t* v4=u4+n;
const int8_t* v5=u5+n;
const int8_t* v6=u6+n;
const int8_t* v7=u7+n;
for (int g=0;g<ng;g++){
for (int tt=0;tt<64;tt+=32){
__m256i VA[7];
__m256i a0=LD(u0,64*g+tt);
__m256i a1=LD(u1,64*g+tt);
__m256i a2=LD(u2,64*g+tt);
__m256i a3=LD(u3,64*g+tt);
__m256i a4=LD(u4,64*g+tt);
__m256i a5=LD(u5,64*g+tt);
__m256i a6=LD(u6,64*g+tt);
__m256i a7=LD(u7,64*g+tt);
VA[0]=A8(A8(A8(a0,a1),a2),a3);
VA[1]=A8(a0,a1);
VA[2]=S8(S8(A8(a4,a5),a2),a3);
VA[3]=A8(A8(S8(S8(zero,a0),a1),a6),a7);
VA[4]=A8(a2,a3);
VA[5]=A8(A8(A8(a0,a4),a1),a5);
VA[6]=A8(A8(A8(a6,a2),a7),a3);
a0=LD(v0,64*g+tt);
a1=LD(v1,64*g+tt);
a2=LD(v2,64*g+tt);
a3=LD(v3,64*g+tt);
a4=LD(v4,64*g+tt);
a5=LD(v5,64*g+tt);
a6=LD(v6,64*g+tt);
a7=LD(v7,64*g+tt);
{ __m256i zvb=A8(A8(A8(a0,a1),a2),a3); BSTORE(VA[0],zvb,0) }
{ __m256i zvb=A8(a0,a1); BSTORE(VA[1],zvb,1) }
{ __m256i zvb=S8(S8(A8(a4,a5),a2),a3); BSTORE(VA[2],zvb,2) }
{ __m256i zvb=A8(A8(S8(S8(zero,a0),a1),a6),a7); BSTORE(VA[3],zvb,3) }
{ __m256i zvb=A8(a2,a3); BSTORE(VA[4],zvb,4) }
{ __m256i zvb=A8(A8(A8(a0,a4),a1),a5); BSTORE(VA[5],zvb,5) }
{ __m256i zvb=A8(A8(A8(a6,a2),a7),a3); BSTORE(VA[6],zvb,6) }
}
}
u0+=2*n;
u1+=2*n;
u2+=2*n;
u3+=2*n;
u4+=2*n;
u5+=2*n;
u6+=2*n;
u7+=2*n;
}
return; }
case 6: {
const int8_t* u0=X+(size_t)(2*h)*n+(size_t)0*h;
const int8_t* u1=X+(size_t)(2*h)*n+(size_t)2*h;
const int8_t* u2=X+(size_t)(3*h)*n+(size_t)1*h;
const int8_t* u3=X+(size_t)(3*h)*n+(size_t)3*h;
const int8_t* u4=X+(size_t)(2*h)*n+(size_t)1*h;
const int8_t* u5=X+(size_t)(2*h)*n+(size_t)3*h;
const int8_t* u6=X+(size_t)(3*h)*n+(size_t)0*h;
const int8_t* u7=X+(size_t)(3*h)*n+(size_t)2*h;
if (RIG_USE & 8) { const int8_t* z=u0; u0=u1; u1=z; }
for (int p=0;p<h/2;p++){
const int8_t* v0=u0+n;
const int8_t* v1=u1+n;
const int8_t* v2=u2+n;
const int8_t* v3=u3+n;
const int8_t* v4=u4+n;
const int8_t* v5=u5+n;
const int8_t* v6=u6+n;
const int8_t* v7=u7+n;
for (int g=0;g<ng;g++){
for (int tt=0;tt<64;tt+=32){
__m256i VA[7];
__m256i a0=LD(u0,64*g+tt);
__m256i a1=LD(u1,64*g+tt);
__m256i a2=LD(u2,64*g+tt);
__m256i a3=LD(u3,64*g+tt);
__m256i a4=LD(u4,64*g+tt);
__m256i a5=LD(u5,64*g+tt);
__m256i a6=LD(u6,64*g+tt);
__m256i a7=LD(u7,64*g+tt);
VA[0]=A8(A8(A8(a0,a1),a2),a3);
VA[1]=A8(a0,a1);
VA[2]=S8(S8(A8(a4,a5),a2),a3);
VA[3]=A8(A8(S8(S8(zero,a0),a1),a6),a7);
VA[4]=A8(a2,a3);
VA[5]=A8(A8(A8(a0,a4),a1),a5);
VA[6]=A8(A8(A8(a6,a2),a7),a3);
a0=LD(v0,64*g+tt);
a1=LD(v1,64*g+tt);
a2=LD(v2,64*g+tt);
a3=LD(v3,64*g+tt);
a4=LD(v4,64*g+tt);
a5=LD(v5,64*g+tt);
a6=LD(v6,64*g+tt);
a7=LD(v7,64*g+tt);
{ __m256i zvb=A8(A8(A8(a0,a1),a2),a3); BSTORE(VA[0],zvb,0) }
{ __m256i zvb=A8(a0,a1); BSTORE(VA[1],zvb,1) }
{ __m256i zvb=S8(S8(A8(a4,a5),a2),a3); BSTORE(VA[2],zvb,2) }
{ __m256i zvb=A8(A8(S8(S8(zero,a0),a1),a6),a7); BSTORE(VA[3],zvb,3) }
{ __m256i zvb=A8(a2,a3); BSTORE(VA[4],zvb,4) }
{ __m256i zvb=A8(A8(A8(a0,a4),a1),a5); BSTORE(VA[5],zvb,5) }
{ __m256i zvb=A8(A8(A8(a6,a2),a7),a3); BSTORE(VA[6],zvb,6) }
}
}
u0+=2*n;
u1+=2*n;
u2+=2*n;
u3+=2*n;
u4+=2*n;
u5+=2*n;
u6+=2*n;
u7+=2*n;
}
return; }
}
}
AVX2 static void strassen2v8(int n,const int8_t*A,const int8_t*B,int8_t*C){
const int m=n>>1, h=D2H;
const int KA = (RIG_USE&2)?7:((RIG_USE&16)?7:1);
const int KB = (RIG_USE&1)?7:((RIG_USE&32)?7:1);
GLDA = 2*h + PADA; GLDB = h + PADB;
for (int k1=0;k1<7;k1++){
if (RIG_USE & 1) fuseB(h,n,B,k1);
if (RIG_USE & 2) fuseA(h,n,A,k1);
for (int k2=0;k2<7;k2++){
if (!(RIG_USE&1))
buildB_q3_b(h,n,B,M2B[k1][k2],M2BN[k1][k2],
BI3b+(size_t)(k2%KB)*BS,Bpb+(size_t)(k2%KB)*BW);
if (!(RIG_USE&2))
buildA_q3_b(h,n,A,M2A[k1][k2],M2AN[k1][k2],
A0b+(size_t)(k2%KA)*AS,A1b+(size_t)(k2%KA)*AW);
submm_tail_c(h,MMS2[k2],
A0b+(size_t)(k2%KA)*AS,A1b+(size_t)(k2%KA)*AW,
BI3b+(size_t)(k2%KB)*BS,Bpb+(size_t)(k2%KB)*BW);
}
{ const int8_t* MP[7]; for(int q=0;q<7;q++) MP[q]=MMS2[q];
for (int rb=0; rb<h; rb+=RB){
const int re = (rb+RB<h)? rb+RB : h;
cmb7c(h,MP,rb,re);
fold_top_r(C,n,m,k1,MMSFc-(size_t)rb*512, rb, re);
fold_top_r(C,n,m,k1,MMSFc-(size_t)(rb+h-RB)*512, rb+h, re+h); } }
}
_mm_sfence();
}
AVX2 static void strassen2v2(int n,const int8_t*A,const int8_t*B,int8_t*C){
const int m=n>>1, h=D2H;
GLDA = 2*h + PADA; GLDB = h + PADB;
for (int k1=0;k1<7;k1++){
for (int k2=0;k2<7;k2++){
buildA_q3(h,n,A,M2A[k1][k2],M2AN[k1][k2]);
buildB_q3(h,n,B,M2B[k1][k2],M2BN[k1][k2]);
submm_tail(h,MMS2[k2]);
}
{ const int8_t* MP[7]; for(int q=0;q<7;q++) MP[q]=MMS2[q];
for (int rb=0; rb<h; rb+=RB){
const int re = (rb+RB<h)? rb+RB : h;
cmb7c(h,MP,rb,re);
fold_top_r(C,n,m,k1,MMSFc-(size_t)rb*512, rb, re);
fold_top_r(C,n,m,k1,MMSFc-(size_t)(rb+h-RB)*512, rb+h, re+h); } }
}
_mm_sfence();
}
#endif
#if D2MODE == 15 || D2MODE == 16
#endif
#if D2MODE == 7 || D2MODE == 8
#endif
AVX2 void matrix_multiply(int n, const int8_t *A, const int8_t *B, int8_t *C){
if (n<=0 || (n&255) || n>NMAX) { generic_mm(n,A,B,C); return; }
if (n==1024) { if(!RIG_USE) strassen2v2(n,A,B,C);
else strassen2v8(n,A,B,C); return; }
generic_mm(n,A,B,C);
}
| Compilation | N/A | N/A | Compile OK | Score: N/A | 显示更多 |
| Testcase #1 | 6.799 ms | 2 MB + 196 KB | Accepted | Score: 100 | 显示更多 |