#pragma GCC optimize("O3")
/* mmmc_k1: int8 GEMM C = A*B mod 256, EXACT, AVX2.
Split A = A0 + 128*A1 (A0 = A & 0x7F unsigned, A1 = top bit of each byte).
term1 = sum_k A0[i][k]*B[k][j] -> vpmaddubsw, no saturation (127*128*2 < 32768)
term2 = 128*(A1 . Bpar) mod 256 -> GF(2) matmul, 64 bits/word (Bpar = B&1)
Layout (lanes = columns): accum lane m is column j0+m.
x = vpbroadcastw of the 2-byte pair A0[i][2p..2p+1]
y = 32B load from BI[p] at 2*j0 ; BI[p][2j]=B[2p][j], BI[p][2j+1]=B[2p+1][j]
*/
#include <immintrin.h>
#include <xmmintrin.h>
#include <stdint.h>
#include <string.h>
#include <stdio.h>
#ifndef PFMODE
#define PFMODE 1 /* MC2: BOTH mikro prefetcht0 pairs DELETED. The BI block is 16 KB and L1-resident;
the prefetches are pure issue overhead. Rig: 4 pf +1.31 %, 2 pf 0.00 %, 0 pf -1.102 %. */
#endif
#if PFMODE==0
#define PF_STR "prefetcht0 512(%[bib],%[bidx])\n\t" "prefetcht0 1024(%[bib],%[bidx])\n\t"
#elif PFMODE==1
#define PF_STR ""
#elif PFMODE==2
#define PF_STR "prefetcht0 512(%[bib],%[bidx])\n\t"
#elif PFMODE==3
#define PF_STR "prefetcht0 256(%[bib],%[bidx])\n\t" "prefetcht0 512(%[bib],%[bidx])\n\t"
#elif PFMODE==4
#define PF_STR "prefetcht0 1024(%[bib],%[bidx])\n\t" "prefetcht0 2048(%[bib],%[bidx])\n\t"
#elif PFMODE==5
#define PF_STR "prefetchnta 512(%[bib],%[bidx])\n\t" "prefetchnta 1024(%[bib],%[bidx])\n\t"
#elif PFMODE==6
#define PF_STR "prefetcht0 512(%[bib],%[bidx])\n\t" "prefetcht0 1024(%[bib],%[bidx])\n\t" \
"prefetcht0 1536(%[bib],%[bidx])\n\t" "prefetcht0 2048(%[bib],%[bidx])\n\t"
#endif
#define AVX2 __attribute__((target("avx2")))
#define NMAX 4096
/* == mmmpad_fleet ARM: SCRATCH ROW-STRIDE PAD ==
A0b and PBb are PRIVATE scratch, so their row stride is not semantic -- but it is `n`,
i.e. 4096 B at n=4096, which is EXACTLY the L1 way-period (64 sets x 64 B). Every row of
both buffers then maps its line j into set j, on top of C's rows (also `n`) and the mikro's
two-row window: 6+ streams share one set out of 8 ways. LDA/LDB pad the two scratch
strides. The cell (row i, col j) is still the same value -- only WHERE it lives changes.
The mikro takes the scratch stride as an argument instead of deriving `A0+n`. */
#ifndef PADA
#define PADA 0
#endif
#define LDA ((int)(2*NMAX + PADA))
#ifndef PADB
#define PADB 0
#endif
#define LDB ((int)(NMAX + PADB))
static int8_t A0bs[(size_t)NMAX*LDA] __attribute__((aligned(64)));
static int8_t *A0b = A0bs;
/* BW9: runtime scratch stride, sized to the ACTUAL m (not NMAX) */
static int GLDA = LDA, GLDB = LDB;
static uint64_t A1bs[NMAX*NMAX/8] __attribute__((aligned(64)));
static uint64_t *A1b = A1bs;
static uint64_t Bpbs[NMAX*NMAX/8] __attribute__((aligned(64)));
static uint64_t *Bpb = Bpbs;
static uint64_t Pbb[NMAX*NMAX/8] __attribute__((aligned(64)));
static uint8_t PBb[(size_t)NMAX*LDB] __attribute__((aligned(64)));
static void generic_mm(int n, const int8_t*A, const int8_t*B, int8_t*C){
for (int i=0;i<n;i++) for (int j=0;j<n;j++){
int s=0; for(int k=0;k<n;k++) s += (int)A[(size_t)i*n+k]*(int)B[(size_t)k*n+j];
C[(size_t)i*n+j]=(int8_t)s;
}
}
AVX2 static inline __m256i bcast16(const int8_t*p){
__m256i v; __asm__("vpbroadcastw %1, %0" : "=x"(v) : "m"(*(const short*)p)); return v;
}
AVX2 static inline __m256i packlo8(__m256i a, __m256i b){
const __m256i m=_mm256_set1_epi16(0x00FF);
return _mm256_permute4x64_epi64(
_mm256_packus_epi16(_mm256_and_si256(a,m), _mm256_and_si256(b,m)), 0xD8);
}
/* build_A0() DELETED by lane CH3: dead, and its A0b stride is the pre-duplication one. */
AVX2 static void build_A1bits(int n, const int8_t*A){
int W=n/64;
for (int i=0;i<n;i++){
const int8_t*row=A+(size_t)i*n; uint64_t*w=A1b+(size_t)i*W;
for (int k=0;k<n;k+=64){
uint32_t lo=(uint32_t)_mm256_movemask_epi8(_mm256_load_si256((const __m256i*)(row+k)));
uint32_t hi=(uint32_t)_mm256_movemask_epi8(_mm256_load_si256((const __m256i*)(row+k+32)));
w[k>>6]=(uint64_t)lo|((uint64_t)hi<<32);
}
}
}
AVX2 static void build_Bpbits(int n, const int8_t*B){
const __m256i one=_mm256_set1_epi8(1);
int W=n/64;
for (int k=0;k<n;k++){
const int8_t*row=B+(size_t)k*n; uint64_t*w=Bpb+(size_t)k*W;
for (int j=0;j<n;j+=64){
__m256i a=_mm256_slli_epi16(_mm256_and_si256(_mm256_load_si256((const __m256i*)(row+j)),one),7);
__m256i b=_mm256_slli_epi16(_mm256_and_si256(_mm256_load_si256((const __m256i*)(row+j+32)),one),7);
uint32_t lo=(uint32_t)_mm256_movemask_epi8(a);
uint32_t hi=(uint32_t)_mm256_movemask_epi8(b);
w[j>>6]=(uint64_t)lo|((uint64_t)hi<<32);
}
}
}
/* P[i] = XOR of Bp[k] over k with A1[i][k] set. NY = n/256 ymm per row. */
AVX2 static void corr_bits(int n){
int W=n/64, NY=W>>2;
for (int i=0;i<n;i++){
__m256i acc[16];
for(int t=0;t<NY;t++) acc[t]=_mm256_setzero_si256();
const uint64_t*row=A1b+(size_t)i*W;
for(int w=0;w<W;w++){
uint64_t b=row[w];
while(b){
int t=__builtin_ctzll(b); b&=b-1;
const uint64_t*br=Bpb+(size_t)(w*64+t)*W;
for(int q=0;q<NY;q++)
acc[q]=_mm256_xor_si256(acc[q],_mm256_load_si256((const __m256i*)(br+4*q)));
}
}
uint64_t*o=Pbb+(size_t)i*W;
for(int q=0;q<NY;q++) _mm256_store_si256((__m256i*)(o+4*q),acc[q]);
}
}
static uint64_t A1T[NMAX*NMAX/8] __attribute__((aligned(64)));
AVX2 static void transpose64(uint64_t *a){
int j,k; uint64_t m,t;
for (j=32,m=0xFFFFFFFF00000000ULL; j; j>>=1, m^=m>>j)
for (k=0;k<64;k=(k+j+1)&~j){
t=(a[k]^(a[k+j]<<j))&m;
a[k]^=t; a[k+j]^=(t>>j);
}
}
AVX2 static void build_A1T(int n){
int W=n/64;
for (int kb=0; kb<n; kb+=64)
for (int ib=0; ib<n; ib+=64){
uint64_t blk[64];
for (int i=0;i<64;i++) blk[i]=A1b[(size_t)(ib+i)*W+(kb>>6)];
transpose64(blk);
for (int k=0;k<64;k++) A1T[(size_t)(kb+k)*W+(ib>>6)]=blk[k];
}
}
/* P[i][j] = XOR over k with A1[i][k] set of Bp[k][j].
Per-output-row bit scan (dense 64-bit words -> predictable), K-BLOCKED so the
Bp slice stays in L2. P accumulated in memory across k-blocks. */
AVX2 static void corr_pass3(int n){
int W=n/64, NY=W>>2;
memset(Pbb,0,(size_t)n*W*8);
const int KB=256;
for (int kb=0; kb<n; kb+=KB){
int w0=kb>>6, w1=w0+(KB>>6);
for (int i=0;i<n;i++){
__m256i acc[16];
for (int q=0;q<NY;q++) acc[q]=_mm256_setzero_si256();
const uint64_t*row=A1b+(size_t)i*W;
for (int w=w0;w<w1;w++){
uint64_t b=row[w];
while(b){
int t=__builtin_ctzll(b); b&=b-1;
const uint64_t*br=Bpb+(size_t)(w*64+t)*W;
for (int q=0;q<NY;q++) acc[q]=_mm256_xor_si256(acc[q],_mm256_load_si256((const __m256i*)(br+4*q)));
}
}
uint64_t*o=Pbb+(size_t)i*W;
for (int q=0;q<NY;q++){
__m256i*po=(__m256i*)(o+4*q);
_mm256_store_si256(po,_mm256_xor_si256(_mm256_load_si256((const __m256i*)po),acc[q]));
}
}
}
}
/* specialized for n=4096 (W=64, NY=16): fully unrolled XOR loops */
/* corr16(): fully hand-written per-row GF(2) accumulate. Bit-identical output. */
#define AVX2BMI __attribute__((target("avx2,bmi")))
AVX2BMI static void corr16(void){
const int n=4096, W=64;
memset(Pbb,0,(size_t)n*W*8);
const int KB=64;
for (int kb=0; kb<n; kb+=KB){
int w0=kb>>6;
const char *bp0=(const char*)Bpb+(size_t)w0*32768;
const char *bp1=bp0; (void)bp1;
for (int i=0;i<n;i++){
const uint64_t*row=A1b+(size_t)i*W+w0;
uint64_t b0=row[0], b1=0;
uint64_t *o=Pbb+(size_t)i*W;
asm volatile(
"vpxor %%xmm0,%%xmm0,%%xmm0\n\t"
"vpxor %%xmm1,%%xmm1,%%xmm1\n\t"
"vpxor %%xmm2,%%xmm2,%%xmm2\n\t"
"vpxor %%xmm3,%%xmm3,%%xmm3\n\t"
"vpxor %%xmm4,%%xmm4,%%xmm4\n\t"
"vpxor %%xmm5,%%xmm5,%%xmm5\n\t"
"vpxor %%xmm6,%%xmm6,%%xmm6\n\t"
"vpxor %%xmm7,%%xmm7,%%xmm7\n\t"
"vpxor %%xmm8,%%xmm8,%%xmm8\n\t"
"vpxor %%xmm9,%%xmm9,%%xmm9\n\t"
"vpxor %%xmm10,%%xmm10,%%xmm10\n\t"
"vpxor %%xmm11,%%xmm11,%%xmm11\n\t"
"vpxor %%xmm12,%%xmm12,%%xmm12\n\t"
"vpxor %%xmm13,%%xmm13,%%xmm13\n\t"
"vpxor %%xmm14,%%xmm14,%%xmm14\n\t"
"vpxor %%xmm15,%%xmm15,%%xmm15\n\t"
"testq %[b0], %[b0]\n\t"
"jz 3%=f\n\t"
"1%=:\n\t"
"tzcntq %[b0], %%r11\n\t"
"shlq $9, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"vpxor 64(%%r11), %%ymm2, %%ymm2\n\t"
"vpxor 96(%%r11), %%ymm3, %%ymm3\n\t"
"vpxor 128(%%r11), %%ymm4, %%ymm4\n\t"
"vpxor 160(%%r11), %%ymm5, %%ymm5\n\t"
"vpxor 192(%%r11), %%ymm6, %%ymm6\n\t"
"vpxor 224(%%r11), %%ymm7, %%ymm7\n\t"
"vpxor 256(%%r11), %%ymm8, %%ymm8\n\t"
"vpxor 288(%%r11), %%ymm9, %%ymm9\n\t"
"vpxor 320(%%r11), %%ymm10, %%ymm10\n\t"
"vpxor 352(%%r11), %%ymm11, %%ymm11\n\t"
"vpxor 384(%%r11), %%ymm12, %%ymm12\n\t"
"vpxor 416(%%r11), %%ymm13, %%ymm13\n\t"
"vpxor 448(%%r11), %%ymm14, %%ymm14\n\t"
"vpxor 480(%%r11), %%ymm15, %%ymm15\n\t"
"blsrq %[b0], %[b0]\n\t"
"jnz 1%=b\n\t"
"3%=:\n\t"
"testq %[b1], %[b1]\n\t"
"jz 4%=f\n\t"
"2%=:\n\t"
"tzcntq %[b1], %%r11\n\t"
"shlq $9, %%r11\n\t"
"addq %[bp1], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"vpxor 64(%%r11), %%ymm2, %%ymm2\n\t"
"vpxor 96(%%r11), %%ymm3, %%ymm3\n\t"
"vpxor 128(%%r11), %%ymm4, %%ymm4\n\t"
"vpxor 160(%%r11), %%ymm5, %%ymm5\n\t"
"vpxor 192(%%r11), %%ymm6, %%ymm6\n\t"
"vpxor 224(%%r11), %%ymm7, %%ymm7\n\t"
"vpxor 256(%%r11), %%ymm8, %%ymm8\n\t"
"vpxor 288(%%r11), %%ymm9, %%ymm9\n\t"
"vpxor 320(%%r11), %%ymm10, %%ymm10\n\t"
"vpxor 352(%%r11), %%ymm11, %%ymm11\n\t"
"vpxor 384(%%r11), %%ymm12, %%ymm12\n\t"
"vpxor 416(%%r11), %%ymm13, %%ymm13\n\t"
"vpxor 448(%%r11), %%ymm14, %%ymm14\n\t"
"vpxor 480(%%r11), %%ymm15, %%ymm15\n\t"
"blsrq %[b1], %[b1]\n\t"
"jnz 2%=b\n\t"
"4%=:\n\t"
"vpxor 0(%[o]), %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[o])\n\t"
"vpxor 32(%[o]), %%ymm1, %%ymm1\n\t"
"vmovdqu %%ymm1, 32(%[o])\n\t"
"vpxor 64(%[o]), %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 64(%[o])\n\t"
"vpxor 96(%[o]), %%ymm3, %%ymm3\n\t"
"vmovdqu %%ymm3, 96(%[o])\n\t"
"vpxor 128(%[o]), %%ymm4, %%ymm4\n\t"
"vmovdqu %%ymm4, 128(%[o])\n\t"
"vpxor 160(%[o]), %%ymm5, %%ymm5\n\t"
"vmovdqu %%ymm5, 160(%[o])\n\t"
"vpxor 192(%[o]), %%ymm6, %%ymm6\n\t"
"vmovdqu %%ymm6, 192(%[o])\n\t"
"vpxor 224(%[o]), %%ymm7, %%ymm7\n\t"
"vmovdqu %%ymm7, 224(%[o])\n\t"
"vpxor 256(%[o]), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 256(%[o])\n\t"
"vpxor 288(%[o]), %%ymm9, %%ymm9\n\t"
"vmovdqu %%ymm9, 288(%[o])\n\t"
"vpxor 320(%[o]), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 320(%[o])\n\t"
"vpxor 352(%[o]), %%ymm11, %%ymm11\n\t"
"vmovdqu %%ymm11, 352(%[o])\n\t"
"vpxor 384(%[o]), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 384(%[o])\n\t"
"vpxor 416(%[o]), %%ymm13, %%ymm13\n\t"
"vmovdqu %%ymm13, 416(%[o])\n\t"
"vpxor 448(%[o]), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 448(%[o])\n\t"
"vpxor 480(%[o]), %%ymm15, %%ymm15\n\t"
"vmovdqu %%ymm15, 480(%[o])\n\t"
: [b0]"+r"(b0), [b1]"+r"(b1)
: [bp0]"r"(bp0), [bp1]"r"(bp1), [o]"r"(o)
: "r11","cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15");
}
}
}
AVX2 static void expand_bits(int n){
static uint8_t T[256*8]; static int ti=0;
if(!ti){ for(int v=0;v<256;v++) for(int b=0;b<8;b++) T[v*8+b]=((v>>b)&1)?0x80:0x00; ti=1; }
int W=n/64;
/* MC2: the inner (q,s) double loop ran 8 separate 8-byte load+store pairs per 64-bit Pbb word
-- 8192 store uops per call and ~40 instructions per 64 output bytes. The eight T entries
are contiguous 8-byte blocks, so they assemble into two 32-byte stores: 16 uops per 64 bytes,
SAME BYTES (output byte j = T[(v>>(8*(j/8)))*8 + j%8] by construction). */
for(int i=0;i<n;i++){
const uint64_t*w=Pbb+(size_t)i*W; uint8_t*o=PBb+(size_t)i*GLDB;
for(int q=0;q<W;q++){
uint64_t v=w[q];
__m128i a0=_mm_loadl_epi64((const __m128i*)(T+((v>> 0)&0xFF)*8));
__m128i a1=_mm_loadl_epi64((const __m128i*)(T+((v>> 8)&0xFF)*8));
__m128i a2=_mm_loadl_epi64((const __m128i*)(T+((v>>16)&0xFF)*8));
__m128i a3=_mm_loadl_epi64((const __m128i*)(T+((v>>24)&0xFF)*8));
__m128i a4=_mm_loadl_epi64((const __m128i*)(T+((v>>32)&0xFF)*8));
__m128i a5=_mm_loadl_epi64((const __m128i*)(T+((v>>40)&0xFF)*8));
__m128i a6=_mm_loadl_epi64((const __m128i*)(T+((v>>48)&0xFF)*8));
__m128i a7=_mm_loadl_epi64((const __m128i*)(T+((v>>56)&0xFF)*8));
__m128i l01=_mm_unpacklo_epi64(a0,a1), l23=_mm_unpacklo_epi64(a2,a3);
__m128i l45=_mm_unpacklo_epi64(a4,a5), l67=_mm_unpacklo_epi64(a6,a7);
_mm256_storeu_si256((__m256i*)(o+q*64), _mm256_set_m128i(l23,l01));
_mm256_storeu_si256((__m256i*)(o+q*64+32), _mm256_set_m128i(l67,l45));
}
}
}
#define NIB 2
#define NJ 4
/* MC2: constants for inlining the Pbb -> PBb bit expansion into the mikro epilogue.
PSCTRL replicates each of the four bytes of a dword eight times (per 128-bit lane);
PSBIT isolates bit (j%8) of output byte j; PS80 is the value the set bits expand to.
With S = pshufb(broadcast_dword, PSCTRL): S[j] = byte (j/8) of the dword, so
(S & PSBIT) != 0 <=> bit j of the dword is set <=> PBb byte j must be 0x80. */
static const int8_t PSCTRL[32] __attribute__((aligned(32))) =
{0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3};
static const int8_t PSBIT[32] __attribute__((aligned(32))) =
{1,2,4,8,16,32,64,(int8_t)128, 1,2,4,8,16,32,64,(int8_t)128,
1,2,4,8,16,32,64,(int8_t)128, 1,2,4,8,16,32,64,(int8_t)128};
static const int8_t PS80[32] __attribute__((aligned(32))) =
{(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,
(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,
(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,
(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128};
static const int16_t W0FF[16] __attribute__((aligned(32))) =
{0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,
0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF};
static int8_t BI3bs[NMAX*NMAX] __attribute__((aligned(64)));
static int8_t *BI3b = BI3bs;
AVX2 static void build_BI3(int n, const int8_t*B){
int ng=n/64;
for (int p=0;p<n/2;p++){
const int8_t *r0=B+(size_t)(2*p)*n, *r1=r0+n;
for (int g=0;g<ng;g++){
const int8_t *a=r0+64*g, *b=r1+64*g;
int8_t *o=BI3b+((size_t)g*(n/2)+p)*128;
for (int t=0;t<64;t+=32){
__m256i va=_mm256_load_si256((const __m256i*)(a+t));
__m256i vb=_mm256_load_si256((const __m256i*)(b+t));
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb);
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20));
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31));
}
}
}
}
AVX2 static void mikro_u4(int n,const int8_t*A0,const int8_t*BI,int8_t*C,const uint8_t*PB,int jb,int lda,int ldb){
const int8_t *x0=A0;
long nn=(long)n;
const int8_t *x1=A0+lda;
const int8_t *bib=BI+(long)(n>>3)*512; long bidx=-((long)(n>>3)*512);
int8_t *C0=C+jb, *C1=C0+n;
const uint8_t *PB0=PB+jb, *PB1=PB0+ldb;
asm volatile(
"vpxor %%xmm8,%%xmm8,%%xmm8\n\t"
"vpxor %%xmm9,%%xmm9,%%xmm9\n\t"
"vpxor %%xmm10,%%xmm10,%%xmm10\n\t"
"vpxor %%xmm11,%%xmm11,%%xmm11\n\t"
"vpxor %%xmm12,%%xmm12,%%xmm12\n\t"
"vpxor %%xmm13,%%xmm13,%%xmm13\n\t"
"vpxor %%xmm14,%%xmm14,%%xmm14\n\t"
"vpxor %%xmm15,%%xmm15,%%xmm15\n\t"
".p2align 5\n\t"
"testq $8, %[nn]\n\t"
"jz 5f\n\t"
"vpbroadcastd 0(%[x0]), %%ymm0\n\t"
"vpbroadcastd 0(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 0(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 32(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 64(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[x0]), %%ymm0\n\t"
"vpbroadcastd 4(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 160(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 192(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 8(%[x0]), %%ymm0\n\t"
"vpbroadcastd 8(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 288(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 320(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[x0]), %%ymm0\n\t"
"vpbroadcastd 12(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 416(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 448(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
PF_STR
"addq $16, %[x0]\n\t"
"addq $512, %[bidx]\n\t"
"5:\n\t"
"testq %[bidx], %[bidx]\n\t"
"jz 9f\n\t"
".p2align 5\n\t"
"2:\n\t"
"vpbroadcastd 0(%[x0]), %%ymm0\n\t"
"vpbroadcastd 0(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 0(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 32(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 64(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[x0]), %%ymm0\n\t"
"vpbroadcastd 4(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 160(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 192(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 8(%[x0]), %%ymm0\n\t"
"vpbroadcastd 8(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 288(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 320(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[x0]), %%ymm0\n\t"
"vpbroadcastd 12(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 416(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 448(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 16(%[x0]), %%ymm0\n\t"
"vpbroadcastd 16(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 512(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 544(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 576(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 608(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 20(%[x0]), %%ymm0\n\t"
"vpbroadcastd 20(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 640(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 672(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 704(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 736(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 24(%[x0]), %%ymm0\n\t"
"vpbroadcastd 24(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 768(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 800(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 832(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 864(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 28(%[x0]), %%ymm0\n\t"
"vpbroadcastd 28(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 896(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 928(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 960(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 992(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
PF_STR
"addq $32, %[x0]\n\t"
"addq $1024, %[bidx]\n\t"
"jnz 2b\n\t"
"9:\n\t"
"vpand %[mask], %%ymm8, %%ymm2\n\t"
"vpand %[mask], %%ymm9, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpxor %%xmm7, %%xmm7, %%xmm7\n\t"
"vpbroadcastd 0(%[PB0]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 0(%[C0])\n\t"
"vpand %[mask], %%ymm10, %%ymm2\n\t"
"vpand %[mask], %%ymm11, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 4(%[PB0]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 32(%[C0])\n\t"
"vpand %[mask], %%ymm12, %%ymm2\n\t"
"vpand %[mask], %%ymm13, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 0(%[PB1]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 0(%[C1])\n\t"
"vpand %[mask], %%ymm14, %%ymm2\n\t"
"vpand %[mask], %%ymm15, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 4(%[PB1]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 32(%[C1])\n\t"
: [x0]"+r"(x0), [bidx]"+r"(bidx)
: [C0]"r"(C0), [C1]"r"(C1), [PB0]"r"(PB0), [PB1]"r"(PB1), [nn]"r"(nn), [mask]"m"(W0FF), [bib]"r"(bib), [psctrl]"m"(PSCTRL), [psbit]"m"(PSBIT), [ps80]"m"(PS80), [glda]"r"((long)lda)
: "ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15","cc","memory");
}
AVX2 static void mikro_u4r(int n,const int8_t*A0,const int8_t*BI,int8_t*C,const uint8_t*PB,int jb,int lda,int ldb){
const int8_t *x0=A0;
long nn=(long)n;
const int8_t *x1=A0+lda;
const int8_t *bib=BI; /* R1: slice START -- see mk_rev.py */ long bidx=((long)(n>>3)*512)-512; /* R1: start at the LAST 512-byte chunk */
int8_t *C0=C+jb, *C1=C0+n;
const uint8_t *PB0=PB+jb, *PB1=PB0+ldb;
asm volatile(
"vpxor %%xmm8,%%xmm8,%%xmm8\n\t"
"vpxor %%xmm9,%%xmm9,%%xmm9\n\t"
"vpxor %%xmm10,%%xmm10,%%xmm10\n\t"
"vpxor %%xmm11,%%xmm11,%%xmm11\n\t"
"vpxor %%xmm12,%%xmm12,%%xmm12\n\t"
"vpxor %%xmm13,%%xmm13,%%xmm13\n\t"
"vpxor %%xmm14,%%xmm14,%%xmm14\n\t"
"vpxor %%xmm15,%%xmm15,%%xmm15\n\t"
".p2align 5\n\t"
"testq $8, %[nn]\n\t"
"jz 5f\n\t"
"vpbroadcastd 0(%[x0]), %%ymm0\n\t"
"vpbroadcastd 0(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 0(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 32(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 64(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[x0]), %%ymm0\n\t"
"vpbroadcastd 4(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 160(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 192(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 8(%[x0]), %%ymm0\n\t"
"vpbroadcastd 8(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 288(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 320(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[x0]), %%ymm0\n\t"
"vpbroadcastd 12(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 416(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 448(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
PF_STR
"subq $16, %[x0]\n\t"
"subq $512, %[bidx]\n\t"
"5:\n\t"
"testq %[bidx], %[bidx]\n\t"
"jz 9f\n\t"
".p2align 5\n\t"
"2:\n\t"
"vpbroadcastd 0(%[x0]), %%ymm0\n\t"
"vpbroadcastd 0(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 0(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 32(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 64(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[x0]), %%ymm0\n\t"
"vpbroadcastd 4(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 160(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 192(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 8(%[x0]), %%ymm0\n\t"
"vpbroadcastd 8(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 288(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 320(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[x0]), %%ymm0\n\t"
"vpbroadcastd 12(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 416(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 448(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd -16(%[x0]), %%ymm0\n\t"
"vpbroadcastd -16(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa -512(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa -480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa -448(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa -416(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd -12(%[x0]), %%ymm0\n\t"
"vpbroadcastd -12(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa -384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa -352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa -320(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa -288(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd -8(%[x0]), %%ymm0\n\t"
"vpbroadcastd -8(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa -256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa -224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa -192(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa -160(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd -4(%[x0]), %%ymm0\n\t"
"vpbroadcastd -4(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa -128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa -96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa -64(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa -32(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
PF_STR
"subq $32, %[x0]\n\t"
"subq $1024, %[bidx]\n\t"
"jns 2b\n\t"
"9:\n\t"
"vpand %[mask], %%ymm8, %%ymm2\n\t"
"vpand %[mask], %%ymm9, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 0(%[PB0]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vpxor %%xmm7, %%xmm7, %%xmm7\n\t"
"vmovdqu %%ymm2, 0(%[C0])\n\t"
"vpand %[mask], %%ymm10, %%ymm2\n\t"
"vpand %[mask], %%ymm11, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 4(%[PB0]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 32(%[C0])\n\t"
"vpand %[mask], %%ymm12, %%ymm2\n\t"
"vpand %[mask], %%ymm13, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 0(%[PB1]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 0(%[C1])\n\t"
"vpand %[mask], %%ymm14, %%ymm2\n\t"
"vpand %[mask], %%ymm15, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 4(%[PB1]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 32(%[C1])\n\t"
: [x0]"+r"(x0), [bidx]"+r"(bidx)
: [C0]"r"(C0), [C1]"r"(C1), [PB0]"r"(PB0), [PB1]"r"(PB1), [nn]"r"(nn), [mask]"m"(W0FF), [bib]"r"(bib), [psctrl]"m"(PSCTRL), [psbit]"m"(PSBIT), [ps80]"m"(PS80), [glda]"r"((long)lda)
: "ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15","cc","memory");
}
/* FUSED BUILDERS (ported from mmmc2k, pure-n): one pass over A, one over B */
AVX2 static void build_A0A1(int n, const int8_t*A){
const __m256i m=_mm256_set1_epi8(0x7F);
int W=n/64;
for (int i=0;i<n;i++){
const int8_t*row=A+(size_t)i*n;
int8_t*dst=A0b+(size_t)i*GLDA;
uint64_t*w1=A1b+(size_t)i*W;
for (int k=0;k<n;k+=64){
__m256i v0=_mm256_load_si256((const __m256i*)(row+k));
__m256i v1=_mm256_load_si256((const __m256i*)(row+k+32));
{ __m256i _w0=_mm256_and_si256(v0,m), _w1=_mm256_and_si256(v1,m);
__m256i _l0=_mm256_unpacklo_epi16(_w0,_w0), _h0=_mm256_unpackhi_epi16(_w0,_w0);
__m256i _l1=_mm256_unpacklo_epi16(_w1,_w1), _h1=_mm256_unpackhi_epi16(_w1,_w1);
_mm256_store_si256((__m256i*)(dst+2*k), _mm256_permute2x128_si256(_l0,_h0,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+32), _mm256_permute2x128_si256(_l0,_h0,0x31));
_mm256_store_si256((__m256i*)(dst+2*k+64), _mm256_permute2x128_si256(_l1,_h1,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+96), _mm256_permute2x128_si256(_l1,_h1,0x31)); }
w1[k>>6]=(uint64_t)(uint32_t)_mm256_movemask_epi8(v0)
|((uint64_t)(uint32_t)_mm256_movemask_epi8(v1)<<32);
}
}
}
AVX2 static void build_BI3Bp(int n, const int8_t*B){
const __m256i one=_mm256_set1_epi8(1);
int ng=n/64, W=n/64;
for (int p=0;p<n/2;p++){
const int8_t *r0=B+(size_t)(2*p)*n, *r1=r0+n;
uint64_t *w0p=Bpb+(size_t)(2*p)*W, *w1p=Bpb+(size_t)(2*p+1)*W;
for (int g=0;g<ng;g++){
const int8_t *a=r0+64*g, *b=r1+64*g;
int8_t *o=BI3b+((size_t)g*(n/2)+p)*128;
for (int t=0;t<64;t+=32){
__m256i va=_mm256_load_si256((const __m256i*)(a+t));
__m256i vb=_mm256_load_si256((const __m256i*)(b+t));
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb);
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20));
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31));
uint32_t m0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(va,one),7));
uint32_t m1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(vb,one),7));
if(t==0) { w0p[g]=(uint64_t)m0; w1p[g]=(uint64_t)m1; }
else { w0p[g]|=(uint64_t)m0<<32; w1p[g]|=(uint64_t)m1<<32; }
}
}
}
}
/* ================= STRASSEN (exact over Z/256) =================
Every coefficient is +-1 and there is no division, so the identities hold in ANY commutative
ring; int8 arithmetic mod 256 is exactly that ring. The case table and the combine
coefficients are GENERATED from tables that mkstrassen4k.py's self-check brute-forces over
Z/256 before this file is written.
============================================================== */
#define SM 2048
static int8_t SA[SM*SM] __attribute__((aligned(64)));
static int8_t SB[SM*SM] __attribute__((aligned(64)));
static int8_t MMS[7][SM*SM+64] __attribute__((aligned(64)));
/* ============ FUSED QUADRANT PACKERS (--fuse) ============
The unfused path builds S2A/S2B with blk() and hands them to build_A0A1/build_BI3Bp, which
read them straight back: 2 MB written and 2 MB re-read per sub-product, x49. These write the
PACKED forms DIRECTLY from the parent's quadrants. Every stored value is identical by
construction, so this is a pure traffic deletion, not a re-derivation.
buildA_q/buildB_q(int h,int n,X,r0,c0,Y,r1,c1,op) == pack(X quad op Y quad)
========================================================= */
/* ===== FE5 FUSION: 4-TERM PACKERS =====
buildA_q4/buildB_q4 = buildA_q/buildB_q with THREE EXTRA OPTIONAL TERMS. Term 0's coefficient is +1
by construction; terms 1..3 carry s1,s2,s3 in {0,+1,-1} and a 0 SKIPS the load.
WHY THE +-2 COLLISION IS NOT A SPECIAL CASE: v is computed AS int8 and only THEN split (A0b <- v&0x7F,
A1b <- sign bits), so the split is downstream of the sum; a repeated quadrant is just a second
add/sub and int8 mod 256 is an exact ring. THE PACKING BODY IS THE ORIGINAL'S, VERBATIM. */
AVX2 static void buildA_q(int h,int n,const int8_t*X,int r0,int c0,
const int8_t*Y,int r1,int c1,int op){
const __m256i mx=_mm256_set1_epi8(0x7F);
int W=h/64;
for (int i=0;i<h;i++){
const int8_t*xa=X+(size_t)(r0+i)*n+c0, *ya=Y+(size_t)(r1+i)*n+c1;
int8_t*dst=A0b+(size_t)i*GLDA;
uint64_t*w1=A1b+(size_t)i*W;
for (int k=0;k<h;k+=64){
__m256i v0=_mm256_load_si256((const __m256i*)(xa+k));
__m256i v1=_mm256_load_si256((const __m256i*)(xa+k+32));
if (op){ __m256i u0=_mm256_load_si256((const __m256i*)(ya+k));
__m256i u1=_mm256_load_si256((const __m256i*)(ya+k+32));
if(op==1){ v0=_mm256_add_epi8(v0,u0); v1=_mm256_add_epi8(v1,u1); }
else { v0=_mm256_sub_epi8(v0,u0); v1=_mm256_sub_epi8(v1,u1); } }
{ __m256i _w0=_mm256_and_si256(v0,mx), _w1=_mm256_and_si256(v1,mx);
__m256i _l0=_mm256_unpacklo_epi16(_w0,_w0), _h0=_mm256_unpackhi_epi16(_w0,_w0);
__m256i _l1=_mm256_unpacklo_epi16(_w1,_w1), _h1=_mm256_unpackhi_epi16(_w1,_w1);
_mm256_store_si256((__m256i*)(dst+2*k), _mm256_permute2x128_si256(_l0,_h0,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+32), _mm256_permute2x128_si256(_l0,_h0,0x31));
_mm256_store_si256((__m256i*)(dst+2*k+64), _mm256_permute2x128_si256(_l1,_h1,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+96), _mm256_permute2x128_si256(_l1,_h1,0x31)); }
w1[k>>6]=(uint64_t)(uint32_t)_mm256_movemask_epi8(v0)
|((uint64_t)(uint32_t)_mm256_movemask_epi8(v1)<<32);
}
}
}
AVX2 static void buildB_q(int h,int n,const int8_t*X,int r0,int c0,
const int8_t*Y,int r1,int c1,int op){
const __m256i one=_mm256_set1_epi8(1);
int ng=h/64, W=h/64;
for (int p=0;p<h/2;p++){
const int8_t *ra=X+(size_t)(r0+2*p)*n+c0, *rb=X+(size_t)(r0+2*p+1)*n+c0;
const int8_t *sa=Y+(size_t)(r1+2*p)*n+c1, *sb=Y+(size_t)(r1+2*p+1)*n+c1;
uint64_t *w0p=Bpb+(size_t)(2*p)*W, *w1p=Bpb+(size_t)(2*p+1)*W;
for (int g=0;g<ng;g++){
for (int t=0;t<64;t+=32){
__m256i va=_mm256_load_si256((const __m256i*)(ra+64*g+t));
__m256i vb=_mm256_load_si256((const __m256i*)(rb+64*g+t));
if (op){ __m256i pa=_mm256_load_si256((const __m256i*)(sa+64*g+t));
__m256i pb=_mm256_load_si256((const __m256i*)(sb+64*g+t));
if(op==1){ va=_mm256_add_epi8(va,pa); vb=_mm256_add_epi8(vb,pb); }
else { va=_mm256_sub_epi8(va,pa); vb=_mm256_sub_epi8(vb,pb); } }
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb);
int8_t *o=BI3b+((size_t)g*(h/2)+p)*128;
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20));
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31));
uint32_t mm0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(va,one),7));
uint32_t mm1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(vb,one),7));
if(t==0){ w0p[g]=(uint64_t)mm0; w1p[g]=(uint64_t)mm1; }
else { w0p[g]|=(uint64_t)mm0<<32; w1p[g]|=(uint64_t)mm1<<32; }
}
}
}
}
/* ---- LEVEL 2 scratch: the m x m sub-product is itself Strassen-decomposed into 7 of h=m/2 ---- */
static int8_t S2M[7][1024*1024+64] __attribute__((aligned(64)));
static int8_t S3A[1024*1024 + 16384] __attribute__((aligned(64)));
static int8_t S3B[1024*1024 + 16384] __attribute__((aligned(64)));
static int8_t S3M[7][512*512+64] __attribute__((aligned(64)));
static int8_t S4A[512*512] __attribute__((aligned(64)));
static int8_t S4B[512*512] __attribute__((aligned(64)));
static int8_t S4M[7][256*256+64] __attribute__((aligned(64)));
/* Cblk += sum_k s_k * M_k (up to 4 terms; NULL skips a term). dst stride is `n`, block is m x m. */
/* GENERATED by gen3.py from src/base.cpp -- do not edit.
==================== THE CORRECTED FUSED SEVEN-OUTPUT PACKER (two-window form) ==========
THE BUG IT FIXES (lane c4k7_rv, localised at the artifact's own first submm_s5f call):
c4k6 froze ONE seven-pattern set, derived from one pairing of inner case index with outer
tuple. A term's slot is NRB(Aa_i + off, Aa_j + off) = 4*(Aa_i/h + u) + (Aa_j/h + v),
so it depends on the OUTER invocation's quadrant offsets Aa0..Aa3 -- not on the inner case
alone. Measured: at the first call (Aa=(0,0,512,512), Aop=1) inner case 0 matched and
inner cases 1..6 were wrong in ~130 000 of 131 072 bytes.
THE INVARIANT THAT IS TRUE (parsed out of the case lines, asserted by gen2.py):
* terms 0 and 2 read the window based at W1 = 4*(Aa0/h) + (Aa1/h);
* terms 1 and 3 read the window based at W2 = 4*(Aa2/h) + (Aa3/h);
* within a window the offset (u,v) in {0,1}^2 and the coefficient are FIXED per inner
case -- two terms from W1 and two from W2, in every one of the seven;
* every window is asked for ALL FOUR (u,v) offsets across the seven inner cases;
* SQA == 0 <=> W1 == W2 (an SQ==0 invocation touches only FOUR blocks).
=> the seven operands live in TWO 2x2 block windows: 8 source lines (4 when W1 == W2),
against 22-24 shipped line-touches over those same 8 lines.
Loads per (i,k) per side: shipped 2*SUM(terms_k) = 48 (24 when SQ==0) -> fused 16 (8).
THE ARITHMETIC IS IDENTICAL -- same int8 sums, same packed split, same layout; only the
number of times each source line is touched changes.
======================================================================================= */
#define SUMB(a,b) _mm256_add_epi8(a,b)
#define DIFB(a,b) _mm256_sub_epi8(a,b)
#ifdef SLOT_CENSUS
static unsigned long g_rdslot[5][4][2];
#define SLOTL(w,uv,lohi) (g_rdslot[w][uv][lohi]++)
#else
#define SLOTL(w,uv,lohi) ((void)0)
#endif
#define A7H 256
static int8_t A0b7w[7][A7H*512] __attribute__((aligned(64)));
static uint64_t A1b7w[7][A7H*4] __attribute__((aligned(64)));
static int8_t BI37w[7][A7H*A7H] __attribute__((aligned(64)));
static uint64_t Bpb7w[7][A7H*4] __attribute__((aligned(64)));
/* A: LO half -> dst+2k, dst+2k+32 ; HI half -> dst+2k+64, dst+2k+96 (+ the A1b word) */
#define AST_LO(IDX,KK,ACC) { \
__m256i _w0=_mm256_and_si256(ACC,_mx); \
__m256i _l0=_mm256_unpacklo_epi16(_w0,_w0), _h0=_mm256_unpackhi_epi16(_w0,_w0); \
int8_t *_d=A0b7w[IDX]+(size_t)_i*GLDA+2*(KK); \
_mm256_store_si256((__m256i*)(_d), _mm256_permute2x128_si256(_l0,_h0,0x20)); \
_mm256_store_si256((__m256i*)(_d+32), _mm256_permute2x128_si256(_l0,_h0,0x31)); \
_mlo[IDX]=(uint32_t)_mm256_movemask_epi8(ACC); }
#define AST_HI(IDX,KK,ACC) { \
__m256i _w1=_mm256_and_si256(ACC,_mx); \
__m256i _l1=_mm256_unpacklo_epi16(_w1,_w1), _h1=_mm256_unpackhi_epi16(_w1,_w1); \
int8_t *_d=A0b7w[IDX]+(size_t)_i*GLDA+2*(KK); \
_mm256_store_si256((__m256i*)(_d+64), _mm256_permute2x128_si256(_l1,_h1,0x20)); \
_mm256_store_si256((__m256i*)(_d+96), _mm256_permute2x128_si256(_l1,_h1,0x31)); \
A1b7w[IDX][(size_t)_i*_W+((KK)>>6)] = (uint64_t)_mlo[IDX] \
| ((uint64_t)(uint32_t)_mm256_movemask_epi8(ACC)<<32); }
/* B: va accumulates block rows 2p, vb rows 2p+1; the two streams interleave into BI3b */
#define BST(TP,IDX,PP,GG,TT,VA,VB) { \
__m256i lo=_mm256_unpacklo_epi8(VA,VB), hi=_mm256_unpackhi_epi8(VA,VB); \
int8_t *_o=BI37w[IDX]+((size_t)(GG)*(h/2)+(PP))*128; \
_mm256_store_si256((__m256i*)(_o+2*(TT)), _mm256_permute2x128_si256(lo,hi,0x20)); \
_mm256_store_si256((__m256i*)(_o+2*(TT)+32), _mm256_permute2x128_si256(lo,hi,0x31)); \
uint32_t _m0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(VA,_one7),7)); \
uint32_t _m1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(VB,_one7),7)); \
uint64_t *_w0p=Bpb7w[IDX]+(size_t)(2*(PP))*_W, *_w1p=Bpb7w[IDX]+(size_t)(2*(PP)+1)*_W; \
if((TP)==0){ _w0p[(GG)]=(uint64_t)_m0; _w1p[(GG)]=(uint64_t)_m1; } \
else { _w0p[(GG)]|=(uint64_t)_m0<<32; _w1p[(GG)]|=(uint64_t)_m1<<32; } }
/* ===== A side : 8 source lines per unit ===== */
template<int SQ>
AVX2 static void packA7w(const int8_t*P,int h,int W1,int W2){
const size_t PIT=(size_t)(h+1)*(size_t)h;
const int8_t *a00=P+(size_t)W1*PIT, *a01=a00+PIT, *a10=a00+4*PIT, *a11=a00+5*PIT;
const int8_t *b00=P+(size_t)W2*PIT, *b01=b00+PIT, *b10=b00+4*PIT, *b11=b00+5*PIT;
const __m256i _mx=_mm256_set1_epi8(0x7F);
const int _W=h/64;
for(int _i=0;_i<h;_i++){
const int8_t *A00=a00+(size_t)_i*h;
const int8_t *A01=a01+(size_t)_i*h;
const int8_t *A10=a10+(size_t)_i*h;
const int8_t *A11=a11+(size_t)_i*h;
const int8_t *B00=b00+(size_t)_i*h;
const int8_t *B01=b01+(size_t)_i*h;
const int8_t *B10=b10+(size_t)_i*h;
const int8_t *B11=b11+(size_t)_i*h;
for(int k=0;k<h;k+=64){
uint32_t _mlo[7];
/* ---- LO half ---- */
SLOTL(1,0,0); __m256i sA00=_mm256_load_si256((const __m256i*)(A00+k));
SLOTL(1,1,0); __m256i sA01=_mm256_load_si256((const __m256i*)(A01+k));
SLOTL(1,4,0); __m256i sA10=_mm256_load_si256((const __m256i*)(A10+k));
SLOTL(1,5,0); __m256i sA11=_mm256_load_si256((const __m256i*)(A11+k));
SLOTL(2,0,0); __m256i sB00=_mm256_load_si256((const __m256i*)(B00+k));
SLOTL(2,1,0); __m256i sB01=_mm256_load_si256((const __m256i*)(B01+k));
SLOTL(2,4,0); __m256i sB10=_mm256_load_si256((const __m256i*)(B10+k));
SLOTL(2,5,0); __m256i sB11=_mm256_load_si256((const __m256i*)(B11+k));
{
__m256i acc=sA00;
if(SQ>0) acc=SUMB(acc,sB00); else if(SQ<0) acc=DIFB(acc,sB00);
acc=SUMB(acc,sA11);
if(SQ>0) acc=SUMB(acc,sB11); else if(SQ<0) acc=DIFB(acc,sB11);
AST_LO(0,k,acc);
}
{
__m256i acc=sA10;
if(SQ>0) acc=SUMB(acc,sB10); else if(SQ<0) acc=DIFB(acc,sB10);
acc=SUMB(acc,sA11);
if(SQ>0) acc=SUMB(acc,sB11); else if(SQ<0) acc=DIFB(acc,sB11);
AST_LO(1,k,acc);
}
{
__m256i acc=sA00;
if(SQ>0) acc=SUMB(acc,sB00); else if(SQ<0) acc=DIFB(acc,sB00);
AST_LO(2,k,acc);
}
{
__m256i acc=sA11;
if(SQ>0) acc=SUMB(acc,sB11); else if(SQ<0) acc=DIFB(acc,sB11);
AST_LO(3,k,acc);
}
{
__m256i acc=sA00;
if(SQ>0) acc=SUMB(acc,sB00); else if(SQ<0) acc=DIFB(acc,sB00);
acc=SUMB(acc,sA01);
if(SQ>0) acc=SUMB(acc,sB01); else if(SQ<0) acc=DIFB(acc,sB01);
AST_LO(4,k,acc);
}
{
__m256i acc=sA10;
if(SQ>0) acc=SUMB(acc,sB10); else if(SQ<0) acc=DIFB(acc,sB10);
acc=DIFB(acc,sA00);
if(SQ>0) acc=DIFB(acc,sB00); else if(SQ<0) acc=SUMB(acc,sB00);
AST_LO(5,k,acc);
}
{
__m256i acc=sA01;
if(SQ>0) acc=SUMB(acc,sB01); else if(SQ<0) acc=DIFB(acc,sB01);
acc=DIFB(acc,sA11);
if(SQ>0) acc=DIFB(acc,sB11); else if(SQ<0) acc=SUMB(acc,sB11);
AST_LO(6,k,acc);
}
/* ---- HI half ---- */
SLOTL(1,0,1); __m256i tA00=_mm256_load_si256((const __m256i*)(A00+k+32));
SLOTL(1,1,1); __m256i tA01=_mm256_load_si256((const __m256i*)(A01+k+32));
SLOTL(1,4,1); __m256i tA10=_mm256_load_si256((const __m256i*)(A10+k+32));
SLOTL(1,5,1); __m256i tA11=_mm256_load_si256((const __m256i*)(A11+k+32));
SLOTL(2,0,1); __m256i tB00=_mm256_load_si256((const __m256i*)(B00+k+32));
SLOTL(2,1,1); __m256i tB01=_mm256_load_si256((const __m256i*)(B01+k+32));
SLOTL(2,4,1); __m256i tB10=_mm256_load_si256((const __m256i*)(B10+k+32));
SLOTL(2,5,1); __m256i tB11=_mm256_load_si256((const __m256i*)(B11+k+32));
{
__m256i acc=tA00;
if(SQ>0) acc=SUMB(acc,tB00); else if(SQ<0) acc=DIFB(acc,tB00);
acc=SUMB(acc,tA11);
if(SQ>0) acc=SUMB(acc,tB11); else if(SQ<0) acc=DIFB(acc,tB11);
AST_HI(0,k,acc);
}
{
__m256i acc=tA10;
if(SQ>0) acc=SUMB(acc,tB10); else if(SQ<0) acc=DIFB(acc,tB10);
acc=SUMB(acc,tA11);
if(SQ>0) acc=SUMB(acc,tB11); else if(SQ<0) acc=DIFB(acc,tB11);
AST_HI(1,k,acc);
}
{
__m256i acc=tA00;
if(SQ>0) acc=SUMB(acc,tB00); else if(SQ<0) acc=DIFB(acc,tB00);
AST_HI(2,k,acc);
}
{
__m256i acc=tA11;
if(SQ>0) acc=SUMB(acc,tB11); else if(SQ<0) acc=DIFB(acc,tB11);
AST_HI(3,k,acc);
}
{
__m256i acc=tA00;
if(SQ>0) acc=SUMB(acc,tB00); else if(SQ<0) acc=DIFB(acc,tB00);
acc=SUMB(acc,tA01);
if(SQ>0) acc=SUMB(acc,tB01); else if(SQ<0) acc=DIFB(acc,tB01);
AST_HI(4,k,acc);
}
{
__m256i acc=tA10;
if(SQ>0) acc=SUMB(acc,tB10); else if(SQ<0) acc=DIFB(acc,tB10);
acc=DIFB(acc,tA00);
if(SQ>0) acc=DIFB(acc,tB00); else if(SQ<0) acc=SUMB(acc,tB00);
AST_HI(5,k,acc);
}
{
__m256i acc=tA01;
if(SQ>0) acc=SUMB(acc,tB01); else if(SQ<0) acc=DIFB(acc,tB01);
acc=DIFB(acc,tA11);
if(SQ>0) acc=DIFB(acc,tB11); else if(SQ<0) acc=SUMB(acc,tB11);
AST_HI(6,k,acc);
}
}
}
}
/* ===== B side : 8 source lines per unit ===== */
template<int SQ>
AVX2 static void packB7w(const int8_t*P,int h,int W1,int W2){
const size_t PIT=(size_t)(h+1)*(size_t)h;
const int8_t *a00=P+(size_t)W1*PIT, *a01=a00+PIT, *a10=a00+4*PIT, *a11=a00+5*PIT;
const int8_t *b00=P+(size_t)W2*PIT, *b01=b00+PIT, *b10=b00+4*PIT, *b11=b00+5*PIT;
const __m256i _one7=_mm256_set1_epi8(1);
const int _W=h/64, ng=h/64;
for(int p=0;p<h/2;p++){
const int8_t *A00=a00+(size_t)(2*p)*h, *A00b=A00+h;
const int8_t *A01=a01+(size_t)(2*p)*h, *A01b=A01+h;
const int8_t *A10=a10+(size_t)(2*p)*h, *A10b=A10+h;
const int8_t *A11=a11+(size_t)(2*p)*h, *A11b=A11+h;
const int8_t *B00=b00+(size_t)(2*p)*h, *B00b=B00+h;
const int8_t *B01=b01+(size_t)(2*p)*h, *B01b=B01+h;
const int8_t *B10=b10+(size_t)(2*p)*h, *B10b=B10+h;
const int8_t *B11=b11+(size_t)(2*p)*h, *B11b=B11+h;
for(int g=0;g<ng;g++){
for(int t=0;t<64;t+=32){
SLOTL(1,0,0); __m256i sA00a=_mm256_load_si256((const __m256i*)(A00+64*g+t)); __m256i sA00b=_mm256_load_si256((const __m256i*)(A00b+64*g+t));
SLOTL(1,1,0); __m256i sA01a=_mm256_load_si256((const __m256i*)(A01+64*g+t)); __m256i sA01b=_mm256_load_si256((const __m256i*)(A01b+64*g+t));
SLOTL(1,4,0); __m256i sA10a=_mm256_load_si256((const __m256i*)(A10+64*g+t)); __m256i sA10b=_mm256_load_si256((const __m256i*)(A10b+64*g+t));
SLOTL(1,5,0); __m256i sA11a=_mm256_load_si256((const __m256i*)(A11+64*g+t)); __m256i sA11b=_mm256_load_si256((const __m256i*)(A11b+64*g+t));
SLOTL(2,0,0); __m256i sB00a=_mm256_load_si256((const __m256i*)(B00+64*g+t)); __m256i sB00b=_mm256_load_si256((const __m256i*)(B00b+64*g+t));
SLOTL(2,1,0); __m256i sB01a=_mm256_load_si256((const __m256i*)(B01+64*g+t)); __m256i sB01b=_mm256_load_si256((const __m256i*)(B01b+64*g+t));
SLOTL(2,4,0); __m256i sB10a=_mm256_load_si256((const __m256i*)(B10+64*g+t)); __m256i sB10b=_mm256_load_si256((const __m256i*)(B10b+64*g+t));
SLOTL(2,5,0); __m256i sB11a=_mm256_load_si256((const __m256i*)(B11+64*g+t)); __m256i sB11b=_mm256_load_si256((const __m256i*)(B11b+64*g+t));
{
__m256i va=sA00a, vb=sA00b;
if(SQ>0){va=SUMB(va,sB00a); vb=SUMB(vb,sB00b);} else if(SQ<0){va=DIFB(va,sB00a); vb=DIFB(vb,sB00b);}
va=SUMB(va,sA11a); vb=SUMB(vb,sA11b);
if(SQ>0){va=SUMB(va,sB11a); vb=SUMB(vb,sB11b);} else if(SQ<0){va=DIFB(va,sB11a); vb=DIFB(vb,sB11b);}
BST(t/32,0,p,g,t,va,vb);
}
{
__m256i va=sA00a, vb=sA00b;
if(SQ>0){va=SUMB(va,sB00a); vb=SUMB(vb,sB00b);} else if(SQ<0){va=DIFB(va,sB00a); vb=DIFB(vb,sB00b);}
BST(t/32,1,p,g,t,va,vb);
}
{
__m256i va=sA01a, vb=sA01b;
if(SQ>0){va=SUMB(va,sB01a); vb=SUMB(vb,sB01b);} else if(SQ<0){va=DIFB(va,sB01a); vb=DIFB(vb,sB01b);}
va=DIFB(va,sA11a); vb=DIFB(vb,sA11b);
if(SQ>0){va=DIFB(va,sB11a); vb=DIFB(vb,sB11b);} else if(SQ<0){va=SUMB(va,sB11a); vb=SUMB(vb,sB11b);}
BST(t/32,2,p,g,t,va,vb);
}
{
__m256i va=sA10a, vb=sA10b;
if(SQ>0){va=SUMB(va,sB10a); vb=SUMB(vb,sB10b);} else if(SQ<0){va=DIFB(va,sB10a); vb=DIFB(vb,sB10b);}
va=DIFB(va,sA00a); vb=DIFB(vb,sA00b);
if(SQ>0){va=DIFB(va,sB00a); vb=DIFB(vb,sB00b);} else if(SQ<0){va=SUMB(va,sB00a); vb=SUMB(vb,sB00b);}
BST(t/32,3,p,g,t,va,vb);
}
{
__m256i va=sA11a, vb=sA11b;
if(SQ>0){va=SUMB(va,sB11a); vb=SUMB(vb,sB11b);} else if(SQ<0){va=DIFB(va,sB11a); vb=DIFB(vb,sB11b);}
BST(t/32,4,p,g,t,va,vb);
}
{
__m256i va=sA00a, vb=sA00b;
if(SQ>0){va=SUMB(va,sB00a); vb=SUMB(vb,sB00b);} else if(SQ<0){va=DIFB(va,sB00a); vb=DIFB(vb,sB00b);}
va=SUMB(va,sA01a); vb=SUMB(vb,sA01b);
if(SQ>0){va=SUMB(va,sB01a); vb=SUMB(vb,sB01b);} else if(SQ<0){va=DIFB(va,sB01a); vb=DIFB(vb,sB01b);}
BST(t/32,5,p,g,t,va,vb);
}
{
__m256i va=sA10a, vb=sA10b;
if(SQ>0){va=SUMB(va,sB10a); vb=SUMB(vb,sB10b);} else if(SQ<0){va=DIFB(va,sB10a); vb=DIFB(vb,sB10b);}
va=SUMB(va,sA11a); vb=SUMB(vb,sA11b);
if(SQ>0){va=SUMB(va,sB11a); vb=SUMB(vb,sB11b);} else if(SQ<0){va=DIFB(va,sB11a); vb=DIFB(vb,sB11b);}
BST(t/32,6,p,g,t,va,vb);
}
}
}
}
}
/* ===== A side (W1==W2, only valid when the windows coincide) : 4 source lines per unit ===== */
AVX2 static void packA7wS(const int8_t*P,int h,int W1,int W2){
const size_t PIT=(size_t)(h+1)*(size_t)h;
const int8_t *a00=P+(size_t)W1*PIT, *a01=a00+PIT, *a10=a00+4*PIT, *a11=a00+5*PIT;
const int8_t *b00=P+(size_t)W2*PIT, *b01=b00+PIT, *b10=b00+4*PIT, *b11=b00+5*PIT;
const __m256i _mx=_mm256_set1_epi8(0x7F);
const int _W=h/64;
for(int _i=0;_i<h;_i++){
const int8_t *A00=a00+(size_t)_i*h;
const int8_t *A01=a01+(size_t)_i*h;
const int8_t *A10=a10+(size_t)_i*h;
const int8_t *A11=a11+(size_t)_i*h;
for(int k=0;k<h;k+=64){
uint32_t _mlo[7];
/* ---- LO half ---- */
SLOTL(1,0,0); __m256i sA00=_mm256_load_si256((const __m256i*)(A00+k));
SLOTL(1,1,0); __m256i sA01=_mm256_load_si256((const __m256i*)(A01+k));
SLOTL(1,4,0); __m256i sA10=_mm256_load_si256((const __m256i*)(A10+k));
SLOTL(1,5,0); __m256i sA11=_mm256_load_si256((const __m256i*)(A11+k));
{
__m256i acc=sA00;
acc=SUMB(acc,sA11);
AST_LO(0,k,acc);
}
{
__m256i acc=sA10;
acc=SUMB(acc,sA11);
AST_LO(1,k,acc);
}
{
__m256i acc=sA00;
AST_LO(2,k,acc);
}
{
__m256i acc=sA11;
AST_LO(3,k,acc);
}
{
__m256i acc=sA00;
acc=SUMB(acc,sA01);
AST_LO(4,k,acc);
}
{
__m256i acc=sA10;
acc=DIFB(acc,sA00);
AST_LO(5,k,acc);
}
{
__m256i acc=sA01;
acc=DIFB(acc,sA11);
AST_LO(6,k,acc);
}
/* ---- HI half ---- */
SLOTL(1,0,1); __m256i tA00=_mm256_load_si256((const __m256i*)(A00+k+32));
SLOTL(1,1,1); __m256i tA01=_mm256_load_si256((const __m256i*)(A01+k+32));
SLOTL(1,4,1); __m256i tA10=_mm256_load_si256((const __m256i*)(A10+k+32));
SLOTL(1,5,1); __m256i tA11=_mm256_load_si256((const __m256i*)(A11+k+32));
{
__m256i acc=tA00;
acc=SUMB(acc,tA11);
AST_HI(0,k,acc);
}
{
__m256i acc=tA10;
acc=SUMB(acc,tA11);
AST_HI(1,k,acc);
}
{
__m256i acc=tA00;
AST_HI(2,k,acc);
}
{
__m256i acc=tA11;
AST_HI(3,k,acc);
}
{
__m256i acc=tA00;
acc=SUMB(acc,tA01);
AST_HI(4,k,acc);
}
{
__m256i acc=tA10;
acc=DIFB(acc,tA00);
AST_HI(5,k,acc);
}
{
__m256i acc=tA01;
acc=DIFB(acc,tA11);
AST_HI(6,k,acc);
}
}
}
}
/* ===== B side (W1==W2, only valid when the windows coincide) : 4 source lines per unit ===== */
AVX2 static void packB7wS(const int8_t*P,int h,int W1,int W2){
const size_t PIT=(size_t)(h+1)*(size_t)h;
const int8_t *a00=P+(size_t)W1*PIT, *a01=a00+PIT, *a10=a00+4*PIT, *a11=a00+5*PIT;
const int8_t *b00=P+(size_t)W2*PIT, *b01=b00+PIT, *b10=b00+4*PIT, *b11=b00+5*PIT;
const __m256i _one7=_mm256_set1_epi8(1);
const int _W=h/64, ng=h/64;
for(int p=0;p<h/2;p++){
const int8_t *A00=a00+(size_t)(2*p)*h, *A00b=A00+h;
const int8_t *A01=a01+(size_t)(2*p)*h, *A01b=A01+h;
const int8_t *A10=a10+(size_t)(2*p)*h, *A10b=A10+h;
const int8_t *A11=a11+(size_t)(2*p)*h, *A11b=A11+h;
for(int g=0;g<ng;g++){
for(int t=0;t<64;t+=32){
SLOTL(1,0,0); __m256i sA00a=_mm256_load_si256((const __m256i*)(A00+64*g+t)); __m256i sA00b=_mm256_load_si256((const __m256i*)(A00b+64*g+t));
SLOTL(1,1,0); __m256i sA01a=_mm256_load_si256((const __m256i*)(A01+64*g+t)); __m256i sA01b=_mm256_load_si256((const __m256i*)(A01b+64*g+t));
SLOTL(1,4,0); __m256i sA10a=_mm256_load_si256((const __m256i*)(A10+64*g+t)); __m256i sA10b=_mm256_load_si256((const __m256i*)(A10b+64*g+t));
SLOTL(1,5,0); __m256i sA11a=_mm256_load_si256((const __m256i*)(A11+64*g+t)); __m256i sA11b=_mm256_load_si256((const __m256i*)(A11b+64*g+t));
{
__m256i va=sA00a, vb=sA00b;
va=SUMB(va,sA11a); vb=SUMB(vb,sA11b);
BST(t/32,0,p,g,t,va,vb);
}
{
__m256i va=sA00a, vb=sA00b;
BST(t/32,1,p,g,t,va,vb);
}
{
__m256i va=sA01a, vb=sA01b;
va=DIFB(va,sA11a); vb=DIFB(vb,sA11b);
BST(t/32,2,p,g,t,va,vb);
}
{
__m256i va=sA10a, vb=sA10b;
va=DIFB(va,sA00a); vb=DIFB(vb,sA00b);
BST(t/32,3,p,g,t,va,vb);
}
{
__m256i va=sA11a, vb=sA11b;
BST(t/32,4,p,g,t,va,vb);
}
{
__m256i va=sA00a, vb=sA00b;
va=SUMB(va,sA01a); vb=SUMB(vb,sA01b);
BST(t/32,5,p,g,t,va,vb);
}
{
__m256i va=sA10a, vb=sA10b;
va=SUMB(va,sA11a); vb=SUMB(vb,sA11b);
BST(t/32,6,p,g,t,va,vb);
}
}
}
}
}
AVX2 static void cmb(int8_t*C,int n,int r0,int c0,int m,
const int8_t*M0,int s0,const int8_t*M1,int s1,
const int8_t*M2,int s2,const int8_t*M3,int s3){
for (int i=0;i<m;i++){
int8_t *d=C+(size_t)(r0+i)*n+c0;
const int8_t *p0=M0?M0+(size_t)i*m:0, *p1=M1?M1+(size_t)i*m:0;
const int8_t *p2=M2?M2+(size_t)i*m:0, *p3=M3?M3+(size_t)i*m:0;
for (int j=0;j<m;j+=32){
__m256i v;
if (p0){ __m256i t=_mm256_load_si256((const __m256i*)(p0+j));
v = s0>0?t:_mm256_sub_epi8(_mm256_setzero_si256(),t); }
else v = _mm256_setzero_si256();
if (p1){ __m256i t=_mm256_load_si256((const __m256i*)(p1+j));
v = _mm256_add_epi8(v, s1>0?t:_mm256_sub_epi8(_mm256_setzero_si256(),t)); }
if (p2){ __m256i t=_mm256_load_si256((const __m256i*)(p2+j));
v = _mm256_add_epi8(v, s2>0?t:_mm256_sub_epi8(_mm256_setzero_si256(),t)); }
if (p3){ __m256i t=_mm256_load_si256((const __m256i*)(p3+j));
v = _mm256_add_epi8(v, s3>0?t:_mm256_sub_epi8(_mm256_setzero_si256(),t)); }
_mm256_stream_si256((__m256i*)(d+j), v);
}
}
}
/* dst[i*h+j] = X[(r0+i)*n+(c0+j)] op Y[(r1+i)*n+(c1+j)], op: 0 copy 1 add 2 sub (mod 256) */
/* lane c4k5_rv: BLOCK-64KB LAYOUT. dst holds a 4 x 4 grid of B x B blocks, B = h/4, each
block CONTIGUOUS: block (I,J) at (I*4+J)*B*B. The level-4 packers read it that way (see NRB in
submm_s5f), so every packer term is a dense sequential 64 KB stream instead of four
1024-byte-strided 256-byte streams. Byte-identical values; addressing only. */
AVX2 static void blk(int8_t*dst,const int8_t*X,int n,int r0,int c0,int h,
const int8_t*Y,int r1,int c1,int op){
const int B = h>>2, NB = 4;
/* c4k6 padblk: block PITCH is (B+1)*B = B*B + B bytes instead of B*B, so the four
concurrent term streams of a packer no longer sit an exact multiple of the L1 set
span (32 KB) AND the L2 set span (64 KB) apart. B here IS the reader's h (the next
level's sub-block size = h>>2), so the reader's row base k*(h+1) x stride h agrees. */
const size_t BB = (size_t)B*(size_t)B + (size_t)B;
for (int i=0;i<h;i++){
const int8_t *xa=X+(size_t)(r0+i)*n+c0, *ya=Y+(size_t)(r1+i)*n+c1;
int8_t *d0 = dst + (size_t)((i/B)*NB)*BB + (size_t)(i%B)*B;
for (int J=0;J<NB;J++){
int8_t *d = d0 + (size_t)J*BB;
const int8_t *xaj = xa + J*B, *yaj = ya + J*B;
int j=0;
if (op==0) for(;j+32<=B;j+=32) _mm256_store_si256((__m256i*)(d+j),
_mm256_load_si256((const __m256i*)(xaj+j)));
else if (op==1) for(;j+32<=B;j+=32) _mm256_store_si256((__m256i*)(d+j),
_mm256_add_epi8(_mm256_load_si256((const __m256i*)(xaj+j)),
_mm256_load_si256((const __m256i*)(yaj+j))));
else for(;j+32<=B;j+=32) _mm256_store_si256((__m256i*)(d+j),
_mm256_sub_epi8(_mm256_load_si256((const __m256i*)(xaj+j)),
_mm256_load_si256((const __m256i*)(yaj+j))));
if (op) for(;j<B;j++) d[j]=(int8_t)(xaj[j]+(op==1?yaj[j]:-yaj[j]));
}
}
}
AVX2 static void blk_nt(int8_t*dst,const int8_t*X,int n,int r0,int c0,int h,
const int8_t*Y,int r1,int c1,int op){
for (int i=0;i<h;i++){
const int8_t *xa=X+(size_t)(r0+i)*n+c0, *ya=Y+(size_t)(r1+i)*n+c1;
int8_t *d=dst+(size_t)i*h;
int j=0;
if (op==0) for(;j+32<=h;j+=32) _mm256_stream_si256((__m256i*)(d+j),
_mm256_load_si256((const __m256i*)(xa+j)));
else if (op==1) for(;j+32<=h;j+=32) _mm256_stream_si256((__m256i*)(d+j),
_mm256_add_epi8(_mm256_load_si256((const __m256i*)(xa+j)),
_mm256_load_si256((const __m256i*)(ya+j))));
else for(;j+32<=h;j+=32) _mm256_stream_si256((__m256i*)(d+j),
_mm256_sub_epi8(_mm256_load_si256((const __m256i*)(xa+j)),
_mm256_load_si256((const __m256i*)(ya+j))));
if (op) for(;j<h;j++) d[j]=(int8_t)(xa[j]+(op==1?ya[j]:-ya[j]));
}
}
/* The GF(2) + combine tail of a sub-product, i.e. everything after the operand packers. Split
out so the FUSED path can reach it without materialising S2A/S2B first. */
AVX2BMI static void corr1024(void);
/* corr1024(): a hand-unrolled mirror of the SHIPPED corr16 for n=1024 (W=16, NY=4), which
corr16's own `if (n==4096)` dispatch never reaches. corr_pass3's inner `for(q<NY)` is
runtime-bounded, so GCC cannot unroll it; corr16 exists because the unrolled form pays.
Only the constants differ from corr16: 4 accumulators instead of 16, Bpb row stride 128 B
instead of 512 B (shlq $7), and offsets 0/32/64/96 instead of 0..480. */
AVX2BMI static void corr1024(void){
const int n=1024, W=16;
memset(Pbb,0,(size_t)n*W*8);
const int KB=64;
for (int kb=0; kb<n; kb+=KB){
int w0=kb>>6;
const char *bp0=(const char*)Bpb+(size_t)w0*8192;
for (int i=0;i<n;i++){
const uint64_t*row=A1b+(size_t)i*W+w0;
uint64_t b0=row[0], b1=0;
uint64_t *o=Pbb+(size_t)i*W;
asm volatile(
"vpxor %%xmm0,%%xmm0,%%xmm0\n\t"
"vpxor %%xmm1,%%xmm1,%%xmm1\n\t"
"vpxor %%xmm2,%%xmm2,%%xmm2\n\t"
"vpxor %%xmm3,%%xmm3,%%xmm3\n\t"
"testq %[b0], %[b0]\n\t"
"jz 3%=f\n\t"
"1%=:\n\t"
"tzcntq %[b0], %%r11\n\t"
"shlq $7, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"vpxor 64(%%r11), %%ymm2, %%ymm2\n\t"
"vpxor 96(%%r11), %%ymm3, %%ymm3\n\t"
"blsrq %[b0], %[b0]\n\t"
"jnz 1%=b\n\t"
"3%=:\n\t"
"testq %[b1], %[b1]\n\t"
"jz 4%=f\n\t"
"2%=:\n\t"
"tzcntq %[b1], %%r11\n\t"
"shlq $7, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"vpxor 64(%%r11), %%ymm2, %%ymm2\n\t"
"vpxor 96(%%r11), %%ymm3, %%ymm3\n\t"
"blsrq %[b1], %[b1]\n\t"
"jnz 2%=b\n\t"
"4%=:\n\t"
"vpxor 0(%[o]), %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[o])\n\t"
"vpxor 32(%[o]), %%ymm1, %%ymm1\n\t"
"vmovdqu %%ymm1, 32(%[o])\n\t"
"vpxor 64(%[o]), %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 64(%[o])\n\t"
"vpxor 96(%[o]), %%ymm3, %%ymm3\n\t"
"vmovdqu %%ymm3, 96(%[o])\n\t"
: [b0]"+r"(b0), [b1]"+r"(b1)
: [bp0]"r"(bp0), [o]"r"(o)
: "r11","cc","memory","ymm0","ymm1","ymm2","ymm3");
}
}
}
/* ===================== lane CH3: NIBBLE-TABLE GF(2) PRODUCT =====================
P[i][:] = XOR_{k: A1[i][k]=1} Bpb[k][:] -- the same contract as corr512/corr1024,
verified bit-identical against them on the real judge (work/lane_ch3_mmmc/rig/).
Rationale and measurements: work/lane_ch3_mmmc/mk_nibcorr.py docstring.
GROUPS = 16 groups (= one 64-bit A1 word) per k-block; table = 16*16*W words. */
static uint64_t TB[((size_t)NMAX/64)*256] __attribute__((aligned(64)));
/* ===== lane m50 (ported from m47/mmmc1k, macro diffed BYTE-IDENTICAL): VECTORISED SUBSET-XOR
TABLE BUILD. The source loop is `for w { 4 scalar loads; 12 scalar xors; 16 scalar stores }`.
At W%4==0 the w-dimension of `T[j*W+w]` is EXACTLY one ymm, so the honest build is
4 vector loads + 12 vpxor + 16 vector stores = 32 uops, NOT ONE of them on p5.
gcc-9's SLP vectorizer instead materialised a 16x16 64-bit transpose through a 2 KB STACK
buffer (measured here: 896 p5-only shuffles + 632 stack refs in nibcorr256).
Taking each operand AS A VECTOR removes the transpose's reason to exist. */
AVX2BMI static void nibtab_build(const uint64_t *__restrict r, uint64_t *__restrict T, int W){
const int U = W>>2; /* ymm per row-of-table = W/4 */
const __m256i *rp = (const __m256i*)r;
__m256i *tp = (__m256i*)T;
for (int q=0;q<U;q++){
const __m256i a1=rp[q], a2=rp[U+q], a4=rp[2*U+q], a8=rp[3*U+q];
const __m256i x12=_mm256_xor_si256(a1,a2), x14=_mm256_xor_si256(a1,a4),
x18=_mm256_xor_si256(a1,a8);
__m256i *o = tp+q; /* o[j*U] holds T[j*W + q*4 .. +3] */
o[ 0*U]=_mm256_setzero_si256();
o[ 1*U]=a1; o[ 2*U]=a2; o[ 4*U]=a4; o[ 8*U]=a8;
o[ 3*U]=x12; o[ 5*U]=x14;
o[ 6*U]=_mm256_xor_si256(a2,a4); o[ 7*U]=_mm256_xor_si256(x12,a4);
o[ 9*U]=x18; o[10*U]=_mm256_xor_si256(a2,a8);
o[12*U]=_mm256_xor_si256(a4,a8);
o[11*U]=_mm256_xor_si256(a2,x18); o[13*U]=_mm256_xor_si256(a4,x18);
o[14*U]=_mm256_xor_si256(_mm256_xor_si256(a2,a4),a8);
o[15*U]=_mm256_xor_si256(_mm256_xor_si256(x12,a4),a8);
}
}
#define DEF_NIBCORR(TAG, M) \
AVX2BMI static void nibcorr##TAG(void){ \
const int n=M, W=n/64, G=n>>2, GB=16, NY=W>>2; \
memset(Pbb,0,(size_t)n*W*8); \
for (int g0=0; g0<G; g0+=GB){ \
for (int t=0;t<GB;t++){ \
const int g=g0+t; \
uint64_t *T=TB+(size_t)t*16*W; \
const uint64_t *r=Bpb+(size_t)(g*4)*W; \
nibtab_build(r,T,W); \
\
} \
for (int i=0;i<n;i++){ \
const uint64_t *arow=A1b+(size_t)i*W+(g0>>4); \
__m256i acc[NY]; \
for (int q=0;q<NY;q++) acc[q]=_mm256_setzero_si256(); \
const uint64_t *T=TB; \
for (int wb=0; wb<(GB>>4); wb++){ \
const uint64_t word=arow[wb]; \
_Pragma("GCC unroll 16") \
for (int b=0;b<16;b++, T+=(size_t)16*W){ \
const uint64_t *e=T+(size_t)(((word>>(4*b))&15u)*W); \
for (int q=0;q<NY;q++) \
acc[q]=_mm256_xor_si256(acc[q],_mm256_load_si256((const __m256i*)(e+4*q))); \
} \
} \
uint64_t *o=Pbb+(size_t)i*W; \
for (int q=0;q<NY;q++){ \
__m256i *po=(__m256i*)(o+4*q); \
_mm256_store_si256(po,_mm256_xor_si256(_mm256_load_si256((const __m256i*)po),acc[q])); \
} \
} \
} \
}
DEF_NIBCORR(512,512)
/* FE5 DEPTH-4: the macro is VALID at M=256 (W=4, G=64, NY=1; the TB row is
16*W*8 = 512 B and the loads are 16 index-slots x 32 B = 512 B, and T[15*W+w]=T[60..63] is in
range). Without this instantiation every depth-4 leaf falls back to corr_pass3 -- the exact
+2.89 % trap that cost 6.55 % at depth 3. */
DEF_NIBCORR(256,256)
DEF_NIBCORR(1024,1024)
AVX2 static void submm_tail(int m,int8_t*Cp){
GLDA = 2*(m) + PADA; GLDB = m + PADB;
/* FE5: nibcorr512 ALREADY EXISTS (DEF_NIBCORR(512,512)) but this dispatch never reached it,
so every depth-3 leaf ran the runtime-bounded corr_pass3 -- exactly the overhead that made
depth 3 lose. corr_pass3's inner `for(q<NY)` is runtime-bounded and GCC cannot unroll it;
the nibcorr forms are hand-unrolled. ONE LINE, EXISTING CODE, NO NEW SCRATCH. */
/* FE5: 256 added alongside the depth-4 level. THE TWO EDITS SHIP TOGETHER BY DESIGN. */
if (m==1024) nibcorr1024(); else if (m==512) nibcorr512();
else if (m==256) nibcorr256(); else corr_pass3(m);
const int GB=1024, IB=1024;
for (int ib=0; ib<m; ib+=IB){
const int ihi = (ib+IB<m)? ib+IB : m;
for (int gb=0; gb<m; gb+=GB){
const int ghi = (gb+GB<m)? gb+GB : m;
for (int g=gb; g<ghi; g+=64){
const int8_t *bi = BI3b+(size_t)(g>>6)*(m/2)*128;
{ int rev=0;
for (int i=ib; i<ihi; i+=2){
/* R1 boustrophedon: every other call walks A and BI BACKWARD, so it starts
where the previous call stopped -- that data is still in L1. */
const int8_t *ap=A0b+(size_t)i*GLDA;
/* MC2: PB now points at the Pbb WORD for row i (Pbb is nibcorr256's output);
the byte expansion is done inline in the epilogue. */
const uint8_t *pw = (const uint8_t*)(Pbb+(size_t)i*(m>>6)+(g>>6));
mikro_u4 (m, ap, bi,
Cp+(size_t)i*m+g, pw, 0, GLDA, (m>>6)*8);
rev^=1; }
}
}
}
}
}
/* one m x m sub-product through the SHIPPED pipeline, NT-stored to Cp (m = 2048 in the
shipped dispatch; `m` is a parameter so the SAME code path gates at smaller m). */
AVX2 static void submm_plain(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp){
GLDA = 2*(m) + PADA; GLDB = m + PADB;
build_A0A1(m,Ap); build_BI3Bp(m,Bp); submm_tail(m,Cp);
}
AVX2 static void submm(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp);
#define S2MIN 1024
/* level 2: 7 sub-products of h=m/2, recombined by the SAME verified identity.
⛔ THE SUB-PRODUCT IS COMPUTED BY `submm_plain`, **NOT** BY `submm`.
Calling `submm(h,...)` re-enters this function at h, and the inner call **overwrites the
parent's own `S2A`/`S2B`/`S2M` scratch** -- so every `S2M[k]` except the last is destroyed
before the combine reads it. That is a 100 %-wrong build, and the 6-second A=I probe at
n=4096 found it (87 % of bytes wrong, all four blocks). If a deeper level is ever wanted,
it needs its own scratch, not the shared pool. */
/* FE5 DEPTH-3: submm_s2's leaves are now full Strassen steps. submm_s4 takes PLAIN X,Y
(produced by blk() in submm_s2) and uses its own S3M scratch, so the parent's S2M is
never clobbered -- that clobber is the 100 %-wrong build base.cpp:926-930 warns about. */
AVX2 static void submm_s4(int m,const int8_t*X,const int8_t*Y,int8_t*dst);
/* FE5 DEPTH-4: submm_s4's leaves are now full Strassen steps with their OWN S4M scratch, so
the parent's S3M is never clobbered -- the clobber base.cpp:926-930 calls a 100 %-wrong build. */
/* FE5 FUSION: forward decl -- submm_s4's junction is now fused into this. */
AVX2 static void submm_s5f(int M,const int8_t*Ap,int Aa0,int Aa1,int Aa2,int Aa3,int Aop,
const int8_t*Bp,int Ba0,int Ba1,int Ba2,int Ba3,int Bop,int8_t*Cp);
AVX2 static void submm_s5f(int M,const int8_t*Ap,int Aa0,int Aa1,int Aa2,int Aa3,int Aop,
const int8_t*Bp,int Ba0,int Ba1,int Ba2,int Ba3,int Bop,int8_t*Cp){
const int h=M>>2, o=h;
const int SQA=(Aop==1)?1:((Aop==2)?-1:0);
const int SQB=(Bop==1)?1:((Bop==2)?-1:0);
const int dz=M>>1;
GLDA = 2*h + PADA; GLDB = h + PADB;
if (h<=A7H){
/* c4k7_rv's two-window fused packers: ONE (i,k) sweep per side emits all seven cases */
{ const int W1=4*(Aa0/h)+(Aa1/h), W2=4*(Aa2/h)+(Aa3/h);
if (SQA==0 && W1==W2) packA7wS(Ap,h,W1,W1);
else if (SQA>0) packA7w< 1>(Ap,h,W1,W2);
else if (SQA<0) packA7w<-1>(Ap,h,W1,W2);
else packA7w< 0>(Ap,h,W1,W2); }
{ const int V1=4*(Ba0/h)+(Ba1/h), V2=4*(Ba2/h)+(Ba3/h);
if (SQB==0 && V1==V2) packB7wS(Bp,h,V1,V1);
else if (SQB>0) packB7w< 1>(Bp,h,V1,V2);
else if (SQB<0) packB7w<-1>(Bp,h,V1,V2);
else packB7w< 0>(Bp,h,V1,V2); }
for (int k=0;k<7;k++){
A0b=A0b7w[k]; A1b=A1b7w[k]; BI3b=BI37w[k]; Bpb=Bpb7w[k];
submm_tail(h,S4M[k]);
}
A0b=A0bs; A1b=A1bs; BI3b=BI3bs; Bpb=Bpbs;
}
_mm_sfence();
_mm_sfence();
cmb(Cp,dz,0,0,h, S4M[0],1,S4M[3],1,S4M[4],-1,S4M[6],1);
cmb(Cp,dz,0,o,h, S4M[2],1,S4M[4],1,0,0,0,0);
cmb(Cp,dz,o,0,h, S4M[1],1,S4M[3],1,0,0,0,0);
cmb(Cp,dz,o,o,h, S4M[0],1,S4M[2],1,S4M[1],-1,S4M[5],1);
_mm_sfence();
}
AVX2 static void submm_s5(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp);
AVX2 static void submm_s5(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp){
GLDA = 2*((m>>1)) + PADA; GLDB = (m>>1) + PADB;
const int h=m>>1, o=h;
for (int k=0;k<7;k++){
switch(k){
case 0: buildA_q(h,m,Ap,0,0,Ap,o,o,1); buildB_q(h,m,Bp,0,0,Bp,o,o,1); submm_tail(h,S4M[0]); break;
case 1: buildA_q(h,m,Ap,o,0,Ap,o,o,1); buildB_q(h,m,Bp,0,0,Bp,0,0,0); submm_tail(h,S4M[1]); break;
case 2: buildA_q(h,m,Ap,0,0,Ap,0,0,0); buildB_q(h,m,Bp,0,o,Bp,o,o,2); submm_tail(h,S4M[2]); break;
case 3: buildA_q(h,m,Ap,o,o,Ap,o,o,0); buildB_q(h,m,Bp,o,0,Bp,0,0,2); submm_tail(h,S4M[3]); break;
case 4: buildA_q(h,m,Ap,0,0,Ap,0,o,1); buildB_q(h,m,Bp,o,o,Bp,o,o,0); submm_tail(h,S4M[4]); break;
case 5: buildA_q(h,m,Ap,o,0,Ap,0,0,2); buildB_q(h,m,Bp,0,0,Bp,0,o,1); submm_tail(h,S4M[5]); break;
case 6: buildA_q(h,m,Ap,0,o,Ap,o,o,2); buildB_q(h,m,Bp,o,0,Bp,o,o,1); submm_tail(h,S4M[6]); break;
}
}
_mm_sfence();
cmb(Cp,m,0,0,h, S4M[0],1,S4M[3],1,S4M[4],-1,S4M[6],1);
cmb(Cp,m,0,o,h, S4M[2],1,S4M[4],1,0,0,0,0);
cmb(Cp,m,o,0,h, S4M[1],1,S4M[3],1,0,0,0,0);
cmb(Cp,m,o,o,h, S4M[0],1,S4M[2],1,S4M[1],-1,S4M[5],1);
_mm_sfence();
}
AVX2 static void submm_s4(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp){
GLDA = 2*((m>>1)) + PADA; GLDB = (m>>1) + PADB;
const int h=m>>1, o=h;
for (int k=0;k<7;k++){
switch(k){
case 0: submm_s5f(m,Ap,0,0,o,o,1, Bp,0,0,o,o,1, S3M[0]); break;
case 1: submm_s5f(m,Ap,o,0,o,o,1, Bp,0,0,0,0,0, S3M[1]); break;
case 2: submm_s5f(m,Ap,0,0,0,0,0, Bp,0,o,o,o,2, S3M[2]); break;
case 3: submm_s5f(m,Ap,o,o,o,o,0, Bp,o,0,0,0,2, S3M[3]); break;
case 4: submm_s5f(m,Ap,0,0,0,o,1, Bp,o,o,o,o,0, S3M[4]); break;
case 5: submm_s5f(m,Ap,o,0,0,0,2, Bp,0,0,0,o,1, S3M[5]); break;
case 6: submm_s5f(m,Ap,0,o,o,o,2, Bp,o,0,o,o,1, S3M[6]); break;
}
}
_mm_sfence();
cmb(Cp,m,0,0,h, S3M[0],1,S3M[3],1,S3M[4],-1,S3M[6],1);
cmb(Cp,m,0,o,h, S3M[2],1,S3M[4],1,0,0,0,0);
cmb(Cp,m,o,0,h, S3M[1],1,S3M[3],1,0,0,0,0);
cmb(Cp,m,o,o,h, S3M[0],1,S3M[2],1,S3M[1],-1,S3M[5],1);
_mm_sfence();
}
AVX2 static void submm_s2(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp){
GLDA = 2*((m>>1)) + PADA; GLDB = (m>>1) + PADB;
const int h=m>>1, o=h;
for (int k=0;k<7;k++){
switch(k){
case 0: blk(S3A,Ap,m,0,0,h,Ap,o,o,1); blk(S3B,Bp,m,0,0,h,Bp,o,o,1); submm_s4(h,S3A,S3B,S2M[0]); break;
case 1: blk(S3A,Ap,m,o,0,h,Ap,o,o,1); blk(S3B,Bp,m,0,0,h,Bp,0,0,0); submm_s4(h,S3A,S3B,S2M[1]); break;
case 2: blk(S3A,Ap,m,0,0,h,Ap,0,0,0); blk(S3B,Bp,m,0,o,h,Bp,o,o,2); submm_s4(h,S3A,S3B,S2M[2]); break;
case 3: blk(S3A,Ap,m,o,o,h,Ap,o,o,0); blk(S3B,Bp,m,o,0,h,Bp,0,0,2); submm_s4(h,S3A,S3B,S2M[3]); break;
case 4: blk(S3A,Ap,m,0,0,h,Ap,0,o,1); blk(S3B,Bp,m,o,o,h,Bp,o,o,0); submm_s4(h,S3A,S3B,S2M[4]); break;
case 5: blk(S3A,Ap,m,o,0,h,Ap,0,0,2); blk(S3B,Bp,m,0,0,h,Bp,0,o,1); submm_s4(h,S3A,S3B,S2M[5]); break;
case 6: blk(S3A,Ap,m,0,o,h,Ap,o,o,2); blk(S3B,Bp,m,o,0,h,Bp,o,o,1); submm_s4(h,S3A,S3B,S2M[6]); break;
}
}
_mm_sfence();
cmb(Cp,m,0,0,h, S2M[0],1,S2M[3],1,S2M[4],-1,S2M[6],1);
cmb(Cp,m,0,o,h, S2M[2],1,S2M[4],1,0,0,0,0);
cmb(Cp,m,o,0,h, S2M[1],1,S2M[3],1,0,0,0,0);
cmb(Cp,m,o,o,h, S2M[0],1,S2M[2],1,S2M[1],-1,S2M[5],1);
_mm_sfence();
}
AVX2 static void submm(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp){
if (m>=S2MIN) { submm_s2(m,Ap,Bp,Cp); return; }
submm_plain(m,Ap,Bp,Cp);
}
AVX2 static void strassen1(int n,const int8_t*A,const int8_t*B,int8_t*C){
const int m=n>>1, o=m;
for (int k=0;k<7;k++){
switch(k){
case 0: blk_nt(SA,A,n,0,0,m,A,o,o,1); blk_nt(SB,B,n,0,0,m,B,o,o,1); break;
case 1: blk_nt(SA,A,n,o,0,m,A,o,o,1); blk_nt(SB,B,n,0,0,m,B,0,0,0); break;
case 2: blk_nt(SA,A,n,0,0,m,A,0,0,0); blk_nt(SB,B,n,0,o,m,B,o,o,2); break;
case 3: blk_nt(SA,A,n,o,o,m,A,o,o,0); blk_nt(SB,B,n,o,0,m,B,0,0,2); break;
case 4: blk_nt(SA,A,n,0,0,m,A,0,o,1); blk_nt(SB,B,n,o,o,m,B,o,o,0); break;
case 5: blk_nt(SA,A,n,o,0,m,A,0,0,2); blk_nt(SB,B,n,0,0,m,B,0,o,1); break;
case 6: blk_nt(SA,A,n,0,o,m,A,o,o,2); blk_nt(SB,B,n,o,0,m,B,o,o,1); break;
}
submm(m,SA,SB,MMS[k]);
}
_mm_sfence();
cmb(C,n,0,0,m, MMS[0],1,MMS[3],1,MMS[4],-1,MMS[6],1);
cmb(C,n,0,o,m, MMS[2],1,MMS[4],1,0,0,0,0);
cmb(C,n,o,0,m, MMS[1],1,MMS[3],1,0,0,0,0);
cmb(C,n,o,o,m, MMS[0],1,MMS[2],1,MMS[1],-1,MMS[5],1);
_mm_sfence();
}
AVX2 void matrix_multiply(int n, const int8_t *A, const int8_t *B, int8_t *C){
if (n<=0 || (n&255) || n>NMAX) { generic_mm(n,A,B,C); return; }
if (n==4096) { strassen1(n,A,B,C); return; }
/* x4: A1T is written by build_A1T and NEVER READ -- dead 16.78 MB + its transpose pass. */
/* FUSED: one pass over A (was two), one over B (was two) */
GLDA = 2*(n) + PADA; GLDB = n + PADB;
build_A0A1(n,A); build_BI3Bp(n,B);
if (n==4096) corr16(); else corr_pass3(n);
/* lane CM1: submm_tail-style IB x GB tiling + R1 boustrophedon (see mk_topblk.py) */
{
const int GB=128, IB=64;
for (int ib=0; ib<n; ib+=IB){
const int ihi = (ib+IB<n)? ib+IB : n;
for (int gb=0; gb<n; gb+=GB){
const int ghi = (gb+GB<n)? gb+GB : n;
for (int g=gb; g<ghi; g+=64){
const int8_t *bi = BI3b+(size_t)(g>>6)*(n/2)*128;
int rev=0;
for (int i=ib; i<ihi; i+=2){
const int8_t *ap = A0b+(size_t)i*GLDA;
const uint8_t *pw = (const uint8_t*)(Pbb+(size_t)i*(n>>6)+(g>>6));
if (rev) mikro_u4r(n, ap+(2*n)-16, bi,
C+(size_t)i*n+g, pw, 0, GLDA, (n>>6)*8);
else mikro_u4 (n, ap, bi,
C+(size_t)i*n+g, pw, 0, GLDA, (n>>6)*8);
rev^=1;
}
}
}
}
}
_mm_sfence();
}
| Compilation | N/A | N/A | Compile OK | Score: N/A | 显示更多 |
| Testcase #1 | 380.052 ms | 64 MB + 676 KB | Accepted | Score: 100 | 显示更多 |