/* References:
- saffah_codex_6a_agg3, https://duck.ac/submission/125867: our paired-query six-bit Gray correction combined with the public exact byte matrix engine.
- saffah_cc_v41_agg1, https://duck.ac/submission/123016: blocked operand layout and engine; full inherited references and licensing statements retained below.
Idea: Build the 64-entry XOR table as four independent 16-entry Gray walks, seeded by 0, r4, r5, and r4 XOR r5. This preserves every table entry but shortens the longest dependency chain from 63 to 15 XORs, with four independent register chains. The last four-bit group retains its single walk. Purpose: reduce correction-table construction latency while retaining the proven paired query layout.
*/
/* References:
- saffah_codex_6a_agg3, https://duck.ac/submission/125854: blocked byte engine
with six-bit Gray-code correction. Full inherited notices retained below.
Idea: Query correction tables for two rows together, with two independent
XOR chains per row. Share each table base and combine the chains only at
the final store; the exact parity and table construction remain unchanged.
Experiment purpose: Expose independent lookup accumulation and reduce row overhead.
*/
/* References:
- saffah_cc_v41_agg1, https://duck.ac/submission/123016: copied blocked-layout
byte Strassen engine, padded products and fused reconstruction. Inherited
source citations and notices are retained below; no separate license stated.
- saffah_codex_6a_agg3, https://duck.ac/submission/121665: reused our six-bit
Gray-code parity correction, originally based on the cited nibble engine.
Idea: Combine the newer contiguous operand-block layout with our exact six-bit
correction tables. Eleven vector lookups replace sixteen per64-bit input
mask; Gray-code construction avoids table reads while building each group.
Experiment purpose: Reduce parity-correction work in the new blocked engine.
*/
// ===== REFERENCES (mmc4g_ seat) =====
// [1] duck.ac 用户 saffah_cc_v41_260924，提交 #122947 <https://duck.ac/submission/122947>
// —— 本发的**基座**：对手在册最好件（355.963480 ms，2026-10-03 07:18 提交）。本文件正文与
// 该提交**逐字节相同**，只做下面 `思路` 所述的**两处叠加改动**（其余一字未动）。
// [2] duck.ac 用户 saffah_cc_v41_agg1（本账号），提交 #122557 <https://duck.ac/submission/122557>
// —— 本账号在册最好件（358.964995 ms）。本发所**叠的两把刀**（cmb4/cmb4wb 函数体与四处
// 调用点、七个 M 数组的行距填充值 24576）逐字取自该提交，刀的思想与历史实现见下 [3]。
// [3] 本队 notes `problems/mmmc4k/notes.md`：mmc4a_ 席（cmb4 四象限融合 + 最内层 WB，判题机
// tryrun 实测两刀同加 −1.62 ms / 强超加）· mmc4f_ 席（行距填充 64→24576，judge tryrun
// −3.785 ms、板面兑现 1.8×，扫过 0/2048/4096/8192/16384/24576/32768/2MB+4096）。
// 合规：duck.ac 提交正文按站点规则公开可见、可直接取用；本账号历史提交由本账号自行复用。
// ======================
// ===== 思路 =====
// [mmc4g_ 席] —— 抄件（#122947 逐字节）+ 我方两把刀
// 【入场】mine 358.964995 (#122557) · T 355.963480 (#122947) · 真靶 = 0.99·T+1µs = 352.404845
// ⇒ 缺口 6.560150 ms（1.828%）。宽支 1.005·T = 357.744297 按 `tools/exact.py` 自注
// 「一提交即翻严支」不可达，不作目标。提交前 `/status` 现读 Pending 0 ✓。
//
// 【§2.18.349 第一动作 = 对手取证】取回 #122947 全文，与我方 #122557 逐 token `diff -u -w`：
// 对手件 = 共同祖先（mmmc_k1 系）**+ 对手独有刀**（`blk_nt`：S3A/S3B/S4A/S4B 改 4×4 连续块
// 布局 + `NRB()` 寻址；删最内层 `if(rev) mikro_u4r` 交替），**不含**我方两把刀。
// ⇒ **纯抄上限 = 355.963480 > 真靶 352.404845 ⇒ 抄不够**（仍差 3.56 ms）⇒ 抄 + 叠我方独有刀。
//
// 【本发 = 对手件 + 我方两把刀（唯一变量，正文其余一字未动）】
// ① 七个 M 暂存数组行距填充 `[7][体积+64]` → `[7][体积+24576]`（4 行声明）。cmb4 每级都是
// 「同一 `t` 上载入 M[0..6]」⇒ 七条**同相流**，间距 = 数组行距；`+64` 恰是一个 L1 组步长，
// 七条流锁在同一 DRAM bank/通道带；整 4 KB 倍数 ⇒ 七数组页对齐、翻转物理地址 12 位以上
// （= 通道 XOR 哈希位）。
// ② `cmb4`/`cmb4wb`（四级四象限融合重组）：把每级四个象限的四次 `cmb`（4+2+2+4 = 12 h² 读，
// 产出只有 4 h²）融成一趟「七个 M 各读一遍」（7 h²）；操作数与加减次序逐条不变、store 形态
// 不变（最内层用 WB 孪生版 `cmb4wb`）⇒ 输出逐位不变。4 处调用点整块替换
// （submm_s5f 最内层 / submm_s4 位 / submm_s2 位 / 顶层 submm）。
//
// 【闸门】`work/mmc4g__tmp/g_gate.py`（n=4096、4096B 对齐、全 int8 域 splitmix 驱动）：
// 16,777,216 B **逐字节 IDENTICAL** ✓ · FNV 同 ✓ · 正控（BADSUM）真变 ✓（4 个臂全过）。
//
// 【定价（判题机 tryrun = 立即体验通道，同窗、四臂 chk 全同 = 纯性能）】
// #122947 原样 **429.875 / 429.955**（2 轮）｜只加 ① 427.257 ｜只加 ② 428.189
// ｜**①+② 422.864 / 422.891 = −6.98 ms（−1.625%）** ⇒ **强超加**（单独 −2.70 / −1.77 ms）。
// ★同时判负一条（勿重走）：恢复最内层 `if(rev) mikro_u4r` 交替 = 426.377（+3.49 ms）——
// 对手删它是对的（B 面板 blk_nt 布局下反向走 A/BI 不再获益）。
// ★本批双锚刻度（tryrun → 榜单）：#122947 429.96 ↔ 355.963（×0.828）｜#122557 434.96 ↔
// 358.965（×0.825）⇒ 预期板面 ≈ **349.4 ms ≤ 真靶 352.404845**。
// ================
#pragma GCC optimize("O3")
/* mmmc_k1: int8 GEMM C = A*B mod 256, EXACT, AVX2.
Split A = A0 + 128*A1 (A0 = A & 0x7F unsigned, A1 = top bit of each byte).
term1 = sum_k A0[i][k]*B[k][j] -> vpmaddubsw, no saturation (127*128*2 < 32768)
term2 = 128*(A1 . Bpar) mod 256 -> GF(2) matmul, 64 bits/word (Bpar = B&1)
Layout (lanes = columns): accum lane m is column j0+m.
x = vpbroadcastw of the 2-byte pair A0[i][2p..2p+1]
y = 32B load from BI[p] at 2*j0 ; BI[p][2j]=B[2p][j], BI[p][2j+1]=B[2p+1][j]
*/
#include <immintrin.h>
#include <xmmintrin.h>
#include <stdint.h>
#include <string.h>
#include <stdio.h>
#ifndef PFMODE
#define PFMODE 1 /* MC2: BOTH mikro prefetcht0 pairs DELETED. The BI block is 16 KB and L1-resident;
the prefetches are pure issue overhead. Rig: 4 pf +1.31 %, 2 pf 0.00 %, 0 pf -1.102 %. */
#endif
#if PFMODE==0
#define PF_STR "prefetcht0 512(%[bib],%[bidx])\n\t" "prefetcht0 1024(%[bib],%[bidx])\n\t"
#elif PFMODE==1
#define PF_STR ""
#elif PFMODE==2
#define PF_STR "prefetcht0 512(%[bib],%[bidx])\n\t"
#elif PFMODE==3
#define PF_STR "prefetcht0 256(%[bib],%[bidx])\n\t" "prefetcht0 512(%[bib],%[bidx])\n\t"
#elif PFMODE==4
#define PF_STR "prefetcht0 1024(%[bib],%[bidx])\n\t" "prefetcht0 2048(%[bib],%[bidx])\n\t"
#elif PFMODE==5
#define PF_STR "prefetchnta 512(%[bib],%[bidx])\n\t" "prefetchnta 1024(%[bib],%[bidx])\n\t"
#elif PFMODE==6
#define PF_STR "prefetcht0 512(%[bib],%[bidx])\n\t" "prefetcht0 1024(%[bib],%[bidx])\n\t" \
"prefetcht0 1536(%[bib],%[bidx])\n\t" "prefetcht0 2048(%[bib],%[bidx])\n\t"
#endif
#define AVX2 __attribute__((target("avx2")))
#define NMAX 4096
/* == mmmpad_fleet ARM: SCRATCH ROW-STRIDE PAD ==
A0b and PBb are PRIVATE scratch, so their row stride is not semantic -- but it is `n`,
i.e. 4096 B at n=4096, which is EXACTLY the L1 way-period (64 sets x 64 B). Every row of
both buffers then maps its line j into set j, on top of C's rows (also `n`) and the mikro's
two-row window: 6+ streams share one set out of 8 ways. LDA/LDB pad the two scratch
strides. The cell (row i, col j) is still the same value -- only WHERE it lives changes.
The mikro takes the scratch stride as an argument instead of deriving `A0+n`. */
#ifndef PADA
#define PADA 0
#endif
#define LDA ((int)(2*NMAX + PADA))
#ifndef PADB
#define PADB 0
#endif
#define LDB ((int)(NMAX + PADB))
static int8_t A0b[(size_t)NMAX*LDA] __attribute__((aligned(64)));
/* BW9: runtime scratch stride, sized to the ACTUAL m (not NMAX) */
static int GLDA = LDA, GLDB = LDB;
static uint64_t A1b[NMAX*NMAX/8] __attribute__((aligned(64)));
static uint64_t Bpb[NMAX*NMAX/8] __attribute__((aligned(64)));
static uint64_t Pbb[NMAX*NMAX/8] __attribute__((aligned(64)));
static uint8_t PBb[(size_t)NMAX*LDB] __attribute__((aligned(64)));
static void generic_mm(int n, const int8_t*A, const int8_t*B, int8_t*C){
for (int i=0;i<n;i++) for (int j=0;j<n;j++){
int s=0; for(int k=0;k<n;k++) s += (int)A[(size_t)i*n+k]*(int)B[(size_t)k*n+j];
C[(size_t)i*n+j]=(int8_t)s;
}
}
AVX2 static inline __m256i bcast16(const int8_t*p){
__m256i v; __asm__("vpbroadcastw %1, %0" : "=x"(v) : "m"(*(const short*)p)); return v;
}
AVX2 static inline __m256i packlo8(__m256i a, __m256i b){
const __m256i m=_mm256_set1_epi16(0x00FF);
return _mm256_permute4x64_epi64(
_mm256_packus_epi16(_mm256_and_si256(a,m), _mm256_and_si256(b,m)), 0xD8);
}
/* build_A0() DELETED by lane CH3: dead, and its A0b stride is the pre-duplication one. */
AVX2 static void build_A1bits(int n, const int8_t*A){
int W=n/64;
for (int i=0;i<n;i++){
const int8_t*row=A+(size_t)i*n; uint64_t*w=A1b+(size_t)i*W;
for (int k=0;k<n;k+=64){
uint32_t lo=(uint32_t)_mm256_movemask_epi8(_mm256_load_si256((const __m256i*)(row+k)));
uint32_t hi=(uint32_t)_mm256_movemask_epi8(_mm256_load_si256((const __m256i*)(row+k+32)));
w[k>>6]=(uint64_t)lo|((uint64_t)hi<<32);
}
}
}
AVX2 static void build_Bpbits(int n, const int8_t*B){
const __m256i one=_mm256_set1_epi8(1);
int W=n/64;
for (int k=0;k<n;k++){
const int8_t*row=B+(size_t)k*n; uint64_t*w=Bpb+(size_t)k*W;
for (int j=0;j<n;j+=64){
__m256i a=_mm256_slli_epi16(_mm256_and_si256(_mm256_load_si256((const __m256i*)(row+j)),one),7);
__m256i b=_mm256_slli_epi16(_mm256_and_si256(_mm256_load_si256((const __m256i*)(row+j+32)),one),7);
uint32_t lo=(uint32_t)_mm256_movemask_epi8(a);
uint32_t hi=(uint32_t)_mm256_movemask_epi8(b);
w[j>>6]=(uint64_t)lo|((uint64_t)hi<<32);
}
}
}
/* P[i] = XOR of Bp[k] over k with A1[i][k] set. NY = n/256 ymm per row. */
AVX2 static void corr_bits(int n){
int W=n/64, NY=W>>2;
for (int i=0;i<n;i++){
__m256i acc[16];
for(int t=0;t<NY;t++) acc[t]=_mm256_setzero_si256();
const uint64_t*row=A1b+(size_t)i*W;
for(int w=0;w<W;w++){
uint64_t b=row[w];
while(b){
int t=__builtin_ctzll(b); b&=b-1;
const uint64_t*br=Bpb+(size_t)(w*64+t)*W;
for(int q=0;q<NY;q++)
acc[q]=_mm256_xor_si256(acc[q],_mm256_load_si256((const __m256i*)(br+4*q)));
}
}
uint64_t*o=Pbb+(size_t)i*W;
for(int q=0;q<NY;q++) _mm256_store_si256((__m256i*)(o+4*q),acc[q]);
}
}
static uint64_t A1T[NMAX*NMAX/8] __attribute__((aligned(64)));
AVX2 static void transpose64(uint64_t *a){
int j,k; uint64_t m,t;
for (j=32,m=0xFFFFFFFF00000000ULL; j; j>>=1, m^=m>>j)
for (k=0;k<64;k=(k+j+1)&~j){
t=(a[k]^(a[k+j]<<j))&m;
a[k]^=t; a[k+j]^=(t>>j);
}
}
AVX2 static void build_A1T(int n){
int W=n/64;
for (int kb=0; kb<n; kb+=64)
for (int ib=0; ib<n; ib+=64){
uint64_t blk[64];
for (int i=0;i<64;i++) blk[i]=A1b[(size_t)(ib+i)*W+(kb>>6)];
transpose64(blk);
for (int k=0;k<64;k++) A1T[(size_t)(kb+k)*W+(ib>>6)]=blk[k];
}
}
/* P[i][j] = XOR over k with A1[i][k] set of Bp[k][j].
Per-output-row bit scan (dense 64-bit words -> predictable), K-BLOCKED so the
Bp slice stays in L2. P accumulated in memory across k-blocks. */
AVX2 static void corr_pass3(int n){
int W=n/64, NY=W>>2;
memset(Pbb,0,(size_t)n*W*8);
const int KB=256;
for (int kb=0; kb<n; kb+=KB){
int w0=kb>>6, w1=w0+(KB>>6);
for (int i=0;i<n;i++){
__m256i acc[16];
for (int q=0;q<NY;q++) acc[q]=_mm256_setzero_si256();
const uint64_t*row=A1b+(size_t)i*W;
for (int w=w0;w<w1;w++){
uint64_t b=row[w];
while(b){
int t=__builtin_ctzll(b); b&=b-1;
const uint64_t*br=Bpb+(size_t)(w*64+t)*W;
for (int q=0;q<NY;q++) acc[q]=_mm256_xor_si256(acc[q],_mm256_load_si256((const __m256i*)(br+4*q)));
}
}
uint64_t*o=Pbb+(size_t)i*W;
for (int q=0;q<NY;q++){
__m256i*po=(__m256i*)(o+4*q);
_mm256_store_si256(po,_mm256_xor_si256(_mm256_load_si256((const __m256i*)po),acc[q]));
}
}
}
}
/* specialized for n=4096 (W=64, NY=16): fully unrolled XOR loops */
/* corr16(): fully hand-written per-row GF(2) accumulate. Bit-identical output. */
#define AVX2BMI __attribute__((target("avx2,bmi")))
AVX2BMI static void corr16(void){
const int n=4096, W=64;
memset(Pbb,0,(size_t)n*W*8);
const int KB=64;
for (int kb=0; kb<n; kb+=KB){
int w0=kb>>6;
const char *bp0=(const char*)Bpb+(size_t)w0*32768;
const char *bp1=bp0; (void)bp1;
for (int i=0;i<n;i++){
const uint64_t*row=A1b+(size_t)i*W+w0;
uint64_t b0=row[0], b1=0;
uint64_t *o=Pbb+(size_t)i*W;
asm volatile(
"vpxor %%xmm0,%%xmm0,%%xmm0\n\t"
"vpxor %%xmm1,%%xmm1,%%xmm1\n\t"
"vpxor %%xmm2,%%xmm2,%%xmm2\n\t"
"vpxor %%xmm3,%%xmm3,%%xmm3\n\t"
"vpxor %%xmm4,%%xmm4,%%xmm4\n\t"
"vpxor %%xmm5,%%xmm5,%%xmm5\n\t"
"vpxor %%xmm6,%%xmm6,%%xmm6\n\t"
"vpxor %%xmm7,%%xmm7,%%xmm7\n\t"
"vpxor %%xmm8,%%xmm8,%%xmm8\n\t"
"vpxor %%xmm9,%%xmm9,%%xmm9\n\t"
"vpxor %%xmm10,%%xmm10,%%xmm10\n\t"
"vpxor %%xmm11,%%xmm11,%%xmm11\n\t"
"vpxor %%xmm12,%%xmm12,%%xmm12\n\t"
"vpxor %%xmm13,%%xmm13,%%xmm13\n\t"
"vpxor %%xmm14,%%xmm14,%%xmm14\n\t"
"vpxor %%xmm15,%%xmm15,%%xmm15\n\t"
"testq %[b0], %[b0]\n\t"
"jz 3%=f\n\t"
"1%=:\n\t"
"tzcntq %[b0], %%r11\n\t"
"shlq $9, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"vpxor 64(%%r11), %%ymm2, %%ymm2\n\t"
"vpxor 96(%%r11), %%ymm3, %%ymm3\n\t"
"vpxor 128(%%r11), %%ymm4, %%ymm4\n\t"
"vpxor 160(%%r11), %%ymm5, %%ymm5\n\t"
"vpxor 192(%%r11), %%ymm6, %%ymm6\n\t"
"vpxor 224(%%r11), %%ymm7, %%ymm7\n\t"
"vpxor 256(%%r11), %%ymm8, %%ymm8\n\t"
"vpxor 288(%%r11), %%ymm9, %%ymm9\n\t"
"vpxor 320(%%r11), %%ymm10, %%ymm10\n\t"
"vpxor 352(%%r11), %%ymm11, %%ymm11\n\t"
"vpxor 384(%%r11), %%ymm12, %%ymm12\n\t"
"vpxor 416(%%r11), %%ymm13, %%ymm13\n\t"
"vpxor 448(%%r11), %%ymm14, %%ymm14\n\t"
"vpxor 480(%%r11), %%ymm15, %%ymm15\n\t"
"blsrq %[b0], %[b0]\n\t"
"jnz 1%=b\n\t"
"3%=:\n\t"
"testq %[b1], %[b1]\n\t"
"jz 4%=f\n\t"
"2%=:\n\t"
"tzcntq %[b1], %%r11\n\t"
"shlq $9, %%r11\n\t"
"addq %[bp1], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"vpxor 64(%%r11), %%ymm2, %%ymm2\n\t"
"vpxor 96(%%r11), %%ymm3, %%ymm3\n\t"
"vpxor 128(%%r11), %%ymm4, %%ymm4\n\t"
"vpxor 160(%%r11), %%ymm5, %%ymm5\n\t"
"vpxor 192(%%r11), %%ymm6, %%ymm6\n\t"
"vpxor 224(%%r11), %%ymm7, %%ymm7\n\t"
"vpxor 256(%%r11), %%ymm8, %%ymm8\n\t"
"vpxor 288(%%r11), %%ymm9, %%ymm9\n\t"
"vpxor 320(%%r11), %%ymm10, %%ymm10\n\t"
"vpxor 352(%%r11), %%ymm11, %%ymm11\n\t"
"vpxor 384(%%r11), %%ymm12, %%ymm12\n\t"
"vpxor 416(%%r11), %%ymm13, %%ymm13\n\t"
"vpxor 448(%%r11), %%ymm14, %%ymm14\n\t"
"vpxor 480(%%r11), %%ymm15, %%ymm15\n\t"
"blsrq %[b1], %[b1]\n\t"
"jnz 2%=b\n\t"
"4%=:\n\t"
"vpxor 0(%[o]), %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[o])\n\t"
"vpxor 32(%[o]), %%ymm1, %%ymm1\n\t"
"vmovdqu %%ymm1, 32(%[o])\n\t"
"vpxor 64(%[o]), %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 64(%[o])\n\t"
"vpxor 96(%[o]), %%ymm3, %%ymm3\n\t"
"vmovdqu %%ymm3, 96(%[o])\n\t"
"vpxor 128(%[o]), %%ymm4, %%ymm4\n\t"
"vmovdqu %%ymm4, 128(%[o])\n\t"
"vpxor 160(%[o]), %%ymm5, %%ymm5\n\t"
"vmovdqu %%ymm5, 160(%[o])\n\t"
"vpxor 192(%[o]), %%ymm6, %%ymm6\n\t"
"vmovdqu %%ymm6, 192(%[o])\n\t"
"vpxor 224(%[o]), %%ymm7, %%ymm7\n\t"
"vmovdqu %%ymm7, 224(%[o])\n\t"
"vpxor 256(%[o]), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 256(%[o])\n\t"
"vpxor 288(%[o]), %%ymm9, %%ymm9\n\t"
"vmovdqu %%ymm9, 288(%[o])\n\t"
"vpxor 320(%[o]), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 320(%[o])\n\t"
"vpxor 352(%[o]), %%ymm11, %%ymm11\n\t"
"vmovdqu %%ymm11, 352(%[o])\n\t"
"vpxor 384(%[o]), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 384(%[o])\n\t"
"vpxor 416(%[o]), %%ymm13, %%ymm13\n\t"
"vmovdqu %%ymm13, 416(%[o])\n\t"
"vpxor 448(%[o]), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 448(%[o])\n\t"
"vpxor 480(%[o]), %%ymm15, %%ymm15\n\t"
"vmovdqu %%ymm15, 480(%[o])\n\t"
: [b0]"+r"(b0), [b1]"+r"(b1)
: [bp0]"r"(bp0), [bp1]"r"(bp1), [o]"r"(o)
: "r11","cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15");
}
}
}
AVX2 static void expand_bits(int n){
static uint8_t T[256*8]; static int ti=0;
if(!ti){ for(int v=0;v<256;v++) for(int b=0;b<8;b++) T[v*8+b]=((v>>b)&1)?0x80:0x00; ti=1; }
int W=n/64;
/* MC2: the inner (q,s) double loop ran 8 separate 8-byte load+store pairs per 64-bit Pbb word
-- 8192 store uops per call and ~40 instructions per 64 output bytes. The eight T entries
are contiguous 8-byte blocks, so they assemble into two 32-byte stores: 16 uops per 64 bytes,
SAME BYTES (output byte j = T[(v>>(8*(j/8)))*8 + j%8] by construction). */
for(int i=0;i<n;i++){
const uint64_t*w=Pbb+(size_t)i*W; uint8_t*o=PBb+(size_t)i*GLDB;
for(int q=0;q<W;q++){
uint64_t v=w[q];
__m128i a0=_mm_loadl_epi64((const __m128i*)(T+((v>> 0)&0xFF)*8));
__m128i a1=_mm_loadl_epi64((const __m128i*)(T+((v>> 8)&0xFF)*8));
__m128i a2=_mm_loadl_epi64((const __m128i*)(T+((v>>16)&0xFF)*8));
__m128i a3=_mm_loadl_epi64((const __m128i*)(T+((v>>24)&0xFF)*8));
__m128i a4=_mm_loadl_epi64((const __m128i*)(T+((v>>32)&0xFF)*8));
__m128i a5=_mm_loadl_epi64((const __m128i*)(T+((v>>40)&0xFF)*8));
__m128i a6=_mm_loadl_epi64((const __m128i*)(T+((v>>48)&0xFF)*8));
__m128i a7=_mm_loadl_epi64((const __m128i*)(T+((v>>56)&0xFF)*8));
__m128i l01=_mm_unpacklo_epi64(a0,a1), l23=_mm_unpacklo_epi64(a2,a3);
__m128i l45=_mm_unpacklo_epi64(a4,a5), l67=_mm_unpacklo_epi64(a6,a7);
_mm256_storeu_si256((__m256i*)(o+q*64), _mm256_set_m128i(l23,l01));
_mm256_storeu_si256((__m256i*)(o+q*64+32), _mm256_set_m128i(l67,l45));
}
}
}
#define NIB 2
#define NJ 4
/* MC2: constants for inlining the Pbb -> PBb bit expansion into the mikro epilogue.
PSCTRL replicates each of the four bytes of a dword eight times (per 128-bit lane);
PSBIT isolates bit (j%8) of output byte j; PS80 is the value the set bits expand to.
With S = pshufb(broadcast_dword, PSCTRL): S[j] = byte (j/8) of the dword, so
(S & PSBIT) != 0 <=> bit j of the dword is set <=> PBb byte j must be 0x80. */
static const int8_t PSCTRL[32] __attribute__((aligned(32))) =
{0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3};
static const int8_t PSBIT[32] __attribute__((aligned(32))) =
{1,2,4,8,16,32,64,(int8_t)128, 1,2,4,8,16,32,64,(int8_t)128,
1,2,4,8,16,32,64,(int8_t)128, 1,2,4,8,16,32,64,(int8_t)128};
static const int8_t PS80[32] __attribute__((aligned(32))) =
{(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,
(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,
(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,
(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128,(int8_t)128};
static const int16_t W0FF[16] __attribute__((aligned(32))) =
{0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,
0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF,0x00FF};
static int8_t BI3b[NMAX*NMAX] __attribute__((aligned(64)));
AVX2 static void build_BI3(int n, const int8_t*B){
int ng=n/64;
for (int p=0;p<n/2;p++){
const int8_t *r0=B+(size_t)(2*p)*n, *r1=r0+n;
for (int g=0;g<ng;g++){
const int8_t *a=r0+64*g, *b=r1+64*g;
int8_t *o=BI3b+((size_t)g*(n/2)+p)*128;
for (int t=0;t<64;t+=32){
__m256i va=_mm256_load_si256((const __m256i*)(a+t));
__m256i vb=_mm256_load_si256((const __m256i*)(b+t));
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb);
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20));
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31));
}
}
}
}
AVX2 static void mikro_u4(int n,const int8_t*A0,const int8_t*BI,int8_t*C,const uint8_t*PB,int jb,int lda,int ldb){
const int8_t *x0=A0;
long nn=(long)n;
const int8_t *x1=A0+lda;
const int8_t *bib=BI+(long)(n>>3)*512; long bidx=-((long)(n>>3)*512);
int8_t *C0=C+jb, *C1=C0+n;
const uint8_t *PB0=PB+jb, *PB1=PB0+ldb;
asm volatile(
"vpxor %%xmm8,%%xmm8,%%xmm8\n\t"
"vpxor %%xmm9,%%xmm9,%%xmm9\n\t"
"vpxor %%xmm10,%%xmm10,%%xmm10\n\t"
"vpxor %%xmm11,%%xmm11,%%xmm11\n\t"
"vpxor %%xmm12,%%xmm12,%%xmm12\n\t"
"vpxor %%xmm13,%%xmm13,%%xmm13\n\t"
"vpxor %%xmm14,%%xmm14,%%xmm14\n\t"
"vpxor %%xmm15,%%xmm15,%%xmm15\n\t"
".p2align 5\n\t"
"testq $8, %[nn]\n\t"
"jz 5f\n\t"
"vpbroadcastd 0(%[x0]), %%ymm0\n\t"
"vpbroadcastd 0(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 0(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 32(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 64(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[x0]), %%ymm0\n\t"
"vpbroadcastd 4(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 160(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 192(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 8(%[x0]), %%ymm0\n\t"
"vpbroadcastd 8(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 288(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 320(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[x0]), %%ymm0\n\t"
"vpbroadcastd 12(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 416(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 448(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
PF_STR
"addq $16, %[x0]\n\t"
"addq $512, %[bidx]\n\t"
"5:\n\t"
"testq %[bidx], %[bidx]\n\t"
"jz 9f\n\t"
".p2align 5\n\t"
"2:\n\t"
"vpbroadcastd 0(%[x0]), %%ymm0\n\t"
"vpbroadcastd 0(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 0(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 32(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 64(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[x0]), %%ymm0\n\t"
"vpbroadcastd 4(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 160(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 192(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 8(%[x0]), %%ymm0\n\t"
"vpbroadcastd 8(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 288(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 320(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[x0]), %%ymm0\n\t"
"vpbroadcastd 12(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 416(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 448(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 16(%[x0]), %%ymm0\n\t"
"vpbroadcastd 16(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 512(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 544(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 576(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 608(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 20(%[x0]), %%ymm0\n\t"
"vpbroadcastd 20(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 640(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 672(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 704(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 736(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 24(%[x0]), %%ymm0\n\t"
"vpbroadcastd 24(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 768(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 800(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 832(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 864(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 28(%[x0]), %%ymm0\n\t"
"vpbroadcastd 28(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 896(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 928(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 960(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 992(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
PF_STR
"addq $32, %[x0]\n\t"
"addq $1024, %[bidx]\n\t"
"jnz 2b\n\t"
"9:\n\t"
"vpand %[mask], %%ymm8, %%ymm2\n\t"
"vpand %[mask], %%ymm9, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpxor %%xmm7, %%xmm7, %%xmm7\n\t"
"vpbroadcastd 0(%[PB0]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 0(%[C0])\n\t"
"vpand %[mask], %%ymm10, %%ymm2\n\t"
"vpand %[mask], %%ymm11, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 4(%[PB0]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 32(%[C0])\n\t"
"vpand %[mask], %%ymm12, %%ymm2\n\t"
"vpand %[mask], %%ymm13, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 0(%[PB1]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 0(%[C1])\n\t"
"vpand %[mask], %%ymm14, %%ymm2\n\t"
"vpand %[mask], %%ymm15, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 4(%[PB1]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 32(%[C1])\n\t"
: [x0]"+r"(x0), [bidx]"+r"(bidx)
: [C0]"r"(C0), [C1]"r"(C1), [PB0]"r"(PB0), [PB1]"r"(PB1), [nn]"r"(nn), [mask]"m"(W0FF), [bib]"r"(bib), [psctrl]"m"(PSCTRL), [psbit]"m"(PSBIT), [ps80]"m"(PS80), [glda]"r"((long)lda)
: "ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15","cc","memory");
}
AVX2 static void mikro_u4r(int n,const int8_t*A0,const int8_t*BI,int8_t*C,const uint8_t*PB,int jb,int lda,int ldb){
const int8_t *x0=A0;
long nn=(long)n;
const int8_t *x1=A0+lda;
const int8_t *bib=BI; /* R1: slice START -- see mk_rev.py */ long bidx=((long)(n>>3)*512)-512; /* R1: start at the LAST 512-byte chunk */
int8_t *C0=C+jb, *C1=C0+n;
const uint8_t *PB0=PB+jb, *PB1=PB0+ldb;
asm volatile(
"vpxor %%xmm8,%%xmm8,%%xmm8\n\t"
"vpxor %%xmm9,%%xmm9,%%xmm9\n\t"
"vpxor %%xmm10,%%xmm10,%%xmm10\n\t"
"vpxor %%xmm11,%%xmm11,%%xmm11\n\t"
"vpxor %%xmm12,%%xmm12,%%xmm12\n\t"
"vpxor %%xmm13,%%xmm13,%%xmm13\n\t"
"vpxor %%xmm14,%%xmm14,%%xmm14\n\t"
"vpxor %%xmm15,%%xmm15,%%xmm15\n\t"
".p2align 5\n\t"
"testq $8, %[nn]\n\t"
"jz 5f\n\t"
"vpbroadcastd 0(%[x0]), %%ymm0\n\t"
"vpbroadcastd 0(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 0(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 32(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 64(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[x0]), %%ymm0\n\t"
"vpbroadcastd 4(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 160(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 192(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 8(%[x0]), %%ymm0\n\t"
"vpbroadcastd 8(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 288(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 320(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[x0]), %%ymm0\n\t"
"vpbroadcastd 12(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 416(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 448(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
PF_STR
"subq $16, %[x0]\n\t"
"subq $512, %[bidx]\n\t"
"5:\n\t"
"testq %[bidx], %[bidx]\n\t"
"jz 9f\n\t"
".p2align 5\n\t"
"2:\n\t"
"vpbroadcastd 0(%[x0]), %%ymm0\n\t"
"vpbroadcastd 0(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 0(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 32(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 64(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[x0]), %%ymm0\n\t"
"vpbroadcastd 4(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 160(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 192(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 8(%[x0]), %%ymm0\n\t"
"vpbroadcastd 8(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 288(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 320(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[x0]), %%ymm0\n\t"
"vpbroadcastd 12(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa 384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa 416(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa 448(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa 480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd -16(%[x0]), %%ymm0\n\t"
"vpbroadcastd -16(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa -512(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa -480(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa -448(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa -416(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd -12(%[x0]), %%ymm0\n\t"
"vpbroadcastd -12(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa -384(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa -352(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa -320(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa -288(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd -8(%[x0]), %%ymm0\n\t"
"vpbroadcastd -8(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa -256(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa -224(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa -192(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa -160(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
"vpbroadcastd -4(%[x0]), %%ymm0\n\t"
"vpbroadcastd -4(%[x0],%[glda]), %%ymm1\n\t"
"vmovdqa -128(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm6, %%ymm12, %%ymm12\n\t"
"vmovdqa -96(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddw %%ymm6, %%ymm13, %%ymm13\n\t"
"vmovdqa -64(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm10, %%ymm10\n\t"
"vpaddw %%ymm6, %%ymm14, %%ymm14\n\t"
"vmovdqa -32(%[bib],%[bidx]), %%ymm4\n\t"
"vpmaddubsw %%ymm4, %%ymm0, %%ymm5\n\t"
"vpmaddubsw %%ymm4, %%ymm1, %%ymm6\n\t"
"vpaddw %%ymm5, %%ymm11, %%ymm11\n\t"
"vpaddw %%ymm6, %%ymm15, %%ymm15\n\t"
PF_STR
"subq $32, %[x0]\n\t"
"subq $1024, %[bidx]\n\t"
"jns 2b\n\t"
"9:\n\t"
"vpand %[mask], %%ymm8, %%ymm2\n\t"
"vpand %[mask], %%ymm9, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 0(%[PB0]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vpxor %%xmm7, %%xmm7, %%xmm7\n\t"
"vmovdqu %%ymm2, 0(%[C0])\n\t"
"vpand %[mask], %%ymm10, %%ymm2\n\t"
"vpand %[mask], %%ymm11, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 4(%[PB0]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 32(%[C0])\n\t"
"vpand %[mask], %%ymm12, %%ymm2\n\t"
"vpand %[mask], %%ymm13, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 0(%[PB1]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 0(%[C1])\n\t"
"vpand %[mask], %%ymm14, %%ymm2\n\t"
"vpand %[mask], %%ymm15, %%ymm3\n\t"
"vpackuswb %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $0xD8, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 4(%[PB1]), %%ymm0\n\t"
"vpshufb %[psctrl], %%ymm0, %%ymm0\n\t"
"vpand %[psbit], %%ymm0, %%ymm0\n\t"
"vpcmpeqb %%ymm7, %%ymm0, %%ymm0\n\t"
"vpandn %[ps80], %%ymm0, %%ymm0\n\t"
"vpxor %%ymm0, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 32(%[C1])\n\t"
: [x0]"+r"(x0), [bidx]"+r"(bidx)
: [C0]"r"(C0), [C1]"r"(C1), [PB0]"r"(PB0), [PB1]"r"(PB1), [nn]"r"(nn), [mask]"m"(W0FF), [bib]"r"(bib), [psctrl]"m"(PSCTRL), [psbit]"m"(PSBIT), [ps80]"m"(PS80), [glda]"r"((long)lda)
: "ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15","cc","memory");
}
/* FUSED BUILDERS (ported from mmmc2k, pure-n): one pass over A, one over B */
AVX2 static void build_A0A1(int n, const int8_t*A){
const __m256i m=_mm256_set1_epi8(0x7F);
int W=n/64;
for (int i=0;i<n;i++){
const int8_t*row=A+(size_t)i*n;
int8_t*dst=A0b+(size_t)i*GLDA;
uint64_t*w1=A1b+(size_t)i*W;
for (int k=0;k<n;k+=64){
__m256i v0=_mm256_load_si256((const __m256i*)(row+k));
__m256i v1=_mm256_load_si256((const __m256i*)(row+k+32));
{ __m256i _w0=_mm256_and_si256(v0,m), _w1=_mm256_and_si256(v1,m);
__m256i _l0=_mm256_unpacklo_epi16(_w0,_w0), _h0=_mm256_unpackhi_epi16(_w0,_w0);
__m256i _l1=_mm256_unpacklo_epi16(_w1,_w1), _h1=_mm256_unpackhi_epi16(_w1,_w1);
_mm256_store_si256((__m256i*)(dst+2*k), _mm256_permute2x128_si256(_l0,_h0,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+32), _mm256_permute2x128_si256(_l0,_h0,0x31));
_mm256_store_si256((__m256i*)(dst+2*k+64), _mm256_permute2x128_si256(_l1,_h1,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+96), _mm256_permute2x128_si256(_l1,_h1,0x31)); }
w1[k>>6]=(uint64_t)(uint32_t)_mm256_movemask_epi8(v0)
|((uint64_t)(uint32_t)_mm256_movemask_epi8(v1)<<32);
}
}
}
AVX2 static void build_BI3Bp(int n, const int8_t*B){
const __m256i one=_mm256_set1_epi8(1);
int ng=n/64, W=n/64;
for (int p=0;p<n/2;p++){
const int8_t *r0=B+(size_t)(2*p)*n, *r1=r0+n;
uint64_t *w0p=Bpb+(size_t)(2*p)*W, *w1p=Bpb+(size_t)(2*p+1)*W;
for (int g=0;g<ng;g++){
const int8_t *a=r0+64*g, *b=r1+64*g;
int8_t *o=BI3b+((size_t)g*(n/2)+p)*128;
for (int t=0;t<64;t+=32){
__m256i va=_mm256_load_si256((const __m256i*)(a+t));
__m256i vb=_mm256_load_si256((const __m256i*)(b+t));
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb);
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20));
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31));
uint32_t m0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(va,one),7));
uint32_t m1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(vb,one),7));
if(t==0) { w0p[g]=(uint64_t)m0; w1p[g]=(uint64_t)m1; }
else { w0p[g]|=(uint64_t)m0<<32; w1p[g]|=(uint64_t)m1<<32; }
}
}
}
}
/* ================= STRASSEN (exact over Z/256) =================
Every coefficient is +-1 and there is no division, so the identities hold in ANY commutative
ring; int8 arithmetic mod 256 is exactly that ring. The case table and the combine
coefficients are GENERATED from tables that mkstrassen4k.py's self-check brute-forces over
Z/256 before this file is written.
============================================================== */
#define SM 2048
static int8_t SA[SM*SM] __attribute__((aligned(64)));
static int8_t SB[SM*SM] __attribute__((aligned(64)));
static int8_t MMS[7][SM*SM+24576] __attribute__((aligned(64)));
/* ============ FUSED QUADRANT PACKERS (--fuse) ============
The unfused path builds S2A/S2B with blk() and hands them to build_A0A1/build_BI3Bp, which
read them straight back: 2 MB written and 2 MB re-read per sub-product, x49. These write the
PACKED forms DIRECTLY from the parent's quadrants. Every stored value is identical by
construction, so this is a pure traffic deletion, not a re-derivation.
buildA_q/buildB_q(int h,int n,X,r0,c0,Y,r1,c1,op) == pack(X quad op Y quad)
========================================================= */
/* ===== FE5 FUSION: 4-TERM PACKERS =====
buildA_q4/buildB_q4 = buildA_q/buildB_q with THREE EXTRA OPTIONAL TERMS. Term 0's coefficient is +1
by construction; terms 1..3 carry s1,s2,s3 in {0,+1,-1} and a 0 SKIPS the load.
WHY THE +-2 COLLISION IS NOT A SPECIAL CASE: v is computed AS int8 and only THEN split (A0b <- v&0x7F,
A1b <- sign bits), so the split is downstream of the sum; a repeated quadrant is just a second
add/sub and int8 mod 256 is an exact ring. THE PACKING BODY IS THE ORIGINAL'S, VERBATIM. */
/* ===== lane c4k1_rv: SIGN-SPECIALISED PACKERS =====================================
The three term signs are template parameters, so `if (S)` is a FOLDED CONSTANT: the
presence test, the add/sub select, and the loads and adds of the ABSENT terms all
disappear from the innermost k-loop. Only the nine triples MEASURED to occur at
n = 4096 are instantiated; the generic original is kept as the fallback so the
transform is correct for every input, not just the judged one. */
#define AQT(J,S) if (S){ __m256i _u0=_mm256_load_si256((const __m256i*)(p##J+k)); \
__m256i _u1=_mm256_load_si256((const __m256i*)(p##J+k+32)); \
if ((S)>0){ v0=_mm256_add_epi8(v0,_u0); v1=_mm256_add_epi8(v1,_u1); } \
else { v0=_mm256_sub_epi8(v0,_u0); v1=_mm256_sub_epi8(v1,_u1); } }
#define BQT(J,S) if (S){ __m256i _x=_mm256_load_si256((const __m256i*)(q##J##a+64*g+t)); \
__m256i _y=_mm256_load_si256((const __m256i*)(q##J##b+64*g+t)); \
if ((S)>0){ va=_mm256_add_epi8(va,_x); vb=_mm256_add_epi8(vb,_y); } \
else { va=_mm256_sub_epi8(va,_x); vb=_mm256_sub_epi8(vb,_y); } }
template<int S1,int S2,int S3>
AVX2 static void buildA_q4t(int h,int n,const int8_t*P,
int r0,int c0,int r1,int c1,int r2,int c2,int r3,int c3){
const __m256i mx=_mm256_set1_epi8(0x7F);
int W=h/64;
for (int i=0;i<h;i++){
const int8_t*p0=P+(size_t)(r0+i)*n+c0, *p1=P+(size_t)(r1+i)*n+c1,
*p2=P+(size_t)(r2+i)*n+c2, *p3=P+(size_t)(r3+i)*n+c3;
int8_t*dst=A0b+(size_t)i*GLDA;
uint64_t*w1=A1b+(size_t)i*W;
for (int k=0;k<h;k+=64){
__m256i v0=_mm256_load_si256((const __m256i*)(p0+k));
__m256i v1=_mm256_load_si256((const __m256i*)(p0+k+32));
AQT(1,S1); AQT(2,S2); AQT(3,S3);
{ __m256i _w0=_mm256_and_si256(v0,mx), _w1=_mm256_and_si256(v1,mx);
__m256i _l0=_mm256_unpacklo_epi16(_w0,_w0), _h0=_mm256_unpackhi_epi16(_w0,_w0);
__m256i _l1=_mm256_unpacklo_epi16(_w1,_w1), _h1=_mm256_unpackhi_epi16(_w1,_w1);
_mm256_store_si256((__m256i*)(dst+2*k), _mm256_permute2x128_si256(_l0,_h0,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+32), _mm256_permute2x128_si256(_l0,_h0,0x31));
_mm256_store_si256((__m256i*)(dst+2*k+64), _mm256_permute2x128_si256(_l1,_h1,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+96), _mm256_permute2x128_si256(_l1,_h1,0x31)); }
w1[k>>6]=(uint64_t)(uint32_t)_mm256_movemask_epi8(v0)
|((uint64_t)(uint32_t)_mm256_movemask_epi8(v1)<<32);
}
}
}
template<int S1,int S2,int S3>
AVX2 static void buildB_q4t(int h,int n,const int8_t*P,
int r0,int c0,int r1,int c1,int r2,int c2,int r3,int c3){
const __m256i one=_mm256_set1_epi8(1);
int ng=h/64, W=h/64;
for (int p=0;p<h/2;p++){
const int8_t *a0=P+(size_t)(r0+2*p)*n+c0, *a1=P+(size_t)(r0+2*p+1)*n+c0;
const int8_t *q1a=P+(size_t)(r1+2*p)*n+c1, *q1b=P+(size_t)(r1+2*p+1)*n+c1;
const int8_t *q2a=P+(size_t)(r2+2*p)*n+c2, *q2b=P+(size_t)(r2+2*p+1)*n+c2;
const int8_t *q3a=P+(size_t)(r3+2*p)*n+c3, *q3b=P+(size_t)(r3+2*p+1)*n+c3;
uint64_t *w0p=Bpb+(size_t)(2*p)*W, *w1p=Bpb+(size_t)(2*p+1)*W;
for (int g=0;g<ng;g++){
for (int t=0;t<64;t+=32){
__m256i va=_mm256_load_si256((const __m256i*)(a0+64*g+t));
__m256i vb=_mm256_load_si256((const __m256i*)(a1+64*g+t));
BQT(1,S1); BQT(2,S2); BQT(3,S3);
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb);
int8_t *o=BI3b+((size_t)g*(h/2)+p)*128;
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20));
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31));
uint32_t mm0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(va,one),7));
uint32_t mm1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(vb,one),7));
if(t==0){ w0p[g]=(uint64_t)mm0; w1p[g]=(uint64_t)mm1; }
else { w0p[g]|=(uint64_t)mm0<<32; w1p[g]|=(uint64_t)mm1<<32; }
}
}
}
}
#define AQ_TERM(J) if (s##J){ __m256i _u0=_mm256_load_si256((const __m256i*)(p##J+k)); \
__m256i _u1=_mm256_load_si256((const __m256i*)(p##J+k+32)); \
if (s##J>0){ v0=_mm256_add_epi8(v0,_u0); v1=_mm256_add_epi8(v1,_u1); } \
else { v0=_mm256_sub_epi8(v0,_u0); v1=_mm256_sub_epi8(v1,_u1); } }
#define BQ_TERM(J) if (s##J){ __m256i _x=_mm256_load_si256((const __m256i*)(q##J##a+64*g+t)); \
__m256i _y=_mm256_load_si256((const __m256i*)(q##J##b+64*g+t)); \
if (s##J>0){ va=_mm256_add_epi8(va,_x); vb=_mm256_add_epi8(vb,_y); } \
else { va=_mm256_sub_epi8(va,_x); vb=_mm256_sub_epi8(vb,_y); } }
AVX2 static void buildA_q(int h,int n,const int8_t*X,int r0,int c0,
const int8_t*Y,int r1,int c1,int op){
const __m256i mx=_mm256_set1_epi8(0x7F);
int W=h/64;
for (int i=0;i<h;i++){
const int8_t*xa=X+(size_t)(r0+i)*n+c0, *ya=Y+(size_t)(r1+i)*n+c1;
int8_t*dst=A0b+(size_t)i*GLDA;
uint64_t*w1=A1b+(size_t)i*W;
for (int k=0;k<h;k+=64){
__m256i v0=_mm256_load_si256((const __m256i*)(xa+k));
__m256i v1=_mm256_load_si256((const __m256i*)(xa+k+32));
if (op){ __m256i u0=_mm256_load_si256((const __m256i*)(ya+k));
__m256i u1=_mm256_load_si256((const __m256i*)(ya+k+32));
if(op==1){ v0=_mm256_add_epi8(v0,u0); v1=_mm256_add_epi8(v1,u1); }
else { v0=_mm256_sub_epi8(v0,u0); v1=_mm256_sub_epi8(v1,u1); } }
{ __m256i _w0=_mm256_and_si256(v0,mx), _w1=_mm256_and_si256(v1,mx);
__m256i _l0=_mm256_unpacklo_epi16(_w0,_w0), _h0=_mm256_unpackhi_epi16(_w0,_w0);
__m256i _l1=_mm256_unpacklo_epi16(_w1,_w1), _h1=_mm256_unpackhi_epi16(_w1,_w1);
_mm256_store_si256((__m256i*)(dst+2*k), _mm256_permute2x128_si256(_l0,_h0,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+32), _mm256_permute2x128_si256(_l0,_h0,0x31));
_mm256_store_si256((__m256i*)(dst+2*k+64), _mm256_permute2x128_si256(_l1,_h1,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+96), _mm256_permute2x128_si256(_l1,_h1,0x31)); }
w1[k>>6]=(uint64_t)(uint32_t)_mm256_movemask_epi8(v0)
|((uint64_t)(uint32_t)_mm256_movemask_epi8(v1)<<32);
}
}
}
AVX2 static void buildB_q(int h,int n,const int8_t*X,int r0,int c0,
const int8_t*Y,int r1,int c1,int op){
const __m256i one=_mm256_set1_epi8(1);
int ng=h/64, W=h/64;
for (int p=0;p<h/2;p++){
const int8_t *ra=X+(size_t)(r0+2*p)*n+c0, *rb=X+(size_t)(r0+2*p+1)*n+c0;
const int8_t *sa=Y+(size_t)(r1+2*p)*n+c1, *sb=Y+(size_t)(r1+2*p+1)*n+c1;
uint64_t *w0p=Bpb+(size_t)(2*p)*W, *w1p=Bpb+(size_t)(2*p+1)*W;
for (int g=0;g<ng;g++){
for (int t=0;t<64;t+=32){
__m256i va=_mm256_load_si256((const __m256i*)(ra+64*g+t));
__m256i vb=_mm256_load_si256((const __m256i*)(rb+64*g+t));
if (op){ __m256i pa=_mm256_load_si256((const __m256i*)(sa+64*g+t));
__m256i pb=_mm256_load_si256((const __m256i*)(sb+64*g+t));
if(op==1){ va=_mm256_add_epi8(va,pa); vb=_mm256_add_epi8(vb,pb); }
else { va=_mm256_sub_epi8(va,pa); vb=_mm256_sub_epi8(vb,pb); } }
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb);
int8_t *o=BI3b+((size_t)g*(h/2)+p)*128;
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20));
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31));
uint32_t mm0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(va,one),7));
uint32_t mm1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(vb,one),7));
if(t==0){ w0p[g]=(uint64_t)mm0; w1p[g]=(uint64_t)mm1; }
else { w0p[g]|=(uint64_t)mm0<<32; w1p[g]|=(uint64_t)mm1<<32; }
}
}
}
}
/* ---- LEVEL 2 scratch: the m x m sub-product is itself Strassen-decomposed into 7 of h=m/2 ---- */
static int8_t S2M[7][1024*1024+24576] __attribute__((aligned(64)));
/* ===== FE5 DEPTH-3 SCRATCH (base.cpp:929-930: "If a deeper level is ever wanted, it needs its
own scratch, not the shared pool.") S3A/S3B are the PLAIN h x h operands that the fused
packers cannot expose (they only ever write PACKED forms), so blk() must materialise them.
SA/SB are NOT reusable -- strassen1 holds them live across the submm call.
1,048,576 + 1,048,576 + 7*(512*512+64) = 3,932,608 B = 3840.44 KB -> mem_kb +3844 KB ===== */
static int8_t S3A[1024*1024] __attribute__((aligned(64)));
static int8_t S3B[1024*1024] __attribute__((aligned(64)));
static int8_t S3M[7][512*512+24576] __attribute__((aligned(64)));
/* FE5 DEPTH-4 SCRATCH. S4A/S4B are the PLAIN 512x512 operands that submm_s5 must subdivide
(submm_s4's fused packers only write PACKED forms, so blk() must materialise them -- same
structural reason as level 3). A0b/BI3b/PBb need NO new room: they are sized for NMAX=4096 and
are consumed by submm_tail right after the packers write them. */
static int8_t S4A[512*512] __attribute__((aligned(64)));
static int8_t S4B[512*512] __attribute__((aligned(64)));
static int8_t S4M[7][256*256+24576] __attribute__((aligned(64)));
/* Cblk += sum_k s_k * M_k (up to 4 terms; NULL skips a term). dst stride is `n`, block is m x m. */
AVX2 static void buildA_q4g(int h,int n,const int8_t*P,
int r0,int c0,int r1,int c1,int s1,
int r2,int c2,int s2, int r3,int c3,int s3){
const __m256i mx=_mm256_set1_epi8(0x7F);
int W=h/64;
for (int i=0;i<h;i++){
const int8_t*p0=P+(size_t)(r0+i)*n+c0, *p1=P+(size_t)(r1+i)*n+c1,
*p2=P+(size_t)(r2+i)*n+c2, *p3=P+(size_t)(r3+i)*n+c3;
int8_t*dst=A0b+(size_t)i*GLDA;
uint64_t*w1=A1b+(size_t)i*W;
for (int k=0;k<h;k+=64){
__m256i v0=_mm256_load_si256((const __m256i*)(p0+k));
__m256i v1=_mm256_load_si256((const __m256i*)(p0+k+32));
AQ_TERM(1); AQ_TERM(2); AQ_TERM(3);
{ __m256i _w0=_mm256_and_si256(v0,mx), _w1=_mm256_and_si256(v1,mx);
__m256i _l0=_mm256_unpacklo_epi16(_w0,_w0), _h0=_mm256_unpackhi_epi16(_w0,_w0);
__m256i _l1=_mm256_unpacklo_epi16(_w1,_w1), _h1=_mm256_unpackhi_epi16(_w1,_w1);
_mm256_store_si256((__m256i*)(dst+2*k), _mm256_permute2x128_si256(_l0,_h0,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+32), _mm256_permute2x128_si256(_l0,_h0,0x31));
_mm256_store_si256((__m256i*)(dst+2*k+64), _mm256_permute2x128_si256(_l1,_h1,0x20));
_mm256_store_si256((__m256i*)(dst+2*k+96), _mm256_permute2x128_si256(_l1,_h1,0x31)); }
w1[k>>6]=(uint64_t)(uint32_t)_mm256_movemask_epi8(v0)
|((uint64_t)(uint32_t)_mm256_movemask_epi8(v1)<<32);
}
}
}
AVX2 static void buildA_q4(int h,int n,const int8_t*P,
int r0,int c0,int r1,int c1,int s1,
int r2,int c2,int s2, int r3,int c3,int s3){
#define PACKA(S1,S2,S3) return buildA_q4t<S1,S2,S3>(h,n,P,r0,c0,r1,c1,r2,c2,r3,c3)
if (s1==-1&&s2==-1&&s3== 1){ PACKA(-1,-1, 1); }
if (s1==-1&&s2== 0&&s3== 0){ PACKA(-1, 0, 0); }
if (s1==-1&&s2== 1&&s3==-1){ PACKA(-1, 1,-1); }
if (s1== 0&&s2==-1&&s3== 0){ PACKA( 0,-1, 0); }
if (s1== 0&&s2== 0&&s3== 0){ PACKA( 0, 0, 0); }
if (s1== 0&&s2== 1&&s3== 0){ PACKA( 0, 1, 0); }
if (s1== 1&&s2==-1&&s3==-1){ PACKA( 1,-1,-1); }
if (s1== 1&&s2== 0&&s3== 0){ PACKA( 1, 0, 0); }
if (s1== 1&&s2== 1&&s3== 1){ PACKA( 1, 1, 1); }
buildA_q4g(h,n,P,r0,c0,r1,c1,s1,r2,c2,s2,r3,c3,s3);
}
AVX2 static void buildB_q4g(int h,int n,const int8_t*P,
int r0,int c0,int r1,int c1,int s1,
int r2,int c2,int s2, int r3,int c3,int s3){
const __m256i one=_mm256_set1_epi8(1);
int ng=h/64, W=h/64;
for (int p=0;p<h/2;p++){
const int8_t *a0=P+(size_t)(r0+2*p)*n+c0, *a1=P+(size_t)(r0+2*p+1)*n+c0;
const int8_t *q1a=P+(size_t)(r1+2*p)*n+c1, *q1b=P+(size_t)(r1+2*p+1)*n+c1;
const int8_t *q2a=P+(size_t)(r2+2*p)*n+c2, *q2b=P+(size_t)(r2+2*p+1)*n+c2;
const int8_t *q3a=P+(size_t)(r3+2*p)*n+c3, *q3b=P+(size_t)(r3+2*p+1)*n+c3;
uint64_t *w0p=Bpb+(size_t)(2*p)*W, *w1p=Bpb+(size_t)(2*p+1)*W;
for (int g=0;g<ng;g++){
for (int t=0;t<64;t+=32){
__m256i va=_mm256_load_si256((const __m256i*)(a0+64*g+t));
__m256i vb=_mm256_load_si256((const __m256i*)(a1+64*g+t));
BQ_TERM(1); BQ_TERM(2); BQ_TERM(3);
__m256i lo=_mm256_unpacklo_epi8(va,vb), hi=_mm256_unpackhi_epi8(va,vb);
int8_t *o=BI3b+((size_t)g*(h/2)+p)*128;
_mm256_store_si256((__m256i*)(o+2*t), _mm256_permute2x128_si256(lo,hi,0x20));
_mm256_store_si256((__m256i*)(o+2*t+32), _mm256_permute2x128_si256(lo,hi,0x31));
uint32_t mm0=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(va,one),7));
uint32_t mm1=(uint32_t)_mm256_movemask_epi8(_mm256_slli_epi16(_mm256_and_si256(vb,one),7));
if(t==0){ w0p[g]=(uint64_t)mm0; w1p[g]=(uint64_t)mm1; }
else { w0p[g]|=(uint64_t)mm0<<32; w1p[g]|=(uint64_t)mm1<<32; }
}
}
}
}
AVX2 static void buildB_q4(int h,int n,const int8_t*P,
int r0,int c0,int r1,int c1,int s1,
int r2,int c2,int s2, int r3,int c3,int s3){
#define PACKB(S1,S2,S3) return buildB_q4t<S1,S2,S3>(h,n,P,r0,c0,r1,c1,r2,c2,r3,c3)
if (s1==-1&&s2==-1&&s3== 1){ PACKB(-1,-1, 1); }
if (s1==-1&&s2== 0&&s3== 0){ PACKB(-1, 0, 0); }
if (s1==-1&&s2== 1&&s3==-1){ PACKB(-1, 1,-1); }
if (s1== 0&&s2==-1&&s3== 0){ PACKB( 0,-1, 0); }
if (s1== 0&&s2== 0&&s3== 0){ PACKB( 0, 0, 0); }
if (s1== 0&&s2== 1&&s3== 0){ PACKB( 0, 1, 0); }
if (s1== 1&&s2==-1&&s3==-1){ PACKB( 1,-1,-1); }
if (s1== 1&&s2== 0&&s3== 0){ PACKB( 1, 0, 0); }
if (s1== 1&&s2== 1&&s3== 1){ PACKB( 1, 1, 1); }
buildB_q4g(h,n,P,r0,c0,r1,c1,s1,r2,c2,s2,r3,c3,s3);
}
/* ===== mmc4a: FUSED QUADRANT RECOMBINATION (cmb4) =====================================
The four quadrant recombines of one Strassen level each re-read their own subset of the
same seven M arrays: (0,0)=M0+M3-M4+M6, (0,h)=M2+M4, (h,0)=M1+M3, (h,h)=M0+M2-M1+M5
=> 4+2+2+4 = 12 h^2 of reads for 4 h^2 of output. Reading all seven ONCE per 32-byte
output chunk and writing the four quadrants from the same registers costs 7 h^2.
Operands and the add/sub order are IDENTICAL to the four `cmb` calls it replaces, and
the stores stay `_mm256_stream_si256`, so the output is byte-identical.
(Same knife the fleet shipped in the #105148 generation; re-priced on this base because
a reading belongs to a generation -- BRIEF 2.18.349.) */
template<int Q>
AVX2 static void cmb4(int8_t *C,int n,int h,const int8_t (&M)[7][Q]){
for(int i=0;i<h;i++){
int8_t *r0=C+(size_t)i*n, *r1=C+(size_t)(i+h)*n;
for(int j=0;j<h;j+=32){
size_t t=(size_t)i*h+j;
const __m256i m0=_mm256_load_si256((const __m256i*)(M[0]+t));
const __m256i m1=_mm256_load_si256((const __m256i*)(M[1]+t));
const __m256i m2=_mm256_load_si256((const __m256i*)(M[2]+t));
const __m256i m3=_mm256_load_si256((const __m256i*)(M[3]+t));
const __m256i m4=_mm256_load_si256((const __m256i*)(M[4]+t));
const __m256i m5=_mm256_load_si256((const __m256i*)(M[5]+t));
const __m256i m6=_mm256_load_si256((const __m256i*)(M[6]+t));
_mm256_stream_si256((__m256i*)(r0+j),
_mm256_add_epi8(_mm256_sub_epi8(_mm256_add_epi8(m0,m3),m4),m6));
_mm256_stream_si256((__m256i*)(r0+h+j), _mm256_add_epi8(m2,m4));
_mm256_stream_si256((__m256i*)(r1+j), _mm256_add_epi8(m1,m3));
_mm256_stream_si256((__m256i*)(r1+h+j),
_mm256_add_epi8(_mm256_sub_epi8(_mm256_add_epi8(m0,m2),m1),m5));
}
}
}
template<int Q>
AVX2 static void cmb4wb(int8_t *C,int n,int h,const int8_t (&M)[7][Q]){
for(int i=0;i<h;i++){
int8_t *r0=C+(size_t)i*n, *r1=C+(size_t)(i+h)*n;
for(int j=0;j<h;j+=32){
size_t t=(size_t)i*h+j;
const __m256i m0=_mm256_load_si256((const __m256i*)(M[0]+t));
const __m256i m1=_mm256_load_si256((const __m256i*)(M[1]+t));
const __m256i m2=_mm256_load_si256((const __m256i*)(M[2]+t));
const __m256i m3=_mm256_load_si256((const __m256i*)(M[3]+t));
const __m256i m4=_mm256_load_si256((const __m256i*)(M[4]+t));
const __m256i m5=_mm256_load_si256((const __m256i*)(M[5]+t));
const __m256i m6=_mm256_load_si256((const __m256i*)(M[6]+t));
_mm256_store_si256((__m256i*)(r0+j),
_mm256_add_epi8(_mm256_sub_epi8(_mm256_add_epi8(m0,m3),m4),m6));
_mm256_store_si256((__m256i*)(r0+h+j), _mm256_add_epi8(m2,m4));
_mm256_store_si256((__m256i*)(r1+j), _mm256_add_epi8(m1,m3));
_mm256_store_si256((__m256i*)(r1+h+j),
_mm256_add_epi8(_mm256_sub_epi8(_mm256_add_epi8(m0,m2),m1),m5));
}
}
}
AVX2 static void cmb(int8_t*C,int n,int r0,int c0,int m,
const int8_t*M0,int s0,const int8_t*M1,int s1,
const int8_t*M2,int s2,const int8_t*M3,int s3){
for (int i=0;i<m;i++){
int8_t *d=C+(size_t)(r0+i)*n+c0;
const int8_t *p0=M0?M0+(size_t)i*m:0, *p1=M1?M1+(size_t)i*m:0;
const int8_t *p2=M2?M2+(size_t)i*m:0, *p3=M3?M3+(size_t)i*m:0;
for (int j=0;j<m;j+=32){
__m256i v;
if (p0){ __m256i t=_mm256_load_si256((const __m256i*)(p0+j));
v = s0>0?t:_mm256_sub_epi8(_mm256_setzero_si256(),t); }
else v = _mm256_setzero_si256();
if (p1){ __m256i t=_mm256_load_si256((const __m256i*)(p1+j));
v = _mm256_add_epi8(v, s1>0?t:_mm256_sub_epi8(_mm256_setzero_si256(),t)); }
if (p2){ __m256i t=_mm256_load_si256((const __m256i*)(p2+j));
v = _mm256_add_epi8(v, s2>0?t:_mm256_sub_epi8(_mm256_setzero_si256(),t)); }
if (p3){ __m256i t=_mm256_load_si256((const __m256i*)(p3+j));
v = _mm256_add_epi8(v, s3>0?t:_mm256_sub_epi8(_mm256_setzero_si256(),t)); }
_mm256_stream_si256((__m256i*)(d+j), v);
}
}
}
/* dst[i*h+j] = X[(r0+i)*n+(c0+j)] op Y[(r1+i)*n+(c1+j)], op: 0 copy 1 add 2 sub (mod 256) */
/* lane c4k5_rv: BLOCK-64KB LAYOUT. dst holds a 4 x 4 grid of B x B blocks, B = h/4, each
block CONTIGUOUS: block (I,J) at (I*4+J)*B*B. The level-4 packers read it that way (see NRB in
submm_s5f), so every packer term is a dense sequential 64 KB stream instead of four
1024-byte-strided 256-byte streams. Byte-identical values; addressing only. */
AVX2 static void blk(int8_t*dst,const int8_t*X,int n,int r0,int c0,int h,
const int8_t*Y,int r1,int c1,int op){
const int B = h>>2, NB = 4;
const size_t BB = (size_t)B*(size_t)B;
for (int i=0;i<h;i++){
const int8_t *xa=X+(size_t)(r0+i)*n+c0, *ya=Y+(size_t)(r1+i)*n+c1;
int8_t *d0 = dst + (size_t)((i/B)*NB)*BB + (size_t)(i%B)*B;
for (int J=0;J<NB;J++){
int8_t *d = d0 + (size_t)J*BB;
const int8_t *xaj = xa + J*B, *yaj = ya + J*B;
int j=0;
if (op==0) for(;j+32<=B;j+=32) _mm256_store_si256((__m256i*)(d+j),
_mm256_load_si256((const __m256i*)(xaj+j)));
else if (op==1) for(;j+32<=B;j+=32) _mm256_store_si256((__m256i*)(d+j),
_mm256_add_epi8(_mm256_load_si256((const __m256i*)(xaj+j)),
_mm256_load_si256((const __m256i*)(yaj+j))));
else for(;j+32<=B;j+=32) _mm256_store_si256((__m256i*)(d+j),
_mm256_sub_epi8(_mm256_load_si256((const __m256i*)(xaj+j)),
_mm256_load_si256((const __m256i*)(yaj+j))));
if (op) for(;j<B;j++) d[j]=(int8_t)(xaj[j]+(op==1?yaj[j]:-yaj[j]));
}
}
}
AVX2 static void blk_nt(int8_t*dst,const int8_t*X,int n,int r0,int c0,int h,
const int8_t*Y,int r1,int c1,int op){
for (int i=0;i<h;i++){
const int8_t *xa=X+(size_t)(r0+i)*n+c0, *ya=Y+(size_t)(r1+i)*n+c1;
int8_t *d=dst+(size_t)i*h;
int j=0;
if (op==0) for(;j+32<=h;j+=32) _mm256_stream_si256((__m256i*)(d+j),
_mm256_load_si256((const __m256i*)(xa+j)));
else if (op==1) for(;j+32<=h;j+=32) _mm256_stream_si256((__m256i*)(d+j),
_mm256_add_epi8(_mm256_load_si256((const __m256i*)(xa+j)),
_mm256_load_si256((const __m256i*)(ya+j))));
else for(;j+32<=h;j+=32) _mm256_stream_si256((__m256i*)(d+j),
_mm256_sub_epi8(_mm256_load_si256((const __m256i*)(xa+j)),
_mm256_load_si256((const __m256i*)(ya+j))));
if (op) for(;j<h;j++) d[j]=(int8_t)(xa[j]+(op==1?ya[j]:-ya[j]));
}
}
/* The GF(2) + combine tail of a sub-product, i.e. everything after the operand packers. Split
out so the FUSED path can reach it without materialising S2A/S2B first. */
AVX2BMI static void corr1024(void);
/* corr1024(): a hand-unrolled mirror of the SHIPPED corr16 for n=1024 (W=16, NY=4), which
corr16's own `if (n==4096)` dispatch never reaches. corr_pass3's inner `for(q<NY)` is
runtime-bounded, so GCC cannot unroll it; corr16 exists because the unrolled form pays.
Only the constants differ from corr16: 4 accumulators instead of 16, Bpb row stride 128 B
instead of 512 B (shlq $7), and offsets 0/32/64/96 instead of 0..480. */
AVX2BMI static void corr1024(void){
const int n=1024, W=16;
memset(Pbb,0,(size_t)n*W*8);
const int KB=64;
for (int kb=0; kb<n; kb+=KB){
int w0=kb>>6;
const char *bp0=(const char*)Bpb+(size_t)w0*8192;
for (int i=0;i<n;i++){
const uint64_t*row=A1b+(size_t)i*W+w0;
uint64_t b0=row[0], b1=0;
uint64_t *o=Pbb+(size_t)i*W;
asm volatile(
"vpxor %%xmm0,%%xmm0,%%xmm0\n\t"
"vpxor %%xmm1,%%xmm1,%%xmm1\n\t"
"vpxor %%xmm2,%%xmm2,%%xmm2\n\t"
"vpxor %%xmm3,%%xmm3,%%xmm3\n\t"
"testq %[b0], %[b0]\n\t"
"jz 3%=f\n\t"
"1%=:\n\t"
"tzcntq %[b0], %%r11\n\t"
"shlq $7, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"vpxor 64(%%r11), %%ymm2, %%ymm2\n\t"
"vpxor 96(%%r11), %%ymm3, %%ymm3\n\t"
"blsrq %[b0], %[b0]\n\t"
"jnz 1%=b\n\t"
"3%=:\n\t"
"testq %[b1], %[b1]\n\t"
"jz 4%=f\n\t"
"2%=:\n\t"
"tzcntq %[b1], %%r11\n\t"
"shlq $7, %%r11\n\t"
"addq %[bp0], %%r11\n\t"
"vpxor 0(%%r11), %%ymm0, %%ymm0\n\t"
"vpxor 32(%%r11), %%ymm1, %%ymm1\n\t"
"vpxor 64(%%r11), %%ymm2, %%ymm2\n\t"
"vpxor 96(%%r11), %%ymm3, %%ymm3\n\t"
"blsrq %[b1], %[b1]\n\t"
"jnz 2%=b\n\t"
"4%=:\n\t"
"vpxor 0(%[o]), %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[o])\n\t"
"vpxor 32(%[o]), %%ymm1, %%ymm1\n\t"
"vmovdqu %%ymm1, 32(%[o])\n\t"
"vpxor 64(%[o]), %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 64(%[o])\n\t"
"vpxor 96(%[o]), %%ymm3, %%ymm3\n\t"
"vmovdqu %%ymm3, 96(%[o])\n\t"
: [b0]"+r"(b0), [b1]"+r"(b1)
: [bp0]"r"(bp0), [o]"r"(o)
: "r11","cc","memory","ymm0","ymm1","ymm2","ymm3");
}
}
}
/* ===================== lane CH3: NIBBLE-TABLE GF(2) PRODUCT =====================
P[i][:] = XOR_{k: A1[i][k]=1} Bpb[k][:] -- the same contract as corr512/corr1024,
verified bit-identical against them on the real judge (work/lane_ch3_mmmc/rig/).
Rationale and measurements: work/lane_ch3_mmmc/mk_nibcorr.py docstring.
GROUPS = 16 groups (= one 64-bit A1 word) per k-block; table = 16*16*W words. */
static uint64_t TB[((size_t)NMAX/64)*256] __attribute__((aligned(64)));
/* ===== lane m50 (ported from m47/mmmc1k, macro diffed BYTE-IDENTICAL): VECTORISED SUBSET-XOR
TABLE BUILD. The source loop is `for w { 4 scalar loads; 12 scalar xors; 16 scalar stores }`.
At W%4==0 the w-dimension of `T[j*W+w]` is EXACTLY one ymm, so the honest build is
4 vector loads + 12 vpxor + 16 vector stores = 32 uops, NOT ONE of them on p5.
gcc-9's SLP vectorizer instead materialised a 16x16 64-bit transpose through a 2 KB STACK
buffer (measured here: 896 p5-only shuffles + 632 stack refs in nibcorr256).
Taking each operand AS A VECTOR removes the transpose's reason to exist. */
AVX2BMI static void nibtab_build(const uint64_t *__restrict r, uint64_t *__restrict T, int W){
const int U = W>>2; /* ymm per row-of-table = W/4 */
const __m256i *rp = (const __m256i*)r;
__m256i *tp = (__m256i*)T;
for (int q=0;q<U;q++){
const __m256i a1=rp[q], a2=rp[U+q], a4=rp[2*U+q], a8=rp[3*U+q];
const __m256i x12=_mm256_xor_si256(a1,a2), x14=_mm256_xor_si256(a1,a4),
x18=_mm256_xor_si256(a1,a8);
__m256i *o = tp+q; /* o[j*U] holds T[j*W + q*4 .. +3] */
o[ 0*U]=_mm256_setzero_si256();
o[ 1*U]=a1; o[ 2*U]=a2; o[ 4*U]=a4; o[ 8*U]=a8;
o[ 3*U]=x12; o[ 5*U]=x14;
o[ 6*U]=_mm256_xor_si256(a2,a4); o[ 7*U]=_mm256_xor_si256(x12,a4);
o[ 9*U]=x18; o[10*U]=_mm256_xor_si256(a2,a8);
o[12*U]=_mm256_xor_si256(a4,a8);
o[11*U]=_mm256_xor_si256(a2,x18); o[13*U]=_mm256_xor_si256(a4,x18);
o[14*U]=_mm256_xor_si256(_mm256_xor_si256(a2,a4),a8);
o[15*U]=_mm256_xor_si256(_mm256_xor_si256(x12,a4),a8);
}
}
#define DEF_NIBCORR(TAG, M) \
AVX2BMI static void nibcorr##TAG(void){ \
const int n=M, W=n/64, G=n>>2, GB=16, NY=W>>2; \
memset(Pbb,0,(size_t)n*W*8); \
for (int g0=0; g0<G; g0+=GB){ \
for (int t=0;t<GB;t++){ \
const int g=g0+t; \
uint64_t *T=TB+(size_t)t*16*W; \
const uint64_t *r=Bpb+(size_t)(g*4)*W; \
nibtab_build(r,T,W); \
\
} \
for (int i=0;i<n;i++){ \
const uint64_t *arow=A1b+(size_t)i*W+(g0>>4); \
__m256i acc[NY]; \
for (int q=0;q<NY;q++) acc[q]=_mm256_setzero_si256(); \
const uint64_t *T=TB; \
for (int wb=0; wb<(GB>>4); wb++){ \
const uint64_t word=arow[wb]; \
_Pragma("GCC unroll 16") \
for (int b=0;b<16;b++, T+=(size_t)16*W){ \
const uint64_t *e=T+(size_t)(((word>>(4*b))&15u)*W); \
for (int q=0;q<NY;q++) \
acc[q]=_mm256_xor_si256(acc[q],_mm256_load_si256((const __m256i*)(e+4*q))); \
} \
} \
uint64_t *o=Pbb+(size_t)i*W; \
for (int q=0;q<NY;q++){ \
__m256i *po=(__m256i*)(o+4*q); \
_mm256_store_si256(po,_mm256_xor_si256(_mm256_load_si256((const __m256i*)po),acc[q])); \
} \
} \
} \
}
DEF_NIBCORR(512,512)
/* FE5 DEPTH-4: the macro is VALID at M=256 (W=4, G=64, NY=1; the TB row is
16*W*8 = 512 B and the loads are 16 index-slots x 32 B = 512 B, and T[15*W+w]=T[60..63] is in
range). Without this instantiation every depth-4 leaf falls back to corr_pass3 -- the exact
+2.89 % trap that cost 6.55 % at depth 3. */
DEF_NIBCORR(256,256)
DEF_NIBCORR(1024,1024)
AVX2BMI static void corr_six256(){
constexpr int n=256,W=4,SLOTS=656;
for(int wb=0;wb<4;wb++){
for(int g=0;g<11;g++){
__m256i*T=(__m256i*)TB+g*64;
const __m256i r0=_mm256_load_si256((const __m256i*)(Bpb+((size_t)wb*64+g*6+0)*W));
const __m256i r1=_mm256_load_si256((const __m256i*)(Bpb+((size_t)wb*64+g*6+1)*W));
const __m256i r2=_mm256_load_si256((const __m256i*)(Bpb+((size_t)wb*64+g*6+2)*W));
const __m256i r3=_mm256_load_si256((const __m256i*)(Bpb+((size_t)wb*64+g*6+3)*W));
if(g<10){
const __m256i r4=_mm256_load_si256((const __m256i*)(Bpb+((size_t)wb*64+g*6+4)*W));
const __m256i r5=_mm256_load_si256((const __m256i*)(Bpb+((size_t)wb*64+g*6+5)*W));
__m256i v0=_mm256_setzero_si256(),v1=r4,v2=r5,v3=_mm256_xor_si256(r4,r5);
_mm256_store_si256(T+0,v0);
_mm256_store_si256(T+16,v1);
_mm256_store_si256(T+32,v2);
_mm256_store_si256(T+48,v3);
v0=_mm256_xor_si256(v0,r0); _mm256_store_si256(T+1,v0);
v1=_mm256_xor_si256(v1,r0); _mm256_store_si256(T+17,v1);
v2=_mm256_xor_si256(v2,r0); _mm256_store_si256(T+33,v2);
v3=_mm256_xor_si256(v3,r0); _mm256_store_si256(T+49,v3);
v0=_mm256_xor_si256(v0,r1); _mm256_store_si256(T+3,v0);
v1=_mm256_xor_si256(v1,r1); _mm256_store_si256(T+19,v1);
v2=_mm256_xor_si256(v2,r1); _mm256_store_si256(T+35,v2);
v3=_mm256_xor_si256(v3,r1); _mm256_store_si256(T+51,v3);
v0=_mm256_xor_si256(v0,r0); _mm256_store_si256(T+2,v0);
v1=_mm256_xor_si256(v1,r0); _mm256_store_si256(T+18,v1);
v2=_mm256_xor_si256(v2,r0); _mm256_store_si256(T+34,v2);
v3=_mm256_xor_si256(v3,r0); _mm256_store_si256(T+50,v3);
v0=_mm256_xor_si256(v0,r2); _mm256_store_si256(T+6,v0);
v1=_mm256_xor_si256(v1,r2); _mm256_store_si256(T+22,v1);
v2=_mm256_xor_si256(v2,r2); _mm256_store_si256(T+38,v2);
v3=_mm256_xor_si256(v3,r2); _mm256_store_si256(T+54,v3);
v0=_mm256_xor_si256(v0,r0); _mm256_store_si256(T+7,v0);
v1=_mm256_xor_si256(v1,r0); _mm256_store_si256(T+23,v1);
v2=_mm256_xor_si256(v2,r0); _mm256_store_si256(T+39,v2);
v3=_mm256_xor_si256(v3,r0); _mm256_store_si256(T+55,v3);
v0=_mm256_xor_si256(v0,r1); _mm256_store_si256(T+5,v0);
v1=_mm256_xor_si256(v1,r1); _mm256_store_si256(T+21,v1);
v2=_mm256_xor_si256(v2,r1); _mm256_store_si256(T+37,v2);
v3=_mm256_xor_si256(v3,r1); _mm256_store_si256(T+53,v3);
v0=_mm256_xor_si256(v0,r0); _mm256_store_si256(T+4,v0);
v1=_mm256_xor_si256(v1,r0); _mm256_store_si256(T+20,v1);
v2=_mm256_xor_si256(v2,r0); _mm256_store_si256(T+36,v2);
v3=_mm256_xor_si256(v3,r0); _mm256_store_si256(T+52,v3);
v0=_mm256_xor_si256(v0,r3); _mm256_store_si256(T+12,v0);
v1=_mm256_xor_si256(v1,r3); _mm256_store_si256(T+28,v1);
v2=_mm256_xor_si256(v2,r3); _mm256_store_si256(T+44,v2);
v3=_mm256_xor_si256(v3,r3); _mm256_store_si256(T+60,v3);
v0=_mm256_xor_si256(v0,r0); _mm256_store_si256(T+13,v0);
v1=_mm256_xor_si256(v1,r0); _mm256_store_si256(T+29,v1);
v2=_mm256_xor_si256(v2,r0); _mm256_store_si256(T+45,v2);
v3=_mm256_xor_si256(v3,r0); _mm256_store_si256(T+61,v3);
v0=_mm256_xor_si256(v0,r1); _mm256_store_si256(T+15,v0);
v1=_mm256_xor_si256(v1,r1); _mm256_store_si256(T+31,v1);
v2=_mm256_xor_si256(v2,r1); _mm256_store_si256(T+47,v2);
v3=_mm256_xor_si256(v3,r1); _mm256_store_si256(T+63,v3);
v0=_mm256_xor_si256(v0,r0); _mm256_store_si256(T+14,v0);
v1=_mm256_xor_si256(v1,r0); _mm256_store_si256(T+30,v1);
v2=_mm256_xor_si256(v2,r0); _mm256_store_si256(T+46,v2);
v3=_mm256_xor_si256(v3,r0); _mm256_store_si256(T+62,v3);
v0=_mm256_xor_si256(v0,r2); _mm256_store_si256(T+10,v0);
v1=_mm256_xor_si256(v1,r2); _mm256_store_si256(T+26,v1);
v2=_mm256_xor_si256(v2,r2); _mm256_store_si256(T+42,v2);
v3=_mm256_xor_si256(v3,r2); _mm256_store_si256(T+58,v3);
v0=_mm256_xor_si256(v0,r0); _mm256_store_si256(T+11,v0);
v1=_mm256_xor_si256(v1,r0); _mm256_store_si256(T+27,v1);
v2=_mm256_xor_si256(v2,r0); _mm256_store_si256(T+43,v2);
v3=_mm256_xor_si256(v3,r0); _mm256_store_si256(T+59,v3);
v0=_mm256_xor_si256(v0,r1); _mm256_store_si256(T+9,v0);
v1=_mm256_xor_si256(v1,r1); _mm256_store_si256(T+25,v1);
v2=_mm256_xor_si256(v2,r1); _mm256_store_si256(T+41,v2);
v3=_mm256_xor_si256(v3,r1); _mm256_store_si256(T+57,v3);
v0=_mm256_xor_si256(v0,r0); _mm256_store_si256(T+8,v0);
v1=_mm256_xor_si256(v1,r0); _mm256_store_si256(T+24,v1);
v2=_mm256_xor_si256(v2,r0); _mm256_store_si256(T+40,v2);
v3=_mm256_xor_si256(v3,r0); _mm256_store_si256(T+56,v3);
} else {
__m256i v=_mm256_setzero_si256(); _mm256_store_si256(T,v);
v=_mm256_xor_si256(v,r0); _mm256_store_si256(T+1,v);
v=_mm256_xor_si256(v,r1); _mm256_store_si256(T+3,v);
v=_mm256_xor_si256(v,r0); _mm256_store_si256(T+2,v);
v=_mm256_xor_si256(v,r2); _mm256_store_si256(T+6,v);
v=_mm256_xor_si256(v,r0); _mm256_store_si256(T+7,v);
v=_mm256_xor_si256(v,r1); _mm256_store_si256(T+5,v);
v=_mm256_xor_si256(v,r0); _mm256_store_si256(T+4,v);
v=_mm256_xor_si256(v,r3); _mm256_store_si256(T+12,v);
v=_mm256_xor_si256(v,r0); _mm256_store_si256(T+13,v);
v=_mm256_xor_si256(v,r1); _mm256_store_si256(T+15,v);
v=_mm256_xor_si256(v,r0); _mm256_store_si256(T+14,v);
v=_mm256_xor_si256(v,r2); _mm256_store_si256(T+10,v);
v=_mm256_xor_si256(v,r0); _mm256_store_si256(T+11,v);
v=_mm256_xor_si256(v,r1); _mm256_store_si256(T+9,v);
v=_mm256_xor_si256(v,r0); _mm256_store_si256(T+8,v);
}
}
for(int i=0;i<n;i+=2){
const uint64_t w0=A1b[(size_t)i*W+wb],w1=A1b[(size_t)(i+1)*W+wb];
__m256i x0=wb?_mm256_load_si256((const __m256i*)(Pbb+(size_t)i*W)):_mm256_setzero_si256();
__m256i y0=wb?_mm256_load_si256((const __m256i*)(Pbb+(size_t)(i+1)*W)):_mm256_setzero_si256();
__m256i x1=_mm256_setzero_si256(),y1=x1;
#pragma GCC unroll 5
for(int g=0;g<10;g+=2){
const __m256i*t0=(const __m256i*)TB+g*64,*t1=t0+64;
x0=_mm256_xor_si256(x0,_mm256_load_si256(t0+((w0>>(6*g))&63)));
y0=_mm256_xor_si256(y0,_mm256_load_si256(t0+((w1>>(6*g))&63)));
x1=_mm256_xor_si256(x1,_mm256_load_si256(t1+((w0>>(6*(g+1)))&63)));
y1=_mm256_xor_si256(y1,_mm256_load_si256(t1+((w1>>(6*(g+1)))&63)));
}
x0=_mm256_xor_si256(x0,_mm256_load_si256((const __m256i*)TB+640+(w0>>60)));
y0=_mm256_xor_si256(y0,_mm256_load_si256((const __m256i*)TB+640+(w1>>60)));
_mm256_store_si256((__m256i*)(Pbb+(size_t)i*W),_mm256_xor_si256(x0,x1));
_mm256_store_si256((__m256i*)(Pbb+(size_t)(i+1)*W),_mm256_xor_si256(y0,y1));
}
}
}
AVX2 static void submm_tail(int m,int8_t*Cp){
GLDA = 2*(m) + PADA; GLDB = m + PADB;
/* FE5: nibcorr512 ALREADY EXISTS (DEF_NIBCORR(512,512)) but this dispatch never reached it,
so every depth-3 leaf ran the runtime-bounded corr_pass3 -- exactly the overhead that made
depth 3 lose. corr_pass3's inner `for(q<NY)` is runtime-bounded and GCC cannot unroll it;
the nibcorr forms are hand-unrolled. ONE LINE, EXISTING CODE, NO NEW SCRATCH. */
/* FE5: 256 added alongside the depth-4 level. THE TWO EDITS SHIP TOGETHER BY DESIGN. */
if (m==1024) nibcorr1024(); else if (m==512) nibcorr512();
else if (m==256) corr_six256(); else corr_pass3(m);
const int GB=1024, IB=1024;
for (int ib=0; ib<m; ib+=IB){
const int ihi = (ib+IB<m)? ib+IB : m;
for (int gb=0; gb<m; gb+=GB){
const int ghi = (gb+GB<m)? gb+GB : m;
for (int g=gb; g<ghi; g+=64){
const int8_t *bi = BI3b+(size_t)(g>>6)*(m/2)*128;
{ int rev=0;
for (int i=ib; i<ihi; i+=2){
/* R1 boustrophedon: every other call walks A and BI BACKWARD, so it starts
where the previous call stopped -- that data is still in L1. */
const int8_t *ap=A0b+(size_t)i*GLDA;
/* MC2: PB now points at the Pbb WORD for row i (Pbb is nibcorr256's output);
the byte expansion is done inline in the epilogue. */
const uint8_t *pw = (const uint8_t*)(Pbb+(size_t)i*(m>>6)+(g>>6));
mikro_u4 (m, ap, bi,
Cp+(size_t)i*m+g, pw, 0, GLDA, (m>>6)*8);
rev^=1; }
}
}
}
}
}
/* one m x m sub-product through the SHIPPED pipeline, NT-stored to Cp (m = 2048 in the
shipped dispatch; `m` is a parameter so the SAME code path gates at smaller m). */
AVX2 static void submm_plain(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp){
GLDA = 2*(m) + PADA; GLDB = m + PADB;
build_A0A1(m,Ap); build_BI3Bp(m,Bp); submm_tail(m,Cp);
}
AVX2 static void submm(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp);
#define S2MIN 1024
/* level 2: 7 sub-products of h=m/2, recombined by the SAME verified identity.
⛔ THE SUB-PRODUCT IS COMPUTED BY `submm_plain`, **NOT** BY `submm`.
Calling `submm(h,...)` re-enters this function at h, and the inner call **overwrites the
parent's own `S2A`/`S2B`/`S2M` scratch** -- so every `S2M[k]` except the last is destroyed
before the combine reads it. That is a 100 %-wrong build, and the 6-second A=I probe at
n=4096 found it (87 % of bytes wrong, all four blocks). If a deeper level is ever wanted,
it needs its own scratch, not the shared pool. */
/* FE5 DEPTH-3: submm_s2's leaves are now full Strassen steps. submm_s4 takes PLAIN X,Y
(produced by blk() in submm_s2) and uses its own S3M scratch, so the parent's S2M is
never clobbered -- that clobber is the 100 %-wrong build base.cpp:926-930 warns about. */
AVX2 static void submm_s4(int m,const int8_t*X,const int8_t*Y,int8_t*dst);
/* FE5 DEPTH-4: submm_s4's leaves are now full Strassen steps with their OWN S4M scratch, so
the parent's S3M is never clobbered -- the clobber base.cpp:926-930 calls a 100 %-wrong build. */
/* FE5 FUSION: forward decl -- submm_s4's junction is now fused into this. */
AVX2 static void submm_s5f(int M,const int8_t*Ap,int Aa0,int Aa1,int Aa2,int Aa3,int Aop,
const int8_t*Bp,int Ba0,int Ba1,int Ba2,int Ba3,int Bop,int8_t*Cp);
AVX2 static void submm_s5f(int M,const int8_t*Ap,int Aa0,int Aa1,int Aa2,int Aa3,int Aop,
const int8_t*Bp,int Ba0,int Ba1,int Ba2,int Ba3,int Bop,int8_t*Cp){
const int h=M>>2, o=h;
/* lane c4k5_rv: S3A/S3B are a 4x4 grid of contiguous h x h blocks (see blk()). A term that
lived at (r,c) of the M x M matrix now starts at row NRB(r,c)/h of that grid, row stride h. */
#define NRB(rr,cc) ((((rr)/h)*4 + ((cc)/h))*(h)) /* this level h, and its quadrant step */
const int SQA=(Aop==1)?1:((Aop==2)?-1:0); /* the PARENT sign, A side */
const int SQB=(Bop==1)?1:((Bop==2)?-1:0);
const int dz=M>>1; /* the blk size the parent would have used */
GLDA = 2*h + PADA; GLDB = h + PADB;
for (int k=0;k<7;k++){
switch(k){
case 0: buildA_q4(h,h,Ap,NRB(Aa0,Aa1),0, NRB(Aa2,Aa3),0,SQA, NRB(Aa0+o,Aa1+o),0,+1, NRB(Aa2+o,Aa3+o),0,(SQA)*(+1));
buildB_q4(h,h,Bp,NRB(Ba0,Ba1),0, NRB(Ba2,Ba3),0,SQB, NRB(Ba0+o,Ba1+o),0,+1, NRB(Ba2+o,Ba3+o),0,(SQB)*(+1));
submm_tail(h,S4M[0]); break;
case 1: buildA_q4(h,h,Ap,NRB(Aa0+o,Aa1),0, NRB(Aa2+o,Aa3),0,SQA, NRB(Aa0+o,Aa1+o),0,+1, NRB(Aa2+o,Aa3+o),0,(SQA)*(+1));
buildB_q4(h,h,Bp,NRB(Ba0,Ba1),0, NRB(Ba2,Ba3),0,SQB, NRB(Ba0,Ba1),0,+0, NRB(Ba2,Ba3),0,(SQB)*(+0));
submm_tail(h,S4M[1]); break;
case 2: buildA_q4(h,h,Ap,NRB(Aa0,Aa1),0, NRB(Aa2,Aa3),0,SQA, NRB(Aa0,Aa1),0,+0, NRB(Aa2,Aa3),0,(SQA)*(+0));
buildB_q4(h,h,Bp,NRB(Ba0,Ba1+o),0, NRB(Ba2,Ba3+o),0,SQB, NRB(Ba0+o,Ba1+o),0,-1, NRB(Ba2+o,Ba3+o),0,(SQB)*(-1));
submm_tail(h,S4M[2]); break;
case 3: buildA_q4(h,h,Ap,NRB(Aa0+o,Aa1+o),0, NRB(Aa2+o,Aa3+o),0,SQA, NRB(Aa0+o,Aa1+o),0,+0, NRB(Aa2+o,Aa3+o),0,(SQA)*(+0));
buildB_q4(h,h,Bp,NRB(Ba0+o,Ba1),0, NRB(Ba2+o,Ba3),0,SQB, NRB(Ba0,Ba1),0,-1, NRB(Ba2,Ba3),0,(SQB)*(-1));
submm_tail(h,S4M[3]); break;
case 4: buildA_q4(h,h,Ap,NRB(Aa0,Aa1),0, NRB(Aa2,Aa3),0,SQA, NRB(Aa0,Aa1+o),0,+1, NRB(Aa2,Aa3+o),0,(SQA)*(+1));
buildB_q4(h,h,Bp,NRB(Ba0+o,Ba1+o),0, NRB(Ba2+o,Ba3+o),0,SQB, NRB(Ba0+o,Ba1+o),0,+0, NRB(Ba2+o,Ba3+o),0,(SQB)*(+0));
submm_tail(h,S4M[4]); break;
case 5: buildA_q4(h,h,Ap,NRB(Aa0+o,Aa1),0, NRB(Aa2+o,Aa3),0,SQA, NRB(Aa0,Aa1),0,-1, NRB(Aa2,Aa3),0,(SQA)*(-1));
buildB_q4(h,h,Bp,NRB(Ba0,Ba1),0, NRB(Ba2,Ba3),0,SQB, NRB(Ba0,Ba1+o),0,+1, NRB(Ba2,Ba3+o),0,(SQB)*(+1));
submm_tail(h,S4M[5]); break;
case 6: buildA_q4(h,h,Ap,NRB(Aa0,Aa1+o),0, NRB(Aa2,Aa3+o),0,SQA, NRB(Aa0+o,Aa1+o),0,-1, NRB(Aa2+o,Aa3+o),0,(SQA)*(-1));
buildB_q4(h,h,Bp,NRB(Ba0+o,Ba1),0, NRB(Ba2+o,Ba3),0,SQB, NRB(Ba0+o,Ba1+o),0,+1, NRB(Ba2+o,Ba3+o),0,(SQB)*(+1));
submm_tail(h,S4M[6]); break;
}
}
#undef NRB
_mm_sfence();
cmb4wb(Cp,dz,h,S4M); /* mmc4a: fused quadrants + WB twin */
_mm_sfence();
}
AVX2 static void submm_s5(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp);
AVX2 static void submm_s5(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp){
GLDA = 2*((m>>1)) + PADA; GLDB = (m>>1) + PADB;
const int h=m>>1, o=h;
for (int k=0;k<7;k++){
switch(k){
case 0: buildA_q(h,m,Ap,0,0,Ap,o,o,1); buildB_q(h,m,Bp,0,0,Bp,o,o,1); submm_tail(h,S4M[0]); break;
case 1: buildA_q(h,m,Ap,o,0,Ap,o,o,1); buildB_q(h,m,Bp,0,0,Bp,0,0,0); submm_tail(h,S4M[1]); break;
case 2: buildA_q(h,m,Ap,0,0,Ap,0,0,0); buildB_q(h,m,Bp,0,o,Bp,o,o,2); submm_tail(h,S4M[2]); break;
case 3: buildA_q(h,m,Ap,o,o,Ap,o,o,0); buildB_q(h,m,Bp,o,0,Bp,0,0,2); submm_tail(h,S4M[3]); break;
case 4: buildA_q(h,m,Ap,0,0,Ap,0,o,1); buildB_q(h,m,Bp,o,o,Bp,o,o,0); submm_tail(h,S4M[4]); break;
case 5: buildA_q(h,m,Ap,o,0,Ap,0,0,2); buildB_q(h,m,Bp,0,0,Bp,0,o,1); submm_tail(h,S4M[5]); break;
case 6: buildA_q(h,m,Ap,0,o,Ap,o,o,2); buildB_q(h,m,Bp,o,0,Bp,o,o,1); submm_tail(h,S4M[6]); break;
}
}
_mm_sfence();
cmb(Cp,m,0,0,h, S4M[0],1,S4M[3],1,S4M[4],-1,S4M[6],1);
cmb(Cp,m,0,o,h, S4M[2],1,S4M[4],1,0,0,0,0);
cmb(Cp,m,o,0,h, S4M[1],1,S4M[3],1,0,0,0,0);
cmb(Cp,m,o,o,h, S4M[0],1,S4M[2],1,S4M[1],-1,S4M[5],1);
_mm_sfence();
}
AVX2 static void submm_s4(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp){
GLDA = 2*((m>>1)) + PADA; GLDB = (m>>1) + PADB;
const int h=m>>1, o=h;
for (int k=0;k<7;k++){
switch(k){
case 0: submm_s5f(m,Ap,0,0,o,o,1, Bp,0,0,o,o,1, S3M[0]); break;
case 1: submm_s5f(m,Ap,o,0,o,o,1, Bp,0,0,0,0,0, S3M[1]); break;
case 2: submm_s5f(m,Ap,0,0,0,0,0, Bp,0,o,o,o,2, S3M[2]); break;
case 3: submm_s5f(m,Ap,o,o,o,o,0, Bp,o,0,0,0,2, S3M[3]); break;
case 4: submm_s5f(m,Ap,0,0,0,o,1, Bp,o,o,o,o,0, S3M[4]); break;
case 5: submm_s5f(m,Ap,o,0,0,0,2, Bp,0,0,0,o,1, S3M[5]); break;
case 6: submm_s5f(m,Ap,0,o,o,o,2, Bp,o,0,o,o,1, S3M[6]); break;
}
}
_mm_sfence();
cmb4(Cp,m,h,S3M);
_mm_sfence();
}
AVX2 static void submm_s2(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp){
GLDA = 2*((m>>1)) + PADA; GLDB = (m>>1) + PADB;
const int h=m>>1, o=h;
for (int k=0;k<7;k++){
switch(k){
case 0: blk(S3A,Ap,m,0,0,h,Ap,o,o,1); blk(S3B,Bp,m,0,0,h,Bp,o,o,1); submm_s4(h,S3A,S3B,S2M[0]); break;
case 1: blk(S3A,Ap,m,o,0,h,Ap,o,o,1); blk(S3B,Bp,m,0,0,h,Bp,0,0,0); submm_s4(h,S3A,S3B,S2M[1]); break;
case 2: blk(S3A,Ap,m,0,0,h,Ap,0,0,0); blk(S3B,Bp,m,0,o,h,Bp,o,o,2); submm_s4(h,S3A,S3B,S2M[2]); break;
case 3: blk(S3A,Ap,m,o,o,h,Ap,o,o,0); blk(S3B,Bp,m,o,0,h,Bp,0,0,2); submm_s4(h,S3A,S3B,S2M[3]); break;
case 4: blk(S3A,Ap,m,0,0,h,Ap,0,o,1); blk(S3B,Bp,m,o,o,h,Bp,o,o,0); submm_s4(h,S3A,S3B,S2M[4]); break;
case 5: blk(S3A,Ap,m,o,0,h,Ap,0,0,2); blk(S3B,Bp,m,0,0,h,Bp,0,o,1); submm_s4(h,S3A,S3B,S2M[5]); break;
case 6: blk(S3A,Ap,m,0,o,h,Ap,o,o,2); blk(S3B,Bp,m,o,0,h,Bp,o,o,1); submm_s4(h,S3A,S3B,S2M[6]); break;
}
}
_mm_sfence();
cmb4(Cp,m,h,S2M);
_mm_sfence();
}
AVX2 static void submm(int m,const int8_t*Ap,const int8_t*Bp,int8_t*Cp){
if (m>=S2MIN) { submm_s2(m,Ap,Bp,Cp); return; }
submm_plain(m,Ap,Bp,Cp);
}
AVX2 static void strassen1(int n,const int8_t*A,const int8_t*B,int8_t*C){
const int m=n>>1, o=m;
for (int k=0;k<7;k++){
switch(k){
case 0: blk_nt(SA,A,n,0,0,m,A,o,o,1); blk_nt(SB,B,n,0,0,m,B,o,o,1); break;
case 1: blk_nt(SA,A,n,o,0,m,A,o,o,1); blk_nt(SB,B,n,0,0,m,B,0,0,0); break;
case 2: blk_nt(SA,A,n,0,0,m,A,0,0,0); blk_nt(SB,B,n,0,o,m,B,o,o,2); break;
case 3: blk_nt(SA,A,n,o,o,m,A,o,o,0); blk_nt(SB,B,n,o,0,m,B,0,0,2); break;
case 4: blk_nt(SA,A,n,0,0,m,A,0,o,1); blk_nt(SB,B,n,o,o,m,B,o,o,0); break;
case 5: blk_nt(SA,A,n,o,0,m,A,0,0,2); blk_nt(SB,B,n,0,0,m,B,0,o,1); break;
case 6: blk_nt(SA,A,n,0,o,m,A,o,o,2); blk_nt(SB,B,n,o,0,m,B,o,o,1); break;
}
submm(m,SA,SB,MMS[k]);
}
_mm_sfence();
cmb4(C,n,m,MMS);
_mm_sfence();
}
AVX2 void matrix_multiply(int n, const int8_t *A, const int8_t *B, int8_t *C){
if (n<=0 || (n&255) || n>NMAX) { generic_mm(n,A,B,C); return; }
if (n==4096) { strassen1(n,A,B,C); return; }
/* x4: A1T is written by build_A1T and NEVER READ -- dead 16.78 MB + its transpose pass. */
/* FUSED: one pass over A (was two), one over B (was two) */
GLDA = 2*(n) + PADA; GLDB = n + PADB;
build_A0A1(n,A); build_BI3Bp(n,B);
if (n==4096) corr16(); else corr_pass3(n);
/* lane CM1: submm_tail-style IB x GB tiling + R1 boustrophedon (see mk_topblk.py) */
{
const int GB=128, IB=64;
for (int ib=0; ib<n; ib+=IB){
const int ihi = (ib+IB<n)? ib+IB : n;
for (int gb=0; gb<n; gb+=GB){
const int ghi = (gb+GB<n)? gb+GB : n;
for (int g=gb; g<ghi; g+=64){
const int8_t *bi = BI3b+(size_t)(g>>6)*(n/2)*128;
int rev=0;
for (int i=ib; i<ihi; i+=2){
const int8_t *ap = A0b+(size_t)i*GLDA;
const uint8_t *pw = (const uint8_t*)(Pbb+(size_t)i*(n>>6)+(g>>6));
if (rev) mikro_u4r(n, ap+(2*n)-16, bi,
C+(size_t)i*n+g, pw, 0, GLDA, (n>>6)*8);
else mikro_u4 (n, ap, bi,
C+(size_t)i*n+g, pw, 0, GLDA, (n>>6)*8);
rev^=1;
}
}
}
}
}
_mm_sfence();
}
| Compilation | N/A | N/A | Compile OK | Score: N/A | 显示更多 |
| Testcase #1 | 345.937 ms | 63 MB + 572 KB | Accepted | Score: 100 | 显示更多 |