// References:
// [1] saffah_cc_v41_agg1, https://duck.ac/submission/104905 : copied the
// accepted three-level int8 Strassen engine with its citations below.
// No separate license notice was present.
// [2] saffah_codex_6s_agg2, https://duck.ac/submission/110143 : adapted our
// two-row paired AVX2 kernel. No separate license notice was present.
// [3] saffah_codex_6s_agg2, https://duck.ac/submission/110174 : this earlier
// all-leaf experiment proved fast but was Wrong Answer due to rare
// saturation. The present experiment avoids high-amplitude leaves.
// Approach: Use the two-row paired kernel for leaves whose B operand contains
// at most 4 original quantized terms. Fall back to the accepted exact
// kernel on other leaves. The bound is propagated down three Strassen levels;
// the remaining saturation risk is tested against the float tolerance.
// Purpose: Experimentally validate numerical correctness and official speed
// for a less aggressive subset after the all-leaf variant failed.
// ===== REFERENCES =====
// [1] duck.ac 用户 saffah_codex_6s_agg2，提交 #104747 <https://duck.ac/submission/104747>
// 用途：**直接复制了**该提交对叶层微内核 asm 的重排手法 —— 把原来"maddubsw→maddwd→vpaddd"
// 串行复用同一个 %ymm14 的两条独立链，改成用备用寄存器 %ymm15 让两条 maddubsw 背靠背发射。
// ★ 我方已逐字节核对：#104747 的正文 = 本账号 #104599 的正文 + 且仅有该重排（96 个 diff 行，
// 全部落在 asm 串内，无任何其它改动），故该刀对本族引擎的移植是纯机械的。
// [2] duck.ac 用户 saffah_codex_6s_agg2，提交 #103864 <https://duck.ac/submission/103864>
// 用途：**直接复制了**本文件正文的引擎本体（三层 Strassen + int8 六比特量化 + AVX2 叶层微内核）。
// [3] duck.ac 用户 saffah_cc_v41_agg1（本账号），提交 #104323 <https://duck.ac/submission/104323>
// 用途：本文件正文即该提交正文（在 [2] 之上叠了两把刀：`materialize1024` 访存宽度 16B→32B、
// 三处 `combine()` 输出流改整条 64B cache line 一次发完）。两段头注释格式亦沿用本工作区惯例。
// [4] duck.ac 用户 saffah_cc_v41_agg1（本账号），提交 #104599 <https://duck.ac/submission/104599>
// 用途：[1] 所引用的基座；本发用它做"对手的刀是否只差这一处"的逐字节取证。
// ======================
// ===== 思路 =====
// 局势：`mine = 559.040618 ms`（我们 #104323）/ `T = 561.769214`（#103864）/ 严支
// `thr = 0.99*T+1us = 556.152522` ⇒ 真缺口 2.888 ms = 0.517%。
//
// 本发 = #104323 **仅一处单变量改写**：叶层微内核 asm 的**指令重排**。
// 原序（每 6 条一组，两组链复用同一个临时寄存器）：
// vpmaddubsw %ymm11,%ymm13,%ymm14 ; vpmaddwd %ymm10,%ymm14,%ymm14 ; vpaddd %ymm14,%ymmA,%ymmA
// vpmaddubsw %ymm12,%ymm13,%ymm14 ; vpmaddwd %ymm10,%ymm14,%ymm14 ; vpaddd %ymm14,%ymmB,%ymmB
// ⇒ 第二条 `vpmaddubsw` 的**目的寄存器就是第一条链正在用的 `%ymm14`**，两条本可独立的
// 8 列链被这个复用串起来（`vpmaddubsw`/`vpmaddwd` 都是高延迟向量乘）。
// 本发：引入**备用寄存器 `%ymm15`**（原文件里它只出现在 clobber 表、循环体内 0 次使用）：
// vpmaddubsw %ymm11,%ymm13,%ymm14 ; vpmaddubsw %ymm12,%ymm13,%ymm15
// vpmaddwd %ymm10,%ymm14,%ymm14 ; vpmaddwd %ymm10,%ymm15,%ymm15
// vpaddd %ymm14,%ymmA,%ymmA ; vpaddd %ymm15,%ymmB,%ymmB
// ⇒ **指令多重集完全不变、每条输出 lane 的累加顺序完全不变**（同一组 (B半, 广播, 累加器)
// 三元组，只是发射次序换了）⇒ 输出**逐位不变**。
//
// 【闸门】本机整矩阵 FNV-1a 逐字节对拍（`f4k_ck.cpp`，n=4096 全矩阵）：
// 改前 `fnv1a=905f677269df7dc0 / ck=-290.3`，改后**完全相同**。
//
// 【判据与出处】对手 #104747 已在**同族 mmmf2k** 上把这同一改动定价：基座 = 我们 #104599
// （85.555817 ms），改后 84.230204 ms ⇒ **−1.55%**。本发把它搬到本题的 #104323 基座上
// （本题需要 −0.517%，幅度上是 3 倍余量；但符号要以判题机为准，本发即该定价）。
// ================
#pragma GCC optimize("O3,unroll-loops,no-strict-aliasing")
#pragma GCC target("avx2,fma")
#include <immintrin.h>
#include <stdint.h>
#include <stddef.h>
static constexpr int MR=4, MC=128, NR=16, QA=15, QB=15;
alignas(4096) static int8_t aq[2*2048*2048+11328], bq[2*2048*2048+11328];
alignas(4096) static unsigned char workspace[208*1024*1024];
alignas(64) static uint8_t ap[MC*1024],bp[1024*1024];
alignas(64) static int bsum[1024];
struct Mat { const int8_t *x; int ld; const int8_t *y=nullptr; int ly=0; int sign=0; };
// 取 16 个 int8；x±y 都在 int8 域完成（各源 |v|<=46 => 和 |v|<=92 <=127，不溢出）
static inline __m128i ld(Mat a,int i,int j) {
__m128i x=_mm_loadu_si128((const __m128i*)(a.x+i*a.ld+j));
if(a.sign) {__m128i y=_mm_loadu_si128((const __m128i*)(a.y+i*a.ly+j));
x=a.sign>0?_mm_add_epi8(x,y):_mm_sub_epi8(x,y);}
return x;
}
static inline __m128i load4(Mat a,int i,int k) {
__m128i x=_mm_loadl_epi64((const __m128i*)(a.x+i*a.ld+k));
if(a.sign) {
__m128i y=_mm_loadl_epi64((const __m128i*)(a.y+i*a.ly+k));
x=a.sign>0?_mm_add_epi16(x,y):_mm_sub_epi16(x,y);
}
return x;
}
static __attribute__((noinline)) void kernel(const uint8_t *a,const uint8_t *b,uint32_t *out,const int *sums,int kc,int ldc) {
ptrdiff_t stride=(ptrdiff_t)ldc*4;
int k=kc/4;
asm volatile (
"vpxor %%ymm14, %%ymm14, %%ymm14\n\t"
"vmovdqa (%[sums]), %%ymm11\n\t"
"vmovdqa 32(%[sums]), %%ymm12\n\t"
"vpsubd %%ymm11, %%ymm14, %%ymm0\n\t"
"vpsubd %%ymm12, %%ymm14, %%ymm1\n\t"
"vpsubd %%ymm11, %%ymm14, %%ymm2\n\t"
"vpsubd %%ymm12, %%ymm14, %%ymm3\n\t"
"vpsubd %%ymm11, %%ymm14, %%ymm4\n\t"
"vpsubd %%ymm12, %%ymm14, %%ymm5\n\t"
"vpsubd %%ymm11, %%ymm14, %%ymm6\n\t"
"vpsubd %%ymm12, %%ymm14, %%ymm7\n\t"
"vpcmpeqw %%ymm10, %%ymm10, %%ymm10\n\t"
"vpsrlw $15, %%ymm10, %%ymm10\n\t"
".p2align 5\n\t"
"1:\n\t"
"vmovdqa 0(%[b]), %%ymm11\n\t"
"vmovdqa 32(%[b]), %%ymm12\n\t"
"vpbroadcastd 0(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 4(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 8(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm15, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 12(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vmovdqa 64(%[b]), %%ymm11\n\t"
"vmovdqa 96(%[b]), %%ymm12\n\t"
"vpbroadcastd 16(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 20(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 24(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm15, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 28(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vmovdqa 128(%[b]), %%ymm11\n\t"
"vmovdqa 160(%[b]), %%ymm12\n\t"
"vpbroadcastd 32(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 36(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 40(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm15, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 44(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vmovdqa 192(%[b]), %%ymm11\n\t"
"vmovdqa 224(%[b]), %%ymm12\n\t"
"vpbroadcastd 48(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 52(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 56(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm15, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 60(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vmovdqa 256(%[b]), %%ymm11\n\t"
"vmovdqa 288(%[b]), %%ymm12\n\t"
"vpbroadcastd 64(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 68(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 72(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm15, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 76(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vmovdqa 320(%[b]), %%ymm11\n\t"
"vmovdqa 352(%[b]), %%ymm12\n\t"
"vpbroadcastd 80(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 84(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 88(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm15, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 92(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vmovdqa 384(%[b]), %%ymm11\n\t"
"vmovdqa 416(%[b]), %%ymm12\n\t"
"vpbroadcastd 96(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 100(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 104(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm15, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 108(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vmovdqa 448(%[b]), %%ymm11\n\t"
"vmovdqa 480(%[b]), %%ymm12\n\t"
"vpbroadcastd 112(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 116(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 120(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm15, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 124(%[a]), %%ymm13\n\t"
"vpmaddubsw %%ymm11, %%ymm13, %%ymm14\n\t"
"vpmaddubsw %%ymm12, %%ymm13, %%ymm15\n\t"
"vpmaddwd %%ymm10, %%ymm14, %%ymm14\n\t"
"vpmaddwd %%ymm10, %%ymm15, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"add $128, %[a]\n\t"
"add $512, %[b]\n\t"
"sub $8, %[k]\n\t"
"jnz 1b\n\t"
"vmovntdq %%ymm0, (%[out])\n\t"
"vmovntdq %%ymm1, 32(%[out])\n\t"
"add %[stride], %[out]\n\t"
"vmovntdq %%ymm2, (%[out])\n\t"
"vmovntdq %%ymm3, 32(%[out])\n\t"
"add %[stride], %[out]\n\t"
"vmovntdq %%ymm4, (%[out])\n\t"
"vmovntdq %%ymm5, 32(%[out])\n\t"
"add %[stride], %[out]\n\t"
"vmovntdq %%ymm6, (%[out])\n\t"
"vmovntdq %%ymm7, 32(%[out])\n\t"
: [a] "+&r"(a), [b] "+&r"(b), [out] "+&r"(out), [k] "+&r"(k)
: [stride] "r"(stride), [sums] "r"(sums)
: "memory", "cc", "ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15");
}
static __attribute__((noinline)) void kernel_pair2(const uint8_t *a,const uint8_t *b,uint32_t *out,const int *sums,int kc,int ldc) {
ptrdiff_t stride=(ptrdiff_t)ldc*4;
int k=kc/4;
asm volatile (
"vpxor %%ymm14, %%ymm14, %%ymm14\n\t"
"vmovdqa (%[sums]), %%ymm11\n\t"
"vmovdqa 32(%[sums]), %%ymm12\n\t"
"vpsubd %%ymm11, %%ymm14, %%ymm0\n\t"
"vpsubd %%ymm12, %%ymm14, %%ymm1\n\t"
"vpsubd %%ymm11, %%ymm14, %%ymm2\n\t"
"vpsubd %%ymm12, %%ymm14, %%ymm3\n\t"
"vpcmpeqw %%ymm15, %%ymm15, %%ymm15\n\t"
"vpsrlw $15, %%ymm15, %%ymm15\n\t"
".p2align 5\n\t"
"1:\n\t"
"vmovdqa 0(%[b]), %%ymm11\n\t"
"vmovdqa 32(%[b]), %%ymm12\n\t"
"vmovdqa 64(%[b]), %%ymm13\n\t"
"vmovdqa 96(%[b]), %%ymm14\n\t"
"vpbroadcastd 0(%[a]), %%ymm10\n\t"
"vpmaddubsw %%ymm11, %%ymm10, %%ymm8\n\t"
"vpmaddubsw %%ymm12, %%ymm10, %%ymm9\n\t"
"vpbroadcastd 16(%[a]), %%ymm10\n\t"
"vpmaddubsw %%ymm13, %%ymm10, %%ymm4\n\t"
"vpmaddubsw %%ymm14, %%ymm10, %%ymm5\n\t"
"vpaddw %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm15, %%ymm8, %%ymm8\n\t"
"vpmaddwd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm8, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm9, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 4(%[a]), %%ymm10\n\t"
"vpmaddubsw %%ymm11, %%ymm10, %%ymm8\n\t"
"vpmaddubsw %%ymm12, %%ymm10, %%ymm9\n\t"
"vpbroadcastd 20(%[a]), %%ymm10\n\t"
"vpmaddubsw %%ymm13, %%ymm10, %%ymm4\n\t"
"vpmaddubsw %%ymm14, %%ymm10, %%ymm5\n\t"
"vpaddw %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm15, %%ymm8, %%ymm8\n\t"
"vpmaddwd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm8, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm9, %%ymm3, %%ymm3\n\t"
"vmovdqa 128(%[b]), %%ymm11\n\t"
"vmovdqa 160(%[b]), %%ymm12\n\t"
"vmovdqa 192(%[b]), %%ymm13\n\t"
"vmovdqa 224(%[b]), %%ymm14\n\t"
"vpbroadcastd 32(%[a]), %%ymm10\n\t"
"vpmaddubsw %%ymm11, %%ymm10, %%ymm8\n\t"
"vpmaddubsw %%ymm12, %%ymm10, %%ymm9\n\t"
"vpbroadcastd 48(%[a]), %%ymm10\n\t"
"vpmaddubsw %%ymm13, %%ymm10, %%ymm4\n\t"
"vpmaddubsw %%ymm14, %%ymm10, %%ymm5\n\t"
"vpaddw %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm15, %%ymm8, %%ymm8\n\t"
"vpmaddwd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm8, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm9, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 36(%[a]), %%ymm10\n\t"
"vpmaddubsw %%ymm11, %%ymm10, %%ymm8\n\t"
"vpmaddubsw %%ymm12, %%ymm10, %%ymm9\n\t"
"vpbroadcastd 52(%[a]), %%ymm10\n\t"
"vpmaddubsw %%ymm13, %%ymm10, %%ymm4\n\t"
"vpmaddubsw %%ymm14, %%ymm10, %%ymm5\n\t"
"vpaddw %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddw %%ymm5, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm15, %%ymm8, %%ymm8\n\t"
"vpmaddwd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm8, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm9, %%ymm3, %%ymm3\n\t"
"add $64, %[a]\n\t"
"add $256, %[b]\n\t"
"sub $4, %[k]\n\t"
"jnz 1b\n\t"
"vmovntdq %%ymm0, (%[out])\n\t"
"vmovntdq %%ymm1, 32(%[out])\n\t"
"add %[stride], %[out]\n\t"
"vmovntdq %%ymm2, (%[out])\n\t"
"vmovntdq %%ymm3, 32(%[out])\n\t"
: [a] "+&r"(a), [b] "+&r"(b), [out] "+&r"(out), [k] "+&r"(k)
: [stride] "r"(stride), [sums] "r"(sums)
: "memory", "cc", "ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15");
}
static void leaf(Mat a,Mat b,uint32_t *c,int n,int ldc,int bcount) {
{
const int pc=0,kc=n;
for(int j=0;j<n;j+=8) _mm256_store_si256((__m256i*)(bsum+j),_mm256_setzero_si256());
for(int k=0;k<kc;k+=4) {
for(int j=0;j<n;j+=16) {
__m256i s0=_mm256_load_si256((const __m256i*)(bsum+j));
__m256i s1=_mm256_load_si256((const __m256i*)(bsum+j+8));
__m128i v0=ld(b,pc+k,j),v1=ld(b,pc+k+1,j),v2=ld(b,pc+k+2,j),v3=ld(b,pc+k+3,j);
__m128i wa=_mm_add_epi16(_mm_add_epi16(_mm_cvtepi8_epi16(v0),_mm_cvtepi8_epi16(v1)),
_mm_add_epi16(_mm_cvtepi8_epi16(v2),_mm_cvtepi8_epi16(v3)));
__m128i wb=_mm_add_epi16(_mm_add_epi16(_mm_cvtepi8_epi16(_mm_srli_si128(v0,8)),_mm_cvtepi8_epi16(_mm_srli_si128(v1,8))),
_mm_add_epi16(_mm_cvtepi8_epi16(_mm_srli_si128(v2,8)),_mm_cvtepi8_epi16(_mm_srli_si128(v3,8))));
s0=_mm256_add_epi32(s0,_mm256_cvtepi16_epi32(wa));
s1=_mm256_add_epi32(s1,_mm256_cvtepi16_epi32(wb));
__m128i t0=_mm_unpacklo_epi8(v0,v1),t1=_mm_unpackhi_epi8(v0,v1);
__m128i t2=_mm_unpacklo_epi8(v2,v3),t3=_mm_unpackhi_epi8(v2,v3);
uint8_t *p=bp+j*kc+k*16;
_mm_store_si128((__m128i*)(p), _mm_unpacklo_epi16(t0,t2));
_mm_store_si128((__m128i*)(p+16),_mm_unpackhi_epi16(t0,t2));
_mm_store_si128((__m128i*)(p+32),_mm_unpacklo_epi16(t1,t3));
_mm_store_si128((__m128i*)(p+48),_mm_unpackhi_epi16(t1,t3));
_mm256_store_si256((__m256i*)(bsum+j),s0);
_mm256_store_si256((__m256i*)(bsum+j+8),s1);
}
}
for(int j=0;j<n;j+=8) _mm256_store_si256((__m256i*)(bsum+j),_mm256_mullo_epi32(_mm256_load_si256((const __m256i*)(bsum+j)),_mm256_set1_epi32(120)));
_mm_sfence();
for(int ic=0;ic<n;ic+=MC) {
int mc=n-ic<MC?n-ic:MC;
for(int r=0;r<mc;r+=MR) {
__m128i *p=(__m128i*)(ap+r*kc);
__m128i b88=_mm_set1_epi8(120);
for(int k=0;k<kc;k+=16) {
__m128i v0=ld(a,ic+r,k),v1=ld(a,ic+r+1,k),v2=ld(a,ic+r+2,k),v3=ld(a,ic+r+3,k);
__m128i s1=_mm_unpacklo_epi32(v0,v1),s2=_mm_unpackhi_epi32(v0,v1);
__m128i s3=_mm_unpacklo_epi32(v2,v3),s4=_mm_unpackhi_epi32(v2,v3);
__m128i o0=_mm_unpacklo_epi64(s1,s3),o1=_mm_unpackhi_epi64(s1,s3);
__m128i o2=_mm_unpacklo_epi64(s2,s4),o3=_mm_unpackhi_epi64(s2,s4);
_mm_store_si128(p++,_mm_add_epi8(o0,b88));
_mm_store_si128(p++,_mm_add_epi8(o1,b88));
_mm_store_si128(p++,_mm_add_epi8(o2,b88));
_mm_store_si128(p++,_mm_add_epi8(o3,b88));
}
}
if(bcount<=4) {
for(int j=0;j<n;j+=NR) for(int r=0;r<mc;r+=MR) {
uint32_t *out=c+(ic+r)*ldc+j;
kernel_pair2(ap+r*kc,bp+j*kc,out,bsum+j,kc,ldc);
kernel_pair2(ap+r*kc+8,bp+j*kc,out+2*ldc,bsum+j,kc,ldc);
}
} else {
for(int j=0;j<n;j+=NR) for(int r=0;r<mc;r+=MR)
kernel(ap+r*kc,bp+j*kc,c+(ic+r)*ldc+j,bsum+j,kc,ldc);
}
}
}
}
static void combine(int h,uint32_t *c,int ldc,uint32_t *m) {
size_t z=(size_t)h*h+11328;
for(int i=0;i<h;i++)for(int j=0;j<h;j+=16) {
size_t t=(size_t)i*h+j;
__m256i m1=_mm256_load_si256((__m256i*)(m+t)),m2=_mm256_load_si256((__m256i*)(m+z+t)),
m3=_mm256_load_si256((__m256i*)(m+2*z+t)),m4=_mm256_load_si256((__m256i*)(m+3*z+t)),
m5=_mm256_load_si256((__m256i*)(m+4*z+t)),m6=_mm256_load_si256((__m256i*)(m+5*z+t)),m7=_mm256_load_si256((__m256i*)(m+6*z+t));
__m256i n1=_mm256_load_si256((__m256i*)(m+t+8)),n2=_mm256_load_si256((__m256i*)(m+z+t+8)),
n3=_mm256_load_si256((__m256i*)(m+2*z+t+8)),n4=_mm256_load_si256((__m256i*)(m+3*z+t+8)),
n5=_mm256_load_si256((__m256i*)(m+4*z+t+8)),n6=_mm256_load_si256((__m256i*)(m+5*z+t+8)),n7=_mm256_load_si256((__m256i*)(m+6*z+t+8));
__m256i c11=_mm256_add_epi32(_mm256_sub_epi32(_mm256_add_epi32(m1,m4),m5),m7);
__m256i c12=_mm256_add_epi32(m3,m5),c21=_mm256_add_epi32(m2,m4);
__m256i c22=_mm256_add_epi32(_mm256_add_epi32(_mm256_sub_epi32(m1,m2),m3),m6);
__m256i d11=_mm256_add_epi32(_mm256_sub_epi32(_mm256_add_epi32(n1,n4),n5),n7);
__m256i d12=_mm256_add_epi32(n3,n5),d21=_mm256_add_epi32(n2,n4);
__m256i d22=_mm256_add_epi32(_mm256_add_epi32(_mm256_sub_epi32(n1,n2),n3),n6);
if(h==2048) {
__m256 inv=_mm256_set1_ps(1.0f/(QA*QB));
_mm256_stream_ps((float*)(c+i*ldc+j),_mm256_mul_ps(_mm256_cvtepi32_ps(c11),inv));
_mm256_stream_ps((float*)(c+i*ldc+j+8),_mm256_mul_ps(_mm256_cvtepi32_ps(d11),inv));
_mm256_stream_ps((float*)(c+i*ldc+j+h),_mm256_mul_ps(_mm256_cvtepi32_ps(c12),inv));
_mm256_stream_ps((float*)(c+i*ldc+j+h+8),_mm256_mul_ps(_mm256_cvtepi32_ps(d12),inv));
_mm256_stream_ps((float*)(c+(i+h)*ldc+j),_mm256_mul_ps(_mm256_cvtepi32_ps(c21),inv));
_mm256_stream_ps((float*)(c+(i+h)*ldc+j+8),_mm256_mul_ps(_mm256_cvtepi32_ps(d21),inv));
_mm256_stream_ps((float*)(c+(i+h)*ldc+j+h),_mm256_mul_ps(_mm256_cvtepi32_ps(c22),inv));
_mm256_stream_ps((float*)(c+(i+h)*ldc+j+h+8),_mm256_mul_ps(_mm256_cvtepi32_ps(d22),inv));
} else {
_mm256_stream_si256((__m256i*)(c+i*ldc+j),c11);_mm256_stream_si256((__m256i*)(c+i*ldc+j+8),d11);
_mm256_stream_si256((__m256i*)(c+i*ldc+j+h),c12);_mm256_stream_si256((__m256i*)(c+i*ldc+j+h+8),d12);
_mm256_stream_si256((__m256i*)(c+(i+h)*ldc+j),c21);_mm256_stream_si256((__m256i*)(c+(i+h)*ldc+j+8),d21);
_mm256_stream_si256((__m256i*)(c+(i+h)*ldc+j+h),c22);_mm256_stream_si256((__m256i*)(c+(i+h)*ldc+j+h+8),d22);
}
}
}
static inline __m256i ld32(Mat a,int i,int k) {
__m256i x=_mm256_loadu_si256((const __m256i*)(a.x+(size_t)i*a.ld+k));
if(a.sign) {__m256i y=_mm256_loadu_si256((const __m256i*)(a.y+(size_t)i*a.ly+k));
x=a.sign>0?_mm256_add_epi8(x,y):_mm256_sub_epi8(x,y);}
return x;
}
static void materialize1024(Mat src, int8_t *dst) {
for (int i=0;i<1024;i++)
for (int j=0;j<1024;j+=32)
_mm256_storeu_si256((__m256i*)(dst+(size_t)i*1024+j),ld32(src,i,j));
}
static void product1024(Mat a,Mat b,uint32_t *c,unsigned char *work,int bcount) {
constexpr int h=512;constexpr size_t z=(size_t)h*h+11328;
Mat aa[4]={{a.x,a.ld},{a.x+h,a.ld},{a.x+(size_t)h*a.ld,a.ld},{a.x+(size_t)h*a.ld+h,a.ld}};
Mat bb[4]={{b.x,b.ld},{b.x+h,b.ld},{b.x+(size_t)h*b.ld,b.ld},{b.x+(size_t)h*b.ld+h,b.ld}};
uint32_t *m=(uint32_t*)work;
const int ai[7]={0,2,0,3,0,2,1},aj[7]={3,3,-1,-1,1,0,3},as[7]={1,1,0,0,1,-1,-1};
const int bi[7]={0,0,1,2,3,0,2},bj[7]={3,-1,3,0,-1,1,3},bs[7]={1,0,-1,-1,0,1,1};
for(int r=0;r<7;r++) {
Mat x=aa[ai[r]],y=bb[bi[r]];
if(as[r]) {x.y=aa[aj[r]].x;x.ly=a.ld;x.sign=as[r];}
if(bs[r]) {y.y=bb[bj[r]].x;y.ly=b.ld;y.sign=bs[r];}
leaf(x,y,m+(size_t)r*z,h,512,bcount*(bs[r]?2:1));
}
_mm_sfence();
combine(h,c,1024,m);
_mm_sfence();
}
static void product2048(Mat a,Mat b,uint32_t *c,unsigned char *work,int bcount) {
constexpr int h=1024;constexpr size_t z=size_t(h)*h+11328;
Mat aa[4]={{a.x,a.ld},{a.x+h,a.ld},{a.x+h*a.ld,a.ld},{a.x+h*a.ld+h,a.ld}};
Mat bb[4]={{b.x,b.ld},{b.x+h,b.ld},{b.x+h*b.ld,b.ld},{b.x+h*b.ld+h,b.ld}};
uint32_t *m=(uint32_t*)work;
int8_t *ma=(int8_t*)(m+7*z), *mb=ma+1024*1024;
unsigned char *nested=(unsigned char*)(mb+1024*1024);
const int ai[7]={0,2,0,3,0,2,1},aj[7]={3,3,-1,-1,1,0,3},as[7]={1,1,0,0,1,-1,-1};
const int bi[7]={0,0,1,2,3,0,2},bj[7]={3,-1,3,0,-1,1,3},bs[7]={1,0,-1,-1,0,1,1};
for(int r=0;r<7;r++) {
Mat x=aa[ai[r]],y=bb[bi[r]];
if(as[r]) {x.y=aa[aj[r]].x;x.ly=a.ld;x.sign=as[r];}
if(bs[r]) {y.y=bb[bj[r]].x;y.ly=b.ld;y.sign=bs[r];}
materialize1024(x,ma); materialize1024(y,mb);
product1024({ma,1024},{mb,1024},m+(size_t)r*z,nested,bcount*(bs[r]?2:1));
}
_mm_sfence();
combine(h,c,2048,m);
_mm_sfence();
}
static inline __m256i quant16(const float *a,__m256 scale) {
__m256i x=_mm256_cvtps_epi32(_mm256_mul_ps(_mm256_load_ps(a),scale));
__m256i y=_mm256_cvtps_epi32(_mm256_mul_ps(_mm256_load_ps(a+8),scale));
return _mm256_permute4x64_epi64(_mm256_packs_epi32(x,y),0xd8);
}
static void quant_sums(const float *a,int8_t *dst,int8_t *diag,int q,bool isb) {
constexpr int h=2048,n=4096;constexpr size_t z=size_t(h)*h+11328;
const __m256 scale=_mm256_set1_ps(float(q));
for(int i=0;i<h;i++)for(int j=0;j<h;j+=32) {
__m256i x0a=quant16(a+i*n+j,scale),x0b=quant16(a+i*n+j+16,scale);
__m256i x1a=quant16(a+i*n+j+h,scale),x1b=quant16(a+i*n+j+h+16,scale);
__m256i x2a=quant16(a+(i+h)*n+j,scale),x2b=quant16(a+(i+h)*n+j+16,scale);
__m256i x3a=quant16(a+(i+h)*n+j+h,scale),x3b=quant16(a+(i+h)*n+j+h+16,scale);
size_t t=(size_t)i*h+j;
__m256i v0a=_mm256_add_epi16(x0a,x3a),v0b=_mm256_add_epi16(x0b,x3b);
__m256i v1a=isb?_mm256_sub_epi16(x1a,x3a):_mm256_add_epi16(x2a,x3a);
__m256i v1b=isb?_mm256_sub_epi16(x1b,x3b):_mm256_add_epi16(x2b,x3b);
__m256i v2a=isb?_mm256_sub_epi16(x2a,x0a):_mm256_add_epi16(x0a,x1a);
__m256i v2b=isb?_mm256_sub_epi16(x2b,x0b):_mm256_add_epi16(x0b,x1b);
__m256i v3a=isb?_mm256_add_epi16(x0a,x1a):_mm256_sub_epi16(x2a,x0a);
__m256i v3b=isb?_mm256_add_epi16(x0b,x1b):_mm256_sub_epi16(x2b,x0b);
__m256i v4a=isb?_mm256_add_epi16(x2a,x3a):_mm256_sub_epi16(x1a,x3a);
__m256i v4b=isb?_mm256_add_epi16(x2b,x3b):_mm256_sub_epi16(x1b,x3b);
_mm256_stream_si256((__m256i*)(dst+t), _mm256_permute4x64_epi64(_mm256_packs_epi16(v0a,v0b),0xd8));
_mm256_stream_si256((__m256i*)(dst+z+t), _mm256_permute4x64_epi64(_mm256_packs_epi16(v1a,v1b),0xd8));
_mm256_stream_si256((__m256i*)(dst+2*z+t), _mm256_permute4x64_epi64(_mm256_packs_epi16(v2a,v2b),0xd8));
_mm256_stream_si256((__m256i*)(dst+3*z+t), _mm256_permute4x64_epi64(_mm256_packs_epi16(v3a,v3b),0xd8));
_mm256_stream_si256((__m256i*)(dst+4*z+t), _mm256_permute4x64_epi64(_mm256_packs_epi16(v4a,v4b),0xd8));
_mm256_stream_si256((__m256i*)(diag+t), _mm256_permute4x64_epi64(_mm256_packs_epi16(x0a,x0b),0xd8));
_mm256_stream_si256((__m256i*)(diag+z+t), _mm256_permute4x64_epi64(_mm256_packs_epi16(x3a,x3b),0xd8));
}
}
void matrix_multiply(int n,const float *a,const float *b,float *c) {
(void)n;
constexpr int h=2048;constexpr size_t z=size_t(h)*h+11328;
uint32_t *m=(uint32_t*)workspace;
int8_t *p=(int8_t*)(m+7*z),*q=p+5*z;
unsigned char *sub=(unsigned char*)(q+5*z);
quant_sums(a,p,aq,QA,false);quant_sums(b,q,bq,QB,true);
_mm_sfence();
product2048({p,h},{q,h},m,sub,2);
product2048({p+z,h},{bq,h},m+z,sub,1);
product2048({aq,h},{q+z,h},m+2*z,sub,2);
product2048({aq+z,h},{q+2*z,h},m+3*z,sub,2);
product2048({p+2*z,h},{bq+z,h},m+4*z,sub,1);
product2048({p+3*z,h},{q+3*z,h},m+5*z,sub,2);
product2048({p+4*z,h},{q+4*z,h},m+6*z,sub,2);
combine(h,(uint32_t*)c,4096,m);
_mm_sfence();
}
| Compilation | N/A | N/A | Compile OK | Score: N/A | 显示更多 |
| Testcase #1 | 525.094 ms | 269 MB + 460 KB | Accepted | Score: 100 | 显示更多 |