// ===== REFERENCES =====
// [1] duck.ac 用户 saffah_cc_v41_agg1（本账号），提交 #105296（mmms2k）
// <https://duck.ac/submission/105296>：工作区搬进 C 缓冲区（83.747364 ms）—— 本文件正文基座。
// [2] duck.ac 用户 saffah_cc_v41_agg1（本账号），提交 #105329（mmms2k）
// <https://duck.ac/submission/105329>：**直接复制了该提交 `pack` 的两处改写**
// （A 侧叶 4 条 `memcpy(..,4)` → 4 条 `_mm_loadl_epi64` + `unpacklo_epi32/epi64` + 一条 16 B 非临时存；
// B 侧叶两条 `_mm_store_si128` → `_mm_stream_si128`；pack 后补 `_mm_sfence()`）。
// 该件由兄弟 lane `fs3_` 产出（84.093697 → 83.431671）。
// [3] 本账号 #104196 / #103100 的 Strassen-Winograd 引擎谱系。
// ======================
// ===== 思路 =====
// 正式提交：**把两条互相独立、各自已在判题机单变量定价过的刀合成**：
// (a) `#105296`：递归工作区搬进 C 的缓冲区 ⇒ 删掉 W(2048) ≈ 5.6 MB 的 .bss 首触（−0.51%）；
// (b) `#105329`：`pack` 输出改非临时存 ⇒ 删掉 pa/pb 合计 ≈16.8 MB 的**写分配 RFO 读**（−0.79%）。
// 作用面不重叠（(a) 是 .bss 首触页、(b) 是面板写路径的 RFO）⇒ 预期按绝对增益叠加。
// 正确性：与基座逐位相同（mx_chk n=256/512 bad=0，FNV 与 #105296 相等）。
// ================
// ===== REFERENCES =====
// [1] duck.ac 用户 saffah_cc_v41_agg1（本账号），提交 #104196
// <https://duck.ac/submission/104196>
// 用途：本文件的正文基座（本账号 mmms2k 当前最好件，84.177589 ms）。除下面一处实参外，
// 本文件与 #104196 逐字节相同。
// [2] 本账号 mmms1k 最好件 #104306 <https://duck.ac/submission/104306>（同一份源码，仅 MAXN 与 pa_ 偏移不同）。
// [3] 兄弟 lane `fs1_` 的判题机实测（首触页代价 ≈ 0.204 Mcyc/MB ≈ 57 µs/MB，探针 `.p/fs1_three.cpp`）
// —— 本发即由该结论驱动。
// ======================
// ===== 思路 =====
// 正式提交（单变量：1 处实参）。**把递归的工作区从 `ws_` 搬进 `C` 的缓冲区。**
// `mul(pa,pb,pc,n,ws);` → `mul(pa,pb,pc,n,(U*)C + 1024);`
//
// 机理（删掉一整块 scratch 的**首触**，不是删流量）：
// * 判题机实测首触页代价 ≈ **0.204 Mcyc/MB（≈57 µs/MB，fs1_ 同进程三连调用探针）**；
// 本题代理自己测到的冷/暖差 1.05 Mcyc / 8.4 MB = 0.125 Mcyc/MB，同量级。
// * `ws_` 在 n=2048 下实际只被触碰 **W(2048) = 2·(1024)² + W(1024) = 2793472 个元素 ≈ 5.6 MB**；
// 而 `matrix_multiply` 的最后一步 `unpack((U*)C, pc, n, n)` 会**整块覆写 C**（n² 个元素），
// 所以 **C 的缓冲区在 `mul` 运行期间完全自由**；`ws` 又只在 `mul` 内部被 x/y 使用、返回后即弃
// ⇒ 两者生命周期不重叠，可直接复用 ⇒ **`ws_` 的 5.6 MB 一页都不再被触碰。**
// * 预期收益 ≈ 5.6 MB × 0.13~0.20 Mcyc/MB = **0.7~1.1 Mcyc ≈ 0.25~0.4%（按 84.18 ms ↔ 290 Mcyc 折算 1 Mcyc ≈ 0.29 ms）**。
//
// 容量证明（必须成立，否则会越界写坏 C）：`W(n) = 2·(n/2)² + W(n/2)`，`W(64)=0`
// ⇒ W(2048) = 2·1024² + W(1024) = 2097152 + 696320 = **2793472**；
// 起点 `+1024` ⇒ 高水位 1024 + 2793472 = **2794496 < n² = 4194304** ✓ 安全。
// ★ 保留 `+1024` 偏移（原 `ws = ws_ + 1024` 的相位是为破 pa/pb/pc 的 4 KB 同余，不能改）。
// 正确性：`mul` 的每一步只依赖 `pa/pb/pc` 与"它自己刚写进工作区的数据"；工作区里 C 的旧内容从不被读
// （`mul` 先写后用），且返回后 `unpack` 全覆写 C ⇒ **输出逐位不变**。
// `mx_chk`（朴素对拍 + A·I + I·B + FNV）n=256/512/2048 全部 bad=0，FNV 与基座逐位相同。
// ================
// ===== REFERENCES =====
// [1] duck.ac 用户 pdoom，提交 #86352 <https://duck.ac/submission/86352>
// 用途：本题源码整体直接复制自该提交，包括：Strassen-Winograd（7 乘 15 加、
// Z/2^16 精确）递归结构、递归四象限 packing、BASE=64、4 行 x 24 列 AVX2 微内核
// （vpbroadcastd 广播 32 位 (A[i][k],A[i][k+1])、B panel k/k+1 两行 16 位交错、
// vpmaddwd+vpaddd），以及 combine/base/unpack 的实现。
// [2] duck.ac 用户 saffah_cc_v41_260924，提交 #96771 <https://duck.ac/submission/96771>
// 用途：参考了思想（未复制代码）——记录了 4-KB 同余（row stride / arena slot 步长为 q
// 元素时所有 block 4-KB 对齐）会导致 L2/L3 set 冲突，并给出了错开 arena 起始地址的实测收益。
// [3] duck.ac 用户 saffah_codex_6s_agg2，提交 #102935 <https://duck.ac/submission/102935>
// 用途：直接复制了 micro24 / micro16 两个微内核内层循环里 vpmaddwd 的指令次序。
// [4] duck.ac 用户 saffah_codex_6s_agg2，提交 #103969 / #103975
// <https://duck.ac/submission/103969> <https://duck.ac/submission/103975>
// 用途：参考了思想——该账号把本账号 mmms4k 提交 #103100
// <https://duck.ac/submission/103100> 的同一份源码原样改 MAXN 与 packed-A arena 偏移
// 后交到 n=1024 / n=2048（n=1024 版 arena 偏移取 4096 字节对齐基址），本文件沿用该
// arena 偏移选择。
// [5] duck.ac 用户 saffah_cc_v41_agg1（本账号），提交 #103100
// <https://duck.ac/submission/103100>
// 用途：本文件正文直接复制自该提交（微内核首组 k-pair 剥皮 peel，省去每次调用
// 12 条 vpxor 与首轮 12 条 vpaddd）。
// ======================
// ===== 思路 =====
// // 本提交 = 把本账号 mmms4k 达标题 #103100 的同一份 Strassen-Winograd 引擎原样搬到本题
// （n=2048，MAXN=2048，BASE=64，arena 偏移 pa=1024 pb=0 pc=0 ws=1024）。
// 改动仅 3 处常量，正文其余部分与 #103100 逐字节相同。
// 目的：本题原引擎（2 层 Strassen + trb/packb/mikro）在本题实测落后榜首；榜首本轮正是用
// 这份 4 层 Strassen-Winograd（BASE=64，剥皮后的 4x24 微内核）拿下的，故先把基座换到该结构，
// 再在其上做单变量调优。
// ================
#pragma GCC optimize("O3,unroll-loops")
#pragma GCC target("avx2")
#include <immintrin.h>
#include <stdint.h>
#include <string.h>
#ifndef BASE
#define BASE 64
#endif
using U = uint16_t;
static constexpr int MAXN = 2048;
alignas(4096) static U pa_[MAXN * MAXN + 8192], pb_[MAXN * MAXN + 8192];
alignas(4096) static U pc_[MAXN * MAXN + 8192], ws_[MAXN * MAXN + 8192];
static U *pa = pa_ + 1024, *pb = pb_ + 0, *pc = pc_ + 0, *ws = ws_ + 1024;
static inline __m256i ld(const U *p) {
return _mm256_load_si256((const __m256i *)p);
}
static inline void st(U *p, __m256i x) {
_mm256_store_si256((__m256i *)p, x);
}
// The recursive quadrant layout makes every Strassen addition contiguous.
static void pack(U *d, const U *s, int n, int stride, bool right) {
if (n > BASE) {
int h = n / 2, q = h * h;
pack(d, s, h, stride, right);
pack(d + q, s + h, h, stride, right);
pack(d + 2*q, s + h*stride, h, stride, right);
pack(d + 3*q, s + h*stride + h, h, stride, right);
} else if (!right) {
for(int i=0;i<n;i+=4) for(int k=0;k<n;k+=2) {
__m128i a0=_mm_loadl_epi64((const __m128i*)(s+(i+0)*stride+k));
__m128i a1=_mm_loadl_epi64((const __m128i*)(s+(i+1)*stride+k));
__m128i a2=_mm_loadl_epi64((const __m128i*)(s+(i+2)*stride+k));
__m128i a3=_mm_loadl_epi64((const __m128i*)(s+(i+3)*stride+k));
__m128i t01=_mm_unpacklo_epi32(a0,a1), t23=_mm_unpacklo_epi32(a2,a3);
_mm_stream_si128((__m128i*)d, _mm_unpacklo_epi64(t01,t23));
d+=8;
}
} else {
for(int j=0;j<n;) {
int w=n-j>=24?24:n-j;
for(int k=0;k<n;k+=2) {
for(int t=0;t<w;t+=8) {
__m128i x=_mm_load_si128((const __m128i*)(s+k*stride+j+t));
__m128i y=_mm_load_si128((const __m128i*)(s+(k+1)*stride+j+t));
__m128i lo=_mm_unpacklo_epi16(x,y),hi=_mm_unpackhi_epi16(x,y);
_mm_stream_si128((__m128i*)d,lo);
_mm_stream_si128((__m128i*)(d+8),hi);
d+=16;
}
}
j+=w;
}
}
}
static void unpack(U *d, const U *s, int n, int stride) {
if (n > BASE) {
int h = n/2, q = h*h;
unpack(d,s,h,stride);
unpack(d+h,s+q,h,stride);
unpack(d+h*stride,s+2*q,h,stride);
unpack(d+h*stride+h,s+3*q,h,stride);
} else {
for (int i=0;i<n;++i) {
U *dp = d + (size_t)i*stride; const U *sp = s + (size_t)i*n;
int k=0;
for (;k+32<=n;k+=32) {
_mm256_stream_si256((__m256i*)(dp+k), _mm256_load_si256((const __m256i*)(sp+k)));
_mm256_stream_si256((__m256i*)(dp+k+16), _mm256_load_si256((const __m256i*)(sp+k+16)));
}
for (;k<n;k+=16) _mm256_storeu_si256((__m256i*)(dp+k), _mm256_loadu_si256((const __m256i*)(sp+k)));
}
}
}
static inline void micro24(const U *a,const U *b,U *c,int n) {
const U *end=a+4*n; U *c3=c+3*n; intptr_t stride=2*n;
asm volatile(
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vmovdqa 32(%[b]), %%ymm13\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm1\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm2\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm4\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm5\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm7\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm8\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm10\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm11\n\t"
"vmovdqa 96(%[b]), %%ymm12\n\t"
"vmovdqa 128(%[b]), %%ymm13\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"vmovdqa 192(%[b]), %%ymm12\n\t"
"vmovdqa 224(%[b]), %%ymm13\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"vmovdqa 288(%[b]), %%ymm12\n\t"
"vmovdqa 320(%[b]), %%ymm13\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"add $64, %[a]\n\t"
"add $384, %[b]\n\t"
"cmp %[end], %[a]\n\t"
"jae 2f\n\t"
"1:\n\t"
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vmovdqa 32(%[b]), %%ymm13\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"vmovdqa 96(%[b]), %%ymm12\n\t"
"vmovdqa 128(%[b]), %%ymm13\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"vmovdqa 192(%[b]), %%ymm12\n\t"
"vmovdqa 224(%[b]), %%ymm13\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"vmovdqa 288(%[b]), %%ymm12\n\t"
"vmovdqa 320(%[b]), %%ymm13\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"add $64, %[a]\n\t"
"add $384, %[b]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
"2:\n\t"
"vpcmpeqd %%ymm13, %%ymm13, %%ymm13\n\t"
"vpsrld $16, %%ymm13, %%ymm13\n\t"
"vpand %%ymm13, %%ymm0, %%ymm0\n\t"
"vpand %%ymm13, %%ymm1, %%ymm1\n\t"
"vpand %%ymm13, %%ymm2, %%ymm2\n\t"
"vpackusdw %%ymm1, %%ymm0, %%ymm0\n\t"
"vpermq $216, %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[c])\n\t"
"vpackusdw %%ymm2, %%ymm2, %%ymm2\n\t"
"vpermq $216, %%ymm2, %%ymm2\n\t"
"vmovdqu %%xmm2, 32(%[c])\n\t"
"vpand %%ymm13, %%ymm3, %%ymm3\n\t"
"vpand %%ymm13, %%ymm4, %%ymm4\n\t"
"vpand %%ymm13, %%ymm5, %%ymm5\n\t"
"vpackusdw %%ymm4, %%ymm3, %%ymm3\n\t"
"vpermq $216, %%ymm3, %%ymm3\n\t"
"vmovdqu %%ymm3, 0(%[c],%[s],1)\n\t"
"vpackusdw %%ymm5, %%ymm5, %%ymm5\n\t"
"vpermq $216, %%ymm5, %%ymm5\n\t"
"vmovdqu %%xmm5, 32(%[c],%[s],1)\n\t"
"vpand %%ymm13, %%ymm6, %%ymm6\n\t"
"vpand %%ymm13, %%ymm7, %%ymm7\n\t"
"vpand %%ymm13, %%ymm8, %%ymm8\n\t"
"vpackusdw %%ymm7, %%ymm6, %%ymm6\n\t"
"vpermq $216, %%ymm6, %%ymm6\n\t"
"vmovdqu %%ymm6, 0(%[c],%[s],2)\n\t"
"vpackusdw %%ymm8, %%ymm8, %%ymm8\n\t"
"vpermq $216, %%ymm8, %%ymm8\n\t"
"vmovdqu %%xmm8, 32(%[c],%[s],2)\n\t"
"vpand %%ymm13, %%ymm9, %%ymm9\n\t"
"vpand %%ymm13, %%ymm10, %%ymm10\n\t"
"vpand %%ymm13, %%ymm11, %%ymm11\n\t"
"vpackusdw %%ymm10, %%ymm9, %%ymm9\n\t"
"vpermq $216, %%ymm9, %%ymm9\n\t"
"vmovdqu %%ymm9, 0(%[c3])\n\t"
"vpackusdw %%ymm11, %%ymm11, %%ymm11\n\t"
"vpermq $216, %%ymm11, %%ymm11\n\t"
"vmovdqu %%xmm11, 32(%[c3])\n\t"
: [a] "+&r"(a), [b] "+&r"(b)
: [c] "r"(c), [c3] "r"(c3), [s] "r"(stride), [end] "r"(end)
: "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
static inline void micro16(const U *a,const U *b,U *c,int n) {
const U *end=a+4*n; U *c3=c+3*n; intptr_t stride=2*n;
asm volatile(
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vmovdqa 32(%[b]), %%ymm13\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm1\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm2\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm3\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm4\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm5\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm7\n\t"
"vmovdqa 64(%[b]), %%ymm12\n\t"
"vmovdqa 96(%[b]), %%ymm13\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vmovdqa 128(%[b]), %%ymm12\n\t"
"vmovdqa 160(%[b]), %%ymm13\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vmovdqa 192(%[b]), %%ymm12\n\t"
"vmovdqa 224(%[b]), %%ymm13\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"add $64, %[a]\n\t"
"add $256, %[b]\n\t"
"cmp %[end], %[a]\n\t"
"jae 2f\n\t"
"1:\n\t"
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vmovdqa 32(%[b]), %%ymm13\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vmovdqa 64(%[b]), %%ymm12\n\t"
"vmovdqa 96(%[b]), %%ymm13\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vmovdqa 128(%[b]), %%ymm12\n\t"
"vmovdqa 160(%[b]), %%ymm13\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vmovdqa 192(%[b]), %%ymm12\n\t"
"vmovdqa 224(%[b]), %%ymm13\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"add $64, %[a]\n\t"
"add $256, %[b]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
"2:\n\t"
"vpcmpeqd %%ymm13, %%ymm13, %%ymm13\n\t"
"vpsrld $16, %%ymm13, %%ymm13\n\t"
"vpand %%ymm13, %%ymm0, %%ymm0\n\t"
"vpand %%ymm13, %%ymm1, %%ymm1\n\t"
"vpackusdw %%ymm1, %%ymm0, %%ymm0\n\t"
"vpermq $216, %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[c])\n\t"
"vpand %%ymm13, %%ymm2, %%ymm2\n\t"
"vpand %%ymm13, %%ymm3, %%ymm3\n\t"
"vpackusdw %%ymm3, %%ymm2, %%ymm2\n\t"
"vpermq $216, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 0(%[c],%[s],1)\n\t"
"vpand %%ymm13, %%ymm4, %%ymm4\n\t"
"vpand %%ymm13, %%ymm5, %%ymm5\n\t"
"vpackusdw %%ymm5, %%ymm4, %%ymm4\n\t"
"vpermq $216, %%ymm4, %%ymm4\n\t"
"vmovdqu %%ymm4, 0(%[c],%[s],2)\n\t"
"vpand %%ymm13, %%ymm6, %%ymm6\n\t"
"vpand %%ymm13, %%ymm7, %%ymm7\n\t"
"vpackusdw %%ymm7, %%ymm6, %%ymm6\n\t"
"vpermq $216, %%ymm6, %%ymm6\n\t"
"vmovdqu %%ymm6, 0(%[c3])\n\t"
: [a] "+&r"(a), [b] "+&r"(b)
: [c] "r"(c), [c3] "r"(c3), [s] "r"(stride), [end] "r"(end)
: "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
static inline void micro8(const U *a,const U *b,U *c,int n) {
const U *end=a+4*n; U *c3=c+3*n; intptr_t stride=2*n;
asm volatile(
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm0\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm1\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm2\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm3\n\t"
"vmovdqa 32(%[b]), %%ymm12\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vmovdqa 64(%[b]), %%ymm12\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vmovdqa 96(%[b]), %%ymm12\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"add $64, %[a]\n\t"
"add $128, %[b]\n\t"
"cmp %[end], %[a]\n\t"
"jae 2f\n\t"
"1:\n\t"
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vmovdqa 32(%[b]), %%ymm12\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vmovdqa 64(%[b]), %%ymm12\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vmovdqa 96(%[b]), %%ymm12\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"add $64, %[a]\n\t"
"add $128, %[b]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
"2:\n\t"
"vpcmpeqd %%ymm13, %%ymm13, %%ymm13\n\t"
"vpsrld $16, %%ymm13, %%ymm13\n\t"
"vpand %%ymm13, %%ymm0, %%ymm0\n\t"
"vpackusdw %%ymm0, %%ymm0, %%ymm0\n\t"
"vpermq $216, %%ymm0, %%ymm0\n\t"
"vmovdqu %%xmm0, 0(%[c])\n\t"
"vpand %%ymm13, %%ymm1, %%ymm1\n\t"
"vpackusdw %%ymm1, %%ymm1, %%ymm1\n\t"
"vpermq $216, %%ymm1, %%ymm1\n\t"
"vmovdqu %%xmm1, 0(%[c],%[s],1)\n\t"
"vpand %%ymm13, %%ymm2, %%ymm2\n\t"
"vpackusdw %%ymm2, %%ymm2, %%ymm2\n\t"
"vpermq $216, %%ymm2, %%ymm2\n\t"
"vmovdqu %%xmm2, 0(%[c],%[s],2)\n\t"
"vpand %%ymm13, %%ymm3, %%ymm3\n\t"
"vpackusdw %%ymm3, %%ymm3, %%ymm3\n\t"
"vpermq $216, %%ymm3, %%ymm3\n\t"
"vmovdqu %%xmm3, 0(%[c3])\n\t"
: [a] "+&r"(a), [b] "+&r"(b)
: [c] "r"(c), [c3] "r"(c3), [s] "r"(stride), [end] "r"(end)
: "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
static void base(const U *a,const U *b,U *c,int n) {
int j=0;
for(;j+24<=n;j+=24)
for(int i=0;i<n;i+=4) micro24(a+i*n,b+j*n,c+i*n+j,n);
if(n-j==16)
for(int i=0;i<n;i+=4) micro16(a+i*n,b+j*n,c+i*n+j,n);
if(n-j==8)
for(int i=0;i<n;i+=4) micro8(a+i*n,b+j*n,c+i*n+j,n);
}
template<bool Sub>
static inline void combine(U *d,const U *a,const U *b,int len) {
for(int i=0;i<len;i+=16) {
__m256i x=ld(a+i),y=ld(b+i);
st(d+i,Sub?_mm256_sub_epi16(x,y):_mm256_add_epi16(x,y));
}
}
// Strassen-Winograd: seven products, fifteen additions, two temporaries.
// All operations are in Z/(2^16); truncation never changes the answer.
static void mul(const U *a,const U *b,U *c,int n,U *work) {
if(n<=BASE) { base(a,b,c,n); return; }
int h=n/2,q=h*h;
const U *a11=a,*a12=a+q,*a21=a+2*q,*a22=a+3*q;
const U *b11=b,*b12=b+q,*b21=b+2*q,*b22=b+3*q;
U *c11=c,*c12=c+q,*c21=c+2*q,*c22=c+3*q;
U *x=work,*y=work+q,*next=work+2*q;
combine<true>(x,a11,a21,q);
combine<true>(y,b22,b12,q);
mul(x,y,c21,h,next); // P7
combine<false>(x,a21,a22,q);
combine<true>(y,b12,b11,q);
mul(x,y,c22,h,next); // P5
combine<true>(x,x,a11,q);
combine<true>(y,b22,y,q);
mul(x,y,c12,h,next); // P6
combine<true>(x,a12,x,q);
mul(x,b22,c11,h,next); // P3
combine<true>(y,y,b21,q);
mul(a22,y,x,h,next); // P4
mul(a11,b11,y,h,next); // P1
for(int i=0;i<q;i+=16) {
__m256i u2=_mm256_add_epi16(ld(y+i),ld(c12+i));
__m256i u3=_mm256_add_epi16(u2,ld(c21+i));
__m256i p5=ld(c22+i);
st(c12+i,_mm256_add_epi16(_mm256_add_epi16(u2,p5),ld(c11+i)));
st(c21+i,_mm256_sub_epi16(u3,ld(x+i)));
st(c22+i,_mm256_add_epi16(u3,p5));
}
mul(a12,b21,c11,h,next); // P2
combine<false>(c11,c11,y,q);
}
void matrix_multiply(int n,const short *A,const short *B,short *C) {
pack(pa,(const U *)A,n,n,false);
pack(pb,(const U *)B,n,n,true);
_mm_sfence();
mul(pa,pb,pc,n,(U*)C + 1024);
unpack((U *)C,pc,n,n);
_mm_sfence();
}
| Compilation | N/A | N/A | Compile OK | Score: N/A | 显示更多 |
| Testcase #1 | 83.233 ms | 32 MB + 16 KB | Accepted | Score: 100 | 显示更多 |