// ===== REFERENCES =====
// [1] duck.ac 本账号 saffah_cc_v41_agg1，提交 **#117301** <https://duck.ac/submission/117301>
// （**11.756837 ms** = 本题现役 `mine`）—— 本文件正文 = 该件所在引擎（#117078，`work/s1y6_sub_epi2.cpp`）
// **逐字节** + 下方"思路"里的唯一改动（该件与 #117078 同为同码锚副本，`.text` 20059 B /
// md5 `240732562f55ae088dba5e4ed4d91ed9`）。
// [2] duck.ac 用户 saffah_codex_6s_agg2，提交 **#105778** <https://duck.ac/submission/105778>
// （11.864668 ms = 本题 T，**早于我方**）⇒ 严支 `0.99*T+1us = 11.74702132 ms`；现缺 **9.816 µs（0.083%）**。
// [3] duck.ac/problem/mmms1k 题面；以上均为公开提交，正文按站点规则公开可见，按 RULES §3 署名。
// ======================
// ===== 思路 =====
// 【本发只有一个变量：删除 `micro24` / `micro16` 汇编**入口处的一对死守卫** `cmp %[end],%[a]` + `jae 2f`】
// 为什么它是死代码（逐条推导，非猜测）：
// `base(a,b,c,n)` 只在 `j+24<=n` 时调 `micro24(a+i*n,b+j*n,c+i*n+j,n)` ⇒ micro24 的 n ≥ 24；
// `micro16` 只在 `n-j==16` 时调用 ⇒ 其 n == 16。两者入口在跑完一整层后执行 `add $64,%[a]`，
// 而本次调用的 a 跨度为 `4n` **个元素** = `8n` 字节 ≥ 128 B（n≥16）⇒ `a0+64 < end` 恒成立
// ⇒ `jae 2f` **永不被采用**（守卫只在 n≤8 时才需要，而那种情况由 `micro8` 走，本发未动 micro8）。
// ⇒ 每条 micro24 调用 −2 条指令、每条 micro16 调用 −2 条（共 −2×76832 −2×38416 = −230 496 条/跑）。
// 定价依据（本席判题机 custom-test 同程序交替面板，`work/s1zd_tmp/p_*`）：
// 入口站点 1 条死 NOP ≈ **+0.22 tick/执行**（`ent32` 32 条 → +544 900 tick @3.44 GHz）；且本席
// 同站点**同条数对照臂** `dead_ctl`（把这一对换成同长度死 NOP：`nopl 0x0(%rax)`+`nop`）实测
// **比真删慢 +52 Ktick** ⇒ 这 2 条的存在本身是正价，删除是负价（方向与剂量律一致）。
// 先验：−230 496 条 × ≈0.15 tick ≈ **−35 Ktick ≈ −10 µs**（≈ 缺口 9.816 µs）。
// 反例警示（在档）：`del3`（尾段/入口删 3 条）在**判题机榜面**上 +46 µs。区别：del3 一并删掉了**入口
// `prefetchnta`**（其后被判为"不是冗余"，删它每调用 ~+50 µs）；本发**不碰任何预取、不改任何访存**，
// 只删一对恒不跳转的比较+跳转。
// 闸门：判题同款 `ref/gcc9/g9.sh -O2 -std=c++17 -U_FORTIFY_SOURCE` rc=0；`.text` 双证；ck 五点
// (n=64/128/256/512/1024) 与现役件**逐字同**；`tools/submit.py --dry` 先过；TMPDIR 题内、零 /tmp。
// ======================
#pragma GCC optimize("O3,unroll-loops")
#pragma GCC target("avx2")
#include <immintrin.h>
#include <stdint.h>
#include <string.h>
#ifndef BASE
#define BASE 64
#endif
using U = uint16_t;
alignas(32) static const unsigned short low16_mask[16] = {0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0};
alignas(32) static const unsigned char single_shuffle_mask[32] = {0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128,0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128};
static constexpr int MAXN = 1024;
alignas(4096) static U pa_[MAXN * MAXN + 8192], pb_[MAXN * MAXN + 8192];
alignas(4096) static U pc_[MAXN * MAXN + 8192], ws_[MAXN * MAXN + 8192];
static U *pa = pa_ + 0, *pb = pb_ + 0, *pc = pc_ + 0, *ws = ws_ + 1024;
static inline __m256i ld(const U *p) {
return _mm256_load_si256((const __m256i *)p);
}
static inline void st(U *p, __m256i x) {
_mm256_store_si256((__m256i *)p, x);
}
// The recursive quadrant layout makes every Strassen addition contiguous.
template<int STRIDE> static void packT(U *d, const U *s, int n, int rstride, bool right);
static void pack(U *d, const U *s, int n, int stride, bool right) {
if (n == 1024 && stride == 1024) { packT<1024>(d, s, n, stride, right); return; }
packT<0>(d, s, n, stride, right);
}
template<int STRIDE> static void packT(U *d, const U *s, int n, int rstride, bool right) {
const int stride = (STRIDE > 0) ? STRIDE : rstride;
if (n > BASE) {
int h = n / 2, q = h * h;
packT<STRIDE>(d, s, h, stride, right);
packT<STRIDE>(d + q, s + h, h, stride, right);
packT<STRIDE>(d + 2*q, s + h*stride, h, stride, right);
packT<STRIDE>(d + 3*q, s + h*stride + h, h, stride, right);
} else if (!right) {
/* 每次读 8 字节 = 4 个 k（2 个 k-pair），两个 k-pair 的输出都由这一组 load 得到：
低 64 位 = k-pair p；高 64 位 = k-pair p+1 ⇒ 每 2 个 k-pair 只付 4 条 load、2 条 unpacklo_epi32。 */
for(int i=0;i<n;i+=4) for(int k=0;k<n;k+=4) {
__m128i a0=_mm_loadl_epi64((const __m128i*)(s+(i+0)*stride+k));
__m128i a1=_mm_loadl_epi64((const __m128i*)(s+(i+1)*stride+k));
__m128i a2=_mm_loadl_epi64((const __m128i*)(s+(i+2)*stride+k));
__m128i a3=_mm_loadl_epi64((const __m128i*)(s+(i+3)*stride+k));
__m128i t01=_mm_unpacklo_epi32(a0,a1), t23=_mm_unpacklo_epi32(a2,a3);
_mm_stream_si128((__m128i*)d, _mm_unpacklo_epi64(t01,t23));
_mm_stream_si128((__m128i*)(d+8),_mm_unpackhi_epi64(t01,t23));
d+=16;
}
} else {
/* B 侧叶：一次处理 4 行（k 步长 2→4）⇒ 每趟 4 条 16 B 载入同时在飞（原来只有 2 条）。
输出寻址推导：某 (k,t) 对的输出块在 `((k/2)*(w/8) + t/8)*16` 元素处；
故行 k,k+1 的块在 `db`、行 k+2,k+3 的块在 `db + (w/8)*16`，两者每 t 步各 +16。 */
for(int j=0;j<n;) {
int w=n-j>=24?24:n-j;
U *db=d;
for(int k=0;k<n;k+=4) {
U *dp=db, *dp2=db+(size_t)(w/8)*16;
int t=0;
/* [s1y6_] 每 16 列跨度按 ss=[0-3, 8-11, 4-7, 12-15] 存（hi 与下一个 lo 对调） */
for(;t+16<=w;t+=16) {
__m128i x0=_mm_load_si128((const __m128i*)(s+(size_t)k*stride+j+t));
__m128i y0=_mm_load_si128((const __m128i*)(s+(size_t)(k+1)*stride+j+t));
__m128i z0=_mm_load_si128((const __m128i*)(s+(size_t)(k+2)*stride+j+t));
__m128i u0=_mm_load_si128((const __m128i*)(s+(size_t)(k+3)*stride+j+t));
__m128i x1=_mm_load_si128((const __m128i*)(s+(size_t)k*stride+j+t+8));
__m128i y1=_mm_load_si128((const __m128i*)(s+(size_t)(k+1)*stride+j+t+8));
__m128i z1=_mm_load_si128((const __m128i*)(s+(size_t)(k+2)*stride+j+t+8));
__m128i u1=_mm_load_si128((const __m128i*)(s+(size_t)(k+3)*stride+j+t+8));
__m128i lo0=_mm_unpacklo_epi16(x0,y0),hi0=_mm_unpackhi_epi16(x0,y0);
__m128i lo1=_mm_unpacklo_epi16(x1,y1),hi1=_mm_unpackhi_epi16(x1,y1);
__m128i lo2=_mm_unpacklo_epi16(z0,u0),hi2=_mm_unpackhi_epi16(z0,u0);
__m128i lo3=_mm_unpacklo_epi16(z1,u1),hi3=_mm_unpackhi_epi16(z1,u1);
_mm_stream_si128((__m128i*)(dp+0),lo0);
_mm_stream_si128((__m128i*)(dp+8),lo1);
_mm_stream_si128((__m128i*)(dp+16),hi0);
_mm_stream_si128((__m128i*)(dp+24),hi1);
_mm_stream_si128((__m128i*)(dp2+0),lo2);
_mm_stream_si128((__m128i*)(dp2+8),lo3);
_mm_stream_si128((__m128i*)(dp2+16),hi2);
_mm_stream_si128((__m128i*)(dp2+24),hi3);
dp+=32; dp2+=32;
}
for(;t<w;t+=8) {
__m128i x=_mm_load_si128((const __m128i*)(s+(size_t)k*stride+j+t));
__m128i y=_mm_load_si128((const __m128i*)(s+(size_t)(k+1)*stride+j+t));
__m128i z=_mm_load_si128((const __m128i*)(s+(size_t)(k+2)*stride+j+t));
__m128i u=_mm_load_si128((const __m128i*)(s+(size_t)(k+3)*stride+j+t));
__m128i lo=_mm_unpacklo_epi16(x,y),hi=_mm_unpackhi_epi16(x,y);
__m128i lo2=_mm_unpacklo_epi16(z,u),hi2=_mm_unpackhi_epi16(z,u);
_mm_stream_si128((__m128i*)dp,lo);
_mm_stream_si128((__m128i*)(dp+8),hi);
_mm_stream_si128((__m128i*)dp2,lo2);
_mm_stream_si128((__m128i*)(dp2+8),hi2);
dp+=16; dp2+=16;
}
db+=(size_t)(w/8)*32;
}
d+=(size_t)(w/8)*16*(size_t)(n/2); /* 本 j 组共 n/2 个 k-pair 组 x (w/8) 块 x 16 元素 */
j+=w;
}
}
}
template<int STRIDE> static void unpackT(U *d, const U *s, int n, int rstride);
static void unpack(U *d, const U *s, int n, int stride) {
if (n == 1024 && stride == 1024) { unpackT<1024>(d, s, n, stride); return; }
unpackT<0>(d, s, n, stride);
}
template<int STRIDE> static void unpackT(U *d, const U *s, int n, int rstride) {
const int stride = (STRIDE > 0) ? STRIDE : rstride;
if (n > BASE) {
int h = n/2, q = h*h;
unpackT<STRIDE>(d,s,h,stride);
unpackT<STRIDE>(d+h,s+q,h,stride);
unpackT<STRIDE>(d+h*stride,s+2*q,h,stride);
unpackT<STRIDE>(d+h*stride+h,s+3*q,h,stride);
} else {
for (int i=0;i<n;++i) {
U *dp = d + (size_t)i*stride; const U *sp = s + (size_t)i*n;
int k=0;
for (;k+32<=n;k+=32) {
_mm256_stream_si256((__m256i*)(dp+k), _mm256_load_si256((const __m256i*)(sp+k)));
_mm256_stream_si256((__m256i*)(dp+k+16), _mm256_load_si256((const __m256i*)(sp+k+16)));
}
for (;k<n;k+=16) _mm256_storeu_si256((__m256i*)(dp+k), _mm256_loadu_si256((const __m256i*)(sp+k)));
}
}
}
static inline void micro24(const U *a,const U *b,U *c,int n) {
const U *end=a+4*n; U *c3=c+3*n; intptr_t stride=2*n;
asm volatile(
"prefetchnta 768(%[b])\n\t"
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vmovdqa 32(%[b]), %%ymm13\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm1\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm2\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm4\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm5\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm7\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm8\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm10\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm11\n\t"
"vmovdqa 96(%[b]), %%ymm12\n\t"
"vmovdqa 128(%[b]), %%ymm13\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"vmovdqa 192(%[b]), %%ymm12\n\t"
"vmovdqa 224(%[b]), %%ymm13\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"vmovdqa 288(%[b]), %%ymm12\n\t"
"vmovdqa 320(%[b]), %%ymm13\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"add $64, %[a]\n\t"
"add $384, %[b]\n\t"
".p2align 4\n\t"
"1:\n\t"
"prefetchnta 768(%[b])\n\t"
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vmovdqa 32(%[b]), %%ymm13\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 64(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"vmovdqa 96(%[b]), %%ymm12\n\t"
"vmovdqa 128(%[b]), %%ymm13\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 160(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"vmovdqa 192(%[b]), %%ymm12\n\t"
"vmovdqa 224(%[b]), %%ymm13\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 256(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"vmovdqa 288(%[b]), %%ymm12\n\t"
"vmovdqa 320(%[b]), %%ymm13\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpaddd %%ymm14, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm9, %%ymm9\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm15\n\t"
"vpmaddwd 352(%[b]), %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm14, %%ymm11, %%ymm11\n\t"
"add $64, %[a]\n\t"
"add $384, %[b]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
"2:\n\t"
"vmovdqa %[low16], %%ymm13\n\t"
"vmovdqa %[shuf], %%ymm12\n\t"
"vpand %%ymm13, %%ymm0, %%ymm0\n\t"
"vpand %%ymm13, %%ymm1, %%ymm1\n\t"
"vpackusdw %%ymm1, %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[c])\n\t"
"vpshufb %%ymm12, %%ymm2, %%ymm2\n\t"
"vpermq $216, %%ymm2, %%ymm2\n\t"
"vmovdqu %%xmm2, 32(%[c])\n\t"
"vpand %%ymm13, %%ymm3, %%ymm3\n\t"
"vpand %%ymm13, %%ymm4, %%ymm4\n\t"
"vpackusdw %%ymm4, %%ymm3, %%ymm3\n\t"
"vmovdqu %%ymm3, 0(%[c],%[s],1)\n\t"
"vpshufb %%ymm12, %%ymm5, %%ymm5\n\t"
"vpermq $216, %%ymm5, %%ymm5\n\t"
"vmovdqu %%xmm5, 32(%[c],%[s],1)\n\t"
"vpand %%ymm13, %%ymm6, %%ymm6\n\t"
"vpand %%ymm13, %%ymm7, %%ymm7\n\t"
"vpackusdw %%ymm7, %%ymm6, %%ymm6\n\t"
"vmovdqu %%ymm6, 0(%[c],%[s],2)\n\t"
"vpshufb %%ymm12, %%ymm8, %%ymm8\n\t"
"vpermq $216, %%ymm8, %%ymm8\n\t"
"vmovdqu %%xmm8, 32(%[c],%[s],2)\n\t"
"vpand %%ymm13, %%ymm9, %%ymm9\n\t"
"vpand %%ymm13, %%ymm10, %%ymm10\n\t"
"vpackusdw %%ymm10, %%ymm9, %%ymm9\n\t"
"vmovdqu %%ymm9, 0(%[c3])\n\t"
"vpshufb %%ymm12, %%ymm11, %%ymm11\n\t"
"vpermq $216, %%ymm11, %%ymm11\n\t"
"vmovdqu %%xmm11, 32(%[c3])\n\t"
: [a] "+&r"(a), [b] "+&r"(b)
: [c] "r"(c), [c3] "r"(c3), [s] "r"(stride), [end] "r"(end), [shuf] "m"(single_shuffle_mask), [low16] "m"(low16_mask)
: "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
static inline void micro16(const U *a,const U *b,U *c,int n) {
const U *end=a+4*n; U *c3=c+3*n; intptr_t stride=2*n;
asm volatile(
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vmovdqa 32(%[b]), %%ymm13\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm0\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm1\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm2\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm3\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm4\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm5\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm6\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm7\n\t"
"vmovdqa 64(%[b]), %%ymm12\n\t"
"vmovdqa 96(%[b]), %%ymm13\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vmovdqa 128(%[b]), %%ymm12\n\t"
"vmovdqa 160(%[b]), %%ymm13\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vmovdqa 192(%[b]), %%ymm12\n\t"
"vmovdqa 224(%[b]), %%ymm13\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"add $64, %[a]\n\t"
"add $256, %[b]\n\t"
".p2align 3\n\t"
"1:\n\t"
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vmovdqa 32(%[b]), %%ymm13\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vmovdqa 64(%[b]), %%ymm12\n\t"
"vmovdqa 96(%[b]), %%ymm13\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vmovdqa 128(%[b]), %%ymm12\n\t"
"vmovdqa 160(%[b]), %%ymm13\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vmovdqa 192(%[b]), %%ymm12\n\t"
"vmovdqa 224(%[b]), %%ymm13\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpaddd %%ymm14, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpaddd %%ymm14, %%ymm3, %%ymm3\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm4, %%ymm4\n\t"
"vpaddd %%ymm14, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpmaddwd %%ymm13, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm15, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"add $64, %[a]\n\t"
"add $256, %[b]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
"2:\n\t"
"vmovdqa %[low16], %%ymm13\n\t"
"vpand %%ymm13, %%ymm0, %%ymm0\n\t"
"vpand %%ymm13, %%ymm1, %%ymm1\n\t"
"vpackusdw %%ymm1, %%ymm0, %%ymm0\n\t"
"vmovdqu %%ymm0, 0(%[c])\n\t"
"vpand %%ymm13, %%ymm2, %%ymm2\n\t"
"vpand %%ymm13, %%ymm3, %%ymm3\n\t"
"vpackusdw %%ymm3, %%ymm2, %%ymm2\n\t"
"vmovdqu %%ymm2, 0(%[c],%[s],1)\n\t"
"vpand %%ymm13, %%ymm4, %%ymm4\n\t"
"vpand %%ymm13, %%ymm5, %%ymm5\n\t"
"vpackusdw %%ymm5, %%ymm4, %%ymm4\n\t"
"vmovdqu %%ymm4, 0(%[c],%[s],2)\n\t"
"vpand %%ymm13, %%ymm6, %%ymm6\n\t"
"vpand %%ymm13, %%ymm7, %%ymm7\n\t"
"vpackusdw %%ymm7, %%ymm6, %%ymm6\n\t"
"vmovdqu %%ymm6, 0(%[c3])\n\t"
: [a] "+&r"(a), [b] "+&r"(b)
: [c] "r"(c), [c3] "r"(c3), [s] "r"(stride), [end] "r"(end), [shuf] "m"(single_shuffle_mask), [low16] "m"(low16_mask)
: "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
static inline void micro8(const U *a,const U *b,U *c,int n) {
const U *end=a+4*n; U *c3=c+3*n; intptr_t stride=2*n;
asm volatile(
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm0\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm1\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm2\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm3\n\t"
"vmovdqa 32(%[b]), %%ymm12\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vmovdqa 64(%[b]), %%ymm12\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vmovdqa 96(%[b]), %%ymm12\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"add $64, %[a]\n\t"
"add $128, %[b]\n\t"
"cmp %[end], %[a]\n\t"
"jae 2f\n\t"
".p2align 5\n\t"
"1:\n\t"
"vmovdqa 0(%[b]), %%ymm12\n\t"
"vpbroadcastd 0(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 4(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 8(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 12(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vmovdqa 32(%[b]), %%ymm12\n\t"
"vpbroadcastd 16(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 20(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 24(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 28(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vmovdqa 64(%[b]), %%ymm12\n\t"
"vpbroadcastd 32(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 36(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 40(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 44(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"vmovdqa 96(%[b]), %%ymm12\n\t"
"vpbroadcastd 48(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm0, %%ymm0\n\t"
"vpbroadcastd 52(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm1, %%ymm1\n\t"
"vpbroadcastd 56(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm2, %%ymm2\n\t"
"vpbroadcastd 60(%[a]), %%ymm14\n\t"
"vpmaddwd %%ymm12, %%ymm14, %%ymm15\n\t"
"vpaddd %%ymm15, %%ymm3, %%ymm3\n\t"
"add $64, %[a]\n\t"
"add $128, %[b]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
"2:\n\t"
"vpcmpeqd %%ymm13, %%ymm13, %%ymm13\n\t"
"vpsrld $16, %%ymm13, %%ymm13\n\t"
"vpand %%ymm13, %%ymm0, %%ymm0\n\t"
"vpackusdw %%ymm0, %%ymm0, %%ymm0\n\t"
"vpermq $216, %%ymm0, %%ymm0\n\t"
"vmovdqu %%xmm0, 0(%[c])\n\t"
"vpand %%ymm13, %%ymm1, %%ymm1\n\t"
"vpackusdw %%ymm1, %%ymm1, %%ymm1\n\t"
"vpermq $216, %%ymm1, %%ymm1\n\t"
"vmovdqu %%xmm1, 0(%[c],%[s],1)\n\t"
"vpand %%ymm13, %%ymm2, %%ymm2\n\t"
"vpackusdw %%ymm2, %%ymm2, %%ymm2\n\t"
"vpermq $216, %%ymm2, %%ymm2\n\t"
"vmovdqu %%xmm2, 0(%[c],%[s],2)\n\t"
"vpand %%ymm13, %%ymm3, %%ymm3\n\t"
"vpackusdw %%ymm3, %%ymm3, %%ymm3\n\t"
"vpermq $216, %%ymm3, %%ymm3\n\t"
"vmovdqu %%xmm3, 0(%[c3])\n\t"
: [a] "+&r"(a), [b] "+&r"(b)
: [c] "r"(c), [c3] "r"(c3), [s] "r"(stride), [end] "r"(end)
: "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
static void base(const U *a,const U *b,U *c,int n) {
int j=0;
for(;j+24<=n;j+=24)
for(int i=0;i<n;i+=4) micro24(a+i*n,b+j*n,c+i*n+j,n);
if(n-j==16)
for(int i=0;i<n;i+=4) micro16(a+i*n,b+j*n,c+i*n+j,n);
if(n-j==8)
for(int i=0;i<n;i+=4) micro8(a+i*n,b+j*n,c+i*n+j,n);
}
template<bool Sub>
static inline void combine(U *d,const U *a,const U *b,int len) {
for(int i=0;i<len;i+=16) {
__m256i x=ld(a+i),y=ld(b+i);
st(d+i,Sub?_mm256_sub_epi16(x,y):_mm256_add_epi16(x,y));
}
}
// Strassen-Winograd: seven products, fifteen additions, two temporaries.
// All operations are in Z/(2^16); truncation never changes the answer.
static void mul_sub(const U *a,const U *b,U *c,int n,U *work,const U *ysub);
static void mul(const U *a,const U *b,U *c,int n,U *work) {
if(n<=BASE) { base(a,b,c,n); return; }
int h=n/2,q=h*h;
const U *a11=a,*a12=a+q,*a21=a+2*q,*a22=a+3*q;
const U *b11=b,*b12=b+q,*b21=b+2*q,*b22=b+3*q;
U *c11=c,*c12=c+q,*c21=c+2*q,*c22=c+3*q;
U *x=work,*y=work+q,*next=work+2*q;
combine<true>(x,a11,a21,q);
combine<true>(y,b22,b12,q);
mul(x,y,c21,h,next); // P7
combine<false>(x,a21,a22,q);
combine<true>(y,b12,b11,q);
mul(x,y,c22,h,next); // P5
combine<true>(x,x,a11,q);
combine<true>(y,b22,y,q);
mul(x,y,c12,h,next); // P6
combine<true>(x,a12,x,q);
mul(x,b22,c11,h,next); // P3
combine<true>(y,y,b21,q);
mul(a22,y,x,h,next); // P4
mul(a11,b11,y,h,next); // P1
for(int i=0;i<q;i+=16) {
__m256i u2=_mm256_add_epi16(ld(y+i),ld(c12+i));
__m256i u3=_mm256_add_epi16(u2,ld(c21+i));
__m256i p5=ld(c22+i);
st(c12+i,_mm256_add_epi16(_mm256_add_epi16(u2,p5),ld(c11+i)));
st(c21+i,_mm256_sub_epi16(u3,ld(x+i)));
st(c22+i,_mm256_add_epi16(u3,p5));
}
{ if (h>BASE) mul_sub(a12,b21,c11,h,next,y);
else { mul(a12,b21,c11,h,next); combine<false>(c11,c11,y,q); } }
}
static void mul_sub(const U *a,const U *b,U *c,int n,U *work,const U *ysub) {
int h=n/2,q=h*h;
const U *a11=a,*a12=a+q,*a21=a+2*q,*a22=a+3*q;
const U *b11=b,*b12=b+q,*b21=b+2*q,*b22=b+3*q;
U *c11=c,*c12=c+q,*c21=c+2*q,*c22=c+3*q;
U *x=work,*y=work+q,*next=work+2*q;
combine<true>(x,a11,a21,q);
combine<true>(y,b22,b12,q);
mul(x,y,c21,h,next); // P7
combine<false>(x,a21,a22,q);
combine<true>(y,b12,b11,q);
mul(x,y,c22,h,next); // P5
combine<true>(x,x,a11,q);
combine<true>(y,b22,y,q);
mul(x,y,c12,h,next); // P6
combine<true>(x,a12,x,q);
mul(x,b22,c11,h,next); // P3
combine<true>(y,y,b21,q);
mul(a22,y,x,h,next); // P4
mul(a11,b11,y,h,next); // P1
{
const U *s12=ysub+q,*s21=ysub+2*q,*s22=ysub+3*q;
for(int i=0;i<q;i+=16) {
__m256i u2=_mm256_add_epi16(ld(y+i),ld(c12+i));
__m256i u3=_mm256_add_epi16(u2,ld(c21+i));
__m256i p5=ld(c22+i);
st(c12+i,_mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(u2,p5),ld(c11+i)),ld(s12+i)));
st(c21+i,_mm256_add_epi16(_mm256_sub_epi16(u3,ld(x+i)),ld(s21+i)));
st(c22+i,_mm256_add_epi16(_mm256_add_epi16(u3,p5),ld(s22+i)));
}
}
mul(a12,b21,c11,h,next); // P2
for(int i=0;i<q;i+=16)
st(c11+i,_mm256_add_epi16(_mm256_add_epi16(ld(c11+i),ld(y+i)),ld(ysub+i)));
}
void matrix_multiply(int n,const short *A,const short *B,short *C) {
pack(pa,(const U *)A,n,n,false);
pack(pb,(const U *)B,n,n,true);
_mm_sfence();
mul(pa,pb,pc,n,(U*)C + 1024); /* ★ 工作区搬进 C 的缓冲区（unpack 会整块覆写 C） */
unpack((U *)C,pc,n,n);
_mm_sfence();
}
| Compilation | N/A | N/A | Compile OK | Score: N/A | 显示更多 |
| Testcase #1 | 11.77 ms | 8 MB + 12 KB | Accepted | Score: 100 | 显示更多 |