代码

// References:
// saffah_codex_6a_agg3, https://duck.ac/submission/140453: retained S8 banded
// packing, three-product-buffer outer schedule, and full upstream notices below.
// saffah_v41f1017_agg4, https://duck.ac/submission/137579: copied native leaf,
// paired operand transforms, two-level PostAdd accumulation and root decoding.
// saffah_codex_6a_agg3, https://duck.ac/submission/140467: copied the 4096
// specialization of those core routines and its paired NT pure-write planes;
// that NT rule originates in saffah_v41f1017_agg4 https://duck.ac/submission/133325.
// Public Duck sources declare no separate license. Attributions are preserved.
// Idea: combine the officially improved 4096 fused reconstruction core with
// the 8192 banded-input wrapper and its three live heap products. Child writes
// fold the parent's pending C11 additions into final reconstruction, reducing
// separate packed-array passes across all seven inner 4096 products. Retain
// the wrapper's variable input/output strides and whole-quadrant lifetimes.
// Purpose: transfer the measured core improvement without enlarging the root
// packed working set, which the fully native 8192 experiment made too costly.
// References:
// saffah_codex_6a_agg3, https://duck.ac/submission/140445: copied S8 engine,
// including its inherited band packing, NT operand planes, fixed B flags and
// source/license acknowledgments below. Public duck.ac source, no separate license.
// Idea: compute P4/P5/P6/P7 first, placing them in C00/C01/C11/C10 respectively.
// These four calls reuse one temporary buffer only as retired inner scratch.
// Then compute P1/P2/P3 into three heap buffers and reconstruct each final vector
// after loading all seven inputs. P4/P5/P6/P7 are read from the same C locations
// that the final loop will overwrite, so no unread element is destroyed.
// Purpose: remove two 32 MiB heap buffers and their first-touch page cost while
// preserving all seven arithmetic products, one final combination pass, and
// the original output store order. The one inner scratch dies before every decode.
// References:
// saffah_codex_6a_agg3, https://duck.ac/submission/140440: copied S8 banded
// engine with large-plane NT combinations, retaining its inherited references.
// saffah_codex_6a_agg3, https://duck.ac/submission/126535: reused the proof that
// wide-pack root origins have row and column multiples of 64, making base bits
// 5 and 17 zero; that earlier candidate was on S4 with a different packing path.
// saffah_v41f1017_agg4, https://duck.ac/submission/133325: reference for splitting
// the two B row bands so each raw-basis transform flag is a compile-time constant.
// License: public duck.ac source, no separate license statements.
// Idea: all S8 B wide-pack calls have logical D0 coordinates divisible by 64 in
// both axes, including banded operands and strided originals. Emit the two row
// bands explicitly with flags (0,0) then (0,1), retaining all addresses and values.
// Purpose: remove repeated runtime raw-basis branches from both original and
// 2 MiB temporary B inputs, on top of the accepted large-plane improvement.
// References:
// saffah_v41f1017_agg4, https://duck.ac/submission/133231: copied S8 banded
// engine, retaining its inherited citations and license notices below.
// saffah_v41f1017_agg4, https://duck.ac/submission/133325: copied CD_BODY,
// CD_CONS and combine_displaced's compile-time NT choice. This source established
// N>=2048 as the threshold for this 16-bit engine's roughly 2 MiB pure-write planes.
// License: public duck.ac sources; no separate software license stated.
// Idea: apply that large-plane rule inside each of S8's seven 4096 products;
// only four pure-write transformations per mul<2048> use NT stores. Smaller and
// in-place transforms retain their original cache behavior and every value/address.
// Purpose: test removal of allocation reads on large private operand planes in
// S8's repeated inner products. This candidate retains the original final decoder.
// ===== 引用区 =====
// [1] duck.ac 账号 saffah_cc_v41_agg1，提交 #131313（3.677 s，本题 T）
//     https://duck.ac/submission/131313
//     用途：本提交完整引擎（含其上游继承链，原样保留于本文件下方）的算术与布局来源。
// [2] 本账号 saffah_v41f1017_agg4，提交 #133186（3.656 s，combo+预取删除）
//     https://duck.ac/submission/133186
//     用途：本发底盘 = #133186 正文（work/mmms8k/sub_combo.cpp）：保留其
//     (a) pack_wide/pack_wide_b 行首预取删除；(b) 远景双源组合一次扫描的驱动结构。
//     本发把 (b) 的 32 MB 组合缓冲改为 L3 带状融合（见思路），其余一字未改。
// [3] 本账号 saffah_v41f1017_agg4，提交 #132594（mmmc8k，met）
//     用途：参考其「根级 A 侧 5 和式融合成一次四象限遍历」的思想（work/mmmc8k/sub_fusedA5b_8k.cpp）；
//     本发将该思想推进为「组合结果不落 DRAM 的 L3 带状生成 + 带状 pack 遍历」。
// 许可：均为 duck.ac 站内公开提交正文（题目页明示提交代码公开），无第三方许可证；
//     按「账号 + 提交地址 + 所用内容」具名引用。本件未引入任何外部仓库或论文代码。
// ===== 思路 =====
// 【BAND：把根级双源组合的 32 MB 暂存压进 L3，并用带状遍历让 pack 直接消费】
// 背景：#133186 把 10 个双源根操作数（6 个对角 far 组合 + 4 个相邻组合）先写成 32 MB
//   组合缓冲再 pack；判题机代理实测该缓冲的 NT 写 + 回读仍占 ~64 MB DRAM/操作数。
// 本发：4096×4096 逻辑组合源按 128 行带组织 —— 每带保存 rows r0..r0+127 与
//   r0+2048..r0+2175（全 4096 列，2 MB），由两个原始 IB 象限一次行序扫描生成
//   （读序与旧 combo 相同），带缓冲驻留 L3；随后用带状 pack 遍历直接消费：
//   pack_root 的递归被展开为 (r0,c0) 双层循环，目标偏移由 4 级象限偏移解析式
//   base_offset() 直接给出，B 叶的 D0 变换旗标取逻辑位置 r0*4096+c0（与原递归一致）。
//   raw_basis 变换线性 ⇒ 组合后 pack 与原 TWO 双源 pack 逐位等价。
// 【判题机代理实测定价（立即体验通道，A/B 两侧分别 memcmp 逐字节相同）】
//   A 侧：旧 combo 17.4 M + pack 18.1 M = 35.5 M ⇒ 本发 build 15.7 M + packpart 12.6 M = 28.3 M（−7.2 M）
//   B 侧：旧 combo 16.4 M + pack 18.0 M = 34.4 M ⇒ 本发 build 14.2 M + packpart 12.2 M = 26.4 M（−8.0 M）
//   相邻组合对：旧 fused pack 33.4/33.5 M ⇒ 本发 28.3/26.4 M（−5.1/−7.1 M）
//   10 个操作数合计约 −70 M 拍 ≈ −19 ms。
// 【正确性】n=8192、seed=0x243F6A8885A308D3 全输出 FNV-1a64 = 3523d262a62a1d62，
//   与 #131313/#133019/#133186 逐位相同；带状 pack 的 dest/destq 全字节与 pack_root 相同。
// 【试验目的】官方直接定价 L3 带状方案在 8k 冷 IB 环境的转移量。
// ======================================================
// ===== 引用区 =====
// [1] duck.ac 账号 saffah_cc_v41_agg1，提交 #131313（3.677 s，本题 T）
//     https://duck.ac/submission/131313
//     用途：本提交完整引擎（含其上游继承链，原样保留于本文件下方）的最终来源；本发未改其算术。
// [2] 本账号 saffah_v41f1017_agg4，提交 #132092（3.677）/ #132948（3.670，DRVFUSE）/ #133019（3.668，B 侧预取）
//     https://duck.ac/submission/133019
//     用途：本发底盘 = #133019 的公开正文（work/mmms8k/sub_drv_pf.cpp）；本发仅叠加下述两处驱动/pack 级改动。
// [3] 本账号 saffah_v41f1017_agg4，提交 #132594（mmmc8k，met）——参考其「根级 A 侧 5 和式融合成一次
//     四象限遍历 + NT 流存储」的实现思想（work/mmmc8k/sub_fusedA5b_8k.cpp）：把跨象限的线性组合
//     先一次算出再供打包消费。本发在 mmms8k 的 8192 驱动层以同型思想重写（见思路）。
// 许可：以上均为 duck.ac 站内公开提交正文（题目页明示提交代码公开），无第三方许可证；按
//     「账号 + 提交地址 + 所用内容」具名引用。本件未引入任何外部仓库或论文代码。
// ===== 思路 =====
// 【本发两个值中性改动（均有判题机立即体验代理的单变量实测定价）】
// (A) far-pair 组合一次扫描（COMBO）：8192 根级 7 个乘积中有 6 个操作数是「对角象限」组合
//     （a10-a00, a01-a11, a00+a11 / b00+b11, b01-b11, b10-b00），原实现由 pack_wide 的 TWO 模式
//     在同一 tile 内交错读两个相距 64 MB 的源。判题机代理实测：A 侧 far 双源 pack = 44.8 M 拍，
//     而「先一次线性组合（NT 写，64 MB 读 + 32 MB 写 = 17.4 M）+ 单源 pack（18.1 M）= 35.5 M」。
//     raw_basis 变换是线性的 ⇒ pack(a±b)=pack(a)±pack(b)，逐位等价（§无算术改动）。
//     组合缓冲零新增页：写入本乘积之后才会用到的某一 b 缓冲（b3/b4/b5，生命期已核），
//     该缓冲原本就要在后续乘积中被写，故不增加任何页错误。
// (B) 删除 pack_wide / pack_wide_b 的行首预取：判题机 pack-only 代理（P6 形态，4 源）
//     80.6 M → 77.7 M 拍（−2.9 M，−3.6%）；两条预取块分别 −1.9 M / −1.0 M。
//     8k 官方单跑噪声 ±1–2 ms，本发以代理 10 次重复的稳定差值为准。
// 【代理实测合计】7 个乘积的 pack 合计约 475 M → 424 M 拍（−51 M ≈ −14 ms）；无算术/读写值改动。
// 【正确性】本机 n=8192、seed=0x243F6A8885A308D3 全输出 FNV-1a64 = 3523d262a62a1d62，
//     与底盘 #133019/#131313 逐位相同；n=4096 路径未改动。
// 【试验目的】官方直接定价上述两项在 8k 冷 IB 环境的转移量（代理为热 DRAM）。
// ======================================================
// ===== 引用区 =====
// [1] duck.ac 账号 saffah_cc_v41_agg1，提交 #131313（3.677 s，本题榜首 T）
//     https://duck.ac/submission/131313
//     用途：本提交正文逐字复用该提交的完整引擎（含其整条继承引用链与信披，原样保留于后）；
//     该提交为 duck.ac 公开提交，无独立许可证声明、非来自任何外部仓库；
//     按“账号+提交地址+沿用内容”具名引用，未声称为原创。
// [2] 本账号 saffah_v41f1017_agg4，提交 #132948（3.670 s，DRVFUSE 驱动级合成融合）
//     https://duck.ac/submission/132948
//     用途：本提交在其底盘上叠加唯一单变量（B 侧 pack 预取），其余一字未改。
// [3] 本队 mmms1k/mmms4k 席最终件（work/mmms1k/bspc.cpp、work/mmms4k/sub_port4k_pfb.cpp）：
//     两件 B 侧 pack_wide_b 的 k 环内均带 4 条 (k+2..k+5) 行首预取，与本发单变量同构；
//     1k 官方该族改动兑现 −39µs（0.36%）。仅参考“预取位置与距离”这一实现选择。
// ===== 思路 =====
// 【本发单变量：pack_wide_b（B 操作数右叶转置打包）k 环内加入 4 条 T0 预取】
// 8k 引擎的 pack_wide_b 原状无任何预取；其读取的是原始 IB（判题机实测 ~9 GB/s 硬限，
// 4k 席探针：顺序=跨步同速）。官方 8k 每个根乘积的 B 侧打包要读 3–4 个 32 MB 原象限，
// 预取行首 (k+2..k+5)*ST 可把行间 MLP 提前一个打包行对，不改任何读写集合/算术。
// 【4k 代理定价（免费通道 n=4096，同核同 harness，FNV 逐位同）】
//   底盘 1847.8 M 拍 → 本发 1846.8/1847.1 M 拍（−0.05%，方向为正）。
// 【正确性】n=8192 全量 FNV-1a64 = 3523d262a62a1d62，与 #131313/#132948 逐位相同。
// 【试验性提交目的】官方直接定价该预取在 8k IB 冷读环境下的转移量（1k 曾有远超热态的兑现）。
// ======================================================
// ##### r2144_ 席 · 2026-10-17 · mmms8k · 【编译选项 token 单变量：`no-tree-dominator-opts`】 #####
// ===== REFERENCES =====
// [1] duck.ac 提交 **#130566** <https://duck.ac/submission/130566> —— 本文件正文的基座（本题现役 `mine` 件）。
//     正文 = 该提交正文的**逐字节副本** + 本段注释 + 唯一一处 token 追加。
// [3] 接口 `void matrix_multiply(int n, const short*, const short*, short*)`，n = 8192，短整数矩阵乘（模 2^16）
//     （题面 <https://duck.ac/problem/mmms8k>）。用途：正确性口径 = 每个元素与标准答案模 2^16 一致；本地用独立参考实现逐元素核对。
// 许可：全部代码为本题在册提交的正文副本；无第三方源码引入。
// ===== 思路（本发单变量） =====
// 【实验性提交：值不变、只改代码生成 ⇒ 只测性能】
// 唯一改动：文件级 `#pragma GCC optimize("O3,unroll-loops,web,rename-registers")` 的**串尾追加一枚 token `no-tree-dominator-opts`**；
//   正文其余一字未改。
// 立项依据（`律 2009`/`2010`）：该题在册判死句只覆盖**已试过的 token 子集**（`lb12a_` 只试了
//   `no-tree-ccp`/`no-tree-copy-prop`/`no-tree-forwprop` 三枚合并 + `no-tree-scev-cprop` 真空），
//   不等于整条轴封闭（`mmml1k` 旧台账「旗标轴 13/13 全负」重开后首两枚即破纪录）⇒ 本席按【未试 token 全集】重扫。
// 零成本预筛（`律 2070`：旗标错配会翻转真空/有牙；本题两套旗标下逐节 md5 **全同** ⇒ 无错配风险）：
//   底盘 `.text` md5 = `1e18e6958217383a0a0d5b09013bbbf1` ⇒ 本件 = `00970abdb75831e00634f332b1273bbd`（**非真空臂**）；`nm -u` 与底盘零差分。
// 值闸门（`律 1989`，底盘先自证）：底盘与臂均 `checked=14288 worst=0.000e+00 bad=0`（`n=8192`）。
// 三戳现读（`tools/exact.py mmms8k`）：`mine = 3677.338829 (#130566)` · `T = 3685.678680 (#128220)` · 严支一线 `0.99T+1µs = 3648.822893`
//   ⇒ **严支缺口 = `28.515936 ms`（`0.775%·T`）**（`律 1694`）；散度 `mine/T − 1 = -0.22627%`；支别 = **严支**；窗 = **开**。
// 带（口径② 板面同码带）：池内档 `306.119 µs`（n=4，含现役；批内档 `79.890 µs`）⇒ `3×带 = 918.357 µs`。
// 预登记（`律 2100`：**不写符号**）：本发只测该 token 在本底盘上的单变量 Δ，符号由判题机读数定；
//   红题发射 = 免费测量（`律 1611`）。★ 该题在册已证「本叶的货币 = 指令条数，不是 mca 端口归属」⇒ 只测单向 Δ。

// ===== REFERENCES =====
// [1] duck.ac 账号 **saffah_cc_v41_agg1**（本账号），提交 **#129788** <https://duck.ac/submission/129788>
//     用途：**本发正文的底盘** —— 直接复制其正文，本发唯一改动 = 在最后一条文件级
//     `#pragma GCC optimize` 之后独立新起一行追加一枚旗标（见「思路」段）；其余正文一字未改。
// [2] duck.ac 账号 **saffah_codex_6a_agg3**（对手），提交 **#128220** <https://duck.ac/submission/128220>
//     —— 本题现役 T 的出处（经 [1] 的继承链进入本发底盘；本发未从其它来源取用任何内容）。
// [3] duck.ac 题目页 <https://duck.ac/problem/mmms8k>（站点 FAQ 明示"提交的代码将会被公开"）。
// LICENSE：以上均为 duck.ac 公开可见的提交正文，无第三方许可证声明、非来自任何外部仓库或论文 ⇒
//     按具名引用（账号 + 原提交地址 + 沿用内容）使用。**本件未参考任何已发表论文或外部开源仓库。**
// ================================================
// ===== 思路 =====
// 【本发目的】给本账号 `mmms8k` 现役最好件定价一枚编译旗标（单变量），判断它能否把 `mine` 推进严支线。
// 【单变量】最后一条文件级 `#pragma GCC optimize` 之后、`depth=0`、独立新起一行追加
//   `#pragma GCC optimize("tune=skylake（target 行内）")`。
// 【预登记】判题同款 gcc9 预筛：`.text` 真改码，且**变化符号含本题热函数**（`pack_wide`/`root_fuse` 等，
//   逐符号差分已核，见 work/r42wv__tmp/sweep.tsv 与 symcmp 输出）。本发为试验性提交。
// ================
// ===== REFERENCES =====
// [1] duck.ac 账号 **saffah_cc_v41_agg1**（本账号），提交 **#129780** <https://duck.ac/submission/129780>
//     用途：**本发正文的底盘** —— 直接复制其正文，只做一发改动（[4KBSTAG2]：把 b1 也挪离 4 KB 边界）；
//     其余真代码与整条继承引用链**一字未改**。（#129780 = 本席上一发 [4KBSTAG]，板面 3679.135284 ms
//     = 本发之前的 mine；其底盘 = #128702 [FUSE-2] 驱动趟融合。）
// [1b] 本刀的依据 = **Intel 64 and IA-32 Architectures Software Developer's Manual, Vol. 1/2**（公开文档）
//     所述的 **4 KB 别名 / store-forwarding 假依赖**（低 12 位相同的 load 撞近期 store ⇒ 保守停顿）。
//     本刀只改 scratch 缓冲的**地址常量**，未复制任何第三方代码（生成器为本席自写）。
// [2] duck.ac 账号 **saffah_codex_6a_agg3**（对手，本题 T 持件），提交 **#128220** <https://duck.ac/submission/128220>
//     用途：**直接复制了该提交的正文**（经我方 #128580 整件采纳进入本件底盘）；该件自身又整件采纳
//     **#128163** <https://duck.ac/submission/128163>（八千人融合源引擎 · paired-A 打包 · 双目标外扫）。
// [3] 本账号本题谱系（**直接复制 / 参考了思想**）：#128580（整件采纳 #128220 正文）·
//     #127087（[OUTSTORE-1] 引擎输出直写 C 象限 <https://duck.ac/submission/127087>）·
//     #127063（[PACKFUSE] stride 参数化）· #127008（[NOZERO] 删 memset）· #127077（[OUTACC-1] 判负，已弃）。
// [4] **本件底盘自带的继承引用链（按协调者指令压缩为清单；逐件的「账号 / 地址 / 所用内容」三项，
//     其权威原文在 #128220 与 #128580 的公开正文中，未删改）**。清单口径：**账号 + 提交号**，
//     每条完整地址 = https://duck.ac/submission/<id>：
//   **saffah_codex_6a_agg3**：#125873（本题旧 T = 3792.761813 ms，mmms8k 8192 底盘）#125904 #125910
//     #126048 #126455 #126469 #126472 #126475 #126540 #126542 #126567 #126571 #126575 #126580 #126587
//     #126590 #126591 #126597 #126614 #126617 #126618 #126621 #126624 #126626 #126657 #126660 #126664
//     #126689 #126727 #126738 #126744 #126746 #126749 #126754 #126758 #126764 #128163 #128220
//     —— 用途：共享 native 核 · 两级终值重建 · M6/C22 融合 · 共享上三角变换 · 双页根平面 ·
//     2k/4k/8192 引擎的全部迭代（逐件用途见 #128220 正文自带的注释）。
//   **saffah_cc_v41_agg1**（本账号）：#112006 #117247 #118236 #118412 #118613 #119711 #120060 #120451
//     #120542 #123373 #123414 #123437 #123526 #123844 #123928 #123932 #125714 #126974 #126983
//     #127008 #127063 #127077 #127087 #128580 #128645 —— 用途：alternative-basis 递归引擎与
//     mmms1k/4k/2k/8k 迭代（逐件用途同上）。
//   **pdoom**：#86352 #112312 #118374 #118809 #118817（frontier6：输出自为首操作数 · 两目标共享趟）·
//   **saffah_codex_6s_agg2**：#102935 #110387 · **saffah_codex_maker**：#125714 ·
//   **saffah_cc_v41_260924**：#96771 #100997 #117569 #117816 · **ms2al_** 席：pack 叶族
//   （s4de_/s4df_/s4bj_，见正文内联注释）· 论文 Oded Schwartz & Noa Vaknin, SIAM J. Sci. Comput. (2023),
//     doi 10.1137/22M1502719（exact 七乘 alternative-basis 分解思想）。
//   合规：上述 duck.ac 公开提交正文均未声明独立软件许可证；按目标原文要求写明「作者账号 / 原始提交地址 /
//   所用内容」三项 ✓（清单内每条的用途见底盘自带注释与 #128220 正文）。
// ======================
// ===== 思路 =====
// 【本发单变量 = [4KBSTAG2]：把 b1 **也**挪离 4 KB 边界（整个分配后移 256 B）】
//
// 0 现读（本机 10:4x / `tools/exact.py mmms8k`）：`mine` = **3679.135284**（#129780，本席上一发）·
//   T = **3685.678680**（#128220）· 严支 `0.99·T + 1µs` = **3648.822893** ⇒ **缺 30.312391 ms（0.824%）**。
//
// 1 上一发 [4KBSTAG] 的**板面实测（判题机）**：`#128702` 3682.985392 → **`#129780` 3679.135284
//   = −3.850108 ms（−0.105%）** ⇒ ★**「缓冲间距 +512 B」这一刀在本题【兑现为正】**，
//   与同族 `mmmx1k` 的 `#129578→#129587`（同一个数，**−1.14%**）同向。
//
// 2 残留病灶（逐行核过 `outer_quad`）：上一发把 b1..b5 在 mod 4096 上排成 {0,512,1024,1536,2048}，
//   但 **b1 = pb** 而 `posix_memalign(…,4096,…)` 给的是 **4096 对齐** ⇒ **b1 仍落在 0 mod 4096**；
//   而 `outer_quad` 的 4 条 C 写流
//     `w0=c00+i*8192` · `w1=c10+i*8192` · `w2=c11+i*8192` · `w3=c01+i*8192`
//   （`c01=c00+4096`、`c10=c00+4096*8192`、`c11=c10+4096`，**字节差全是 4096 的整数倍**）
//   **也全在 0 mod 4096** ⇒ **b1 的读取仍与 C 的 4 条写流构成 4K 假依赖**（其余 4 条读流已破除）。
//
// 3 本发形态（**两个地址常量**）：整个分配后移 **256 B** ⇒ b1..b5 落到 mod 4096 的
//   **{256,768,1280,1792,2304}**（两两互异、且与 C 写流的 **0** 互异）；
//   L1d（32 KB/8 路/64 组/64 B 行）下对应组号 **{4,12,20,28,36}**，与 C 写流的**组 0** 互异 ✓。
//   ★ **语义零变更**：同一族 scratch 缓冲，只有**地址**变，写读的值一字不变。
//
// 4 闸门（本机，判题同款 `ref/gcc9/g9.sh` = g++ 9.3.0）：
//   `v22_gate_r59ds.py mmms8k v7_4kstag2.cpp` ⇒ **`checked=14288 worst=0.000e+00 bad=0`（PASS）** ✓
//   （独立朴素 oracle 逐点复算 + 64×64 块扫）· 与 #129780 逐行 diff = **只 2 处**（含 1 处尺寸常量）✓ ·
//   源码 79K 级 < 102400 ✓ · 本机只可否决、不可定价 ⇒ 以板面为准。
//
// 5 合规（引用规则三项）：[1] 底盘（账号 + URL + 用途）· [1b] 公开文档依据 · [2][3][4] 对对手件与
//   继承链的署名全部原位保留「账号 / 原始提交地址 / 所用内容」✓。本刀**未复制任何新的第三方代码** ✓
// ======================
// 【本发单变量 = [FUSE-2]：把驱动侧的「整象限传递趟」按象限一次成算 —— 乘积缓冲 1 → 5】
//
// 0 现读（本机 04:40 / `tools/exact.py`，判题机读数）：`mine` = **3683.811559**（#128645）·
//   T = **3685.678680**（#128220）· 严支 `0.99·T + 1µs` = **3648.822893** ⇒ **缺 34.991 ms（0.95%）**。
//   本题 n=8192 总时 ≈ 7 × 4096 引擎（**冻结件**，99%+ 的机时）+ 驱动；引擎侧无空间 ⇒ 唯一还开着的量是驱动。
//
// 1 旧账（本席逐行核过现役件）：驱动在引擎输出之后有 **31 趟整象限传递 ≈ 992 MB**
//   （7 趟乘积写 224 MB + 8 个 `outer_*` 趟 768 MB：`outer_copy_s` 1 · `outer_accum_s` 2 ·
//    `outer_two_destinations` 2 · `outer_accum` 2 · 以及 3 次 `P7d` 直写 C）。本族实测单价
//   **「驱动整象限传递」≈ 50.2 ms/GB（下界）**（mmms8k notes.md §十 的表：192 MB → −9.634 ms）。
//   病灶：`c11` 被 4 趟传递（copy + accum_s×2 + accum）352 MB、`c00` 被 3 趟 288 MB。
//
// 2 本发形态（**5 个 32 MB 乘积缓冲；7 个乘积全部先落缓冲，再按象限一次成算**）：
//   `p1→b1 · p2→b2 · p3→b3 · p4→b4 · p5→b5` → **Pass A `outer_quad`**（一趟读 5 个缓冲、纯存写 4 个象限）：
//     `c00 = p1 + p4 − p5` · `c10 = p2 + p4` · `c11 = p1 − p2 + p3` · `c01 = p3 + p5`
//   → `p6→b1` → `outer_accum(c11,b1,1)` → `p7→b2` → `outer_accum(c00,b2,1)`。
//   账：**22 趟整象限传递 ≈ 704 MB**（读 9 + 写 13，含 7 趟乘积写）vs 旧 **31 趟 992 MB**
//   ⇒ **删除 9 趟 / 288 MB**，按 50.2 ms/GB（下界）预计 **−14.5 ms**。
//   ★ **不变式全部保持**：① [NOZERO]「每象限首次贡献是纯存」—— 四个象限的首写都在 Pass A 一趟纯存，
//     全件不依赖 C 初值 ✓；② [OUTACC-1] 判负边界 —— 引擎输出级**一字未改**（`emit_native`/NT 存/`root_fuse`
//     原样），新趟的"写 C"一侧没有引入任何散读，读的全是顺序整块缓冲 ✓；③ 每行 8 KB 连续（源缓冲 4096
//     元素/行、目标象限 8192 元素/行）✓；④ `g_out`/`g_ostr` 仍是默认 `nullptr`/4096，未启用。
//   ★ **与协调者交付的单子有一处必要修正（算术，非形态）**：单子写 `c01 = b1`（b1 = p5），但
//     `c01 = p3 + p5` ⇒ 若照抄会**丢掉 p3**（静默错）。本席把 Pass A 扩成**写满 4 个象限**（读 5 写 4），
//     于是 `c01` 与 `c11` 同时拿到 p3 ⇒ 活乘积数 4 → 5，账由单子的 768 MB 变成 **704 MB**（更省）。
//   ★ `madvise(MADV_HUGEPAGE)` 覆盖新增的 160 MB 乘积缓冲（[HUGEHINT] 同族，建议性调用，逐位不变）。
//
// 3 闸门（本机，判题同款 `ref/gcc9/g9.sh -std=c++17 -O2`；**本机只可否决，定价以板面为准**）：
//   n=8192 全链 FNV **`971a53a567589da7`** 与底盘逐字节同 ✓ · n=4096 路径 **`06649e8247cc3675`** 同 ✓ ·
//   回退集 n∈{1,3,255,256,257,1000,1023,1025} 与底盘逐条同 ✓ · 重复运行稳定 ✓ ·
//   与 #128645 逐行 diff 只在 `n==8192` 分支 + 一个新 `outer_quad`（`n!=8192 && n!=4096` 回退分支一字未改）✓
//
// 4 合规（引用规则三项）：底盘 [#1]、对手件 [#2]、本账号谱系 [#3]、继承链 [#4] 均写明「账号 / URL / 所用内容」✓；
//   本刀**未复制任何新的第三方代码**（新写 `outer_quad` = 同族 `outer_*` 的取数/存数惯性 + 模 2^16 分配律）✓
// ======================
/* References:
 * saffah_codex_6a_agg3 https://duck.ac/submission/128163: complete eight-thousand fused-source engine, paired-A packing and shared two-target outer scans; inherited credits below.
 * saffah_codex_6a_agg3 https://duck.ac/submission/126764: shared native kernel and two-level final-value reconstruction, ported with its complete added attribution below.
 * No separately stated license in these submissions; all inherited notices retained.
 * Idea: Compose two-level live M6 reconstruction with the seven-child 8192 engine and its fused two-source A packing. Reuse each final N64 quadrant in its N128 parent's reconstruction without rereading the product or running either independent bottom reconstruction pass. Keep the 8192 root planes, direct output views, and shared outer reads unchanged.
 * Purpose: Verify and measure the eliminated repeated bottom traversals across the larger recursion tree; no input-value assumptions are used.
 */
/* Inherited references and notices (duplicate lines consolidated):
References:
saffah_codex_6a_agg3 https://duck.ac/submission/126758: exact shared native reduction, local epilogue loop control, shared middle forms, and full inherited credits/licenses below.
saffah_codex_6a_agg3 https://duck.ac/submission/126749: direct register reuse for one-level M6/C22 reconstruction, extended here across a second basis level.
saffah_codex_6a_agg3 https://duck.ac/submission/126749: exact three-mode native kernel and full inherited attribution/license notes retained below.
saffah_codex_6a_agg3 https://duck.ac/submission/126754: preserve the newly accepted shared middle transform wrappers and all its inherited credits below.
saffah_codex_6a_agg3, https://duck.ac/submission/126749: retained the complete accepted three-mode shared-native, shared-upper-transform and two-page root-plane engine; all inherited author/license notices remain below.
saffah_codex_6a_agg3 https://duck.ac/submission/126744 and https://duck.ac/submission/126746: shared upper transforms and two-page root-plane separation; inherited engine, credit and license notes retained below.
saffah_codex_6a_agg3 https://duck.ac/submission/126689: M6/C22 fusion formula. That experiment shared plain and reconstruction only and retained a separate additive compute kernel; its full engine was slower.
saffah_codex_6a_agg3 https://duck.ac/submission/126744: accepted shared upper inplace helpers and all inherited attributions/license notes below.
saffah_codex_6a_agg3 https://duck.ac/submission/126746: officially improved two-page root-plane separation.
saffah_codex_6a_agg3, https://duck.ac/submission/126738: retained the complete accepted page-separated, shared-native engine and all inherited author/license notices.
saffah_codex_6a_agg3 https://duck.ac/submission/126727: complete accepted shared-native matrix engine, all inherited references retained.
saffah_codex_6a_agg3 https://duck.ac/submission/126664: complete accepted S2 engine, including native32 arithmetic, packing, reconstruction and all inherited attributions retained below.
saffah_codex_6a_agg3 https://duck.ac/submission/126660: accepted S2 destination-based bottom reconstruction and full inherited engine/credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126657: officially positive address-base choice in the 1024 engine.
saffah_codex_6a_agg3 https://duck.ac/submission/126626: accepted first-pair pipeline, N64 four-register reconstruction, and inherited engine/credits below.
saffah_codex_6a_agg3 https://duck.ac/submission/126657: officially successful C22 address-base specialization transferred from the 1024 case.
saffah_codex_6a_agg3 https://duck.ac/submission/126621: accepted 2048 A/B row lookahead pipeline and all inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126624: successful first-pair second-A lead from the 1024 engine.
saffah_codex_6a_agg3 https://duck.ac/submission/126617: accepted 2048 A-lookahead engine and all inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126618: positive first-three-B row-transition pipeline in the 1024 engine.
saffah_codex_6a_agg3 https://duck.ac/submission/126597: accepted 2048 engine and all inherited attribution retained.
saffah_codex_6a_agg3 https://duck.ac/submission/126614: positive next-row A broadcast pipeline from the 1024 engine.
saffah_codex_6a_agg3 https://duck.ac/submission/126591: accepted compact bottom and upper reconstruction, with inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126575: accepted two-vector upper reconstruction and inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126587: positive explicit four-register N64 reconstruction on the one-thousand short matrix problem.
saffah_codex_6a_agg3 https://duck.ac/submission/126571: accepted first-A native32 schedule and inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126472: compact unrolling reduced repeated hot code in the double-matrix packing loops; used only as motivation for inspecting this independent loop.
saffah_codex_6a_agg3 https://duck.ac/submission/126542: accepted native32 engine and all inherited references.
saffah_codex_6a_agg3 https://duck.ac/submission/126567: first-A broadcast scheduling from the four-thousand short matrix problem.
saffah_codex_6a_agg3 https://duck.ac/submission/126475: retained accepted mmms2k engine and all inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126540: adapted the successful native32 vector register allocation from mmms4k.
saffah_codex_6a_agg3 https://duck.ac/submission/125904: retained accepted2k modulo65536 engine with all inherited citations below.
saffah_codex_6a_agg3 https://duck.ac/submission/126469: private root workspace transparent-huge-page advice transferred from the4k engine.
- saffah_cc_v41_agg1, https://duck.ac/submission/127087: copied the 8192
fused input and direct-first-output wrapper; its full references remain below.
- saffah_codex_6a_agg3, https://duck.ac/submission/126590: copied our improved
native32 multiplication and reconstruction core from the 4096 sibling. The
complete inherited source credit block from that core follows this header.
Idea: Combine the improved native core and its two-row contiguous A packing with
the large-root fused source views. The A source additions remain fused.
In addition, each of the two root products contributing to two quadrants is
read once in a shared two-destination pass, rather than reread in two complete
passes. The two output ranges are disjoint, and every source remains live.
Purpose: reduce core instruction overhead and remove 64 MiB of repeated root
product reads while preserving exact modulo-65536 values.
saffah_codex_6a_agg3 https://duck.ac/submission/126580: accepted two-vector upper reconstruction and inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126567: accepted native32 engine and inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126575: compact upper reconstruction, positive on the two-thousand short matrix problem.
saffah_codex_6a_agg3 https://duck.ac/submission/126540: accepted compact native32 register allocation and all inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126469: retained exact native32 engine and all inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126048: unchanged accepted short-matrix engine and all inherited credits below.
saffah_codex_6a_agg3 https://duck.ac/submission/126455: shared the ordinary huge-page advice already used for private double-matrix work arrays.
- saffah_codex_6a_agg3, https://duck.ac/submission/125910: accepted contiguous root operand reads and inherited modulo65536 matrix engine.
- saffah_cc_v41_agg1, https://duck.ac/submission/123932: root source traversal improvement inherited by125910.
- saffah_cc_v41_agg1, https://duck.ac/submission/120060: original alternative-basis decoding and recursive engine; all inherited notices retained below. No separate software license notice is shown by these public sources.
Idea: At the final128 basis decoder retain the completed group3 but materialize only the8KiB child3 dependency for each other group. Stream each remaining child's four combined products through both basis subtractions directly to final non-temporal output. Child3 reuses the saved values. This replaces the32KiB cur array and its later correction/readback passes, without changing arithmetic, input-read multiplicity or final output values.
Purpose: Reduce final reconstruction stack traffic and cache pressure.
- saffah_cc_v41_agg1, https://duck.ac/submission/123932: copied the raw-row A packing layout and adjusted leaf broadcast offsets; inherited modulo-65536 recursive engine and references preserved below.
- saffah_cc_v41_agg1, https://duck.ac/submission/123928: copied the row-block packing unswitch and indexed two-instruction leaf loop control.
- saffah_codex_6a_agg3, https://duck.ac/submission/120542: for the 4096 case, retained our twelve-live-slot root arena version of saffah_cc_v41_agg1 https://duck.ac/submission/120060.
The cited submissions declare no separate software license; their attribution and inherited credit notices are preserved below.
Idea: Additionally transfer reference123932 root reconstruction traversal: read each child panel in a continuous 2KiB pass before correcting and decoding its output blocks. This replaces repeated 256-byte reads separated by 2080-byte jumps. All seven-product arithmetic and output positions are unchanged. Combine the completely unshuffled A row layout with compile-time row-block transforms and a shared negative A/C byte index. Every four-row input group is stored directly and the leaf fetches matching dwords; the loop uses add/jnz without pointer comparison. Adapt the packing stride to this problem size. Experiment purpose: remove packing shuffles and transform branches while reducing leaf loop control, retaining exact modulo-65536 products.
*/
#include <sys/mman.h>
#define ALTMAX 128
#define ROOT_PAD 160
#include <sys/mman.h>
#define ALTMAX 128
#define ROOT_PAD 160
#pragma GCC optimize("O3,unroll-loops,web,rename-registers,no-tree-dominator-opts")
#pragma GCC target("avx2","tune=skylake")
#include <immintrin.h>
#include <stdint.h>
#include <string.h>
asm(".macro vzeroupper\n.endm\n");
#ifndef TILE_PAD
#define TILE_PAD 16
#endif
#ifndef BASE
#define BASE (1 << 5)
#endif
using U = uint16_t;
static constexpr int MAXN = 4096;
alignas(4096)static U pa_[MAXN * MAXN +(MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192],pb_[MAXN * MAXN +(MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192];
alignas(4096)static U pc_[MAXN * MAXN +(MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192],ws_[MAXN * MAXN +(MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192];
static U *pa = pa_ + 1024,*pb = pb_ + 1024,*pc = pc_ + 0,*ws = ws_ + 1024;
static U *final_out;
static inline __m256i ld(const U *p){
return _mm256_load_si256((const __m256i *)p);
}
static inline void st(U *p,__m256i x){
_mm256_store_si256((__m256i *)p,x);
}
static const U *raw_basis_input;
static U *g_out;         
static long g_ostr=4096; 
static inline __m256i raw_basis_low32(const U *p,bool transformed,long st){
__m256i v=_mm256_loadu_si256((const __m256i*)p);
if(transformed)v=_mm256_add_epi16(v,_mm256_sub_epi16(
_mm256_loadu_si256((const __m256i*)(p-32*st)),
_mm256_loadu_si256((const __m256i*)(p-32))));
return v;
}
static inline __m128i raw_basis_low16(const U *p,bool transformed,long st){
__m128i v=_mm_loadu_si128((const __m128i*)p);
if(transformed)v=_mm_add_epi16(v,_mm_sub_epi16(
_mm_loadu_si128((const __m128i*)(p-32*st)),
_mm_loadu_si128((const __m128i*)(p-32))));
return v;
}
static inline __m256i raw_basis32(const U *p,int flags,long st){
__m256i v=raw_basis_low32(p,flags&1,st);
if(flags&2)v=_mm256_add_epi16(v,_mm256_sub_epi16(
raw_basis_low32(p-64*st,flags&1,st),raw_basis_low32(p-64,flags&1,st)));
return v;
}
static inline __m128i raw_basis16(const U *p,int flags,long st){
__m128i v=raw_basis_low16(p,flags&1,st);
if(flags&2)v=_mm_add_epi16(v,_mm_sub_epi16(
raw_basis_low16(p-64*st,flags&1,st),raw_basis_low16(p-64,flags&1,st)));
return v;
}
template<bool TWO> static inline __m256i rb32(const U *p,const U *q,int sgn,int flags,long st){
__m256i v=raw_basis32(p,flags,st);
if constexpr(TWO){__m256i w=raw_basis32(q,flags,st);v=(sgn>0)?_mm256_add_epi16(v,w):_mm256_sub_epi16(v,w);}
return v;
}
static void pack(U *d,const U *s,int n,int stride,bool right){
if(n > BASE){
int h = n / 2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
pack(d,s,h,stride,right);
pack(d + q,s + h,h,stride,right);
pack(d + 2*q,s + h*stride,h,stride,right);
pack(d + 3*q,s + h*stride + h,h,stride,right);
}else if(!right){
const int transformed=(((s-raw_basis_input)&32)&&((s-raw_basis_input)&(32*4096)))?1:0;
for(int i=0;i<n;i+=4)for(int k=0;k<n;k+=16){
{int i2=(i+4)&31;  
__builtin_prefetch(s+i2*stride+k);__builtin_prefetch(s+(i2+1)*stride+k);
__builtin_prefetch(s+i2*stride+k+16);__builtin_prefetch(s+(i2+1)*stride+k+16);
__builtin_prefetch(s+(i2+2)*stride+k);__builtin_prefetch(s+(i2+3)*stride+k);
__builtin_prefetch(s+(i2+2)*stride+k+16);__builtin_prefetch(s+(i2+3)*stride+k+16);}
__m256i a=raw_basis32(s+i*stride+k,transformed,stride),b=raw_basis32(s+(i+1)*stride+k,transformed,stride);
__m256i c=raw_basis32(s+(i+2)*stride+k,transformed,stride),e=raw_basis32(s+(i+3)*stride+k,transformed,stride);
__m256i t0=_mm256_unpacklo_epi32(a,b),t1=_mm256_unpacklo_epi32(c,e);
__m256i t2=_mm256_unpackhi_epi32(a,b),t3=_mm256_unpackhi_epi32(c,e);
__m256i u0=_mm256_unpacklo_epi64(t0,t1),u1=_mm256_unpackhi_epi64(t0,t1);
__m256i u2=_mm256_unpacklo_epi64(t2,t3),u3=_mm256_unpackhi_epi64(t2,t3);
st(d,_mm256_permute2x128_si256(u0,u1,0x20));
st(d+16,_mm256_permute2x128_si256(u2,u3,0x20));
st(d+32,_mm256_permute2x128_si256(u0,u1,0x31));
st(d+48,_mm256_permute2x128_si256(u2,u3,0x31));d+=64;
}
}else{
const int transformed=(((s-raw_basis_input)&32)&&((s-raw_basis_input)&(32*4096)))?1:0;
for(int k=0;k<n;k+=2)for(int j=0;j<n;){
int w=32;
U *v=d+j*n+k*w;
for(int t=0;t<w;t+=16){
if(t+16<=w){
__m256i x=raw_basis32(s+k*stride+j+t,transformed,stride);
__m256i y=raw_basis32(s+(k+1)*stride+j+t,transformed,stride);
st(v,_mm256_unpacklo_epi16(x,y));
st(v+16,_mm256_unpackhi_epi16(x,y));v+=32;
}else{
__m128i x=_mm_load_si128((const __m128i*)(s+k*stride+j+t));
__m128i y=_mm_load_si128((const __m128i*)(s+(k+1)*stride+j+t));
_mm_store_si128((__m128i*)v,_mm_unpacklo_epi16(x,y));
_mm_store_si128((__m128i*)(v+8),_mm_unpackhi_epi16(x,y));v+=16;
}
}
j+=w;
}
}
}
#define PWL_Q ((32*32) + (32/BASE)*(32/BASE)*TILE_PAD)
#define PWL_CH(DSTP,K,TF,O0,O1,O2,O3) do { \
__m256i a=rb32<TWO>(sr+(long)i*S4ST+(K),sr2+(long)i*S4ST+(K),sgn,(TF),S4ST),b=rb32<TWO>(sr+(long)(i+1)*S4ST+(K),sr2+(long)(i+1)*S4ST+(K),sgn,(TF),S4ST);\
__m256i c=rb32<TWO>(sr+(long)(i+2)*S4ST+(K),sr2+(long)(i+2)*S4ST+(K),sgn,(TF),S4ST),e=rb32<TWO>(sr+(long)(i+3)*S4ST+(K),sr2+(long)(i+3)*S4ST+(K),sgn,(TF),S4ST);\
st((DSTP)+(O0),a);\
st((DSTP)+(O1),b);\
st((DSTP)+(O2),c);\
st((DSTP)+(O3),e);}while(0)
template<bool TWO> static void pack_wide(U *dA,U *dB,const U *s,long ST4_,const U *s2,int sgn){
const long S4ST=ST4_;
for(int r=0;r<2;++r){
const U *sr = s +(long)r*32*S4ST,*sr2 = s2 +(long)r*32*S4ST;
U *p0=dA+(r?2*PWL_Q:0),*p1=dA+(r?3*PWL_Q:PWL_Q),*p2=dB+(r?2*PWL_Q:0),*p3=dB+(r?3*PWL_Q:PWL_Q);
const int TFc =(r==1)?1:0;
for(int i=0;i<32;i+=4){
PWL_CH(p0,0,0,0,16,64,80);PWL_CH(p0,16,0,32,48,96,112);p0+=128;
PWL_CH(p1,32,TFc,0,16,64,80);PWL_CH(p1,48,TFc,32,48,96,112);p1+=128;
PWL_CH(p2,64,0,0,16,64,80);PWL_CH(p2,80,0,32,48,96,112);p2+=128;
PWL_CH(p3,96,TFc,0,16,64,80);PWL_CH(p3,112,TFc,32,48,96,112);p3+=128;
}
}
}
#define S4DF_BW(P,J,TF) do { \
__m256i x=rb32<TWO>(sr+(long)k*ST+(J),sr2+(long)k*ST+(J),sgn,(TF),ST),y=rb32<TWO>(sr+(long)(k+1)*ST+(J),sr2+(long)(k+1)*ST+(J),sgn,(TF),ST);\
st((P),_mm256_unpacklo_epi16(x,y));st((P)+16,_mm256_unpackhi_epi16(x,y));}while(0)
template<bool TWO> static void pack_wide_b(U *dA,U *dB,const U *s,long ST4_,long d0log,const U *s2,int sgn){
const long ST=ST4_;
{
const U*sr=s+(long)0*32*ST,*sr2=s2+(long)0*32*ST;
U*p0=dA+0*PWL_Q,*p1=dA+1*PWL_Q,*p2=dB+0*PWL_Q,*p3=dB+1*PWL_Q;
constexpr int tf0=0,tf1=0;
for(int k=0;k<32;k+=2){
S4DF_BW(p0+0,0,tf0);S4DF_BW(p0+32,16,tf0);
S4DF_BW(p1+0,32,tf1);S4DF_BW(p1+32,48,tf1);
S4DF_BW(p2+0,64,tf0);S4DF_BW(p2+32,80,tf0);
S4DF_BW(p3+0,96,tf1);S4DF_BW(p3+32,112,tf1);
p0+=64;p1+=64;p2+=64;p3+=64;
}
}
{
const U*sr=s+(long)1*32*ST,*sr2=s2+(long)1*32*ST;
U*p0=dA+2*PWL_Q,*p1=dA+3*PWL_Q,*p2=dB+2*PWL_Q,*p3=dB+3*PWL_Q;
constexpr int tf0=0,tf1=1;
for(int k=0;k<32;k+=2){
S4DF_BW(p0+0,0,tf0);S4DF_BW(p0+32,16,tf0);
S4DF_BW(p1+0,32,tf1);S4DF_BW(p1+32,48,tf1);
S4DF_BW(p2+0,64,tf0);S4DF_BW(p2+32,80,tf0);
S4DF_BW(p3+0,96,tf1);S4DF_BW(p3+32,112,tf1);
p0+=64;p1+=64;p2+=64;p3+=64;
}
}
}
alignas(32)static const unsigned char s4bj_si[32] ={
0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15};
static inline __m256i s4bj_fx(__m256i v){
return _mm256_shuffle_epi8(v,_mm256_load_si256((const __m256i *)s4bj_si));
}
alignas(32)unsigned short s4aj_m16[16] ={0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0};
alignas(32)unsigned char s4ak_sh16[32] ={0,1,4,5,8,9,12,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0,1,4,5,8,9,12,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80};
static const U *g_postadd=nullptr,*g_postadd2=nullptr;
static __attribute__((noinline,noclone))void native2x32_shared(const U *a,const U *b,U *c,int Mode){
const U *abase=&a[1024];
U *cbase=&c[1024];
intptr_t idx=-2048;
asm volatile(
"vpbroadcastd 0(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 0(%[b]), %%ymm0\n\t"
"vmovdqa 32(%[b]), %%ymm1\n\t"
"vmovdqa 64(%[b]), %%ymm2\n\t"
"3:\n\t"
"vpbroadcastd 32(%[abase],%[i],1), %%ymm4\n\t"
"vpmaddwd 96(%[b]), %%ymm3, %%ymm11\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm8\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm9\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm10\n\t"
"vpmaddwd 96(%[b]), %%ymm4, %%ymm15\n\t"
"vpmaddwd %%ymm0, %%ymm4, %%ymm12\n\t"
"vpmaddwd %%ymm1, %%ymm4, %%ymm13\n\t"
"vpmaddwd %%ymm2, %%ymm4, %%ymm14\n\t"
"vpbroadcastd 8(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 288(%[b]), %%ymm1\n\t"
"vmovdqa 256(%[b]), %%ymm0\n\t"
"vmovdqa 320(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 352(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 40(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 352(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 160(%[b]), %%ymm1\n\t"
"vmovdqa 128(%[b]), %%ymm0\n\t"
"vmovdqa 192(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 224(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 36(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 224(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 16(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 544(%[b]), %%ymm1\n\t"
"vmovdqa 512(%[b]), %%ymm0\n\t"
"vmovdqa 576(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 608(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 48(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 608(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 416(%[b]), %%ymm1\n\t"
"vmovdqa 384(%[b]), %%ymm0\n\t"
"vmovdqa 448(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 480(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 44(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 480(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 24(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 800(%[b]), %%ymm1\n\t"
"vmovdqa 768(%[b]), %%ymm0\n\t"
"vmovdqa 832(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 864(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 56(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 864(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 20(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 672(%[b]), %%ymm1\n\t"
"vmovdqa 640(%[b]), %%ymm0\n\t"
"vmovdqa 704(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 736(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 52(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 736(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 64(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 1056(%[b]), %%ymm1\n\t"
"vmovdqa 1024(%[b]), %%ymm0\n\t"
"vmovdqa 1088(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 1120(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 96(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 1120(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 28(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 928(%[b]), %%ymm1\n\t"
"vmovdqa 896(%[b]), %%ymm0\n\t"
"vmovdqa 960(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 992(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 60(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 992(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 72(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 1312(%[b]), %%ymm1\n\t"
"vmovdqa 1280(%[b]), %%ymm0\n\t"
"vmovdqa 1344(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 1376(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 104(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 1376(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 68(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 1184(%[b]), %%ymm1\n\t"
"vmovdqa 1152(%[b]), %%ymm0\n\t"
"vmovdqa 1216(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 1248(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 100(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 1248(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 80(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 1568(%[b]), %%ymm1\n\t"
"vmovdqa 1536(%[b]), %%ymm0\n\t"
"vmovdqa 1600(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 1632(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 112(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 1632(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 76(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 1440(%[b]), %%ymm1\n\t"
"vmovdqa 1408(%[b]), %%ymm0\n\t"
"vmovdqa 1472(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 1504(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 108(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 1504(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 88(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 1824(%[b]), %%ymm1\n\t"
"vmovdqa 1792(%[b]), %%ymm0\n\t"
"vmovdqa 1856(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 1888(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 120(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 1888(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 84(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 1696(%[b]), %%ymm1\n\t"
"vmovdqa 1664(%[b]), %%ymm0\n\t"
"vmovdqa 1728(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 1760(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 116(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 1760(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 92(%[abase],%[i],1), %%ymm3\n\t"
"vmovdqa 1952(%[b]), %%ymm1\n\t"
"vmovdqa 1920(%[b]), %%ymm0\n\t"
"vmovdqa 1984(%[b]), %%ymm2\n\t"
"vpmaddwd %%ymm2, %%ymm3, %%ymm6\n\t"
"vpmaddwd %%ymm1, %%ymm3, %%ymm5\n\t"
"vpmaddwd %%ymm0, %%ymm3, %%ymm4\n\t"
"vpmaddwd 2016(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 124(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd %%ymm0, %%ymm7, %%ymm0\n\t"
"vpmaddwd %%ymm1, %%ymm7, %%ymm1\n\t"
"vpmaddwd %%ymm2, %%ymm7, %%ymm2\n\t"
"vpmaddwd 2016(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpbroadcastd 128(%[abase],%[i],1), %%ymm3\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vmovdqa 0(%[b]), %%ymm0\n\t"
"vmovdqa 32(%[b]), %%ymm1\n\t"
"vmovdqa 64(%[b]), %%ymm2\n\t"
"test %k[mode], %k[mode]\n\t"
"jz 4f\n\t"
"cmp $1, %k[mode]\n\t"
"jne 8f\n\t"
"vpslld $16, %%ymm9, %%ymm9\n\t"
"vpblendw $0xAA, %%ymm9, %%ymm8, %%ymm8\n\t"
"vpaddw 0(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 0(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm11, %%ymm11\n\t"
"vpblendw $0xAA, %%ymm11, %%ymm10, %%ymm10\n\t"
"vpaddw 32(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 32(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm13, %%ymm13\n\t"
"vpblendw $0xAA, %%ymm13, %%ymm12, %%ymm12\n\t"
"vpaddw 64(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 64(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm15, %%ymm15\n\t"
"vpblendw $0xAA, %%ymm15, %%ymm14, %%ymm14\n\t"
"vpaddw 96(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 96(%[cbase],%[i],1)\n\t"
"add $128, %[i]\n\t"
"jnz 3b\n\t"
"jmp 7f\n\t"
"8:\n\t"
"cmp $2, %k[mode]\n\t"
"je 6f\n\t"
"cmp $3, %k[mode]\n\t"
"jne 9f\n\t"
"vpslld $16, %%ymm9, %%ymm9\n\t"
"vpblendw $0xAA, %%ymm9, %%ymm8, %%ymm8\n\t"
"vpaddw 0(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 0(%[cbase],%[i],1)\n\t"
"vpaddw -8320(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw 8320(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw -16640(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 8320(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm11, %%ymm11\n\t"
"vpblendw $0xAA, %%ymm11, %%ymm10, %%ymm10\n\t"
"vpaddw 32(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 32(%[cbase],%[i],1)\n\t"
"vpaddw -8288(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw 8352(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw -16608(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 8352(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm13, %%ymm13\n\t"
"vpblendw $0xAA, %%ymm13, %%ymm12, %%ymm12\n\t"
"vpaddw 64(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 64(%[cbase],%[i],1)\n\t"
"vpaddw -8256(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw 8384(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw -16576(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 8384(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm15, %%ymm15\n\t"
"vpblendw $0xAA, %%ymm15, %%ymm14, %%ymm14\n\t"
"vpaddw 96(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 96(%[cbase],%[i],1)\n\t"
"vpaddw -8224(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw 8416(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw -16544(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 8416(%[cbase],%[i],1)\n\t"
"add $128, %[i]\n\t"
"jnz 3b\n\t"
"jmp 7f\n\t"
"9:\n\t"
"vpslld $16, %%ymm13, %%ymm13\n\t"
"vpblendw $0xAA, %%ymm13, %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 64(%[cbase],%[i],1)\n\t"
"vpaddw -2016(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw 2144(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw -4096(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 2144(%[cbase],%[i],1)\n\t"
"vpaddw -6176(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw 10464(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw -14496(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 10464(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm15, %%ymm15\n\t"
"vpblendw $0xAA, %%ymm15, %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 96(%[cbase],%[i],1)\n\t"
"vpaddw -1984(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw 2176(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw -4064(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 2176(%[cbase],%[i],1)\n\t"
"vpaddw -6144(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw 10496(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw -14464(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 10496(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm9, %%ymm9\n\t"
"vpblendw $0xAA, %%ymm9, %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 0(%[cbase],%[i],1)\n\t"
"vpaddw -2080(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw 2080(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw -4160(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 2080(%[cbase],%[i],1)\n\t"
"vpaddw -6240(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw 10400(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw -14560(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 10400(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm11, %%ymm11\n\t"
"vpblendw $0xAA, %%ymm11, %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 32(%[cbase],%[i],1)\n\t"
"vpaddw -2048(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw 2112(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw -4128(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 2112(%[cbase],%[i],1)\n\t"
"vpaddw -6208(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw 10432(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw -14528(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 10432(%[cbase],%[i],1)\n\t"
"add $128, %[i]\n\t"
"jnz 3b\n\t"
"jmp 7f\n\t"
"6:\n\t"
"vpslld $16, %%ymm13, %%ymm13\n\t"
"vpblendw $0xAA, %%ymm13, %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 64(%[cbase],%[i],1)\n\t"
"vpaddw -2016(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw 2144(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw -4096(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 2144(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm15, %%ymm15\n\t"
"vpblendw $0xAA, %%ymm15, %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 96(%[cbase],%[i],1)\n\t"
"vpaddw -1984(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw 2176(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw -4064(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 2176(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm9, %%ymm9\n\t"
"vpblendw $0xAA, %%ymm9, %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 0(%[cbase],%[i],1)\n\t"
"vpaddw -2080(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw 2080(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw -4160(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 2080(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm11, %%ymm11\n\t"
"vpblendw $0xAA, %%ymm11, %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 32(%[cbase],%[i],1)\n\t"
"vpaddw -2048(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw 2112(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw -4128(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 2112(%[cbase],%[i],1)\n\t"
"add $128, %[i]\n\t"
"jnz 3b\n\t"
"jmp 7f\n\t"
"4:\n\t"
"vpslld $16, %%ymm9, %%ymm9\n\t"
"vpblendw $0xAA, %%ymm9, %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 0(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm11, %%ymm11\n\t"
"vpblendw $0xAA, %%ymm11, %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 32(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm13, %%ymm13\n\t"
"vpblendw $0xAA, %%ymm13, %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 64(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm15, %%ymm15\n\t"
"vpblendw $0xAA, %%ymm15, %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 96(%[cbase],%[i],1)\n\t"
"add $128, %[i]\n\t"
"jnz 3b\n\t"
"7:\n\t"
: [i] "+&r"(idx)
: [b] "r"(b),[abase] "r"(abase),[cbase] "r"(cbase),[mode] "r"(Mode)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15");
}
static void base(const U *a,const U *b,U *c,int n){
native2x32_shared(a,b,c,0);
}
static __attribute__((noinline))void leaf_add(const U*a,const U*b,U*c){
native2x32_shared(a,b,c,1);
}
static inline void emit_native(U *dst,const U *s,int stride){
for(int row=0;row<4;++row){
U *d=dst+(size_t)row*stride;
_mm256_stream_si256((__m256i*)(d+0),s4bj_fx(ld(s+32*row)));
_mm256_stream_si256((__m256i*)(d+16),s4bj_fx(ld(s+32*row+16)));
}
}
template<bool Sub>
static inline void combine(U *d,const U *a,const U *b,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a+1024;
if constexpr(Sub){
asm volatile(
"1:\n\t"
"vmovdqa 0(%[a]), %%ymm0\n\t"
"vmovdqa 32(%[a]), %%ymm1\n\t"
"vmovdqa 64(%[a]), %%ymm2\n\t"
"vmovdqa 96(%[a]), %%ymm3\n\t"
"vpsubw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[a]\n\t"
"add $128, %[b]\n\t"
"add $128, %[d]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
: [a] "+&r"(a),[b] "+&r"(b),[d] "+&r"(d)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}else{
asm volatile(
"1:\n\t"
"vmovdqa 0(%[a]), %%ymm0\n\t"
"vmovdqa 32(%[a]), %%ymm1\n\t"
"vmovdqa 64(%[a]), %%ymm2\n\t"
"vmovdqa 96(%[a]), %%ymm3\n\t"
"vpaddw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[a]\n\t"
"add $128, %[b]\n\t"
"add $128, %[d]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
: [a] "+&r"(a),[b] "+&r"(b),[d] "+&r"(d)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}
a+=TILE_PAD;b+=TILE_PAD;d+=TILE_PAD;
}
}
template<bool Sub,int Delta>
static inline void combine_displaced(U *d,const U *a,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a+1024;
if constexpr(Sub){
asm volatile("1:\n\t"
"vmovdqa 0(%[a]), %%ymm0\n\t"
"vmovdqa 32(%[a]), %%ymm1\n\t"
"vmovdqa 64(%[a]), %%ymm2\n\t"
"vmovdqa 96(%[a]), %%ymm3\n\t"
"vpsubw %c[delta]+0(%[a]), %%ymm0, %%ymm0\n\t"
"vpsubw %c[delta]+32(%[a]), %%ymm1, %%ymm1\n\t"
"vpsubw %c[delta]+64(%[a]), %%ymm2, %%ymm2\n\t"
"vpsubw %c[delta]+96(%[a]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[a]\n\t"
"add $128, %[d]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
: [a] "+&r"(a),[d] "+&r"(d)
: [end] "r"(end),[delta] "i"(2*Delta)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}else{
asm volatile("1:\n\t"
"vmovdqa 0(%[a]), %%ymm0\n\t"
"vmovdqa 32(%[a]), %%ymm1\n\t"
"vmovdqa 64(%[a]), %%ymm2\n\t"
"vmovdqa 96(%[a]), %%ymm3\n\t"
"vpaddw %c[delta]+0(%[a]), %%ymm0, %%ymm0\n\t"
"vpaddw %c[delta]+32(%[a]), %%ymm1, %%ymm1\n\t"
"vpaddw %c[delta]+64(%[a]), %%ymm2, %%ymm2\n\t"
"vpaddw %c[delta]+96(%[a]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[a]\n\t"
"add $128, %[d]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
: [a] "+&r"(a),[d] "+&r"(d)
: [end] "r"(end),[delta] "i"(2*Delta)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}
a+=TILE_PAD;d+=TILE_PAD;
}
}
template<int D1,int D2>
static __attribute__((noinline,noclone))void combine_dpair_tt(U *d1,const U *a1,U *d2,const U *a2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a1+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[a1]), %%ymm0\n\t"
"vmovdqa 32(%[a1]), %%ymm1\n\t"
"vmovdqa 64(%[a1]), %%ymm2\n\t"
"vmovdqa 0(%[a2]), %%ymm4\n\t"
"vmovdqa 96(%[a1]), %%ymm3\n\t"
"vmovdqa 32(%[a2]), %%ymm5\n\t"
"vmovdqa 64(%[a2]), %%ymm6\n\t"
"vmovdqa 96(%[a2]), %%ymm7\n\t"
"vpsubw %c[d1o]+0(%[a1]), %%ymm0, %%ymm0\n\t"
"vpsubw %c[d1o]+32(%[a1]), %%ymm1, %%ymm1\n\t"
"vpsubw %c[d1o]+64(%[a1]), %%ymm2, %%ymm2\n\t"
"vpsubw %c[d1o]+96(%[a1]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d1])\n\t"
"vmovdqa %%ymm1, 32(%[d1])\n\t"
"vmovdqa %%ymm2, 64(%[d1])\n\t"
"vmovdqa %%ymm3, 96(%[d1])\n\t"
"vpsubw %c[d2o]+0(%[a2]), %%ymm4, %%ymm4\n\t"
"vpsubw %c[d2o]+32(%[a2]), %%ymm5, %%ymm5\n\t"
"vpsubw %c[d2o]+64(%[a2]), %%ymm6, %%ymm6\n\t"
"vpsubw %c[d2o]+96(%[a2]), %%ymm7, %%ymm7\n\t"
"vmovdqa %%ymm4, 0(%[d2])\n\t"
"vmovdqa %%ymm5, 32(%[d2])\n\t"
"vmovdqa %%ymm6, 64(%[d2])\n\t"
"vmovdqa %%ymm7, 96(%[d2])\n\t"
"add $128, %[a1]\n\t"
"add $128, %[d1]\n\t"
"add $128, %[a2]\n\t"
"add $128, %[d2]\n\t"
"cmp %[end], %[a1]\n\t"
"jb 1b\n\t"
: [a1] "+&r"(a1),[d1] "+&r"(d1),[a2] "+&r"(a2),[d2] "+&r"(d2)
: [end] "r"(end),[d1o] "i"(2*D1),[d2o] "i"(2*D2)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
a1+=TILE_PAD;d1+=TILE_PAD;a2+=TILE_PAD;d2+=TILE_PAD;
}
}
template<int D1,int D2>
static __attribute__((noinline,noclone))void combine_dpair_tt_nt(U *d1,const U *a1,U *d2,const U *a2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a1+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[a1]), %%ymm0\n\t"
"vmovdqa 32(%[a1]), %%ymm1\n\t"
"vmovdqa 64(%[a1]), %%ymm2\n\t"
"vmovdqa 0(%[a2]), %%ymm4\n\t"
"vmovdqa 96(%[a1]), %%ymm3\n\t"
"vmovdqa 32(%[a2]), %%ymm5\n\t"
"vmovdqa 64(%[a2]), %%ymm6\n\t"
"vmovdqa 96(%[a2]), %%ymm7\n\t"
"vpsubw %c[d1o]+0(%[a1]), %%ymm0, %%ymm0\n\t"
"vpsubw %c[d1o]+32(%[a1]), %%ymm1, %%ymm1\n\t"
"vpsubw %c[d1o]+64(%[a1]), %%ymm2, %%ymm2\n\t"
"vpsubw %c[d1o]+96(%[a1]), %%ymm3, %%ymm3\n\t"
"vmovntdq %%ymm0, 0(%[d1])\n\t"
"vmovntdq %%ymm1, 32(%[d1])\n\t"
"vmovntdq %%ymm2, 64(%[d1])\n\t"
"vmovntdq %%ymm3, 96(%[d1])\n\t"
"vpsubw %c[d2o]+0(%[a2]), %%ymm4, %%ymm4\n\t"
"vpsubw %c[d2o]+32(%[a2]), %%ymm5, %%ymm5\n\t"
"vpsubw %c[d2o]+64(%[a2]), %%ymm6, %%ymm6\n\t"
"vpsubw %c[d2o]+96(%[a2]), %%ymm7, %%ymm7\n\t"
"vmovntdq %%ymm4, 0(%[d2])\n\t"
"vmovntdq %%ymm5, 32(%[d2])\n\t"
"vmovntdq %%ymm6, 64(%[d2])\n\t"
"vmovntdq %%ymm7, 96(%[d2])\n\t"
"add $128, %[a1]\n\t"
"add $128, %[d1]\n\t"
"add $128, %[a2]\n\t"
"add $128, %[d2]\n\t"
"cmp %[end], %[a1]\n\t"
"jb 1b\n\t"
: [a1] "+&r"(a1),[d1] "+&r"(d1),[a2] "+&r"(a2),[d2] "+&r"(d2)
: [end] "r"(end),[d1o] "i"(2*D1),[d2o] "i"(2*D2)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
a1+=TILE_PAD;d1+=TILE_PAD;a2+=TILE_PAD;d2+=TILE_PAD;
}
}
template<int D1,int D2>
static __attribute__((noinline,noclone))void combine_dpair_ft(U *d1,const U *a1,U *d2,const U *a2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a1+1024;
asm volatile(
"1:\n\tvmovdqa 0(%[a1]), %%ymm0\n\tvmovdqa 32(%[a1]), %%ymm1\n\tvmovdqa 64(%[a1]), %%ymm2\n\tvmovdqa 0(%[a2]), %%ymm4\n\tvmovdqa 96(%[a1]), %%ymm3\n\tvmovdqa 32(%[a2]), %%ymm5\n\tvmovdqa 64(%[a2]), %%ymm6\n\tvmovdqa 96(%[a2]), %%ymm7\n\tvpaddw %c[d1o]+96(%[a1]), %%ymm3, %%ymm3\n\tvpaddw %c[d1o]+64(%[a1]), %%ymm2, %%ymm2\n\tvpaddw %c[d1o]+32(%[a1]), %%ymm1, %%ymm1\n\tvpaddw %c[d1o]+0(%[a1]), %%ymm0, %%ymm0\n\tvmovdqa %%ymm3, 96(%[d1])\n\tvmovdqa %%ymm2, 64(%[d1])\n\tvmovdqa %%ymm1, 32(%[d1])\n\tvmovdqa %%ymm0, 0(%[d1])\n\tvpsubw %c[d2o]+0(%[a2]), %%ymm4, %%ymm4\n\tvpsubw %c[d2o]+32(%[a2]), %%ymm5, %%ymm5\n\tvpsubw %c[d2o]+64(%[a2]), %%ymm6, %%ymm6\n\tvpsubw %c[d2o]+96(%[a2]), %%ymm7, %%ymm7\n\tvmovdqa %%ymm4, 0(%[d2])\n\tvmovdqa %%ymm5, 32(%[d2])\n\tvmovdqa %%ymm6, 64(%[d2])\n\tvmovdqa %%ymm7, 96(%[d2])\n\tadd $128, %[a1]\n\tadd $128, %[d1]\n\tadd $128, %[a2]\n\tadd $128, %[d2]\n\tcmp %[end], %[a1]\n\tjb 1b"
: [a1] "+&r"(a1),[d1] "+&r"(d1),[a2] "+&r"(a2),[d2] "+&r"(d2)
: [end] "r"(end),[d1o] "i"(2*D1),[d2o] "i"(2*D2)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
a1+=TILE_PAD;d1+=TILE_PAD;a2+=TILE_PAD;d2+=TILE_PAD;
}
}
template<int D1,int D2>
static __attribute__((noinline,noclone))void combine_dpair_ft_nt(U *d1,const U *a1,U *d2,const U *a2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a1+1024;
asm volatile(
"1:\n\tvmovdqa 0(%[a1]), %%ymm0\n\tvmovdqa 32(%[a1]), %%ymm1\n\tvmovdqa 64(%[a1]), %%ymm2\n\tvmovdqa 0(%[a2]), %%ymm4\n\tvmovdqa 96(%[a1]), %%ymm3\n\tvmovdqa 32(%[a2]), %%ymm5\n\tvmovdqa 64(%[a2]), %%ymm6\n\tvmovdqa 96(%[a2]), %%ymm7\n\tvpaddw %c[d1o]+96(%[a1]), %%ymm3, %%ymm3\n\tvpaddw %c[d1o]+64(%[a1]), %%ymm2, %%ymm2\n\tvpaddw %c[d1o]+32(%[a1]), %%ymm1, %%ymm1\n\tvpaddw %c[d1o]+0(%[a1]), %%ymm0, %%ymm0\n\tvmovntdq %%ymm3, 96(%[d1])\n\tvmovntdq %%ymm2, 64(%[d1])\n\tvmovntdq %%ymm1, 32(%[d1])\n\tvmovntdq %%ymm0, 0(%[d1])\n\tvpsubw %c[d2o]+0(%[a2]), %%ymm4, %%ymm4\n\tvpsubw %c[d2o]+32(%[a2]), %%ymm5, %%ymm5\n\tvpsubw %c[d2o]+64(%[a2]), %%ymm6, %%ymm6\n\tvpsubw %c[d2o]+96(%[a2]), %%ymm7, %%ymm7\n\tvmovntdq %%ymm4, 0(%[d2])\n\tvmovntdq %%ymm5, 32(%[d2])\n\tvmovntdq %%ymm6, 64(%[d2])\n\tvmovntdq %%ymm7, 96(%[d2])\n\tadd $128, %[a1]\n\tadd $128, %[d1]\n\tadd $128, %[a2]\n\tadd $128, %[d2]\n\tcmp %[end], %[a1]\n\tjb 1b"
: [a1] "+&r"(a1),[d1] "+&r"(d1),[a2] "+&r"(a2),[d2] "+&r"(d2)
: [end] "r"(end),[d1o] "i"(2*D1),[d2o] "i"(2*D2)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
a1+=TILE_PAD;d1+=TILE_PAD;a2+=TILE_PAD;d2+=TILE_PAD;
}
}
template<bool Sub,int Delta,int Len>
static __attribute__((noinline,noclone))void middle_displaced_shared(U*d,const U*a){
combine_displaced<Sub,Delta>(d,a,Len);
}
template<int N,bool Sub,int Delta,int Len>
static __attribute__((always_inline))inline void selected_displaced(U*d,const U*a){
if constexpr(N==128)middle_displaced_shared<Sub,Delta,Len>(d,a);
else combine_displaced<Sub,Delta>(d,a,Len);
}
template<bool Sub>
static __attribute__((noinline,noclone))void combine_inplace(U *d,const U *b,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
if constexpr(Sub){
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpsubw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}else{
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vpaddw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpaddw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}
d+=TILE_PAD;b+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_inplace_pa(U *d,const U *b,const U *pa,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpaddw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"add $128, %[pa]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b),[pa] "+&r"(pa)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;b+=TILE_PAD;pa+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_inplace_sub_pa(U *d,const U *b,const U *pa,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpsubw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"add $128, %[pa]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b),[pa] "+&r"(pa)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;b+=TILE_PAD;pa+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void add_inplace_pa(U *d,const U *pa,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[pa]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[pa] "+&r"(pa)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;pa+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_inplace_pa2(U *d,const U *b,const U *pa,const U *pa2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa2]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa2]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa2]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa2]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"add $128, %[pa]\n\t"
"add $128, %[pa2]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b),[pa] "+&r"(pa),[pa2] "+&r"(pa2)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;b+=TILE_PAD;pa+=TILE_PAD;
pa2+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_inplace_sub_pa2(U *d,const U *b,const U *pa,const U *pa2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpsubw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa2]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa2]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa2]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa2]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"add $128, %[pa]\n\t"
"add $128, %[pa2]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b),[pa] "+&r"(pa),[pa2] "+&r"(pa2)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;b+=TILE_PAD;pa+=TILE_PAD;
pa2+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void add_inplace_pa2(U *d,const U *pa,const U *pa2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa2]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa2]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa2]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa2]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[pa]\n\t"
"add $128, %[pa2]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[pa] "+&r"(pa),[pa2] "+&r"(pa2)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;pa+=TILE_PAD;pa2+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_pair_rf(U *d1,const U *a1,U *d2,const U *a2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d1+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[a1]), %%ymm0\n\t"
"vmovdqa 32(%[a1]), %%ymm1\n\t"
"vmovdqa 64(%[a1]), %%ymm2\n\t"
"vmovdqa 96(%[a1]), %%ymm3\n\t"
"vpsubw 0(%[d1]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[d1]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[d1]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[d1]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d1])\n\t"
"vmovdqa %%ymm1, 32(%[d1])\n\t"
"vmovdqa %%ymm2, 64(%[d1])\n\t"
"vmovdqa %%ymm3, 96(%[d1])\n\t"
"vmovdqa 0(%[d2]), %%ymm4\n\t"
"vmovdqa 32(%[d2]), %%ymm5\n\t"
"vmovdqa 64(%[d2]), %%ymm6\n\t"
"vmovdqa 96(%[d2]), %%ymm7\n\t"
"vpsubw 0(%[a2]), %%ymm4, %%ymm4\n\t"
"vpsubw 32(%[a2]), %%ymm5, %%ymm5\n\t"
"vpsubw 64(%[a2]), %%ymm6, %%ymm6\n\t"
"vpsubw 96(%[a2]), %%ymm7, %%ymm7\n\t"
"vmovdqa %%ymm4, 0(%[d2])\n\t"
"vmovdqa %%ymm5, 32(%[d2])\n\t"
"vmovdqa %%ymm6, 64(%[d2])\n\t"
"vmovdqa %%ymm7, 96(%[d2])\n\t"
"add $128, %[d1]\n\t"
"add $128, %[a1]\n\t"
"add $128, %[d2]\n\t"
"add $128, %[a2]\n\t"
"cmp %[end], %[d1]\n\t"
"jb 1b\n\t"
: [d1] "+&r"(d1),[a1] "+&r"(a1),[d2] "+&r"(d2),[a2] "+&r"(a2)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
d1+=TILE_PAD;a1+=TILE_PAD;d2+=TILE_PAD;a2+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_reverse_inplace(U *d,const U *b,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile("1:\n\t"
"vmovdqa 0(%[b]), %%ymm0\n\t"
"vmovdqa 32(%[b]), %%ymm1\n\t"
"vmovdqa 64(%[b]), %%ymm2\n\t"
"vmovdqa 96(%[b]), %%ymm3\n\t"
"vpsubw 0(%[d]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[d]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[d]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[d]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;b+=TILE_PAD;
}
}
template<int N>
static __attribute__((noinline))void combine3_unpack(
U *d12,U *d21,U *d22,
const U *p6,const U *p7,const U *p5,
const U *p1,const U *p4,const U *p3,int stride){
if constexpr(N > BASE){
constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
combine3_unpack<h>(d12,d21,d22,p6,p7,p5,p1,p4,p3,stride);
combine3_unpack<h>(d12+h,d21+h,d22+h,p6+q,p7+q,p5+q,p1+q,p4+q,p3+q,stride);
combine3_unpack<h>(d12+h*stride,d21+h*stride,d22+h*stride,
p6+2*q,p7+2*q,p5+2*q,p1+2*q,p4+2*q,p3+2*q,stride);
combine3_unpack<h>(d12+h*stride+h,d21+h*stride+h,d22+h*stride+h,
p6+3*q,p7+3*q,p5+3*q,p1+3*q,p4+3*q,p3+3*q,stride);
}else{
alignas(32)U temp[3*128];
for(int block=0;block<1024;block=block+128){
for(int v=0;v<128;v+=16){
int idx=block+v;
__m256i u2=_mm256_add_epi16(ld(p1+idx),ld(p6+idx));
__m256i u3=_mm256_add_epi16(u2,ld(p7+idx));
__m256i v5=ld(p5+idx);
st(temp+v,_mm256_add_epi16(_mm256_add_epi16(u2,v5),ld(p3+idx)));
st(temp+128+v,_mm256_sub_epi16(u3,ld(p4+idx)));
st(temp+256+v,_mm256_add_epi16(u3,v5));
}
size_t off=(size_t)(block/32)*stride;
emit_native(d12+off,temp,stride);
emit_native(d21+off,temp+128,stride);
emit_native(d22+off,temp+256,stride);
}
}
}
template<int N>
static __attribute__((noinline))void unpack_add(U *d,const U *p2,const U *p1,int stride){
if constexpr(N>BASE){
constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
unpack_add<h>(d,p2,p1,stride);
unpack_add<h>(d+h,p2+q,p1+q,stride);
unpack_add<h>(d+h*stride,p2+2*q,p1+2*q,stride);
unpack_add<h>(d+h*stride+h,p2+3*q,p1+3*q,stride);
}else{
alignas(32)U temp[128];
for(int block=0;block<1024;block+=128){
for(int v=0;v<128;v+=16)
st(temp+v,_mm256_add_epi16(ld(p2+block+v),ld(p1+block+v)));
emit_native(d+(size_t)(block/32)*stride,temp,stride);
}
}
}
template<int N>
static __attribute__((noinline))void root_fuse(
U *d11,U *d12,U *d21,U *d22,
const U *p1,const U *p2,const U *p3,const U *p4,const U *p5,const U *p6,const U *p7,int stride){
if constexpr(N==128){
constexpr int q=BASE*BASE+TILE_PAD;
root_fuse<64>(d11,d12,d21,d22,p1,p2,p3,p4,p5,p6,p7,stride);
alignas(32)U keep[4*8*512],cur[4*8*512];
U *dest[4]={d11,d12,d21,d22};
auto make_all=[&](U *tmp,int group){
for(int child=0;child<4;child++)
for(int b=0;b<8;b++){
int block=b*128;
for(int v=0;v<128;v+=16){
int idx=(4*group+child)*q+block+v;
if(!(v&16)){__builtin_prefetch(p1+idx+128,0,0);__builtin_prefetch(p2+idx+128,0,0);__builtin_prefetch(p3+idx+128,0,0);__builtin_prefetch(p4+idx+128,0,0);__builtin_prefetch(p5+idx+128,0,0);__builtin_prefetch(p6+idx+128,0,0);__builtin_prefetch(p7+idx+128,0,0);}
__m256i v1=ld(p1+idx),v5=ld(p5+idx);
__m256i u2=_mm256_add_epi16(v1,ld(p6+idx));
__m256i u3=_mm256_add_epi16(u2,ld(p7+idx));
U *t=tmp+(child*8+b)*512;
st(t+v,_mm256_add_epi16(v1,ld(p2+idx)));
st(t+256+v,_mm256_sub_epi16(u3,ld(p4+idx)));
st(t+384+v,_mm256_add_epi16(u3,v5));
st(t+128+v,_mm256_add_epi16(_mm256_add_epi16(u2,v5),ld(p3+idx)));
}
}
for(int b=0;b<8;b++)
for(int v=0;v<512;v+=16){
U *t1=tmp+(1*8+b)*512,*t2=tmp+(2*8+b)*512;
__m256i c22=ld(tmp+(3*8+b)*512+v);
st(t2+v,_mm256_sub_epi16(c22,ld(t2+v)));
st(t1+v,_mm256_sub_epi16(ld(t1+v),c22));
}
};
make_all(keep,3);
for(int child=0;child<4;child++){
int row=64+((child>>1)&1)*32,col=64+(child&1)*32;
for(int b=0;b<8;b++){
size_t off=(size_t)(b*4)*stride;
for(int r=0;r<4;r++)emit_native(dest[r]+off+row*stride+col,keep+(child*8+b)*512+r*128,stride);
}
}
for(int group=1;group<=2;group++){
if(group==2)_mm_sfence();
make_all(cur,group);
for(int child=0;child<4;child++){
int row=((group>>1)&1)*64+((child>>1)&1)*32;
int col=(group&1)*64+(child&1)*32;
for(int b=0;b<8;b++){
size_t off=(size_t)(b*4)*stride;
for(int r=0;r<4;r++)for(int rr=0;rr<4;rr++){
U *d=dest[r]+off+(row+rr)*stride+col;
const U *x=cur+(child*8+b)*512+r*128+rr*32;
const U *y=keep+(child*8+b)*512+r*128+rr*32;
__m256i v0,v1;
if(group==1){
v0=_mm256_sub_epi16(ld(x),ld(y));
v1=_mm256_sub_epi16(ld(x+16),ld(y+16));
}else{
v0=_mm256_sub_epi16(ld(y),ld(x));
v1=_mm256_sub_epi16(ld(y+16),ld(x+16));
}
_mm256_stream_si256((__m256i*)d,s4bj_fx(v0));
_mm256_stream_si256((__m256i*)(d+16),s4bj_fx(v1));
}
}
}
}
}else if constexpr(N==64){
constexpr int q=BASE*BASE+TILE_PAD;
root_fuse<32>(d11,d12,d21,d22,p1,p2,p3,p4,p5,p6,p7,stride);
alignas(32)U tmp[3*8*512];
U *dest[4]={d11,d12,d21,d22};
for(int child=1;child<=3;child++)
for(int b=0;b<8;b++){
int block=b*128;
for(int v=0;v<128;v+=16){
int idx=child*q+block+v;
if(!(v&16)){__builtin_prefetch(p1+idx+128,0,3);__builtin_prefetch(p2+idx+128,0,3);__builtin_prefetch(p3+idx+128,0,3);__builtin_prefetch(p4+idx+128,0,3);__builtin_prefetch(p5+idx+128,0,3);__builtin_prefetch(p6+idx+128,0,3);__builtin_prefetch(p7+idx+128,0,3);}
__m256i v1=ld(p1+idx),v5=ld(p5+idx);
__m256i u2=_mm256_add_epi16(v1,ld(p6+idx));
__m256i u3=_mm256_add_epi16(u2,ld(p7+idx));
U *t=tmp+((child-1)*8+b)*512;
st(t+v,_mm256_add_epi16(v1,ld(p2+idx)));
st(t+128+v,_mm256_add_epi16(_mm256_add_epi16(u2,v5),ld(p3+idx)));
st(t+256+v,_mm256_sub_epi16(u3,ld(p4+idx)));
st(t+384+v,_mm256_add_epi16(u3,v5));
}
}
for(int b=0;b<8;b++)
for(int v=0;v<512;v+=16){
U *t1=tmp+(0*8+b)*512,*t2=tmp+(1*8+b)*512;
__m256i c22=ld(tmp+(2*8+b)*512+v);
st(t1+v,_mm256_sub_epi16(ld(t1+v),c22));
st(t2+v,_mm256_sub_epi16(c22,ld(t2+v)));
}
for(int b=0;b<8;b++){
size_t off=(size_t)(b*4)*stride;
for(int r=0;r<4;r++){
emit_native(dest[r]+off+32,tmp+(0*8+b)*512+r*128,stride);
emit_native(dest[r]+off+32*stride,tmp+(1*8+b)*512+r*128,stride);
emit_native(dest[r]+off+32*stride+32,tmp+(2*8+b)*512+r*128,stride);
}
}
}else if constexpr(N > BASE){
constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
root_fuse<h>(d11,d12,d21,d22,p1,p2,p3,p4,p5,p6,p7,stride);
root_fuse<h>(d11+h,d12+h,d21+h,d22+h,p1+q,p2+q,p3+q,p4+q,p5+q,p6+q,p7+q,stride);
root_fuse<h>(d11+h*stride,d12+h*stride,d21+h*stride,d22+h*stride,
p1+2*q,p2+2*q,p3+2*q,p4+2*q,p5+2*q,p6+2*q,p7+2*q,stride);
root_fuse<h>(d11+h*stride+h,d12+h*stride+h,d21+h*stride+h,d22+h*stride+h,
p1+3*q,p2+3*q,p3+3*q,p4+3*q,p5+3*q,p6+3*q,p7+3*q,stride);
}else{
alignas(32)U temp[4*128];
for(int block=0;block<1024;block+=128){
for(int v=0;v<128;v+=16){
int idx=block+v;
if(!(v&16)){__builtin_prefetch(p1+idx+128,0,3);__builtin_prefetch(p2+idx+128,0,3);__builtin_prefetch(p3+idx+128,0,3);__builtin_prefetch(p4+idx+128,0,3);__builtin_prefetch(p5+idx+128,0,3);__builtin_prefetch(p6+idx+128,0,3);__builtin_prefetch(p7+idx+128,0,3);}
__m256i u2=_mm256_add_epi16(ld(p1+idx),ld(p6+idx));
__m256i u3=_mm256_add_epi16(u2,ld(p7+idx));
__m256i v5=ld(p5+idx);
st(temp+v,_mm256_add_epi16(_mm256_add_epi16(u2,v5),ld(p3+idx)));
st(temp+128+v,_mm256_sub_epi16(u3,ld(p4+idx)));
st(temp+256+v,_mm256_add_epi16(u3,v5));
st(temp+384+v,_mm256_add_epi16(ld(p2+idx),ld(p1+idx)));
}
size_t off=(size_t)(block/32)*stride;
emit_native(d12+off,temp,stride);
emit_native(d21+off,temp+128,stride);
emit_native(d22+off,temp+256,stride);
emit_native(d11+off,temp+384,stride);
}
}
}
static __attribute__((noinline,noclone))void combine_pair_p6(U *x,const U *a11,U *y,const U *b22,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=x+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[x]), %%ymm0\n\t"
"vmovdqa 32(%[x]), %%ymm1\n\t"
"vmovdqa 64(%[x]), %%ymm2\n\t"
"vmovdqa 0(%[b22]), %%ymm4\n\t"
"vmovdqa 96(%[x]), %%ymm3\n\t"
"vmovdqa 32(%[b22]), %%ymm5\n\t"
"vmovdqa 64(%[b22]), %%ymm6\n\t"
"vmovdqa 96(%[b22]), %%ymm7\n\t"
"vpsubw 0(%[a11]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[a11]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[a11]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[a11]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[x])\n\t"
"vmovdqa %%ymm1, 32(%[x])\n\t"
"vmovdqa %%ymm2, 64(%[x])\n\t"
"vmovdqa %%ymm3, 96(%[x])\n\t"
"vpsubw 0(%[y]), %%ymm4, %%ymm4\n\t"
"vpsubw 32(%[y]), %%ymm5, %%ymm5\n\t"
"vpsubw 64(%[y]), %%ymm6, %%ymm6\n\t"
"vpsubw 96(%[y]), %%ymm7, %%ymm7\n\t"
"vmovdqa %%ymm4, 0(%[y])\n\t"
"vmovdqa %%ymm5, 32(%[y])\n\t"
"vmovdqa %%ymm6, 64(%[y])\n\t"
"vmovdqa %%ymm7, 96(%[y])\n\t"
"add $128, %[x]\n\t"
"add $128, %[a11]\n\t"
"add $128, %[y]\n\t"
"add $128, %[b22]\n\t"
"cmp %[end], %[x]\n\t"
"jb 1b\n\t"
: [x] "+&r"(x),[a11] "+&r"(a11),[y] "+&r"(y),[b22] "+&r"(b22)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
x+=TILE_PAD;a11+=TILE_PAD;y+=TILE_PAD;b22+=TILE_PAD;
}
}
template<int N,bool ParentReconstruct=false,bool PostAdd=false,bool PostAdd2=false>
static __attribute__((noinline))void mul(const U *a,const U *b,U *c,U *work){
if constexpr(N<=BASE){
base(a,b,c,N);
}else if constexpr(N<=ALTMAX){
constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
const U *pa = nullptr;if constexpr(PostAdd)pa = g_postadd;
const U *pa2 = nullptr;if constexpr(PostAdd2)pa2 = g_postadd2;
const U *a11=a,*a12=a+q,*a21=a+2*q,*a22=a+3*q;
const U *b11=b,*b12=b+q,*b21=b+2*q,*b22=b+3*q;
U *c11=c,*c12=c+q,*c21=c+2*q,*c22=c+3*q;
U *x=work+0,*y=work+2*q+704,*next=work+4*q+640;
mul<h>(a12,b21,c11,next);                 
mul<h>(a22,b22,c22,next);                 
selected_displaced<N,false,q,q>(x,a21);
selected_displaced<N,false,q,q>(y,b21);
mul<h>(x,y,c12,next);                     
selected_displaced<N,true,-2*q,q>(x,a22);
selected_displaced<N,true,-2*q,q>(y,b22);
if constexpr(N==64)native2x32_shared(x,y,c21,ParentReconstruct?4:2);else mul<h,true>(x,y,c21,next); 
if constexpr(PostAdd2)add_inplace_pa2(c22,pa+3*q,pa2+3*q,q);else if constexpr(PostAdd)add_inplace_pa(c22,pa+3*q,q);
if constexpr(N==64){
if constexpr(ParentReconstruct)native2x32_shared(a11,b11,c11,3);else leaf_add(a11,b11,c11);                  
selected_displaced<N,true,-3*q,q>(y,b22);
if constexpr(ParentReconstruct)native2x32_shared(a21,y,c21,3);else leaf_add(a21,y,c21);                    
selected_displaced<N,true,3*q,q>(x,a11);   
if constexpr(ParentReconstruct)native2x32_shared(x,b12,c12,3);else leaf_add(x,b12,c12);                    
}else{
selected_displaced<N,true,-3*q,q>(y,b22);
mul<h>(a11,b11,x,next);                   
if constexpr(PostAdd2)combine_inplace_pa2(c11,x,pa,pa2,q);else if constexpr(PostAdd)combine_inplace_pa(c11,x,pa,q);else combine_inplace<false>(c11,x,q);
mul<h>(a21,y,x,next);                     
if constexpr(PostAdd2)combine_inplace_pa2(c21,x,pa+2*q,pa2+2*q,q);else if constexpr(PostAdd)combine_inplace_pa(c21,x,pa+2*q,q);else combine_inplace<false>(c21,x,q);
selected_displaced<N,true,-3*q,q>(x,a22);
mul<h>(x,b12,y,next);                     
if constexpr(PostAdd2)combine_inplace_sub_pa2(c12,y,pa+q,pa2+q,q);else if constexpr(PostAdd)combine_inplace_sub_pa(c12,y,pa+q,q);else combine_inplace<true>(c12,y,q);
}
}else{
constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
const U *pa = nullptr;if constexpr(PostAdd)pa = g_postadd;
const U *pa2 = nullptr;if constexpr(PostAdd2)pa2 = g_postadd2;
const U *a11=a,*a12=a+q,*a21=a+2*q,*a22=a+3*q;
const U *b11=b,*b12=b+q,*b21=b+2*q,*b22=b+3*q;
U *c11=c,*c12=c+q,*c21=c+2*q,*c22=c+3*q;
U *x=work+192,*y=work+q+512,*next=work+q+q+512;
if constexpr(N>=2048)combine_dpair_tt_nt<-2*q,2*q>(y,b22,x,a11,q);else combine_dpair_tt<-2*q,2*q>(y,b22,x,a11,q);
mul<h>(x,y,c21,next);                       
if constexpr(N>=2048)combine_dpair_ft_nt<q,-q>(x,a21,y,b12,q);else combine_dpair_ft<q,-q>(x,a21,y,b12,q);
mul<h>(x,y,c22,next);                       
combine_pair_p6(x,a11,y,b22,q);
mul<h>(x,y,c12,next);                       
combine_pair_rf(x,a12,y,b21,q);
mul<h>(x,b22,c11,next);                     
mul<h>(a22,y,x,next);                       
mul<h>(a11,b11,y,next);                     
if constexpr(N == 4096){
combine3_unpack<1024>(final_out+1024,
final_out+(size_t)1024*4096,
final_out+(size_t)1024*4096+1024,
c12,c21,c22,y,x,c11,4096);
}else{
U *rp=c12;const U *xp=x;
const U *pac=pa;
const U *pac2=pa2;
for(int chunk=0;chunk<q;chunk+=1024+TILE_PAD){
const U *end=rp+1024;
if constexpr(PostAdd2){
asm volatile(
"1:\n\tvmovdqa %c[yd]+0(%[x]), %%ymm0\n\tvpaddw 0(%[p]), %%ymm0, %%ymm0\n\tvmovdqa %c[q2]+0(%[p]), %%ymm1\n\tvpaddw %c[q]+0(%[p]), %%ymm0, %%ymm2\n\tvpaddw %c[mq]+0(%[p]), %%ymm1, %%ymm3\n\tvpaddw %%ymm0, %%ymm3, %%ymm3\n\tvpaddw %c[q]+0(%[pa]), %%ymm3, %%ymm3\n\tvpaddw %c[q]+0(%[pa2]), %%ymm3, %%ymm3\n\tvmovdqa %%ymm3, 0(%[p])\n\tvpaddw %%ymm2, %%ymm1, %%ymm1\n\tvpsubw 0(%[x]), %%ymm2, %%ymm2\n\tvpaddw %c[q3]+0(%[pa]), %%ymm1, %%ymm1\n\tvpaddw %c[q3]+0(%[pa2]), %%ymm1, %%ymm1\n\tvmovdqa %%ymm1, %c[q2]+0(%[p])\n\tvpaddw %c[q2]+0(%[pa]), %%ymm2, %%ymm2\n\tvpaddw %c[q2]+0(%[pa2]), %%ymm2, %%ymm2\n\tvmovdqa %c[yd]+32(%[x]), %%ymm4\n\tvpaddw 32(%[p]), %%ymm4, %%ymm4\n\tvmovdqa %%ymm2, %c[q]+0(%[p])\n\tvmovdqa %c[q2]+32(%[p]), %%ymm5\n\tvpaddw %c[q]+32(%[p]), %%ymm4, %%ymm6\n\tvpaddw %c[mq]+32(%[p]), %%ymm5, %%ymm7\n\tvpaddw %%ymm4, %%ymm7, %%ymm7\n\tvpaddw %c[q]+32(%[pa]), %%ymm7, %%ymm7\n\tvpaddw %c[q]+32(%[pa2]), %%ymm7, %%ymm7\n\tvmovdqa %%ymm7, 32(%[p])\n\tvpaddw %%ymm6, %%ymm5, %%ymm5\n\tvpsubw 32(%[x]), %%ymm6, %%ymm6\n\tvpaddw %c[q3]+32(%[pa]), %%ymm5, %%ymm5\n\tvpaddw %c[q3]+32(%[pa2]), %%ymm5, %%ymm5\n\tvmovdqa %%ymm5, %c[q2]+32(%[p])\n\tvpaddw %c[q2]+32(%[pa]), %%ymm6, %%ymm6\n\tvpaddw %c[q2]+32(%[pa2]), %%ymm6, %%ymm6\n\tvmovdqa %%ymm6, %c[q]+32(%[p])\n\tadd $64, %[p]\n\tadd $64, %[x]\n\tadd $64, %[pa]\n\tadd $64, %[pa2]\n\tcmp %[end], %[p]\n\tjb 1b\n\t"
: [p] "+&r"(rp),[x] "+&r"(xp),[pa] "+&r"(pac),[pa2] "+&r"(pac2)
: [end] "r"(end),[q] "i"(2*q),[q2] "i"(4*q),[q3] "i"(6*q),[mq] "i"(-2*q),[yd] "i"(2*(q+320))
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
}else if constexpr(PostAdd){
asm volatile(
"1:\n\tvmovdqa %c[yd]+0(%[x]), %%ymm0\n\tvpaddw 0(%[p]), %%ymm0, %%ymm0\n\tvmovdqa %c[q2]+0(%[p]), %%ymm1\n\tvpaddw %c[q]+0(%[p]), %%ymm0, %%ymm2\n\tvpaddw %c[mq]+0(%[p]), %%ymm1, %%ymm3\n\tvpaddw %%ymm0, %%ymm3, %%ymm3\n\tvpaddw %c[q]+0(%[pa]), %%ymm3, %%ymm3\n\tvmovdqa %%ymm3, 0(%[p])\n\tvpaddw %%ymm2, %%ymm1, %%ymm1\n\tvpsubw 0(%[x]), %%ymm2, %%ymm2\n\tvpaddw %c[q3]+0(%[pa]), %%ymm1, %%ymm1\n\tvmovdqa %%ymm1, %c[q2]+0(%[p])\n\tvpaddw %c[q2]+0(%[pa]), %%ymm2, %%ymm2\n\tvmovdqa %c[yd]+32(%[x]), %%ymm4\n\tvpaddw 32(%[p]), %%ymm4, %%ymm4\n\tvmovdqa %%ymm2, %c[q]+0(%[p])\n\tvmovdqa %c[q2]+32(%[p]), %%ymm5\n\tvpaddw %c[q]+32(%[p]), %%ymm4, %%ymm6\n\tvpaddw %c[mq]+32(%[p]), %%ymm5, %%ymm7\n\tvpaddw %%ymm4, %%ymm7, %%ymm7\n\tvpaddw %c[q]+32(%[pa]), %%ymm7, %%ymm7\n\tvmovdqa %%ymm7, 32(%[p])\n\tvpaddw %%ymm6, %%ymm5, %%ymm5\n\tvpsubw 32(%[x]), %%ymm6, %%ymm6\n\tvpaddw %c[q3]+32(%[pa]), %%ymm5, %%ymm5\n\tvmovdqa %%ymm5, %c[q2]+32(%[p])\n\tvpaddw %c[q2]+32(%[pa]), %%ymm6, %%ymm6\n\tvmovdqa %%ymm6, %c[q]+32(%[p])\n\tadd $64, %[p]\n\tadd $64, %[x]\n\tadd $64, %[pa]\n\tcmp %[end], %[p]\n\tjb 1b\n\t"
: [p] "+&r"(rp),[x] "+&r"(xp),[pa] "+&r"(pac)
: [end] "r"(end),[q] "i"(2*q),[q2] "i"(4*q),[q3] "i"(6*q),[mq] "i"(-2*q),[yd] "i"(2*(q+320))
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
}else{
asm volatile(
"1:\n\tvmovdqa %c[yd]+0(%[x]), %%ymm0\n\tvpaddw 0(%[p]), %%ymm0, %%ymm0\n\tvmovdqa %c[q2]+0(%[p]), %%ymm1\n\tvpaddw %c[q]+0(%[p]), %%ymm0, %%ymm2\n\tvpaddw %c[mq]+0(%[p]), %%ymm1, %%ymm3\n\tvpaddw %%ymm0, %%ymm3, %%ymm3\n\tvmovdqa %%ymm3, 0(%[p])\n\tvpaddw %%ymm2, %%ymm1, %%ymm1\n\tvpsubw 0(%[x]), %%ymm2, %%ymm2\n\tvmovdqa %%ymm1, %c[q2]+0(%[p])\n\tvmovdqa %c[yd]+32(%[x]), %%ymm4\n\tvpaddw 32(%[p]), %%ymm4, %%ymm4\n\tvmovdqa %%ymm2, %c[q]+0(%[p])\n\tvmovdqa %c[q2]+32(%[p]), %%ymm5\n\tvpaddw %c[q]+32(%[p]), %%ymm4, %%ymm6\n\tvpaddw %c[mq]+32(%[p]), %%ymm5, %%ymm7\n\tvpaddw %%ymm4, %%ymm7, %%ymm7\n\tvmovdqa %%ymm7, 32(%[p])\n\tvpaddw %%ymm6, %%ymm5, %%ymm5\n\tvpsubw 32(%[x]), %%ymm6, %%ymm6\n\tvmovdqa %%ymm5, %c[q2]+32(%[p])\n\tvmovdqa %%ymm6, %c[q]+32(%[p])\n\tadd $64, %[p]\n\tadd $64, %[x]\n\tcmp %[end], %[p]\n\tjb 1b\n\t"
: [p] "+&r"(rp),[x] "+&r"(xp)
: [end] "r"(end),[q] "i"(2*q),[q2] "i"(4*q),[mq] "i"(-2*q),[yd] "i"(2*(q+320))
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
}
rp+=TILE_PAD;xp+=TILE_PAD;if constexpr(PostAdd)pac+=TILE_PAD;if constexpr(PostAdd2)pac2+=TILE_PAD;
}
}
if constexpr(!PostAdd){g_postadd=y;mul<h,false,true>(a12,b21,c11,next);g_postadd=nullptr;}
else if constexpr(PostAdd && !PostAdd2){g_postadd=pa;g_postadd2=y;mul<h,false,true,true>(a12,b21,c11,next);g_postadd=nullptr;g_postadd2=nullptr;}
else mul<h>(a12,b21,c11,next);                   
if constexpr(N == 4096)unpack_add<1024>(final_out,c11,y,4096);
else if constexpr(PostAdd2)combine_inplace_pa2(c11,y,pa,pa2,q);
}
}
static constexpr int RQ=2048*2048+(2048/BASE)*(2048/BASE)*TILE_PAD,RS=RQ+ROOT_PAD;
alignas(2*1024*1024)static U root_mem[12*RS+4096];
alignas(4096)static U root_tile[4*(64*64+(64/BASE)*(64/BASE)*TILE_PAD)];
static constexpr int HQ=64*64+(64/BASE)*(64/BASE)*TILE_PAD;
alignas(4096)static U high_left[4*HQ],high_right[4*HQ];
template<bool Right>
static void pack_four(U *tile,const U *s){
pack(tile,s,64,4096,Right);
pack(tile+HQ,s+2048,64,4096,Right);
pack(tile+2*HQ,s+2048*4096,64,4096,Right);
pack(tile+3*HQ,s+2048*4096+2048,64,4096,Right);
}
template<bool Right,bool TWO>
static void pack_wide_root(U *tA,U *tB,const U *s,long STR_,long D0_,const U *s2,int sgn){
if constexpr(!Right){
pack_wide<TWO>(tA+0*HQ,tB+0*HQ,s,STR_,s2,sgn);
pack_wide<TWO>(tA+1*HQ,tB+1*HQ,s+2048,STR_,s2+2048,sgn);
pack_wide<TWO>(tA+2*HQ,tB+2*HQ,s+2048L*STR_,STR_,s2+2048L*STR_,sgn);
pack_wide<TWO>(tA+3*HQ,tB+3*HQ,s+2048L*STR_+2048,STR_,s2+2048L*STR_+2048,sgn);
}else{
pack_wide_b<TWO>(tA+0*HQ,tB+0*HQ,s,STR_,D0_,s2,sgn);
pack_wide_b<TWO>(tA+1*HQ,tB+1*HQ,s+2048,STR_,D0_+2048,s2+2048,sgn);
pack_wide_b<TWO>(tA+2*HQ,tB+2*HQ,s+2048L*STR_,STR_,D0_+2048L*4096,s2+2048L*STR_,sgn);
pack_wide_b<TWO>(tA+3*HQ,tB+3*HQ,s+2048L*STR_+2048,STR_,D0_+2048L*4096+2048,s2+2048L*STR_+2048,sgn);
}
}
template<bool Right,bool High>
static void emit_root64(const U *tile,const U *left,const U *right,int offset,U *dest,U *destq,int voff){
U *r0=destq+offset,*r1=r0+RS,*r2=r1+RS,*v7=dest+offset+voff*RS,*v5=v7+RS,*v6=v5+RS,*v34=v6+RS;
for(int i=0;i<(64*64+(64/BASE)*(64/BASE)*TILE_PAD);i+=32){
__m256i a=ld(tile+i),b=ld(tile+(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i),c=ld(tile+2*(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i),d=ld(tile+3*(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i);
__m256i aa=ld(tile+i+16),bb=ld(tile+(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i+16),cc=ld(tile+2*(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i+16),dd=ld(tile+3*(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i+16);
if constexpr(High){
a=_mm256_add_epi16(a,_mm256_sub_epi16(ld(left+0*HQ+i),ld(right+0*HQ+i)));
b=_mm256_add_epi16(b,_mm256_sub_epi16(ld(left+1*HQ+i),ld(right+1*HQ+i)));
c=_mm256_add_epi16(c,_mm256_sub_epi16(ld(left+2*HQ+i),ld(right+2*HQ+i)));
d=_mm256_add_epi16(d,_mm256_sub_epi16(ld(left+3*HQ+i),ld(right+3*HQ+i)));
aa=_mm256_add_epi16(aa,_mm256_sub_epi16(ld(left+0*HQ+i+16),ld(right+0*HQ+i+16)));
bb=_mm256_add_epi16(bb,_mm256_sub_epi16(ld(left+1*HQ+i+16),ld(right+1*HQ+i+16)));
cc=_mm256_add_epi16(cc,_mm256_sub_epi16(ld(left+2*HQ+i+16),ld(right+2*HQ+i+16)));
dd=_mm256_add_epi16(dd,_mm256_sub_epi16(ld(left+3*HQ+i+16),ld(right+3*HQ+i+16)));
}
_mm256_stream_si256((__m256i*)(r0+i),a);_mm256_stream_si256((__m256i*)(r0+i+16),aa);
_mm256_stream_si256((__m256i*)(r1+i),Right?c:b);_mm256_stream_si256((__m256i*)(r1+i+16),Right?cc:bb);
_mm256_stream_si256((__m256i*)(r2+i),d);_mm256_stream_si256((__m256i*)(r2+i+16),dd);
__m256i x7=Right?_mm256_sub_epi16(d,b):_mm256_sub_epi16(a,c);
__m256i xx7=Right?_mm256_sub_epi16(dd,bb):_mm256_sub_epi16(aa,cc);
_mm256_stream_si256((__m256i*)(v7+i),x7);_mm256_stream_si256((__m256i*)(v7+i+16),xx7);
__m256i x5=Right?_mm256_sub_epi16(b,a):_mm256_add_epi16(c,d);
__m256i xx5=Right?_mm256_sub_epi16(bb,aa):_mm256_add_epi16(cc,dd);
_mm256_stream_si256((__m256i*)(v5+i),x5);_mm256_stream_si256((__m256i*)(v5+i+16),xx5);
__m256i x6=Right?_mm256_sub_epi16(d,x5):_mm256_sub_epi16(x5,a);
__m256i xx6=Right?_mm256_sub_epi16(dd,xx5):_mm256_sub_epi16(xx5,aa);
_mm256_stream_si256((__m256i*)(v6+i),x6);_mm256_stream_si256((__m256i*)(v6+i+16),xx6);
__m256i x34=Right?_mm256_sub_epi16(x6,c):_mm256_sub_epi16(b,x6);
__m256i xx34=Right?_mm256_sub_epi16(xx6,cc):_mm256_sub_epi16(bb,xx6);
_mm256_stream_si256((__m256i*)(v34+i),x34);_mm256_stream_si256((__m256i*)(v34+i+16),xx34);
}
}
template<bool Right,bool TWO=Right>
static void pack_root(const U*s,int n,int offset,U *dest,U *destq,int voff,long STR_=4096,long D0_=0,const U*s2=nullptr,int sgn=0){
if(n>128){int h=n/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
pack_root<Right,TWO>(s,h,offset,dest,destq,voff,STR_,D0_,s2,sgn);
pack_root<Right,TWO>(s+h,h,offset+q,dest,destq,voff,STR_,D0_+h,s2+h,sgn);
pack_root<Right,TWO>(s+h*STR_,h,offset+2*q,dest,destq,voff,STR_,D0_+h*4096L,s2+h*STR_,sgn);
pack_root<Right,TWO>(s+h*STR_+h,h,offset+3*q,dest,destq,voff,STR_,D0_+h*4096L+h,s2+h*STR_+h,sgn);
}else{
pack_wide_root<Right,TWO>(root_tile,high_left,s,STR_,D0_,s2,sgn);
emit_root64<Right,false>(root_tile,nullptr,nullptr,offset,dest,destq,voff);
emit_root64<Right,false>(high_left,nullptr,nullptr,offset+HQ,dest,destq,voff);
pack_wide_root<Right,TWO>(high_right,root_tile,s+64*STR_,STR_,D0_+64*4096L,s2+64*STR_,sgn);
emit_root64<Right,false>(high_right,nullptr,nullptr,offset+2*HQ,dest,destq,voff);
emit_root64<Right,true>(root_tile,high_left,high_right,offset+3*HQ,dest,destq,voff);
}
}
static void short_product4096s(int n,const short*A,const short*B,short*C,long sA,long sB);
static void short_product4096x(int n,const short*A,const short*A2,int sgA,long sA,const short*B,const short*B2,int sgB,long sB,short*C);
static void outer_sept(const unsigned short*b1,const unsigned short*b2,const unsigned short*b3,
const unsigned short*b4,const unsigned short*b5,
unsigned short*c00,unsigned short*c10,unsigned short*c11,unsigned short*c01){
for(int i=0;i<4096;i++){
const unsigned short*r1=b1+(size_t)i*4096,*r2=b2+(size_t)i*4096,*r3=b3+(size_t)i*4096,*r4=b4+(size_t)i*8192,*r5=b5+(size_t)i*8192;
const unsigned short*q6=c11+(size_t)i*8192,*q7=c10+(size_t)i*8192;
unsigned short*w0=c00+(size_t)i*8192,*w1=c10+(size_t)i*8192,*w2=c11+(size_t)i*8192,*w3=c01+(size_t)i*8192;
for(int j=0;j<4096;j+=16){
__m256i x1=_mm256_loadu_si256((const __m256i*)(r1+j));
__m256i x2=_mm256_loadu_si256((const __m256i*)(r2+j));
__m256i x3=_mm256_loadu_si256((const __m256i*)(r3+j));
__m256i x4=_mm256_loadu_si256((const __m256i*)(r4+j));
__m256i x5=_mm256_loadu_si256((const __m256i*)(r5+j));
__m256i z6=_mm256_loadu_si256((const __m256i*)(q6+j));
__m256i z7=_mm256_loadu_si256((const __m256i*)(q7+j));
__m256i y0=_mm256_add_epi16(_mm256_sub_epi16(_mm256_add_epi16(x1,x4),x5),z7);
__m256i y1=_mm256_add_epi16(x2,x4);
__m256i y2=_mm256_add_epi16(_mm256_add_epi16(_mm256_sub_epi16(x1,x2),x3),z6);
__m256i y3=_mm256_add_epi16(x3,x5);
_mm256_stream_si256((__m256i*)(w0+j),y0);
_mm256_stream_si256((__m256i*)(w1+j),y1);
_mm256_stream_si256((__m256i*)(w2+j),y2);
_mm256_stream_si256((__m256i*)(w3+j),y3);
}
}
}
alignas(64)static U g_band[256*4096];
static inline int base_offset(int r0,int c0){
int off=0,r=r0,c=c0;
const int H[4]={1024,512,256,128};
const int Q[4]={1064960,266240,66560,16640};
for(int i=0;i<4;i++){int h=H[i],ci=((r>=h)?2:0)|((c>=h)?1:0);off+=ci*Q[i];r&=h-1;c&=h-1;}
return off;
}
template<bool Right>
static void pack_wide_root_banded(U *tA,U *tB,const U *s,long D0_){
const long FARROW=128L*4096;
if constexpr(!Right){
pack_wide<false>(tA+0*HQ,tB+0*HQ,s,4096,s,0);
pack_wide<false>(tA+1*HQ,tB+1*HQ,s+2048,4096,s+2048,0);
pack_wide<false>(tA+2*HQ,tB+2*HQ,s+FARROW,4096,s+FARROW,0);
pack_wide<false>(tA+3*HQ,tB+3*HQ,s+FARROW+2048,4096,s+FARROW+2048,0);
}else{
pack_wide_b<false>(tA+0*HQ,tB+0*HQ,s,4096,D0_,s,0);
pack_wide_b<false>(tA+1*HQ,tB+1*HQ,s+2048,4096,D0_+2048,s+2048,0);
pack_wide_b<false>(tA+2*HQ,tB+2*HQ,s+FARROW,4096,D0_+2048L*4096,s+FARROW,0);
pack_wide_b<false>(tA+3*HQ,tB+3*HQ,s+FARROW+2048,4096,D0_+2048L*4096+2048,s+FARROW+2048,0);
}
}
template<bool Right>
static void pack_base_banded(const U*s,long D0,int offset,U *dest,U *destq,int voff){
pack_wide_root_banded<Right>(root_tile,high_left,s,D0);
emit_root64<Right,false>(root_tile,nullptr,nullptr,offset,dest,destq,voff);
emit_root64<Right,false>(high_left,nullptr,nullptr,offset+HQ,dest,destq,voff);
pack_wide_root_banded<Right>(high_right,root_tile,s+64*4096,D0+64*4096L);
emit_root64<Right,false>(high_right,nullptr,nullptr,offset+2*HQ,dest,destq,voff);
emit_root64<Right,true>(root_tile,high_left,high_right,offset+3*HQ,dest,destq,voff);
}
static void build_band(const U*a,const U*b,int sg,int r0){
for(int half=0;half<2;half++){
int row0=r0+half*2048;
U *d=g_band+(size_t)half*128*4096;
for(int i=0;i<128;i++){
const U*ra=a+(size_t)(row0+i)*8192,*rb=b+(size_t)(row0+i)*8192;
U*dd=d+(size_t)i*4096;
for(int j=0;j<4096;j+=16){
__m256i x=_mm256_loadu_si256((const __m256i*)(ra+j));
__m256i y=_mm256_loadu_si256((const __m256i*)(rb+j));
_mm256_storeu_si256((__m256i*)(dd+j),sg>0?_mm256_add_epi16(x,y):_mm256_sub_epi16(x,y));
}}}
}
template<bool Right>
static void pack_banded(const U*a,const U*b,int sg,U*dest,U*destq,int voff){
for(int r0=0;r0<2048;r0+=128){
build_band(a,b,sg,r0);
for(int c0=0;c0<2048;c0+=128)
pack_base_banded<Right>(g_band+c0,(long)r0*4096+c0,base_offset(r0,c0),dest,destq,voff);
}
}
static void short_product4096xb(int n,const unsigned short*A,const unsigned short*A2,int sgA,const unsigned short*B,const unsigned short*B2,int sgB,unsigned short*C){
U *cq=(U*)C;
U *ar=root_mem,*br=ar+4*RS,*pr=br+7*RS;
U *work=cq+3*RS+(464*2);
if(A2)pack_banded<false>((const U*)A,(const U*)A2,sgA,ar,cq,0);
else{raw_basis_input=(const U*)A;pack_root<false,false>((const U*)A,2048,0,ar,cq,0,8192,0,nullptr,0);}
if(B2)pack_banded<true>((const U*)B,(const U*)B2,sgB,br,br,3);
else{raw_basis_input=(const U*)B;pack_root<true,false>((const U*)B,2048,0,br,br,3,8192,0,nullptr,0);}
_mm_mfence();
const U *a11=cq,*a12=cq+RS,*a22=cq+2*RS,*s7=ar,*s5=ar+RS,*s6=ar+2*RS,*s3=ar+3*RS;
const U *b11=br,*b21=br+RS,*b22=br+2*RS,*t7=br+3*RS,*t5=br+4*RS,*t6=br+5*RS,*t4=br+6*RS;
U *p1=ar+2*RS,*p2=br+5*RS,*p3=ar+RS,*p4=br+4*RS,*p5=ar,*p6=br+3*RS,*p7=pr;
mul<2048>(s7,t7,p7,work);mul<2048>(s5,t5,p5,work);mul<2048>(s6,t6,p6,work);
mul<2048>(s3,b22,p3,work);mul<2048>(a22,t4,p4,work);mul<2048>(a11,b11,p1,work);mul<2048>(a12,b21,p2,work);
U *out=g_out?g_out:(U*)C;
const long ostr=g_ostr;
root_fuse<2048>(out,out+2048,out+2048*ostr,out+2048*ostr+2048,p1,p2,p3,p4,p5,p6,p7,(int)ostr);_mm_sfence();
asm volatile(".purgem vzeroupper");
_mm256_zeroupper();
}
void matrix_multiply(int n,const short*A,const short*B,short*C){
if(n!=8192){for(int i=0;i<n;i++)for(int j=0;j<n;j++){unsigned z=0;for(int k=0;k<n;k++)z+=(unsigned short)A[(size_t)i*n+k]*(unsigned)(unsigned short)B[(size_t)k*n+j];C[(size_t)i*n+j]=(short)z;}return;}
using V=unsigned short;V *pb;
constexpr size_t z=(size_t)4096*4096;
madvise((void*)C,(size_t)8192*8192*2,MADV_HUGEPAGE);
constexpr size_t zp = z + 256;   
if(posix_memalign((void**)&pb,4096,3*zp*2 + 256))abort();
madvise((void*)pb,3*z*2,MADV_HUGEPAGE);
madvise((void*)root_mem,sizeof(root_mem),MADV_HUGEPAGE);
V*b1=pb+128,*b2=b1+zp,*b3=b1+2*zp;   
const V*a00=(const V*)A,*a01=a00+4096,*a10=a00+4096*8192,*a11=a10+4096;
const V*b00=(const V*)B,*b01=b00+4096,*b10=b00+4096*8192,*b11=b10+4096;
V*c00=(V*)C,*c01=c00+4096,*c10=c00+4096*8192,*c11=c10+4096;
g_ostr=8192;
g_out=c00;
short_product4096xb(4096,a11,nullptr,0,b10,b00,-1,b1); 
g_out=c01;
short_product4096xb(4096,a00,a01,1,b11,nullptr,0,b1); 
g_out=c11;
short_product4096xb(4096,a10,a00,-1,b00,b01,1,b1); 
g_out=c10;
short_product4096xb(4096,a01,a11,-1,b10,b11,1,b1); 
g_out=nullptr;g_ostr=4096;
short_product4096xb(4096,a00,a11,1,b00,b11,1,b1); 
short_product4096xb(4096,a10,a11,1,b00,nullptr,0,b2); 
short_product4096xb(4096,a00,nullptr,0,b01,b11,-1,b3); 
outer_sept(b1,b2,b3,c00,c01,c00,c10,c11,c01);
_mm_sfence();
free(pb);
}

评测结果