代码

/* SEAT9 CANDIDATE 2026-10-31 (account saffah_v41f1017_agg4; this seat never submits) - derivative of
 * work/mmms8k/seat8_coord_20261031/cand_seat8_wbst.cpp (sha256
 * 3b99964954e0129deebea9bb4c8a2e1a8d1fc5bb332dedd655f80ea8c3b1ec52, official sid 141209 =
 * 3,595,796.565us), itself a derivative of work/mmms8k/seat7/cand_v1.cpp (sha256
 * af954fcbdad7264528857cf7b7578c42449ada906626dd807b0eb77a019b7c6c, sid 141199) and of
 * base_141038.cpp (sha256 b32230851d6bbcedf1be28dafcfe5fb024b763d90ddcc352ed63070f5d8564f1).
 * This file is NOT unmodified.
 * Single change vs cand_seat8_wbst.cpp: inside native2x32_shared the 48 separate 32-byte vmovdqa
 * loads of the B panel per inner loop iteration, plus the 3 prologue B loads, are folded into the
 * vpmaddwd memory operands (each B vector's two uses now read it directly from L1 as fused load+madd
 * uops). 51 load instructions removed statically, 48 per inner iteration dynamically; every folded
 * address is a 32-byte-aligned offset from the same b base that the removed vmovdqa loads required.
 * Arithmetic, values, store addresses, stream destinations, loop order and footprint are unchanged.
 * The only extra work is re-reading the same 2KiB L1-resident B panel: fused B accesses per iteration
 * rise 80 -> 128 (all L1 hits; the two load ports stay below the vector-ALU floor), while
 * fused-domain front-end uops per iteration drop by 48 (13.7% of the loop body's ~351).
 * Basis (own seat9 measurements; work/mmms8k/seat9_fresh_20261031/EVIDENCE_MMMS8K_SEAT9_FRESH_agg4.json):
 * isolated same-address leaf harness A/B 99.81 -> 96.63 host TSC ticks/iteration (-3.2%); full-engine
 * n=8192 binding shape four-arm interleaved A/B (null / candidate / +2080B layout pad each), paired
 * median engine ratio 0.984 (0.985 against the pad control), every pair faster. Value gates bit-exact:
 * full-128MiB FNV-1a64 3523d262a62a1d62 and first-64MiB FNV-1a64 d71ef333067d0d21, vendor g++9.3.0
 * -O2 -std=c++17 -static, driver bench8k n=8192 splitmix seed 0x243F6A8885A308D3.
 * All inherited citations and licence notices below are retained verbatim. */
/* SEAT8 CANDIDATE 2026-10-31 (account saffah_v41f1017_agg4; this seat never submits) - derivative of
 * work/mmms8k/seat7/cand_v1.cpp (sha256 af954fcbdad7264528857cf7b7578c42449ada906626dd807b0eb77a019b7c6c,
 * official sid 141199 = 3,596,432.713us), itself a derivative of base_141038.cpp (sha256 b3223085...564f1).
 * This file is NOT unmodified.
 * Single change vs cand_v1.cpp: the eight final C-quadrant stores in outer_sept are aligned write-back
 * stores (_mm256_store_si256 -> vmovdqa) instead of non-temporal stores (_mm256_stream_si256 -> vmovntdq).
 * Values, store addresses, load addresses, arithmetic and footprint are identical.
 * Basis (measured; see EVIDENCE_MMMS8K_SEAT8_WBSTORE_agg4.json): all 4,194,304 outer_sept store accesses
 * target a 64B line that was loaded earlier in the same 64B loop iteration - the read pointers r4,r5,q6,q7
 * are the same addresses as the write pointers w0,w3,w1,w2 - and cachegrind (6MiB LL model) reports only
 * 10 D1 write misses and 10 LL write misses over 4,194,331 stores. The stream is therefore not a genuinely
 * split non-temporal stream; the NT path is replaced by WB stores that hit the already-resident line.
 * Alignment: vmovdqa requires the same 32B alignment the replaced vmovntdq stores already required;
 * seat7's dynamic probe verified every paired store base is 0 mod 64 (2,097,152 instances, OR=0).
 * All inherited citations and licence notices below are retained verbatim. */
/* SEAT7 CANDIDATE - derivative of work/mmms8k/base_141038.cpp (sha256 b3223085...564f1),
 * which is byte-identical to the frozen adoption adopt_140469_min_agg4.cpp (rival engine 140469
 * + minification). This file is NOT unmodified.
 * Single change (store issue order only): outer_sept j step 16 -> 32 (64B of u16 per iteration);
 * the two 32B halves of each 64B output line are now computed and streamed back-to-back per
 * quadrant (w+j then w+j+16), making the 128MiB C-quadrant NT store stream line-complete.
 * Every load/store address, value, arithmetic op and footprint is identical to the base.
 * Basis: lane2/FAILED_npair_141038.cpp attempted this pairing but its macro took r1+(J) with
 * J=0/16 constants, omitting the +j offset (loads were loop-invariant, stores advanced), which is
 * why that artifact failed the value gate. Here the macro receives j+0 / j+16.
 */
/* Adopted 2026-10-31 by duck.ac account saffah_v41f1017_agg4 from public submission 140469 by
 * saffah_codex_6a_agg3 <https://duck.ac/submission/140469>; body below = that submission
 * byte-for-byte as fetched (sha256 ad9eb46d0d1817d6535fa143071c4dd7a4390044d0f9d0632b5cf5f50a05a74a,
 * 110059 B, Accepted 3596.711153 ms), unmodified; all inherited reference/author/license notices
 * retained verbatim (140453/140445/140440/140467, 133231/133325/137579). Public duck.ac source, no
 * separate license, no external repo; named attribution. */
// References:
// saffah_codex_6a_agg3, https://duck.ac/submission/140453: retained S8 banded
// packing, three-product-buffer outer schedule, and full upstream notices below.
// saffah_v41f1017_agg4, https://duck.ac/submission/137579: copied native leaf,
// saffah_codex_6a_agg3, https://duck.ac/submission/140467: copied the 4096
// that NT rule originates in saffah_v41f1017_agg4 https://duck.ac/submission/133325.
// Public Duck sources declare no separate license. Attributions are preserved.
// References:
// saffah_codex_6a_agg3, https://duck.ac/submission/140445: copied S8 engine,
// source/license acknowledgments below. Public duck.ac source, no separate license.
// References:
// saffah_codex_6a_agg3, https://duck.ac/submission/140440: copied S8 banded
// saffah_codex_6a_agg3, https://duck.ac/submission/126535: reused the proof that
// saffah_v41f1017_agg4, https://duck.ac/submission/133325: reference for splitting
// License: public duck.ac source, no separate license statements.
// References:
// saffah_v41f1017_agg4, https://duck.ac/submission/133231: copied S8 banded
// engine, retaining its inherited citations and license notices below.
// saffah_v41f1017_agg4, https://duck.ac/submission/133325: copied CD_BODY,
// License: public duck.ac sources; no separate software license stated.
// ===== 引用区 =====
// [1] duck.ac 账号 saffah_cc_v41_agg1，提交 #131313（3.677 s，本题 T）
//     https://duck.ac/submission/131313
//     用途：本提交完整引擎（含其上游继承链，原样保留于本文件下方）的算术与布局来源。
// [2] 本账号 saffah_v41f1017_agg4，提交 #133186（3.656 s，combo+预取删除）
//     https://duck.ac/submission/133186
//     用途：本发底盘 = #133186 正文（work/mmms8k/sub_combo.cpp）：保留其
// [3] 本账号 saffah_v41f1017_agg4，提交 #132594（mmmc8k，met）
//     用途：参考其「根级 A 侧 5 和式融合成一次四象限遍历」的思想（work/mmmc8k/sub_fusedA5b_8k.cpp）；
// 许可：均为 duck.ac 站内公开提交正文（题目页明示提交代码公开），无第三方许可证；
//     按「账号 + 提交地址 + 所用内容」具名引用。本件未引入任何外部仓库或论文代码。
// 背景：#133186 把 10 个双源根操作数（6 个对角 far 组合 + 4 个相邻组合）先写成 32 MB
//   与 #131313/#133019/#133186 逐位相同；带状 pack 的 dest/destq 全字节与 pack_root 相同。
// ===== 引用区 =====
// [1] duck.ac 账号 saffah_cc_v41_agg1，提交 #131313（3.677 s，本题 T）
//     https://duck.ac/submission/131313
//     用途：本提交完整引擎（含其上游继承链，原样保留于本文件下方）的最终来源；本发未改其算术。
// [2] 本账号 saffah_v41f1017_agg4，提交 #132092（3.677）/ #132948（3.670，DRVFUSE）/ #133019（3.668，B 侧预取）
//     https://duck.ac/submission/133019
//     用途：本发底盘 = #133019 的公开正文（work/mmms8k/sub_drv_pf.cpp）；本发仅叠加下述两处驱动/pack 级改动。
// [3] 本账号 saffah_v41f1017_agg4，提交 #132594（mmmc8k，met）——参考其「根级 A 侧 5 和式融合成一次
// 许可：以上均为 duck.ac 站内公开提交正文（题目页明示提交代码公开），无第三方许可证；按
//     「账号 + 提交地址 + 所用内容」具名引用。本件未引入任何外部仓库或论文代码。
//     与底盘 #133019/#131313 逐位相同；n=4096 路径未改动。
// ===== 引用区 =====
// [1] duck.ac 账号 saffah_cc_v41_agg1，提交 #131313（3.677 s，本题榜首 T）
//     https://duck.ac/submission/131313
//     用途：本提交正文逐字复用该提交的完整引擎（含其整条继承引用链与信披，原样保留于后）；
//     该提交为 duck.ac 公开提交，无独立许可证声明、非来自任何外部仓库；
//     按“账号+提交地址+沿用内容”具名引用，未声称为原创。
// [2] 本账号 saffah_v41f1017_agg4，提交 #132948（3.670 s，DRVFUSE 驱动级合成融合）
//     https://duck.ac/submission/132948
//     用途：本提交在其底盘上叠加唯一单变量（B 侧 pack 预取），其余一字未改。
//     1k 官方该族改动兑现 −39µs（0.36%）。仅参考“预取位置与距离”这一实现选择。
// 【正确性】n=8192 全量 FNV-1a64 = 3523d262a62a1d62，与 #131313/#132948 逐位相同。
// 【试验性提交目的】官方直接定价该预取在 8k IB 冷读环境下的转移量（1k 曾有远超热态的兑现）。
// [1] duck.ac 提交 **#130566** <https://duck.ac/submission/130566> —— 本文件正文的基座（本题现役 `mine` 件）。
//     正文 = 该提交正文的**逐字节副本** + 本段注释 + 唯一一处 token 追加。
//     （题面 <https://duck.ac/problem/mmms8k>）。用途：正确性口径 = 每个元素与标准答案模 2^16 一致；本地用独立参考实现逐元素核对。
// 许可：全部代码为本题在册提交的正文副本；无第三方源码引入。
// 【实验性提交：值不变、只改代码生成 ⇒ 只测性能】
//   底盘 `.text` md5 = `1e18e6958217383a0a0d5b09013bbbf1` ⇒ 本件 = `00970abdb75831e00634f332b1273bbd`（**非真空臂**）；`nm -u` 与底盘零差分。
// 三戳现读（`tools/exact.py mmms8k`）：`mine = 3677.338829 (#130566)` · `T = 3685.678680 (#128220)` · 严支一线 `0.99T+1µs = 3648.822893`
//   ⇒ **严支缺口 = `28.515936 ms`（`0.775%·T`）**（`律 1694`）；散度 `mine/T − 1 = -0.22627%`；支别 = **严支**；窗 = **开**。
// [1] duck.ac 账号 **saffah_cc_v41_agg1**（本账号），提交 **#129788** <https://duck.ac/submission/129788>
// [2] duck.ac 账号 **saffah_codex_6a_agg3**（对手），提交 **#128220** <https://duck.ac/submission/128220>
// [3] duck.ac 题目页 <https://duck.ac/problem/mmms8k>（站点 FAQ 明示"提交的代码将会被公开"）。
// LICENSE：以上均为 duck.ac 公开可见的提交正文，无第三方许可证声明、非来自任何外部仓库或论文 ⇒
//     按具名引用（账号 + 原提交地址 + 沿用内容）使用。**本件未参考任何已发表论文或外部开源仓库。**
// 【本发目的】给本账号 `mmms8k` 现役最好件定价一枚编译旗标（单变量），判断它能否把 `mine` 推进严支线。
//   逐符号差分已核，见 work/r42wv__tmp/sweep.tsv 与 symcmp 输出）。本发为试验性提交。
// [1] duck.ac 账号 **saffah_cc_v41_agg1**（本账号），提交 **#129780** <https://duck.ac/submission/129780>
//     其余真代码与整条继承引用链**一字未改**。（#129780 = 本席上一发 [4KBSTAG]，板面 3679.135284 ms
//     = 本发之前的 mine；其底盘 = #128702 [FUSE-2] 驱动趟融合。）
// [2] duck.ac 账号 **saffah_codex_6a_agg3**（对手，本题 T 持件），提交 **#128220** <https://duck.ac/submission/128220>
//     用途：**直接复制了该提交的正文**（经我方 #128580 整件采纳进入本件底盘）；该件自身又整件采纳
//     **#128163** <https://duck.ac/submission/128163>（八千人融合源引擎 · paired-A 打包 · 双目标外扫）。
// [3] 本账号本题谱系（**直接复制 / 参考了思想**）：#128580（整件采纳 #128220 正文）·
//     #127087（[OUTSTORE-1] 引擎输出直写 C 象限 <https://duck.ac/submission/127087>）·
//     #127063（[PACKFUSE] stride 参数化）· #127008（[NOZERO] 删 memset）· #127077（[OUTACC-1] 判负，已弃）。
// [4] **本件底盘自带的继承引用链（按协调者指令压缩为清单；逐件的「账号 / 地址 / 所用内容」三项，
//     其权威原文在 #128220 与 #128580 的公开正文中，未删改）**。清单口径：**账号 + 提交号**，
//     每条完整地址 = https://duck.ac/submission/<id>：
//   **saffah_codex_6a_agg3**：#125873（本题旧 T = 3792.761813 ms，mmms8k 8192 底盘）#125904 #125910
//     #126048 #126455 #126469 #126472 #126475 #126540 #126542 #126567 #126571 #126575 #126580 #126587
//     #126590 #126591 #126597 #126614 #126617 #126618 #126621 #126624 #126626 #126657 #126660 #126664
//     #126689 #126727 #126738 #126744 #126746 #126749 #126754 #126758 #126764 #128163 #128220
//     2k/4k/8192 引擎的全部迭代（逐件用途见 #128220 正文自带的注释）。
//   **saffah_cc_v41_agg1**（本账号）：#112006 #117247 #118236 #118412 #118613 #119711 #120060 #120451
//     #120542 #123373 #123414 #123437 #123526 #123844 #123928 #123932 #125714 #126974 #126983
//     #127008 #127063 #127077 #127087 #128580 #128645 —— 用途：alternative-basis 递归引擎与
//   **pdoom**：#86352 #112312 #118374 #118809 #118817（frontier6：输出自为首操作数 · 两目标共享趟）·
//   **saffah_codex_6s_agg2**：#102935 #110387 · **saffah_codex_maker**：#125714 ·
//   **saffah_cc_v41_260924**：#96771 #100997 #117569 #117816 · **ms2al_** 席：pack 叶族
//     doi 10.1137/22M1502719（exact 七乘 alternative-basis 分解思想）。
//   合规：上述 duck.ac 公开提交正文均未声明独立软件许可证；按目标原文要求写明「作者账号 / 原始提交地址 /
//   所用内容」三项 ✓（清单内每条的用途见底盘自带注释与 #128220 正文）。
// 0 现读（本机 10:4x / `tools/exact.py mmms8k`）：`mine` = **3679.135284**（#129780，本席上一发）·
//   T = **3685.678680**（#128220）· 严支 `0.99·T + 1µs` = **3648.822893** ⇒ **缺 30.312391 ms（0.824%）**。
// 1 上一发 [4KBSTAG] 的**板面实测（判题机）**：`#128702` 3682.985392 → **`#129780` 3679.135284
//   = −3.850108 ms（−0.105%）** ⇒ ★**「缓冲间距 +512 B」这一刀在本题【兑现为正】**，
//   与同族 `mmmx1k` 的 `#129578→#129587`（同一个数，**−1.14%**）同向。
//   （独立朴素 oracle 逐点复算 + 64×64 块扫）· 与 #129780 逐行 diff = **只 2 处**（含 1 处尺寸常量）✓ ·
//   源码 79K 级 < 102400 ✓ · 本机只可否决、不可定价 ⇒ 以板面为准。
// 5 合规（引用规则三项）：[1] 底盘（账号 + URL + 用途）· [1b] 公开文档依据 · [2][3][4] 对对手件与
//   继承链的署名全部原位保留「账号 / 原始提交地址 / 所用内容」✓。本刀**未复制任何新的第三方代码** ✓
// 0 现读（本机 04:40 / `tools/exact.py`，判题机读数）：`mine` = **3683.811559**（#128645）·
//   T = **3685.678680**（#128220）· 严支 `0.99·T + 1µs` = **3648.822893** ⇒ **缺 34.991 ms（0.95%）**。
//   n=8192 全链 FNV **`971a53a567589da7`** 与底盘逐字节同 ✓ · n=4096 路径 **`06649e8247cc3675`** 同 ✓ ·
//   与 #128645 逐行 diff 只在 `n==8192` 分支 + 一个新 `outer_quad`（`n!=8192 && n!=4096` 回退分支一字未改）✓
// 4 合规（引用规则三项）：底盘 [#1]、对手件 [#2]、本账号谱系 [#3]、继承链 [#4] 均写明「账号 / URL / 所用内容」✓；
/* References:
 * saffah_codex_6a_agg3 https://duck.ac/submission/128163: complete eight-thousand fused-source engine, paired-A packing and shared two-target outer scans; inherited credits below.
 * saffah_codex_6a_agg3 https://duck.ac/submission/126764: shared native kernel and two-level final-value reconstruction, ported with its complete added attribution below.
 * No separately stated license in these submissions; all inherited notices retained.
 */
/* Inherited references and notices (duplicate lines consolidated):
References:
saffah_codex_6a_agg3 https://duck.ac/submission/126758: exact shared native reduction, local epilogue loop control, shared middle forms, and full inherited credits/licenses below.
saffah_codex_6a_agg3 https://duck.ac/submission/126749: direct register reuse for one-level M6/C22 reconstruction, extended here across a second basis level.
saffah_codex_6a_agg3 https://duck.ac/submission/126749: exact three-mode native kernel and full inherited attribution/license notes retained below.
saffah_codex_6a_agg3 https://duck.ac/submission/126754: preserve the newly accepted shared middle transform wrappers and all its inherited credits below.
saffah_codex_6a_agg3, https://duck.ac/submission/126749: retained the complete accepted three-mode shared-native, shared-upper-transform and two-page root-plane engine; all inherited author/license notices remain below.
saffah_codex_6a_agg3 https://duck.ac/submission/126744 and https://duck.ac/submission/126746: shared upper transforms and two-page root-plane separation; inherited engine, credit and license notes retained below.
saffah_codex_6a_agg3 https://duck.ac/submission/126689: M6/C22 fusion formula. That experiment shared plain and reconstruction only and retained a separate additive compute kernel; its full engine was slower.
saffah_codex_6a_agg3 https://duck.ac/submission/126744: accepted shared upper inplace helpers and all inherited attributions/license notes below.
saffah_codex_6a_agg3 https://duck.ac/submission/126746: officially improved two-page root-plane separation.
saffah_codex_6a_agg3, https://duck.ac/submission/126738: retained the complete accepted page-separated, shared-native engine and all inherited author/license notices.
saffah_codex_6a_agg3 https://duck.ac/submission/126727: complete accepted shared-native matrix engine, all inherited references retained.
saffah_codex_6a_agg3 https://duck.ac/submission/126664: complete accepted S2 engine, including native32 arithmetic, packing, reconstruction and all inherited attributions retained below.
saffah_codex_6a_agg3 https://duck.ac/submission/126660: accepted S2 destination-based bottom reconstruction and full inherited engine/credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126657: officially positive address-base choice in the 1024 engine.
saffah_codex_6a_agg3 https://duck.ac/submission/126626: accepted first-pair pipeline, N64 four-register reconstruction, and inherited engine/credits below.
saffah_codex_6a_agg3 https://duck.ac/submission/126657: officially successful C22 address-base specialization transferred from the 1024 case.
saffah_codex_6a_agg3 https://duck.ac/submission/126621: accepted 2048 A/B row lookahead pipeline and all inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126624: successful first-pair second-A lead from the 1024 engine.
saffah_codex_6a_agg3 https://duck.ac/submission/126617: accepted 2048 A-lookahead engine and all inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126618: positive first-three-B row-transition pipeline in the 1024 engine.
saffah_codex_6a_agg3 https://duck.ac/submission/126597: accepted 2048 engine and all inherited attribution retained.
saffah_codex_6a_agg3 https://duck.ac/submission/126614: positive next-row A broadcast pipeline from the 1024 engine.
saffah_codex_6a_agg3 https://duck.ac/submission/126591: accepted compact bottom and upper reconstruction, with inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126575: accepted two-vector upper reconstruction and inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126587: positive explicit four-register N64 reconstruction on the one-thousand short matrix problem.
saffah_codex_6a_agg3 https://duck.ac/submission/126571: accepted first-A native32 schedule and inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126472: compact unrolling reduced repeated hot code in the double-matrix packing loops; used only as motivation for inspecting this independent loop.
saffah_codex_6a_agg3 https://duck.ac/submission/126542: accepted native32 engine and all inherited references.
saffah_codex_6a_agg3 https://duck.ac/submission/126567: first-A broadcast scheduling from the four-thousand short matrix problem.
saffah_codex_6a_agg3 https://duck.ac/submission/126475: retained accepted mmms2k engine and all inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126540: adapted the successful native32 vector register allocation from mmms4k.
saffah_codex_6a_agg3 https://duck.ac/submission/125904: retained accepted2k modulo65536 engine with all inherited citations below.
saffah_codex_6a_agg3 https://duck.ac/submission/126469: private root workspace transparent-huge-page advice transferred from the4k engine.
- saffah_cc_v41_agg1, https://duck.ac/submission/127087: copied the 8192
- saffah_codex_6a_agg3, https://duck.ac/submission/126590: copied our improved
saffah_codex_6a_agg3 https://duck.ac/submission/126580: accepted two-vector upper reconstruction and inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126567: accepted native32 engine and inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126575: compact upper reconstruction, positive on the two-thousand short matrix problem.
saffah_codex_6a_agg3 https://duck.ac/submission/126540: accepted compact native32 register allocation and all inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126469: retained exact native32 engine and all inherited credits.
saffah_codex_6a_agg3 https://duck.ac/submission/126048: unchanged accepted short-matrix engine and all inherited credits below.
saffah_codex_6a_agg3 https://duck.ac/submission/126455: shared the ordinary huge-page advice already used for private double-matrix work arrays.
- saffah_codex_6a_agg3, https://duck.ac/submission/125910: accepted contiguous root operand reads and inherited modulo65536 matrix engine.
- saffah_cc_v41_agg1, https://duck.ac/submission/123932: root source traversal improvement inherited by125910.
- saffah_cc_v41_agg1, https://duck.ac/submission/120060: original alternative-basis decoding and recursive engine; all inherited notices retained below. No separate software license notice is shown by these public sources.
- saffah_cc_v41_agg1, https://duck.ac/submission/123932: copied the raw-row A packing layout and adjusted leaf broadcast offsets; inherited modulo-65536 recursive engine and references preserved below.
- saffah_cc_v41_agg1, https://duck.ac/submission/123928: copied the row-block packing unswitch and indexed two-instruction leaf loop control.
- saffah_codex_6a_agg3, https://duck.ac/submission/120542: for the 4096 case, retained our twelve-live-slot root arena version of saffah_cc_v41_agg1 https://duck.ac/submission/120060.
The cited submissions declare no separate software license; their attribution and inherited credit notices are preserved below.
Idea: Additionally transfer reference123932 root reconstruction traversal: read each child panel in a continuous 2KiB pass before correcting and decoding its output blocks. This replaces repeated 256-byte reads separated by 2080-byte jumps. All seven-product arithmetic and output positions are unchanged. Combine the completely unshuffled A row layout with compile-time row-block transforms and a shared negative A/C byte index. Every four-row input group is stored directly and the leaf fetches matching dwords; the loop uses add/jnz without pointer comparison. Adapt the packing stride to this problem size. Experiment purpose: remove packing shuffles and transform branches while reducing leaf loop control, retaining exact modulo-65536 products.
*/
#include <sys/mman.h>
#define ALTMAX 128
#define ROOT_PAD 160
#include <sys/mman.h>
#define ALTMAX 128
#define ROOT_PAD 160
#pragma GCC optimize("O3,unroll-loops,web,rename-registers,no-tree-dominator-opts")
#pragma GCC target("avx2","tune=skylake")
#include <immintrin.h>
#include <stdint.h>
#include <string.h>
asm(".macro vzeroupper\n.endm\n");
#ifndef TILE_PAD
#define TILE_PAD 16
#endif
#ifndef BASE
#define BASE (1 << 5)
#endif
using U = uint16_t;
static constexpr int MAXN = 4096;
alignas(4096)static U pa_[MAXN * MAXN +(MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192],pb_[MAXN * MAXN +(MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192];
alignas(4096)static U pc_[MAXN * MAXN +(MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192],ws_[MAXN * MAXN +(MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192];
static U *pa = pa_ + 1024,*pb = pb_ + 1024,*pc = pc_ + 0,*ws = ws_ + 1024;
static U *final_out;
static inline __m256i ld(const U *p){
return _mm256_load_si256((const __m256i *)p);
}
static inline void st(U *p,__m256i x){
_mm256_store_si256((__m256i *)p,x);
}
static const U *raw_basis_input;
static U *g_out;         
static long g_ostr=4096; 
static inline __m256i raw_basis_low32(const U *p,bool transformed,long st){
__m256i v=_mm256_loadu_si256((const __m256i*)p);
if(transformed)v=_mm256_add_epi16(v,_mm256_sub_epi16(
_mm256_loadu_si256((const __m256i*)(p-32*st)),
_mm256_loadu_si256((const __m256i*)(p-32))));
return v;
}
static inline __m128i raw_basis_low16(const U *p,bool transformed,long st){
__m128i v=_mm_loadu_si128((const __m128i*)p);
if(transformed)v=_mm_add_epi16(v,_mm_sub_epi16(
_mm_loadu_si128((const __m128i*)(p-32*st)),
_mm_loadu_si128((const __m128i*)(p-32))));
return v;
}
static inline __m256i raw_basis32(const U *p,int flags,long st){
__m256i v=raw_basis_low32(p,flags&1,st);
if(flags&2)v=_mm256_add_epi16(v,_mm256_sub_epi16(
raw_basis_low32(p-64*st,flags&1,st),raw_basis_low32(p-64,flags&1,st)));
return v;
}
static inline __m128i raw_basis16(const U *p,int flags,long st){
__m128i v=raw_basis_low16(p,flags&1,st);
if(flags&2)v=_mm_add_epi16(v,_mm_sub_epi16(
raw_basis_low16(p-64*st,flags&1,st),raw_basis_low16(p-64,flags&1,st)));
return v;
}
template<bool TWO> static inline __m256i rb32(const U *p,const U *q,int sgn,int flags,long st){
__m256i v=raw_basis32(p,flags,st);
if constexpr(TWO){__m256i w=raw_basis32(q,flags,st);v=(sgn>0)?_mm256_add_epi16(v,w):_mm256_sub_epi16(v,w);}
return v;
}
static void pack(U *d,const U *s,int n,int stride,bool right){
if(n > BASE){
int h = n / 2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
pack(d,s,h,stride,right);
pack(d + q,s + h,h,stride,right);
pack(d + 2*q,s + h*stride,h,stride,right);
pack(d + 3*q,s + h*stride + h,h,stride,right);
}else if(!right){
const int transformed=(((s-raw_basis_input)&32)&&((s-raw_basis_input)&(32*4096)))?1:0;
for(int i=0;i<n;i+=4)for(int k=0;k<n;k+=16){
{int i2=(i+4)&31;  
__builtin_prefetch(s+i2*stride+k);__builtin_prefetch(s+(i2+1)*stride+k);
__builtin_prefetch(s+i2*stride+k+16);__builtin_prefetch(s+(i2+1)*stride+k+16);
__builtin_prefetch(s+(i2+2)*stride+k);__builtin_prefetch(s+(i2+3)*stride+k);
__builtin_prefetch(s+(i2+2)*stride+k+16);__builtin_prefetch(s+(i2+3)*stride+k+16);}
__m256i a=raw_basis32(s+i*stride+k,transformed,stride),b=raw_basis32(s+(i+1)*stride+k,transformed,stride);
__m256i c=raw_basis32(s+(i+2)*stride+k,transformed,stride),e=raw_basis32(s+(i+3)*stride+k,transformed,stride);
__m256i t0=_mm256_unpacklo_epi32(a,b),t1=_mm256_unpacklo_epi32(c,e);
__m256i t2=_mm256_unpackhi_epi32(a,b),t3=_mm256_unpackhi_epi32(c,e);
__m256i u0=_mm256_unpacklo_epi64(t0,t1),u1=_mm256_unpackhi_epi64(t0,t1);
__m256i u2=_mm256_unpacklo_epi64(t2,t3),u3=_mm256_unpackhi_epi64(t2,t3);
st(d,_mm256_permute2x128_si256(u0,u1,0x20));
st(d+16,_mm256_permute2x128_si256(u2,u3,0x20));
st(d+32,_mm256_permute2x128_si256(u0,u1,0x31));
st(d+48,_mm256_permute2x128_si256(u2,u3,0x31));d+=64;
}
}else{
const int transformed=(((s-raw_basis_input)&32)&&((s-raw_basis_input)&(32*4096)))?1:0;
for(int k=0;k<n;k+=2)for(int j=0;j<n;){
int w=32;
U *v=d+j*n+k*w;
for(int t=0;t<w;t+=16){
if(t+16<=w){
__m256i x=raw_basis32(s+k*stride+j+t,transformed,stride);
__m256i y=raw_basis32(s+(k+1)*stride+j+t,transformed,stride);
st(v,_mm256_unpacklo_epi16(x,y));
st(v+16,_mm256_unpackhi_epi16(x,y));v+=32;
}else{
__m128i x=_mm_load_si128((const __m128i*)(s+k*stride+j+t));
__m128i y=_mm_load_si128((const __m128i*)(s+(k+1)*stride+j+t));
_mm_store_si128((__m128i*)v,_mm_unpacklo_epi16(x,y));
_mm_store_si128((__m128i*)(v+8),_mm_unpackhi_epi16(x,y));v+=16;
}
}
j+=w;
}
}
}
#define PWL_Q ((32*32) + (32/BASE)*(32/BASE)*TILE_PAD)
#define PWL_CH(DSTP,K,TF,O0,O1,O2,O3) do { \
__m256i a=rb32<TWO>(sr+(long)i*S4ST+(K),sr2+(long)i*S4ST+(K),sgn,(TF),S4ST),b=rb32<TWO>(sr+(long)(i+1)*S4ST+(K),sr2+(long)(i+1)*S4ST+(K),sgn,(TF),S4ST);\
__m256i c=rb32<TWO>(sr+(long)(i+2)*S4ST+(K),sr2+(long)(i+2)*S4ST+(K),sgn,(TF),S4ST),e=rb32<TWO>(sr+(long)(i+3)*S4ST+(K),sr2+(long)(i+3)*S4ST+(K),sgn,(TF),S4ST);\
st((DSTP)+(O0),a);\
st((DSTP)+(O1),b);\
st((DSTP)+(O2),c);\
st((DSTP)+(O3),e);}while(0)
template<bool TWO> static void pack_wide(U *dA,U *dB,const U *s,long ST4_,const U *s2,int sgn){
const long S4ST=ST4_;
for(int r=0;r<2;++r){
const U *sr = s +(long)r*32*S4ST,*sr2 = s2 +(long)r*32*S4ST;
U *p0=dA+(r?2*PWL_Q:0),*p1=dA+(r?3*PWL_Q:PWL_Q),*p2=dB+(r?2*PWL_Q:0),*p3=dB+(r?3*PWL_Q:PWL_Q);
const int TFc =(r==1)?1:0;
for(int i=0;i<32;i+=4){
PWL_CH(p0,0,0,0,16,64,80);PWL_CH(p0,16,0,32,48,96,112);p0+=128;
PWL_CH(p1,32,TFc,0,16,64,80);PWL_CH(p1,48,TFc,32,48,96,112);p1+=128;
PWL_CH(p2,64,0,0,16,64,80);PWL_CH(p2,80,0,32,48,96,112);p2+=128;
PWL_CH(p3,96,TFc,0,16,64,80);PWL_CH(p3,112,TFc,32,48,96,112);p3+=128;
}
}
}
#define S4DF_BW(P,J,TF) do { \
__m256i x=rb32<TWO>(sr+(long)k*ST+(J),sr2+(long)k*ST+(J),sgn,(TF),ST),y=rb32<TWO>(sr+(long)(k+1)*ST+(J),sr2+(long)(k+1)*ST+(J),sgn,(TF),ST);\
st((P),_mm256_unpacklo_epi16(x,y));st((P)+16,_mm256_unpackhi_epi16(x,y));}while(0)
template<bool TWO> static void pack_wide_b(U *dA,U *dB,const U *s,long ST4_,long d0log,const U *s2,int sgn){
const long ST=ST4_;
{
const U*sr=s+(long)0*32*ST,*sr2=s2+(long)0*32*ST;
U*p0=dA+0*PWL_Q,*p1=dA+1*PWL_Q,*p2=dB+0*PWL_Q,*p3=dB+1*PWL_Q;
constexpr int tf0=0,tf1=0;
for(int k=0;k<32;k+=2){
S4DF_BW(p0+0,0,tf0);S4DF_BW(p0+32,16,tf0);
S4DF_BW(p1+0,32,tf1);S4DF_BW(p1+32,48,tf1);
S4DF_BW(p2+0,64,tf0);S4DF_BW(p2+32,80,tf0);
S4DF_BW(p3+0,96,tf1);S4DF_BW(p3+32,112,tf1);
p0+=64;p1+=64;p2+=64;p3+=64;
}
}
{
const U*sr=s+(long)1*32*ST,*sr2=s2+(long)1*32*ST;
U*p0=dA+2*PWL_Q,*p1=dA+3*PWL_Q,*p2=dB+2*PWL_Q,*p3=dB+3*PWL_Q;
constexpr int tf0=0,tf1=1;
for(int k=0;k<32;k+=2){
S4DF_BW(p0+0,0,tf0);S4DF_BW(p0+32,16,tf0);
S4DF_BW(p1+0,32,tf1);S4DF_BW(p1+32,48,tf1);
S4DF_BW(p2+0,64,tf0);S4DF_BW(p2+32,80,tf0);
S4DF_BW(p3+0,96,tf1);S4DF_BW(p3+32,112,tf1);
p0+=64;p1+=64;p2+=64;p3+=64;
}
}
}
alignas(32)static const unsigned char s4bj_si[32] ={
0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15};
static inline __m256i s4bj_fx(__m256i v){
return _mm256_shuffle_epi8(v,_mm256_load_si256((const __m256i *)s4bj_si));
}
alignas(32)unsigned short s4aj_m16[16] ={0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0};
alignas(32)unsigned char s4ak_sh16[32] ={0,1,4,5,8,9,12,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0,1,4,5,8,9,12,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80};
static const U *g_postadd=nullptr,*g_postadd2=nullptr;
static __attribute__((noinline,noclone))void native2x32_shared(const U *a,const U *b,U *c,int Mode){
const U *abase=&a[1024];
U *cbase=&c[1024];
intptr_t idx=-2048;
asm volatile(
"vpbroadcastd 0(%[abase],%[i],1), %%ymm3\n\t"
"3:\n\t"
"vpbroadcastd 32(%[abase],%[i],1), %%ymm4\n\t"
"vpmaddwd 96(%[b]), %%ymm3, %%ymm11\n\t"
"vpmaddwd 0(%[b]), %%ymm3, %%ymm8\n\t"
"vpmaddwd 32(%[b]), %%ymm3, %%ymm9\n\t"
"vpmaddwd 64(%[b]), %%ymm3, %%ymm10\n\t"
"vpmaddwd 96(%[b]), %%ymm4, %%ymm15\n\t"
"vpmaddwd 0(%[b]), %%ymm4, %%ymm12\n\t"
"vpmaddwd 32(%[b]), %%ymm4, %%ymm13\n\t"
"vpmaddwd 64(%[b]), %%ymm4, %%ymm14\n\t"
"vpbroadcastd 8(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 320(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 288(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 256(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 352(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 40(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 256(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 288(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 320(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 352(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 4(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 192(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 160(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 128(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 224(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 36(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 128(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 160(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 192(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 224(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 16(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 576(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 544(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 512(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 608(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 48(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 512(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 544(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 576(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 608(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 12(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 448(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 416(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 384(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 480(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 44(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 384(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 416(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 448(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 480(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 24(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 832(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 800(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 768(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 864(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 56(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 768(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 800(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 832(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 864(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 20(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 704(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 672(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 640(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 736(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 52(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 640(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 672(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 704(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 736(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 64(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 1088(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 1056(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 1024(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 1120(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 96(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 1024(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 1056(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 1088(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 1120(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 28(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 960(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 928(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 896(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 992(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 60(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 896(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 928(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 960(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 992(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 72(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 1344(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 1312(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 1280(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 1376(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 104(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 1280(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 1312(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 1344(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 1376(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 68(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 1216(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 1184(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 1152(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 1248(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 100(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 1152(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 1184(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 1216(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 1248(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 80(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 1600(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 1568(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 1536(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 1632(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 112(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 1536(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 1568(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 1600(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 1632(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 76(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 1472(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 1440(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 1408(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 1504(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 108(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 1408(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 1440(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 1472(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 1504(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 88(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 1856(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 1824(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 1792(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 1888(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 120(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 1792(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 1824(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 1856(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 1888(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 84(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 1728(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 1696(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 1664(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 1760(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 116(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 1664(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 1696(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 1728(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 1760(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"vpbroadcastd 92(%[abase],%[i],1), %%ymm3\n\t"
"vpmaddwd 1984(%[b]), %%ymm3, %%ymm6\n\t"
"vpmaddwd 1952(%[b]), %%ymm3, %%ymm5\n\t"
"vpmaddwd 1920(%[b]), %%ymm3, %%ymm4\n\t"
"vpmaddwd 2016(%[b]), %%ymm3, %%ymm3\n\t"
"vpbroadcastd 124(%[abase],%[i],1), %%ymm7\n\t"
"vpmaddwd 1920(%[b]), %%ymm7, %%ymm0\n\t"
"vpmaddwd 1952(%[b]), %%ymm7, %%ymm1\n\t"
"vpmaddwd 1984(%[b]), %%ymm7, %%ymm2\n\t"
"vpmaddwd 2016(%[b]), %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm4, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm5, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm6, %%ymm10, %%ymm10\n\t"
"vpaddd %%ymm3, %%ymm11, %%ymm11\n\t"
"vpbroadcastd 128(%[abase],%[i],1), %%ymm3\n\t"
"vpaddd %%ymm0, %%ymm12, %%ymm12\n\t"
"vpaddd %%ymm1, %%ymm13, %%ymm13\n\t"
"vpaddd %%ymm2, %%ymm14, %%ymm14\n\t"
"vpaddd %%ymm7, %%ymm15, %%ymm15\n\t"
"test %k[mode], %k[mode]\n\t"
"jz 4f\n\t"
"cmp $1, %k[mode]\n\t"
"jne 8f\n\t"
"vpslld $16, %%ymm9, %%ymm9\n\t"
"vpblendw $0xAA, %%ymm9, %%ymm8, %%ymm8\n\t"
"vpaddw 0(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 0(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm11, %%ymm11\n\t"
"vpblendw $0xAA, %%ymm11, %%ymm10, %%ymm10\n\t"
"vpaddw 32(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 32(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm13, %%ymm13\n\t"
"vpblendw $0xAA, %%ymm13, %%ymm12, %%ymm12\n\t"
"vpaddw 64(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 64(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm15, %%ymm15\n\t"
"vpblendw $0xAA, %%ymm15, %%ymm14, %%ymm14\n\t"
"vpaddw 96(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 96(%[cbase],%[i],1)\n\t"
"add $128, %[i]\n\t"
"jnz 3b\n\t"
"jmp 7f\n\t"
"8:\n\t"
"cmp $2, %k[mode]\n\t"
"je 6f\n\t"
"cmp $3, %k[mode]\n\t"
"jne 9f\n\t"
"vpslld $16, %%ymm9, %%ymm9\n\t"
"vpblendw $0xAA, %%ymm9, %%ymm8, %%ymm8\n\t"
"vpaddw 0(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 0(%[cbase],%[i],1)\n\t"
"vpaddw -8320(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw 8320(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw -16640(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 8320(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm11, %%ymm11\n\t"
"vpblendw $0xAA, %%ymm11, %%ymm10, %%ymm10\n\t"
"vpaddw 32(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 32(%[cbase],%[i],1)\n\t"
"vpaddw -8288(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw 8352(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw -16608(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 8352(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm13, %%ymm13\n\t"
"vpblendw $0xAA, %%ymm13, %%ymm12, %%ymm12\n\t"
"vpaddw 64(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 64(%[cbase],%[i],1)\n\t"
"vpaddw -8256(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw 8384(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw -16576(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 8384(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm15, %%ymm15\n\t"
"vpblendw $0xAA, %%ymm15, %%ymm14, %%ymm14\n\t"
"vpaddw 96(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 96(%[cbase],%[i],1)\n\t"
"vpaddw -8224(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw 8416(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw -16544(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 8416(%[cbase],%[i],1)\n\t"
"add $128, %[i]\n\t"
"jnz 3b\n\t"
"jmp 7f\n\t"
"9:\n\t"
"vpslld $16, %%ymm13, %%ymm13\n\t"
"vpblendw $0xAA, %%ymm13, %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 64(%[cbase],%[i],1)\n\t"
"vpaddw -2016(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw 2144(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw -4096(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 2144(%[cbase],%[i],1)\n\t"
"vpaddw -6176(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw 10464(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw -14496(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 10464(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm15, %%ymm15\n\t"
"vpblendw $0xAA, %%ymm15, %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 96(%[cbase],%[i],1)\n\t"
"vpaddw -1984(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw 2176(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw -4064(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 2176(%[cbase],%[i],1)\n\t"
"vpaddw -6144(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw 10496(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw -14464(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 10496(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm9, %%ymm9\n\t"
"vpblendw $0xAA, %%ymm9, %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 0(%[cbase],%[i],1)\n\t"
"vpaddw -2080(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw 2080(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw -4160(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 2080(%[cbase],%[i],1)\n\t"
"vpaddw -6240(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw 10400(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw -14560(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 10400(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm11, %%ymm11\n\t"
"vpblendw $0xAA, %%ymm11, %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 32(%[cbase],%[i],1)\n\t"
"vpaddw -2048(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw 2112(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw -4128(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 2112(%[cbase],%[i],1)\n\t"
"vpaddw -6208(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw 10432(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw -14528(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 10432(%[cbase],%[i],1)\n\t"
"add $128, %[i]\n\t"
"jnz 3b\n\t"
"jmp 7f\n\t"
"6:\n\t"
"vpslld $16, %%ymm13, %%ymm13\n\t"
"vpblendw $0xAA, %%ymm13, %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 64(%[cbase],%[i],1)\n\t"
"vpaddw -2016(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw 2144(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vpsubw -4096(%[cbase],%[i],1), %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 2144(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm15, %%ymm15\n\t"
"vpblendw $0xAA, %%ymm15, %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 96(%[cbase],%[i],1)\n\t"
"vpaddw -1984(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw 2176(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vpsubw -4064(%[cbase],%[i],1), %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 2176(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm9, %%ymm9\n\t"
"vpblendw $0xAA, %%ymm9, %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 0(%[cbase],%[i],1)\n\t"
"vpaddw -2080(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw 2080(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vpsubw -4160(%[cbase],%[i],1), %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 2080(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm11, %%ymm11\n\t"
"vpblendw $0xAA, %%ymm11, %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 32(%[cbase],%[i],1)\n\t"
"vpaddw -2048(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw 2112(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vpsubw -4128(%[cbase],%[i],1), %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 2112(%[cbase],%[i],1)\n\t"
"add $128, %[i]\n\t"
"jnz 3b\n\t"
"jmp 7f\n\t"
"4:\n\t"
"vpslld $16, %%ymm9, %%ymm9\n\t"
"vpblendw $0xAA, %%ymm9, %%ymm8, %%ymm8\n\t"
"vmovdqu %%ymm8, 0(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm11, %%ymm11\n\t"
"vpblendw $0xAA, %%ymm11, %%ymm10, %%ymm10\n\t"
"vmovdqu %%ymm10, 32(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm13, %%ymm13\n\t"
"vpblendw $0xAA, %%ymm13, %%ymm12, %%ymm12\n\t"
"vmovdqu %%ymm12, 64(%[cbase],%[i],1)\n\t"
"vpslld $16, %%ymm15, %%ymm15\n\t"
"vpblendw $0xAA, %%ymm15, %%ymm14, %%ymm14\n\t"
"vmovdqu %%ymm14, 96(%[cbase],%[i],1)\n\t"
"add $128, %[i]\n\t"
"jnz 3b\n\t"
"7:\n\t"
: [i] "+&r"(idx)
: [b] "r"(b),[abase] "r"(abase),[cbase] "r"(cbase),[mode] "r"(Mode)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15");
}
static void base(const U *a,const U *b,U *c,int n){
native2x32_shared(a,b,c,0);
}
static __attribute__((noinline))void leaf_add(const U*a,const U*b,U*c){
native2x32_shared(a,b,c,1);
}
static inline void emit_native(U *dst,const U *s,int stride){
for(int row=0;row<4;++row){
U *d=dst+(size_t)row*stride;
_mm256_stream_si256((__m256i*)(d+0),s4bj_fx(ld(s+32*row)));
_mm256_stream_si256((__m256i*)(d+16),s4bj_fx(ld(s+32*row+16)));
}
}
template<bool Sub>
static inline void combine(U *d,const U *a,const U *b,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a+1024;
if constexpr(Sub){
asm volatile(
"1:\n\t"
"vmovdqa 0(%[a]), %%ymm0\n\t"
"vmovdqa 32(%[a]), %%ymm1\n\t"
"vmovdqa 64(%[a]), %%ymm2\n\t"
"vmovdqa 96(%[a]), %%ymm3\n\t"
"vpsubw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[a]\n\t"
"add $128, %[b]\n\t"
"add $128, %[d]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
: [a] "+&r"(a),[b] "+&r"(b),[d] "+&r"(d)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}else{
asm volatile(
"1:\n\t"
"vmovdqa 0(%[a]), %%ymm0\n\t"
"vmovdqa 32(%[a]), %%ymm1\n\t"
"vmovdqa 64(%[a]), %%ymm2\n\t"
"vmovdqa 96(%[a]), %%ymm3\n\t"
"vpaddw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[a]\n\t"
"add $128, %[b]\n\t"
"add $128, %[d]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
: [a] "+&r"(a),[b] "+&r"(b),[d] "+&r"(d)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}
a+=TILE_PAD;b+=TILE_PAD;d+=TILE_PAD;
}
}
template<bool Sub,int Delta>
static inline void combine_displaced(U *d,const U *a,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a+1024;
if constexpr(Sub){
asm volatile("1:\n\t"
"vmovdqa 0(%[a]), %%ymm0\n\t"
"vmovdqa 32(%[a]), %%ymm1\n\t"
"vmovdqa 64(%[a]), %%ymm2\n\t"
"vmovdqa 96(%[a]), %%ymm3\n\t"
"vpsubw %c[delta]+0(%[a]), %%ymm0, %%ymm0\n\t"
"vpsubw %c[delta]+32(%[a]), %%ymm1, %%ymm1\n\t"
"vpsubw %c[delta]+64(%[a]), %%ymm2, %%ymm2\n\t"
"vpsubw %c[delta]+96(%[a]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[a]\n\t"
"add $128, %[d]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
: [a] "+&r"(a),[d] "+&r"(d)
: [end] "r"(end),[delta] "i"(2*Delta)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}else{
asm volatile("1:\n\t"
"vmovdqa 0(%[a]), %%ymm0\n\t"
"vmovdqa 32(%[a]), %%ymm1\n\t"
"vmovdqa 64(%[a]), %%ymm2\n\t"
"vmovdqa 96(%[a]), %%ymm3\n\t"
"vpaddw %c[delta]+0(%[a]), %%ymm0, %%ymm0\n\t"
"vpaddw %c[delta]+32(%[a]), %%ymm1, %%ymm1\n\t"
"vpaddw %c[delta]+64(%[a]), %%ymm2, %%ymm2\n\t"
"vpaddw %c[delta]+96(%[a]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[a]\n\t"
"add $128, %[d]\n\t"
"cmp %[end], %[a]\n\t"
"jb 1b\n\t"
: [a] "+&r"(a),[d] "+&r"(d)
: [end] "r"(end),[delta] "i"(2*Delta)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}
a+=TILE_PAD;d+=TILE_PAD;
}
}
template<int D1,int D2>
static __attribute__((noinline,noclone))void combine_dpair_tt(U *d1,const U *a1,U *d2,const U *a2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a1+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[a1]), %%ymm0\n\t"
"vmovdqa 32(%[a1]), %%ymm1\n\t"
"vmovdqa 64(%[a1]), %%ymm2\n\t"
"vmovdqa 0(%[a2]), %%ymm4\n\t"
"vmovdqa 96(%[a1]), %%ymm3\n\t"
"vmovdqa 32(%[a2]), %%ymm5\n\t"
"vmovdqa 64(%[a2]), %%ymm6\n\t"
"vmovdqa 96(%[a2]), %%ymm7\n\t"
"vpsubw %c[d1o]+0(%[a1]), %%ymm0, %%ymm0\n\t"
"vpsubw %c[d1o]+32(%[a1]), %%ymm1, %%ymm1\n\t"
"vpsubw %c[d1o]+64(%[a1]), %%ymm2, %%ymm2\n\t"
"vpsubw %c[d1o]+96(%[a1]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d1])\n\t"
"vmovdqa %%ymm1, 32(%[d1])\n\t"
"vmovdqa %%ymm2, 64(%[d1])\n\t"
"vmovdqa %%ymm3, 96(%[d1])\n\t"
"vpsubw %c[d2o]+0(%[a2]), %%ymm4, %%ymm4\n\t"
"vpsubw %c[d2o]+32(%[a2]), %%ymm5, %%ymm5\n\t"
"vpsubw %c[d2o]+64(%[a2]), %%ymm6, %%ymm6\n\t"
"vpsubw %c[d2o]+96(%[a2]), %%ymm7, %%ymm7\n\t"
"vmovdqa %%ymm4, 0(%[d2])\n\t"
"vmovdqa %%ymm5, 32(%[d2])\n\t"
"vmovdqa %%ymm6, 64(%[d2])\n\t"
"vmovdqa %%ymm7, 96(%[d2])\n\t"
"add $128, %[a1]\n\t"
"add $128, %[d1]\n\t"
"add $128, %[a2]\n\t"
"add $128, %[d2]\n\t"
"cmp %[end], %[a1]\n\t"
"jb 1b\n\t"
: [a1] "+&r"(a1),[d1] "+&r"(d1),[a2] "+&r"(a2),[d2] "+&r"(d2)
: [end] "r"(end),[d1o] "i"(2*D1),[d2o] "i"(2*D2)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
a1+=TILE_PAD;d1+=TILE_PAD;a2+=TILE_PAD;d2+=TILE_PAD;
}
}
template<int D1,int D2>
static __attribute__((noinline,noclone))void combine_dpair_tt_nt(U *d1,const U *a1,U *d2,const U *a2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a1+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[a1]), %%ymm0\n\t"
"vmovdqa 32(%[a1]), %%ymm1\n\t"
"vmovdqa 64(%[a1]), %%ymm2\n\t"
"vmovdqa 0(%[a2]), %%ymm4\n\t"
"vmovdqa 96(%[a1]), %%ymm3\n\t"
"vmovdqa 32(%[a2]), %%ymm5\n\t"
"vmovdqa 64(%[a2]), %%ymm6\n\t"
"vmovdqa 96(%[a2]), %%ymm7\n\t"
"vpsubw %c[d1o]+0(%[a1]), %%ymm0, %%ymm0\n\t"
"vpsubw %c[d1o]+32(%[a1]), %%ymm1, %%ymm1\n\t"
"vpsubw %c[d1o]+64(%[a1]), %%ymm2, %%ymm2\n\t"
"vpsubw %c[d1o]+96(%[a1]), %%ymm3, %%ymm3\n\t"
"vmovntdq %%ymm0, 0(%[d1])\n\t"
"vmovntdq %%ymm1, 32(%[d1])\n\t"
"vmovntdq %%ymm2, 64(%[d1])\n\t"
"vmovntdq %%ymm3, 96(%[d1])\n\t"
"vpsubw %c[d2o]+0(%[a2]), %%ymm4, %%ymm4\n\t"
"vpsubw %c[d2o]+32(%[a2]), %%ymm5, %%ymm5\n\t"
"vpsubw %c[d2o]+64(%[a2]), %%ymm6, %%ymm6\n\t"
"vpsubw %c[d2o]+96(%[a2]), %%ymm7, %%ymm7\n\t"
"vmovntdq %%ymm4, 0(%[d2])\n\t"
"vmovntdq %%ymm5, 32(%[d2])\n\t"
"vmovntdq %%ymm6, 64(%[d2])\n\t"
"vmovntdq %%ymm7, 96(%[d2])\n\t"
"add $128, %[a1]\n\t"
"add $128, %[d1]\n\t"
"add $128, %[a2]\n\t"
"add $128, %[d2]\n\t"
"cmp %[end], %[a1]\n\t"
"jb 1b\n\t"
: [a1] "+&r"(a1),[d1] "+&r"(d1),[a2] "+&r"(a2),[d2] "+&r"(d2)
: [end] "r"(end),[d1o] "i"(2*D1),[d2o] "i"(2*D2)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
a1+=TILE_PAD;d1+=TILE_PAD;a2+=TILE_PAD;d2+=TILE_PAD;
}
}
template<int D1,int D2>
static __attribute__((noinline,noclone))void combine_dpair_ft(U *d1,const U *a1,U *d2,const U *a2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a1+1024;
asm volatile(
"1:\n\tvmovdqa 0(%[a1]), %%ymm0\n\tvmovdqa 32(%[a1]), %%ymm1\n\tvmovdqa 64(%[a1]), %%ymm2\n\tvmovdqa 0(%[a2]), %%ymm4\n\tvmovdqa 96(%[a1]), %%ymm3\n\tvmovdqa 32(%[a2]), %%ymm5\n\tvmovdqa 64(%[a2]), %%ymm6\n\tvmovdqa 96(%[a2]), %%ymm7\n\tvpaddw %c[d1o]+96(%[a1]), %%ymm3, %%ymm3\n\tvpaddw %c[d1o]+64(%[a1]), %%ymm2, %%ymm2\n\tvpaddw %c[d1o]+32(%[a1]), %%ymm1, %%ymm1\n\tvpaddw %c[d1o]+0(%[a1]), %%ymm0, %%ymm0\n\tvmovdqa %%ymm3, 96(%[d1])\n\tvmovdqa %%ymm2, 64(%[d1])\n\tvmovdqa %%ymm1, 32(%[d1])\n\tvmovdqa %%ymm0, 0(%[d1])\n\tvpsubw %c[d2o]+0(%[a2]), %%ymm4, %%ymm4\n\tvpsubw %c[d2o]+32(%[a2]), %%ymm5, %%ymm5\n\tvpsubw %c[d2o]+64(%[a2]), %%ymm6, %%ymm6\n\tvpsubw %c[d2o]+96(%[a2]), %%ymm7, %%ymm7\n\tvmovdqa %%ymm4, 0(%[d2])\n\tvmovdqa %%ymm5, 32(%[d2])\n\tvmovdqa %%ymm6, 64(%[d2])\n\tvmovdqa %%ymm7, 96(%[d2])\n\tadd $128, %[a1]\n\tadd $128, %[d1]\n\tadd $128, %[a2]\n\tadd $128, %[d2]\n\tcmp %[end], %[a1]\n\tjb 1b"
: [a1] "+&r"(a1),[d1] "+&r"(d1),[a2] "+&r"(a2),[d2] "+&r"(d2)
: [end] "r"(end),[d1o] "i"(2*D1),[d2o] "i"(2*D2)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
a1+=TILE_PAD;d1+=TILE_PAD;a2+=TILE_PAD;d2+=TILE_PAD;
}
}
template<int D1,int D2>
static __attribute__((noinline,noclone))void combine_dpair_ft_nt(U *d1,const U *a1,U *d2,const U *a2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=a1+1024;
asm volatile(
"1:\n\tvmovdqa 0(%[a1]), %%ymm0\n\tvmovdqa 32(%[a1]), %%ymm1\n\tvmovdqa 64(%[a1]), %%ymm2\n\tvmovdqa 0(%[a2]), %%ymm4\n\tvmovdqa 96(%[a1]), %%ymm3\n\tvmovdqa 32(%[a2]), %%ymm5\n\tvmovdqa 64(%[a2]), %%ymm6\n\tvmovdqa 96(%[a2]), %%ymm7\n\tvpaddw %c[d1o]+96(%[a1]), %%ymm3, %%ymm3\n\tvpaddw %c[d1o]+64(%[a1]), %%ymm2, %%ymm2\n\tvpaddw %c[d1o]+32(%[a1]), %%ymm1, %%ymm1\n\tvpaddw %c[d1o]+0(%[a1]), %%ymm0, %%ymm0\n\tvmovntdq %%ymm3, 96(%[d1])\n\tvmovntdq %%ymm2, 64(%[d1])\n\tvmovntdq %%ymm1, 32(%[d1])\n\tvmovntdq %%ymm0, 0(%[d1])\n\tvpsubw %c[d2o]+0(%[a2]), %%ymm4, %%ymm4\n\tvpsubw %c[d2o]+32(%[a2]), %%ymm5, %%ymm5\n\tvpsubw %c[d2o]+64(%[a2]), %%ymm6, %%ymm6\n\tvpsubw %c[d2o]+96(%[a2]), %%ymm7, %%ymm7\n\tvmovntdq %%ymm4, 0(%[d2])\n\tvmovntdq %%ymm5, 32(%[d2])\n\tvmovntdq %%ymm6, 64(%[d2])\n\tvmovntdq %%ymm7, 96(%[d2])\n\tadd $128, %[a1]\n\tadd $128, %[d1]\n\tadd $128, %[a2]\n\tadd $128, %[d2]\n\tcmp %[end], %[a1]\n\tjb 1b"
: [a1] "+&r"(a1),[d1] "+&r"(d1),[a2] "+&r"(a2),[d2] "+&r"(d2)
: [end] "r"(end),[d1o] "i"(2*D1),[d2o] "i"(2*D2)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
a1+=TILE_PAD;d1+=TILE_PAD;a2+=TILE_PAD;d2+=TILE_PAD;
}
}
template<bool Sub,int Delta,int Len>
static __attribute__((noinline,noclone))void middle_displaced_shared(U*d,const U*a){
combine_displaced<Sub,Delta>(d,a,Len);
}
template<int N,bool Sub,int Delta,int Len>
static __attribute__((always_inline))inline void selected_displaced(U*d,const U*a){
if constexpr(N==128)middle_displaced_shared<Sub,Delta,Len>(d,a);
else combine_displaced<Sub,Delta>(d,a,Len);
}
template<bool Sub>
static __attribute__((noinline,noclone))void combine_inplace(U *d,const U *b,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
if constexpr(Sub){
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpsubw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}else{
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vpaddw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpaddw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
}
d+=TILE_PAD;b+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_inplace_pa(U *d,const U *b,const U *pa,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpaddw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"add $128, %[pa]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b),[pa] "+&r"(pa)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;b+=TILE_PAD;pa+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_inplace_sub_pa(U *d,const U *b,const U *pa,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpsubw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"add $128, %[pa]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b),[pa] "+&r"(pa)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;b+=TILE_PAD;pa+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void add_inplace_pa(U *d,const U *pa,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[pa]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[pa] "+&r"(pa)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;pa+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_inplace_pa2(U *d,const U *b,const U *pa,const U *pa2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa2]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa2]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa2]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa2]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"add $128, %[pa]\n\t"
"add $128, %[pa2]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b),[pa] "+&r"(pa),[pa2] "+&r"(pa2)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;b+=TILE_PAD;pa+=TILE_PAD;
pa2+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_inplace_sub_pa2(U *d,const U *b,const U *pa,const U *pa2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpsubw 0(%[b]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[b]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[b]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[b]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa2]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa2]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa2]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa2]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"add $128, %[pa]\n\t"
"add $128, %[pa2]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b),[pa] "+&r"(pa),[pa2] "+&r"(pa2)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;b+=TILE_PAD;pa+=TILE_PAD;
pa2+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void add_inplace_pa2(U *d,const U *pa,const U *pa2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile("1:\n\t"
"vmovdqa 0(%[d]), %%ymm0\n\t"
"vmovdqa 32(%[d]), %%ymm1\n\t"
"vmovdqa 64(%[d]), %%ymm2\n\t"
"vmovdqa 96(%[d]), %%ymm3\n\t"
"vpaddw 0(%[pa]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa]), %%ymm3, %%ymm3\n\t"
"vpaddw 0(%[pa2]), %%ymm0, %%ymm0\n\t"
"vpaddw 32(%[pa2]), %%ymm1, %%ymm1\n\t"
"vpaddw 64(%[pa2]), %%ymm2, %%ymm2\n\t"
"vpaddw 96(%[pa2]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[pa]\n\t"
"add $128, %[pa2]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[pa] "+&r"(pa),[pa2] "+&r"(pa2)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;pa+=TILE_PAD;pa2+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_pair_rf(U *d1,const U *a1,U *d2,const U *a2,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d1+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[a1]), %%ymm0\n\t"
"vmovdqa 32(%[a1]), %%ymm1\n\t"
"vmovdqa 64(%[a1]), %%ymm2\n\t"
"vmovdqa 96(%[a1]), %%ymm3\n\t"
"vpsubw 0(%[d1]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[d1]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[d1]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[d1]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d1])\n\t"
"vmovdqa %%ymm1, 32(%[d1])\n\t"
"vmovdqa %%ymm2, 64(%[d1])\n\t"
"vmovdqa %%ymm3, 96(%[d1])\n\t"
"vmovdqa 0(%[d2]), %%ymm4\n\t"
"vmovdqa 32(%[d2]), %%ymm5\n\t"
"vmovdqa 64(%[d2]), %%ymm6\n\t"
"vmovdqa 96(%[d2]), %%ymm7\n\t"
"vpsubw 0(%[a2]), %%ymm4, %%ymm4\n\t"
"vpsubw 32(%[a2]), %%ymm5, %%ymm5\n\t"
"vpsubw 64(%[a2]), %%ymm6, %%ymm6\n\t"
"vpsubw 96(%[a2]), %%ymm7, %%ymm7\n\t"
"vmovdqa %%ymm4, 0(%[d2])\n\t"
"vmovdqa %%ymm5, 32(%[d2])\n\t"
"vmovdqa %%ymm6, 64(%[d2])\n\t"
"vmovdqa %%ymm7, 96(%[d2])\n\t"
"add $128, %[d1]\n\t"
"add $128, %[a1]\n\t"
"add $128, %[d2]\n\t"
"add $128, %[a2]\n\t"
"cmp %[end], %[d1]\n\t"
"jb 1b\n\t"
: [d1] "+&r"(d1),[a1] "+&r"(a1),[d2] "+&r"(d2),[a2] "+&r"(a2)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
d1+=TILE_PAD;a1+=TILE_PAD;d2+=TILE_PAD;a2+=TILE_PAD;
}
}
static __attribute__((noinline,noclone))void combine_reverse_inplace(U *d,const U *b,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=d+1024;
asm volatile("1:\n\t"
"vmovdqa 0(%[b]), %%ymm0\n\t"
"vmovdqa 32(%[b]), %%ymm1\n\t"
"vmovdqa 64(%[b]), %%ymm2\n\t"
"vmovdqa 96(%[b]), %%ymm3\n\t"
"vpsubw 0(%[d]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[d]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[d]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[d]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[d])\n\t"
"vmovdqa %%ymm1, 32(%[d])\n\t"
"vmovdqa %%ymm2, 64(%[d])\n\t"
"vmovdqa %%ymm3, 96(%[d])\n\t"
"add $128, %[d]\n\t"
"add $128, %[b]\n\t"
"cmp %[end], %[d]\n\t"
"jb 1b\n\t"
: [d] "+&r"(d),[b] "+&r"(b)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3");
d+=TILE_PAD;b+=TILE_PAD;
}
}
template<int N>
static __attribute__((noinline))void combine3_unpack(
U *d12,U *d21,U *d22,
const U *p6,const U *p7,const U *p5,
const U *p1,const U *p4,const U *p3,int stride){
if constexpr(N > BASE){
constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
combine3_unpack<h>(d12,d21,d22,p6,p7,p5,p1,p4,p3,stride);
combine3_unpack<h>(d12+h,d21+h,d22+h,p6+q,p7+q,p5+q,p1+q,p4+q,p3+q,stride);
combine3_unpack<h>(d12+h*stride,d21+h*stride,d22+h*stride,
p6+2*q,p7+2*q,p5+2*q,p1+2*q,p4+2*q,p3+2*q,stride);
combine3_unpack<h>(d12+h*stride+h,d21+h*stride+h,d22+h*stride+h,
p6+3*q,p7+3*q,p5+3*q,p1+3*q,p4+3*q,p3+3*q,stride);
}else{
alignas(32)U temp[3*128];
for(int block=0;block<1024;block=block+128){
for(int v=0;v<128;v+=16){
int idx=block+v;
__m256i u2=_mm256_add_epi16(ld(p1+idx),ld(p6+idx));
__m256i u3=_mm256_add_epi16(u2,ld(p7+idx));
__m256i v5=ld(p5+idx);
st(temp+v,_mm256_add_epi16(_mm256_add_epi16(u2,v5),ld(p3+idx)));
st(temp+128+v,_mm256_sub_epi16(u3,ld(p4+idx)));
st(temp+256+v,_mm256_add_epi16(u3,v5));
}
size_t off=(size_t)(block/32)*stride;
emit_native(d12+off,temp,stride);
emit_native(d21+off,temp+128,stride);
emit_native(d22+off,temp+256,stride);
}
}
}
template<int N>
static __attribute__((noinline))void unpack_add(U *d,const U *p2,const U *p1,int stride){
if constexpr(N>BASE){
constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
unpack_add<h>(d,p2,p1,stride);
unpack_add<h>(d+h,p2+q,p1+q,stride);
unpack_add<h>(d+h*stride,p2+2*q,p1+2*q,stride);
unpack_add<h>(d+h*stride+h,p2+3*q,p1+3*q,stride);
}else{
alignas(32)U temp[128];
for(int block=0;block<1024;block+=128){
for(int v=0;v<128;v+=16)
st(temp+v,_mm256_add_epi16(ld(p2+block+v),ld(p1+block+v)));
emit_native(d+(size_t)(block/32)*stride,temp,stride);
}
}
}
template<int N>
static __attribute__((noinline))void root_fuse(
U *d11,U *d12,U *d21,U *d22,
const U *p1,const U *p2,const U *p3,const U *p4,const U *p5,const U *p6,const U *p7,int stride){
if constexpr(N==128){
constexpr int q=BASE*BASE+TILE_PAD;
root_fuse<64>(d11,d12,d21,d22,p1,p2,p3,p4,p5,p6,p7,stride);
alignas(32)U keep[4*8*512],cur[4*8*512];
U *dest[4]={d11,d12,d21,d22};
auto make_all=[&](U *tmp,int group){
for(int child=0;child<4;child++)
for(int b=0;b<8;b++){
int block=b*128;
for(int v=0;v<128;v+=16){
int idx=(4*group+child)*q+block+v;
if(!(v&16)){__builtin_prefetch(p1+idx+128,0,0);__builtin_prefetch(p2+idx+128,0,0);__builtin_prefetch(p3+idx+128,0,0);__builtin_prefetch(p4+idx+128,0,0);__builtin_prefetch(p5+idx+128,0,0);__builtin_prefetch(p6+idx+128,0,0);__builtin_prefetch(p7+idx+128,0,0);}
__m256i v1=ld(p1+idx),v5=ld(p5+idx);
__m256i u2=_mm256_add_epi16(v1,ld(p6+idx));
__m256i u3=_mm256_add_epi16(u2,ld(p7+idx));
U *t=tmp+(child*8+b)*512;
st(t+v,_mm256_add_epi16(v1,ld(p2+idx)));
st(t+256+v,_mm256_sub_epi16(u3,ld(p4+idx)));
st(t+384+v,_mm256_add_epi16(u3,v5));
st(t+128+v,_mm256_add_epi16(_mm256_add_epi16(u2,v5),ld(p3+idx)));
}
}
for(int b=0;b<8;b++)
for(int v=0;v<512;v+=16){
U *t1=tmp+(1*8+b)*512,*t2=tmp+(2*8+b)*512;
__m256i c22=ld(tmp+(3*8+b)*512+v);
st(t2+v,_mm256_sub_epi16(c22,ld(t2+v)));
st(t1+v,_mm256_sub_epi16(ld(t1+v),c22));
}
};
make_all(keep,3);
for(int child=0;child<4;child++){
int row=64+((child>>1)&1)*32,col=64+(child&1)*32;
for(int b=0;b<8;b++){
size_t off=(size_t)(b*4)*stride;
for(int r=0;r<4;r++)emit_native(dest[r]+off+row*stride+col,keep+(child*8+b)*512+r*128,stride);
}
}
for(int group=1;group<=2;group++){
if(group==2)_mm_sfence();
make_all(cur,group);
for(int child=0;child<4;child++){
int row=((group>>1)&1)*64+((child>>1)&1)*32;
int col=(group&1)*64+(child&1)*32;
for(int b=0;b<8;b++){
size_t off=(size_t)(b*4)*stride;
for(int r=0;r<4;r++)for(int rr=0;rr<4;rr++){
U *d=dest[r]+off+(row+rr)*stride+col;
const U *x=cur+(child*8+b)*512+r*128+rr*32;
const U *y=keep+(child*8+b)*512+r*128+rr*32;
__m256i v0,v1;
if(group==1){
v0=_mm256_sub_epi16(ld(x),ld(y));
v1=_mm256_sub_epi16(ld(x+16),ld(y+16));
}else{
v0=_mm256_sub_epi16(ld(y),ld(x));
v1=_mm256_sub_epi16(ld(y+16),ld(x+16));
}
_mm256_stream_si256((__m256i*)d,s4bj_fx(v0));
_mm256_stream_si256((__m256i*)(d+16),s4bj_fx(v1));
}
}
}
}
}else if constexpr(N==64){
constexpr int q=BASE*BASE+TILE_PAD;
root_fuse<32>(d11,d12,d21,d22,p1,p2,p3,p4,p5,p6,p7,stride);
alignas(32)U tmp[3*8*512];
U *dest[4]={d11,d12,d21,d22};
for(int child=1;child<=3;child++)
for(int b=0;b<8;b++){
int block=b*128;
for(int v=0;v<128;v+=16){
int idx=child*q+block+v;
if(!(v&16)){__builtin_prefetch(p1+idx+128,0,3);__builtin_prefetch(p2+idx+128,0,3);__builtin_prefetch(p3+idx+128,0,3);__builtin_prefetch(p4+idx+128,0,3);__builtin_prefetch(p5+idx+128,0,3);__builtin_prefetch(p6+idx+128,0,3);__builtin_prefetch(p7+idx+128,0,3);}
__m256i v1=ld(p1+idx),v5=ld(p5+idx);
__m256i u2=_mm256_add_epi16(v1,ld(p6+idx));
__m256i u3=_mm256_add_epi16(u2,ld(p7+idx));
U *t=tmp+((child-1)*8+b)*512;
st(t+v,_mm256_add_epi16(v1,ld(p2+idx)));
st(t+128+v,_mm256_add_epi16(_mm256_add_epi16(u2,v5),ld(p3+idx)));
st(t+256+v,_mm256_sub_epi16(u3,ld(p4+idx)));
st(t+384+v,_mm256_add_epi16(u3,v5));
}
}
for(int b=0;b<8;b++)
for(int v=0;v<512;v+=16){
U *t1=tmp+(0*8+b)*512,*t2=tmp+(1*8+b)*512;
__m256i c22=ld(tmp+(2*8+b)*512+v);
st(t1+v,_mm256_sub_epi16(ld(t1+v),c22));
st(t2+v,_mm256_sub_epi16(c22,ld(t2+v)));
}
for(int b=0;b<8;b++){
size_t off=(size_t)(b*4)*stride;
for(int r=0;r<4;r++){
emit_native(dest[r]+off+32,tmp+(0*8+b)*512+r*128,stride);
emit_native(dest[r]+off+32*stride,tmp+(1*8+b)*512+r*128,stride);
emit_native(dest[r]+off+32*stride+32,tmp+(2*8+b)*512+r*128,stride);
}
}
}else if constexpr(N > BASE){
constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
root_fuse<h>(d11,d12,d21,d22,p1,p2,p3,p4,p5,p6,p7,stride);
root_fuse<h>(d11+h,d12+h,d21+h,d22+h,p1+q,p2+q,p3+q,p4+q,p5+q,p6+q,p7+q,stride);
root_fuse<h>(d11+h*stride,d12+h*stride,d21+h*stride,d22+h*stride,
p1+2*q,p2+2*q,p3+2*q,p4+2*q,p5+2*q,p6+2*q,p7+2*q,stride);
root_fuse<h>(d11+h*stride+h,d12+h*stride+h,d21+h*stride+h,d22+h*stride+h,
p1+3*q,p2+3*q,p3+3*q,p4+3*q,p5+3*q,p6+3*q,p7+3*q,stride);
}else{
alignas(32)U temp[4*128];
for(int block=0;block<1024;block+=128){
for(int v=0;v<128;v+=16){
int idx=block+v;
if(!(v&16)){__builtin_prefetch(p1+idx+128,0,3);__builtin_prefetch(p2+idx+128,0,3);__builtin_prefetch(p3+idx+128,0,3);__builtin_prefetch(p4+idx+128,0,3);__builtin_prefetch(p5+idx+128,0,3);__builtin_prefetch(p6+idx+128,0,3);__builtin_prefetch(p7+idx+128,0,3);}
__m256i u2=_mm256_add_epi16(ld(p1+idx),ld(p6+idx));
__m256i u3=_mm256_add_epi16(u2,ld(p7+idx));
__m256i v5=ld(p5+idx);
st(temp+v,_mm256_add_epi16(_mm256_add_epi16(u2,v5),ld(p3+idx)));
st(temp+128+v,_mm256_sub_epi16(u3,ld(p4+idx)));
st(temp+256+v,_mm256_add_epi16(u3,v5));
st(temp+384+v,_mm256_add_epi16(ld(p2+idx),ld(p1+idx)));
}
size_t off=(size_t)(block/32)*stride;
emit_native(d12+off,temp,stride);
emit_native(d21+off,temp+128,stride);
emit_native(d22+off,temp+256,stride);
emit_native(d11+off,temp+384,stride);
}
}
}
static __attribute__((noinline,noclone))void combine_pair_p6(U *x,const U *a11,U *y,const U *b22,int len){
#pragma GCC unroll 1
for(int chunk=0;chunk<len;chunk+=1024+TILE_PAD){
const U *end=x+1024;
asm volatile(
"1:\n\t"
"vmovdqa 0(%[x]), %%ymm0\n\t"
"vmovdqa 32(%[x]), %%ymm1\n\t"
"vmovdqa 64(%[x]), %%ymm2\n\t"
"vmovdqa 0(%[b22]), %%ymm4\n\t"
"vmovdqa 96(%[x]), %%ymm3\n\t"
"vmovdqa 32(%[b22]), %%ymm5\n\t"
"vmovdqa 64(%[b22]), %%ymm6\n\t"
"vmovdqa 96(%[b22]), %%ymm7\n\t"
"vpsubw 0(%[a11]), %%ymm0, %%ymm0\n\t"
"vpsubw 32(%[a11]), %%ymm1, %%ymm1\n\t"
"vpsubw 64(%[a11]), %%ymm2, %%ymm2\n\t"
"vpsubw 96(%[a11]), %%ymm3, %%ymm3\n\t"
"vmovdqa %%ymm0, 0(%[x])\n\t"
"vmovdqa %%ymm1, 32(%[x])\n\t"
"vmovdqa %%ymm2, 64(%[x])\n\t"
"vmovdqa %%ymm3, 96(%[x])\n\t"
"vpsubw 0(%[y]), %%ymm4, %%ymm4\n\t"
"vpsubw 32(%[y]), %%ymm5, %%ymm5\n\t"
"vpsubw 64(%[y]), %%ymm6, %%ymm6\n\t"
"vpsubw 96(%[y]), %%ymm7, %%ymm7\n\t"
"vmovdqa %%ymm4, 0(%[y])\n\t"
"vmovdqa %%ymm5, 32(%[y])\n\t"
"vmovdqa %%ymm6, 64(%[y])\n\t"
"vmovdqa %%ymm7, 96(%[y])\n\t"
"add $128, %[x]\n\t"
"add $128, %[a11]\n\t"
"add $128, %[y]\n\t"
"add $128, %[b22]\n\t"
"cmp %[end], %[x]\n\t"
"jb 1b\n\t"
: [x] "+&r"(x),[a11] "+&r"(a11),[y] "+&r"(y),[b22] "+&r"(b22)
: [end] "r"(end)
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
x+=TILE_PAD;a11+=TILE_PAD;y+=TILE_PAD;b22+=TILE_PAD;
}
}
template<int N,bool ParentReconstruct=false,bool PostAdd=false,bool PostAdd2=false>
static __attribute__((noinline))void mul(const U *a,const U *b,U *c,U *work){
if constexpr(N<=BASE){
base(a,b,c,N);
}else if constexpr(N<=ALTMAX){
constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
const U *pa = nullptr;if constexpr(PostAdd)pa = g_postadd;
const U *pa2 = nullptr;if constexpr(PostAdd2)pa2 = g_postadd2;
const U *a11=a,*a12=a+q,*a21=a+2*q,*a22=a+3*q;
const U *b11=b,*b12=b+q,*b21=b+2*q,*b22=b+3*q;
U *c11=c,*c12=c+q,*c21=c+2*q,*c22=c+3*q;
U *x=work+0,*y=work+2*q+704,*next=work+4*q+640;
mul<h>(a12,b21,c11,next);                 
mul<h>(a22,b22,c22,next);                 
selected_displaced<N,false,q,q>(x,a21);
selected_displaced<N,false,q,q>(y,b21);
mul<h>(x,y,c12,next);                     
selected_displaced<N,true,-2*q,q>(x,a22);
selected_displaced<N,true,-2*q,q>(y,b22);
if constexpr(N==64)native2x32_shared(x,y,c21,ParentReconstruct?4:2);else mul<h,true>(x,y,c21,next); 
if constexpr(PostAdd2)add_inplace_pa2(c22,pa+3*q,pa2+3*q,q);else if constexpr(PostAdd)add_inplace_pa(c22,pa+3*q,q);
if constexpr(N==64){
if constexpr(ParentReconstruct)native2x32_shared(a11,b11,c11,3);else leaf_add(a11,b11,c11);                  
selected_displaced<N,true,-3*q,q>(y,b22);
if constexpr(ParentReconstruct)native2x32_shared(a21,y,c21,3);else leaf_add(a21,y,c21);                    
selected_displaced<N,true,3*q,q>(x,a11);   
if constexpr(ParentReconstruct)native2x32_shared(x,b12,c12,3);else leaf_add(x,b12,c12);                    
}else{
selected_displaced<N,true,-3*q,q>(y,b22);
mul<h>(a11,b11,x,next);                   
if constexpr(PostAdd2)combine_inplace_pa2(c11,x,pa,pa2,q);else if constexpr(PostAdd)combine_inplace_pa(c11,x,pa,q);else combine_inplace<false>(c11,x,q);
mul<h>(a21,y,x,next);                     
if constexpr(PostAdd2)combine_inplace_pa2(c21,x,pa+2*q,pa2+2*q,q);else if constexpr(PostAdd)combine_inplace_pa(c21,x,pa+2*q,q);else combine_inplace<false>(c21,x,q);
selected_displaced<N,true,-3*q,q>(x,a22);
mul<h>(x,b12,y,next);                     
if constexpr(PostAdd2)combine_inplace_sub_pa2(c12,y,pa+q,pa2+q,q);else if constexpr(PostAdd)combine_inplace_sub_pa(c12,y,pa+q,q);else combine_inplace<true>(c12,y,q);
}
}else{
constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
const U *pa = nullptr;if constexpr(PostAdd)pa = g_postadd;
const U *pa2 = nullptr;if constexpr(PostAdd2)pa2 = g_postadd2;
const U *a11=a,*a12=a+q,*a21=a+2*q,*a22=a+3*q;
const U *b11=b,*b12=b+q,*b21=b+2*q,*b22=b+3*q;
U *c11=c,*c12=c+q,*c21=c+2*q,*c22=c+3*q;
U *x=work+192,*y=work+q+512,*next=work+q+q+512;
if constexpr(N>=2048)combine_dpair_tt_nt<-2*q,2*q>(y,b22,x,a11,q);else combine_dpair_tt<-2*q,2*q>(y,b22,x,a11,q);
mul<h>(x,y,c21,next);                       
if constexpr(N>=2048)combine_dpair_ft_nt<q,-q>(x,a21,y,b12,q);else combine_dpair_ft<q,-q>(x,a21,y,b12,q);
mul<h>(x,y,c22,next);                       
combine_pair_p6(x,a11,y,b22,q);
mul<h>(x,y,c12,next);                       
combine_pair_rf(x,a12,y,b21,q);
mul<h>(x,b22,c11,next);                     
mul<h>(a22,y,x,next);                       
mul<h>(a11,b11,y,next);                     
if constexpr(N == 4096){
combine3_unpack<1024>(final_out+1024,
final_out+(size_t)1024*4096,
final_out+(size_t)1024*4096+1024,
c12,c21,c22,y,x,c11,4096);
}else{
U *rp=c12;const U *xp=x;
const U *pac=pa;
const U *pac2=pa2;
for(int chunk=0;chunk<q;chunk+=1024+TILE_PAD){
const U *end=rp+1024;
if constexpr(PostAdd2){
asm volatile(
"1:\n\tvmovdqa %c[yd]+0(%[x]), %%ymm0\n\tvpaddw 0(%[p]), %%ymm0, %%ymm0\n\tvmovdqa %c[q2]+0(%[p]), %%ymm1\n\tvpaddw %c[q]+0(%[p]), %%ymm0, %%ymm2\n\tvpaddw %c[mq]+0(%[p]), %%ymm1, %%ymm3\n\tvpaddw %%ymm0, %%ymm3, %%ymm3\n\tvpaddw %c[q]+0(%[pa]), %%ymm3, %%ymm3\n\tvpaddw %c[q]+0(%[pa2]), %%ymm3, %%ymm3\n\tvmovdqa %%ymm3, 0(%[p])\n\tvpaddw %%ymm2, %%ymm1, %%ymm1\n\tvpsubw 0(%[x]), %%ymm2, %%ymm2\n\tvpaddw %c[q3]+0(%[pa]), %%ymm1, %%ymm1\n\tvpaddw %c[q3]+0(%[pa2]), %%ymm1, %%ymm1\n\tvmovdqa %%ymm1, %c[q2]+0(%[p])\n\tvpaddw %c[q2]+0(%[pa]), %%ymm2, %%ymm2\n\tvpaddw %c[q2]+0(%[pa2]), %%ymm2, %%ymm2\n\tvmovdqa %c[yd]+32(%[x]), %%ymm4\n\tvpaddw 32(%[p]), %%ymm4, %%ymm4\n\tvmovdqa %%ymm2, %c[q]+0(%[p])\n\tvmovdqa %c[q2]+32(%[p]), %%ymm5\n\tvpaddw %c[q]+32(%[p]), %%ymm4, %%ymm6\n\tvpaddw %c[mq]+32(%[p]), %%ymm5, %%ymm7\n\tvpaddw %%ymm4, %%ymm7, %%ymm7\n\tvpaddw %c[q]+32(%[pa]), %%ymm7, %%ymm7\n\tvpaddw %c[q]+32(%[pa2]), %%ymm7, %%ymm7\n\tvmovdqa %%ymm7, 32(%[p])\n\tvpaddw %%ymm6, %%ymm5, %%ymm5\n\tvpsubw 32(%[x]), %%ymm6, %%ymm6\n\tvpaddw %c[q3]+32(%[pa]), %%ymm5, %%ymm5\n\tvpaddw %c[q3]+32(%[pa2]), %%ymm5, %%ymm5\n\tvmovdqa %%ymm5, %c[q2]+32(%[p])\n\tvpaddw %c[q2]+32(%[pa]), %%ymm6, %%ymm6\n\tvpaddw %c[q2]+32(%[pa2]), %%ymm6, %%ymm6\n\tvmovdqa %%ymm6, %c[q]+32(%[p])\n\tadd $64, %[p]\n\tadd $64, %[x]\n\tadd $64, %[pa]\n\tadd $64, %[pa2]\n\tcmp %[end], %[p]\n\tjb 1b\n\t"
: [p] "+&r"(rp),[x] "+&r"(xp),[pa] "+&r"(pac),[pa2] "+&r"(pac2)
: [end] "r"(end),[q] "i"(2*q),[q2] "i"(4*q),[q3] "i"(6*q),[mq] "i"(-2*q),[yd] "i"(2*(q+320))
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
}else if constexpr(PostAdd){
asm volatile(
"1:\n\tvmovdqa %c[yd]+0(%[x]), %%ymm0\n\tvpaddw 0(%[p]), %%ymm0, %%ymm0\n\tvmovdqa %c[q2]+0(%[p]), %%ymm1\n\tvpaddw %c[q]+0(%[p]), %%ymm0, %%ymm2\n\tvpaddw %c[mq]+0(%[p]), %%ymm1, %%ymm3\n\tvpaddw %%ymm0, %%ymm3, %%ymm3\n\tvpaddw %c[q]+0(%[pa]), %%ymm3, %%ymm3\n\tvmovdqa %%ymm3, 0(%[p])\n\tvpaddw %%ymm2, %%ymm1, %%ymm1\n\tvpsubw 0(%[x]), %%ymm2, %%ymm2\n\tvpaddw %c[q3]+0(%[pa]), %%ymm1, %%ymm1\n\tvmovdqa %%ymm1, %c[q2]+0(%[p])\n\tvpaddw %c[q2]+0(%[pa]), %%ymm2, %%ymm2\n\tvmovdqa %c[yd]+32(%[x]), %%ymm4\n\tvpaddw 32(%[p]), %%ymm4, %%ymm4\n\tvmovdqa %%ymm2, %c[q]+0(%[p])\n\tvmovdqa %c[q2]+32(%[p]), %%ymm5\n\tvpaddw %c[q]+32(%[p]), %%ymm4, %%ymm6\n\tvpaddw %c[mq]+32(%[p]), %%ymm5, %%ymm7\n\tvpaddw %%ymm4, %%ymm7, %%ymm7\n\tvpaddw %c[q]+32(%[pa]), %%ymm7, %%ymm7\n\tvmovdqa %%ymm7, 32(%[p])\n\tvpaddw %%ymm6, %%ymm5, %%ymm5\n\tvpsubw 32(%[x]), %%ymm6, %%ymm6\n\tvpaddw %c[q3]+32(%[pa]), %%ymm5, %%ymm5\n\tvmovdqa %%ymm5, %c[q2]+32(%[p])\n\tvpaddw %c[q2]+32(%[pa]), %%ymm6, %%ymm6\n\tvmovdqa %%ymm6, %c[q]+32(%[p])\n\tadd $64, %[p]\n\tadd $64, %[x]\n\tadd $64, %[pa]\n\tcmp %[end], %[p]\n\tjb 1b\n\t"
: [p] "+&r"(rp),[x] "+&r"(xp),[pa] "+&r"(pac)
: [end] "r"(end),[q] "i"(2*q),[q2] "i"(4*q),[q3] "i"(6*q),[mq] "i"(-2*q),[yd] "i"(2*(q+320))
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
}else{
asm volatile(
"1:\n\tvmovdqa %c[yd]+0(%[x]), %%ymm0\n\tvpaddw 0(%[p]), %%ymm0, %%ymm0\n\tvmovdqa %c[q2]+0(%[p]), %%ymm1\n\tvpaddw %c[q]+0(%[p]), %%ymm0, %%ymm2\n\tvpaddw %c[mq]+0(%[p]), %%ymm1, %%ymm3\n\tvpaddw %%ymm0, %%ymm3, %%ymm3\n\tvmovdqa %%ymm3, 0(%[p])\n\tvpaddw %%ymm2, %%ymm1, %%ymm1\n\tvpsubw 0(%[x]), %%ymm2, %%ymm2\n\tvmovdqa %%ymm1, %c[q2]+0(%[p])\n\tvmovdqa %c[yd]+32(%[x]), %%ymm4\n\tvpaddw 32(%[p]), %%ymm4, %%ymm4\n\tvmovdqa %%ymm2, %c[q]+0(%[p])\n\tvmovdqa %c[q2]+32(%[p]), %%ymm5\n\tvpaddw %c[q]+32(%[p]), %%ymm4, %%ymm6\n\tvpaddw %c[mq]+32(%[p]), %%ymm5, %%ymm7\n\tvpaddw %%ymm4, %%ymm7, %%ymm7\n\tvmovdqa %%ymm7, 32(%[p])\n\tvpaddw %%ymm6, %%ymm5, %%ymm5\n\tvpsubw 32(%[x]), %%ymm6, %%ymm6\n\tvmovdqa %%ymm5, %c[q2]+32(%[p])\n\tvmovdqa %%ymm6, %c[q]+32(%[p])\n\tadd $64, %[p]\n\tadd $64, %[x]\n\tcmp %[end], %[p]\n\tjb 1b\n\t"
: [p] "+&r"(rp),[x] "+&r"(xp)
: [end] "r"(end),[q] "i"(2*q),[q2] "i"(4*q),[mq] "i"(-2*q),[yd] "i"(2*(q+320))
: "cc","memory","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7");
}
rp+=TILE_PAD;xp+=TILE_PAD;if constexpr(PostAdd)pac+=TILE_PAD;if constexpr(PostAdd2)pac2+=TILE_PAD;
}
}
if constexpr(!PostAdd){g_postadd=y;mul<h,false,true>(a12,b21,c11,next);g_postadd=nullptr;}
else if constexpr(PostAdd && !PostAdd2){g_postadd=pa;g_postadd2=y;mul<h,false,true,true>(a12,b21,c11,next);g_postadd=nullptr;g_postadd2=nullptr;}
else mul<h>(a12,b21,c11,next);                   
if constexpr(N == 4096)unpack_add<1024>(final_out,c11,y,4096);
else if constexpr(PostAdd2)combine_inplace_pa2(c11,y,pa,pa2,q);
}
}
static constexpr int RQ=2048*2048+(2048/BASE)*(2048/BASE)*TILE_PAD,RS=RQ+ROOT_PAD;
alignas(2*1024*1024)static U root_mem[12*RS+4096];
alignas(4096)static U root_tile[4*(64*64+(64/BASE)*(64/BASE)*TILE_PAD)];
static constexpr int HQ=64*64+(64/BASE)*(64/BASE)*TILE_PAD;
alignas(4096)static U high_left[4*HQ],high_right[4*HQ];
template<bool Right>
static void pack_four(U *tile,const U *s){
pack(tile,s,64,4096,Right);
pack(tile+HQ,s+2048,64,4096,Right);
pack(tile+2*HQ,s+2048*4096,64,4096,Right);
pack(tile+3*HQ,s+2048*4096+2048,64,4096,Right);
}
template<bool Right,bool TWO>
static void pack_wide_root(U *tA,U *tB,const U *s,long STR_,long D0_,const U *s2,int sgn){
if constexpr(!Right){
pack_wide<TWO>(tA+0*HQ,tB+0*HQ,s,STR_,s2,sgn);
pack_wide<TWO>(tA+1*HQ,tB+1*HQ,s+2048,STR_,s2+2048,sgn);
pack_wide<TWO>(tA+2*HQ,tB+2*HQ,s+2048L*STR_,STR_,s2+2048L*STR_,sgn);
pack_wide<TWO>(tA+3*HQ,tB+3*HQ,s+2048L*STR_+2048,STR_,s2+2048L*STR_+2048,sgn);
}else{
pack_wide_b<TWO>(tA+0*HQ,tB+0*HQ,s,STR_,D0_,s2,sgn);
pack_wide_b<TWO>(tA+1*HQ,tB+1*HQ,s+2048,STR_,D0_+2048,s2+2048,sgn);
pack_wide_b<TWO>(tA+2*HQ,tB+2*HQ,s+2048L*STR_,STR_,D0_+2048L*4096,s2+2048L*STR_,sgn);
pack_wide_b<TWO>(tA+3*HQ,tB+3*HQ,s+2048L*STR_+2048,STR_,D0_+2048L*4096+2048,s2+2048L*STR_+2048,sgn);
}
}
template<bool Right,bool High>
static void emit_root64(const U *tile,const U *left,const U *right,int offset,U *dest,U *destq,int voff){
U *r0=destq+offset,*r1=r0+RS,*r2=r1+RS,*v7=dest+offset+voff*RS,*v5=v7+RS,*v6=v5+RS,*v34=v6+RS;
for(int i=0;i<(64*64+(64/BASE)*(64/BASE)*TILE_PAD);i+=32){
__m256i a=ld(tile+i),b=ld(tile+(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i),c=ld(tile+2*(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i),d=ld(tile+3*(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i);
__m256i aa=ld(tile+i+16),bb=ld(tile+(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i+16),cc=ld(tile+2*(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i+16),dd=ld(tile+3*(64*64+(64/BASE)*(64/BASE)*TILE_PAD)+i+16);
if constexpr(High){
a=_mm256_add_epi16(a,_mm256_sub_epi16(ld(left+0*HQ+i),ld(right+0*HQ+i)));
b=_mm256_add_epi16(b,_mm256_sub_epi16(ld(left+1*HQ+i),ld(right+1*HQ+i)));
c=_mm256_add_epi16(c,_mm256_sub_epi16(ld(left+2*HQ+i),ld(right+2*HQ+i)));
d=_mm256_add_epi16(d,_mm256_sub_epi16(ld(left+3*HQ+i),ld(right+3*HQ+i)));
aa=_mm256_add_epi16(aa,_mm256_sub_epi16(ld(left+0*HQ+i+16),ld(right+0*HQ+i+16)));
bb=_mm256_add_epi16(bb,_mm256_sub_epi16(ld(left+1*HQ+i+16),ld(right+1*HQ+i+16)));
cc=_mm256_add_epi16(cc,_mm256_sub_epi16(ld(left+2*HQ+i+16),ld(right+2*HQ+i+16)));
dd=_mm256_add_epi16(dd,_mm256_sub_epi16(ld(left+3*HQ+i+16),ld(right+3*HQ+i+16)));
}
_mm256_stream_si256((__m256i*)(r0+i),a);_mm256_stream_si256((__m256i*)(r0+i+16),aa);
_mm256_stream_si256((__m256i*)(r1+i),Right?c:b);_mm256_stream_si256((__m256i*)(r1+i+16),Right?cc:bb);
_mm256_stream_si256((__m256i*)(r2+i),d);_mm256_stream_si256((__m256i*)(r2+i+16),dd);
__m256i x7=Right?_mm256_sub_epi16(d,b):_mm256_sub_epi16(a,c);
__m256i xx7=Right?_mm256_sub_epi16(dd,bb):_mm256_sub_epi16(aa,cc);
_mm256_stream_si256((__m256i*)(v7+i),x7);_mm256_stream_si256((__m256i*)(v7+i+16),xx7);
__m256i x5=Right?_mm256_sub_epi16(b,a):_mm256_add_epi16(c,d);
__m256i xx5=Right?_mm256_sub_epi16(bb,aa):_mm256_add_epi16(cc,dd);
_mm256_stream_si256((__m256i*)(v5+i),x5);_mm256_stream_si256((__m256i*)(v5+i+16),xx5);
__m256i x6=Right?_mm256_sub_epi16(d,x5):_mm256_sub_epi16(x5,a);
__m256i xx6=Right?_mm256_sub_epi16(dd,xx5):_mm256_sub_epi16(xx5,aa);
_mm256_stream_si256((__m256i*)(v6+i),x6);_mm256_stream_si256((__m256i*)(v6+i+16),xx6);
__m256i x34=Right?_mm256_sub_epi16(x6,c):_mm256_sub_epi16(b,x6);
__m256i xx34=Right?_mm256_sub_epi16(xx6,cc):_mm256_sub_epi16(bb,xx6);
_mm256_stream_si256((__m256i*)(v34+i),x34);_mm256_stream_si256((__m256i*)(v34+i+16),xx34);
}
}
template<bool Right,bool TWO=Right>
static void pack_root(const U*s,int n,int offset,U *dest,U *destq,int voff,long STR_=4096,long D0_=0,const U*s2=nullptr,int sgn=0){
if(n>128){int h=n/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
pack_root<Right,TWO>(s,h,offset,dest,destq,voff,STR_,D0_,s2,sgn);
pack_root<Right,TWO>(s+h,h,offset+q,dest,destq,voff,STR_,D0_+h,s2+h,sgn);
pack_root<Right,TWO>(s+h*STR_,h,offset+2*q,dest,destq,voff,STR_,D0_+h*4096L,s2+h*STR_,sgn);
pack_root<Right,TWO>(s+h*STR_+h,h,offset+3*q,dest,destq,voff,STR_,D0_+h*4096L+h,s2+h*STR_+h,sgn);
}else{
pack_wide_root<Right,TWO>(root_tile,high_left,s,STR_,D0_,s2,sgn);
emit_root64<Right,false>(root_tile,nullptr,nullptr,offset,dest,destq,voff);
emit_root64<Right,false>(high_left,nullptr,nullptr,offset+HQ,dest,destq,voff);
pack_wide_root<Right,TWO>(high_right,root_tile,s+64*STR_,STR_,D0_+64*4096L,s2+64*STR_,sgn);
emit_root64<Right,false>(high_right,nullptr,nullptr,offset+2*HQ,dest,destq,voff);
emit_root64<Right,true>(root_tile,high_left,high_right,offset+3*HQ,dest,destq,voff);
}
}
static void short_product4096s(int n,const short*A,const short*B,short*C,long sA,long sB);
static void short_product4096x(int n,const short*A,const short*A2,int sgA,long sA,const short*B,const short*B2,int sgB,long sB,short*C);
static void outer_sept(const unsigned short*b1,const unsigned short*b2,const unsigned short*b3,
const unsigned short*b4,const unsigned short*b5,
unsigned short*c00,unsigned short*c10,unsigned short*c11,unsigned short*c01){
for(int i=0;i<4096;i++){
const unsigned short*r1=b1+(size_t)i*4096,*r2=b2+(size_t)i*4096,*r3=b3+(size_t)i*4096,*r4=b4+(size_t)i*8192,*r5=b5+(size_t)i*8192;
const unsigned short*q6=c11+(size_t)i*8192,*q7=c10+(size_t)i*8192;
unsigned short*w0=c00+(size_t)i*8192,*w1=c10+(size_t)i*8192,*w2=c11+(size_t)i*8192,*w3=c01+(size_t)i*8192;
for(int j=0;j<4096;j+=32){
#define SEPT16(J,A,B,C,D) \
{__m256i x1=_mm256_loadu_si256((const __m256i*)(r1+(J))); \
__m256i x2=_mm256_loadu_si256((const __m256i*)(r2+(J))); \
__m256i x3=_mm256_loadu_si256((const __m256i*)(r3+(J))); \
__m256i x4=_mm256_loadu_si256((const __m256i*)(r4+(J))); \
__m256i x5=_mm256_loadu_si256((const __m256i*)(r5+(J))); \
__m256i z6=_mm256_loadu_si256((const __m256i*)(q6+(J))); \
__m256i z7=_mm256_loadu_si256((const __m256i*)(q7+(J))); \
(A)=_mm256_add_epi16(_mm256_sub_epi16(_mm256_add_epi16(x1,x4),x5),z7); \
(B)=_mm256_add_epi16(x2,x4); \
(C)=_mm256_add_epi16(_mm256_add_epi16(_mm256_sub_epi16(x1,x2),x3),z6); \
(D)=_mm256_add_epi16(x3,x5);}
__m256i a0,a1,a2,a3,b0,b1,b2,b3;
SEPT16(j+0,a0,a1,a2,a3)
SEPT16(j+16,b0,b1,b2,b3)
_mm256_store_si256((__m256i*)(w0+j),a0);
_mm256_store_si256((__m256i*)(w0+j+16),b0);
_mm256_store_si256((__m256i*)(w1+j),a1);
_mm256_store_si256((__m256i*)(w1+j+16),b1);
_mm256_store_si256((__m256i*)(w2+j),a2);
_mm256_store_si256((__m256i*)(w2+j+16),b2);
_mm256_store_si256((__m256i*)(w3+j),a3);
_mm256_store_si256((__m256i*)(w3+j+16),b3);
#undef SEPT16
}
}
}
alignas(64)static U g_band[256*4096];
static inline int base_offset(int r0,int c0){
int off=0,r=r0,c=c0;
const int H[4]={1024,512,256,128};
const int Q[4]={1064960,266240,66560,16640};
for(int i=0;i<4;i++){int h=H[i],ci=((r>=h)?2:0)|((c>=h)?1:0);off+=ci*Q[i];r&=h-1;c&=h-1;}
return off;
}
template<bool Right>
static void pack_wide_root_banded(U *tA,U *tB,const U *s,long D0_){
const long FARROW=128L*4096;
if constexpr(!Right){
pack_wide<false>(tA+0*HQ,tB+0*HQ,s,4096,s,0);
pack_wide<false>(tA+1*HQ,tB+1*HQ,s+2048,4096,s+2048,0);
pack_wide<false>(tA+2*HQ,tB+2*HQ,s+FARROW,4096,s+FARROW,0);
pack_wide<false>(tA+3*HQ,tB+3*HQ,s+FARROW+2048,4096,s+FARROW+2048,0);
}else{
pack_wide_b<false>(tA+0*HQ,tB+0*HQ,s,4096,D0_,s,0);
pack_wide_b<false>(tA+1*HQ,tB+1*HQ,s+2048,4096,D0_+2048,s+2048,0);
pack_wide_b<false>(tA+2*HQ,tB+2*HQ,s+FARROW,4096,D0_+2048L*4096,s+FARROW,0);
pack_wide_b<false>(tA+3*HQ,tB+3*HQ,s+FARROW+2048,4096,D0_+2048L*4096+2048,s+FARROW+2048,0);
}
}
template<bool Right>
static void pack_base_banded(const U*s,long D0,int offset,U *dest,U *destq,int voff){
pack_wide_root_banded<Right>(root_tile,high_left,s,D0);
emit_root64<Right,false>(root_tile,nullptr,nullptr,offset,dest,destq,voff);
emit_root64<Right,false>(high_left,nullptr,nullptr,offset+HQ,dest,destq,voff);
pack_wide_root_banded<Right>(high_right,root_tile,s+64*4096,D0+64*4096L);
emit_root64<Right,false>(high_right,nullptr,nullptr,offset+2*HQ,dest,destq,voff);
emit_root64<Right,true>(root_tile,high_left,high_right,offset+3*HQ,dest,destq,voff);
}
static void build_band(const U*a,const U*b,int sg,int r0){
for(int half=0;half<2;half++){
int row0=r0+half*2048;
U *d=g_band+(size_t)half*128*4096;
for(int i=0;i<128;i++){
const U*ra=a+(size_t)(row0+i)*8192,*rb=b+(size_t)(row0+i)*8192;
U*dd=d+(size_t)i*4096;
for(int j=0;j<4096;j+=16){
__m256i x=_mm256_loadu_si256((const __m256i*)(ra+j));
__m256i y=_mm256_loadu_si256((const __m256i*)(rb+j));
_mm256_storeu_si256((__m256i*)(dd+j),sg>0?_mm256_add_epi16(x,y):_mm256_sub_epi16(x,y));
}}}
}
template<bool Right>
static void pack_banded(const U*a,const U*b,int sg,U*dest,U*destq,int voff){
for(int r0=0;r0<2048;r0+=128){
build_band(a,b,sg,r0);
for(int c0=0;c0<2048;c0+=128)
pack_base_banded<Right>(g_band+c0,(long)r0*4096+c0,base_offset(r0,c0),dest,destq,voff);
}
}
static void short_product4096xb(int n,const unsigned short*A,const unsigned short*A2,int sgA,const unsigned short*B,const unsigned short*B2,int sgB,unsigned short*C){
U *cq=(U*)C;
U *ar=root_mem,*br=ar+4*RS,*pr=br+7*RS;
U *work=cq+3*RS+(464*2);
if(A2)pack_banded<false>((const U*)A,(const U*)A2,sgA,ar,cq,0);
else{raw_basis_input=(const U*)A;pack_root<false,false>((const U*)A,2048,0,ar,cq,0,8192,0,nullptr,0);}
if(B2)pack_banded<true>((const U*)B,(const U*)B2,sgB,br,br,3);
else{raw_basis_input=(const U*)B;pack_root<true,false>((const U*)B,2048,0,br,br,3,8192,0,nullptr,0);}
_mm_mfence();
const U *a11=cq,*a12=cq+RS,*a22=cq+2*RS,*s7=ar,*s5=ar+RS,*s6=ar+2*RS,*s3=ar+3*RS;
const U *b11=br,*b21=br+RS,*b22=br+2*RS,*t7=br+3*RS,*t5=br+4*RS,*t6=br+5*RS,*t4=br+6*RS;
U *p1=ar+2*RS,*p2=br+5*RS,*p3=ar+RS,*p4=br+4*RS,*p5=ar,*p6=br+3*RS,*p7=pr;
mul<2048>(s7,t7,p7,work);mul<2048>(s5,t5,p5,work);mul<2048>(s6,t6,p6,work);
mul<2048>(s3,b22,p3,work);mul<2048>(a22,t4,p4,work);mul<2048>(a11,b11,p1,work);mul<2048>(a12,b21,p2,work);
U *out=g_out?g_out:(U*)C;
const long ostr=g_ostr;
root_fuse<2048>(out,out+2048,out+2048*ostr,out+2048*ostr+2048,p1,p2,p3,p4,p5,p6,p7,(int)ostr);_mm_sfence();
asm volatile(".purgem vzeroupper");
_mm256_zeroupper();
}
void matrix_multiply(int n,const short*A,const short*B,short*C){
if(n!=8192){for(int i=0;i<n;i++)for(int j=0;j<n;j++){unsigned z=0;for(int k=0;k<n;k++)z+=(unsigned short)A[(size_t)i*n+k]*(unsigned)(unsigned short)B[(size_t)k*n+j];C[(size_t)i*n+j]=(short)z;}return;}
using V=unsigned short;V *pb;
constexpr size_t z=(size_t)4096*4096;
madvise((void*)C,(size_t)8192*8192*2,MADV_HUGEPAGE);
constexpr size_t zp = z + 256;   
if(posix_memalign((void**)&pb,4096,3*zp*2 + 256))abort();
madvise((void*)pb,3*z*2,MADV_HUGEPAGE);
madvise((void*)root_mem,sizeof(root_mem),MADV_HUGEPAGE);
V*b1=pb+128,*b2=b1+zp,*b3=b1+2*zp;   
const V*a00=(const V*)A,*a01=a00+4096,*a10=a00+4096*8192,*a11=a10+4096;
const V*b00=(const V*)B,*b01=b00+4096,*b10=b00+4096*8192,*b11=b10+4096;
V*c00=(V*)C,*c01=c00+4096,*c10=c00+4096*8192,*c11=c10+4096;
g_ostr=8192;
g_out=c00;
short_product4096xb(4096,a11,nullptr,0,b10,b00,-1,b1); 
g_out=c01;
short_product4096xb(4096,a00,a01,1,b11,nullptr,0,b1); 
g_out=c11;
short_product4096xb(4096,a10,a00,-1,b00,b01,1,b1); 
g_out=c10;
short_product4096xb(4096,a01,a11,-1,b10,b11,1,b1); 
g_out=nullptr;g_ostr=4096;
short_product4096xb(4096,a00,a11,1,b00,b11,1,b1); 
short_product4096xb(4096,a10,a11,1,b00,nullptr,0,b2); 
short_product4096xb(4096,a00,nullptr,0,b01,b11,-1,b3); 
outer_sept(b1,b2,b3,c00,c01,c00,c10,c11,c01);
_mm_sfence();
free(pb);
}

评测结果