代码

/*
Credit (inherited implementation and ideas):
- saffah_cc_v41_agg1 https://duck.ac/submission/105696: upper defer schedule.
- saffah_codex_6s_agg2 https://duck.ac/submission/104821: immediate engine base.
- saffah_cc_v41_agg1 https://duck.ac/submission/101193: two-level deferred combine.
- saffah_cc_v41_agg1 https://duck.ac/submission/103010: produced-row frontier.
- saffah_codex_6s_agg2 https://duck.ac/submission/103194: eight-K plain kernel.
- saffah_codex_6s_agg2 https://duck.ac/submission/103224: frontier/leaf ordering port.
- saffah_codex_6s_agg2 https://duck.ac/submission/103275: power-of-two addressing.
- saffah_codex_6s_agg2 https://duck.ac/submission/104809: leaf-plane layout.
- saffah_cc_v41_agg1 https://duck.ac/submission/100776: two-row tail.
- saffah_cc_v41_agg1 https://duck.ac/submission/100511 and /100475: fused A packing.
- saffah_cc_v41_260924 https://duck.ac/submission/99846: fused sums/packing.
- saffah_cc_v41_agg1 https://duck.ac/submission/99875 and /99950: NT stores.
- saffah_cc_v41_260924 https://duck.ac/submission/99141 and
  saffah_cc_v41_agg1 https://duck.ac/submission/99167: AVX2 kernel/layout baseline.
- pdoom earlier submissions: Winograd/Strassen scheduling, tail merge,
  delayed combine, page spacing, and the new register epilogue in this file.
*/
/*
Credit: inherited engine and row-frontier sources105696/104821 are retained
below. New here: keep the final Strassen M7 tile in its FMA accumulators,
load the other six products, and reconstruct the four C quadrants directly
in the assembly epilogue. Prefetch those six products before the FMA loop.
This removes the final product's whole-matrix write/read; no precision or
parenthesization changes. This experiment is specialized to BASE512,
MC30, LEAF_PAD1152 and the final-product order ending in M7.
*/
/*
Credit: saffah_cc_v41_agg1 https://duck.ac/submission/105696 and
saffah_codex_6s_agg2 https://duck.ac/submission/104821; full original credits
follow. New here: specialize upper reconstruction geometry. In the fixed
4096 problem, tasks serviced inside leaves always have half-size1024;
the only half-size2048 task is armed after the final leaf and drained at
function exit. Use constant geometry in the hot path and dispatch once at
synchronous finish. Matrix arithmetic and scheduling remain unchanged.
*/
#pragma GCC optimize("O3,unroll-loops,no-strict-aliasing")
#include <sys/mman.h>
#pragma GCC target("avx2,fma,arch=skylake")
#include <immintrin.h>
#ifndef B2M_NO_BARA
#define B2M_NO_BARA 1
#endif
#define MD9_T2BASE 1
#include <stddef.h>
#ifndef BASE
#define BASE 512
#endif
#ifndef MC
#define MC 30
#endif
#ifndef NTMIN
#define NTMIN 512
#endif
#ifndef UPPER_PAGE_PAD
#define UPPER_PAGE_PAD 15872
#endif
#ifndef LEAF_PAD
#define LEAF_PAD 1152
#endif
#ifndef SUMNT
#define SUMNT 1024
#endif
static constexpr int MR=6,NR=8;alignas(4096)static double workspace[48*1024*1024];alignas(64)static double ap[(MC+MR)*(BASE+24)],ap2[(MC+MR)*(BASE+24)],bp[BASE*(BASE+24)];alignas(64)static double pk_zero[BASE+8];alignas(64)static double pk_mp[4]={0.0,0.0,0.0,0.0},pk_mn[4]={-0.0,-0.0,-0.0,-0.0};struct Mat{const double*x;int ld;const double*y=nullptr;int ly=0;int sign=0;};static inline __m256d ld(Mat a,int i,int j){__m256d x=_mm256_load_pd(a.x+(size_t)i*a.ld+j);if(a.sign){__m256d y=_mm256_load_pd(a.y+(size_t)i*a.ly+j);x=a.sign>0?_mm256_add_pd(x,y):_mm256_sub_pd(x,y);}return x;}static inline double scalar(Mat a,int i,int k){double x=a.x[(size_t)i*a.ld+k];if(a.sign)x+=a.sign>0?a.y[(size_t)i*a.ly+k]:-a.y[(size_t)i*a.ly+k];return x;}template<int SG,bool FUSE=false>static __attribute__((always_inline))inline void kernel(int n,const double*a,const double*b,double*c,int ldc,int rows,const double*pks,const double*pky,double*pkd,const double*cm=nullptr){alignas(32)double tmp[MR*8];double*out=rows==MR?c:tmp;ptrdiff_t stride=rows==MR?ldc*8:64;int k=n;ptrdiff_t delta=(ptrdiff_t)BASE*ldc*8;asm volatile("vxorpd %%ymm0, %%ymm0, %%ymm0\n\t""vxorpd %%ymm1, %%ymm1, %%ymm1\n\t""vxorpd %%ymm2, %%ymm2, %%ymm2\n\t""vxorpd %%ymm3, %%ymm3, %%ymm3\n\t""vxorpd %%ymm4, %%ymm4, %%ymm4\n\t""vxorpd %%ymm5, %%ymm5, %%ymm5\n\t""vxorpd %%ymm6, %%ymm6, %%ymm6\n\t""vxorpd %%ymm7, %%ymm7, %%ymm7\n\t""vxorpd %%ymm8, %%ymm8, %%ymm8\n\t""vxorpd %%ymm9, %%ymm9, %%ymm9\n\t""vxorpd %%ymm10, %%ymm10, %%ymm10\n\t""vxorpd %%ymm11, %%ymm11, %%ymm11\n\t"".p2align 6\n\t""1:\n\t"".if %c[sg] != 0\n\t""prefetcht0 512(%[b])\n\t""prefetcht0 576(%[b])\n\t""prefetcht0 640(%[b])\n\t""prefetcht0 704(%[b])\n\t""vmovapd 0(%[b]), %%ymm12\n\t""vmovapd 32(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 64(%[b]), %%ymm12\n\t""vmovapd 96(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 128(%[b]), %%ymm12\n\t""vmovapd 160(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 192(%[b]), %%ymm12\n\t""vmovapd 224(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""prefetcht0 768(%[b])\n\t""prefetcht0 832(%[b])\n\t""prefetcht0 896(%[b])\n\t""prefetcht0 960(%[b])\n\t""vmovapd 256(%[b]), %%ymm12\n\t""vmovapd 288(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 320(%[b]), %%ymm12\n\t""vmovapd 352(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 384(%[b]), %%ymm12\n\t""vmovapd 416(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 448(%[b]), %%ymm12\n\t""vmovapd 480(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovupd 0(%[pks]), %%ymm15\n\t"".if %c[sg] == 1\n\t""vaddpd 0(%[pky]), %%ymm15, %%ymm15\n\t"".elseif %c[sg] == -1\n\t""vsubpd 0(%[pky]), %%ymm15, %%ymm15\n\t"".endif\n\t""vmovupd %%ymm15, 0(%[pkd])\n\t""add $32, %[pks]\n\t""add $32, %[pky]\n\t""add $32, %[pkd]\n\t""add $64, %[a]\n\t""add $512, %[b]\n\t""sub $8, %[k]\n\t"".else\n\t""prefetcht0 512(%[b])\n\t""prefetcht0 576(%[b])\n\t""prefetcht0 640(%[b])\n\t""prefetcht0 704(%[b])\n\t""vmovapd 0(%[b]), %%ymm12\n\t""vmovapd 32(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 64(%[b]), %%ymm12\n\t""vmovapd 96(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 128(%[b]), %%ymm12\n\t""vmovapd 160(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 192(%[b]), %%ymm12\n\t""vmovapd 224(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""add $32, %[a]\n\t""add $256, %[b]\n\t""sub $4, %[k]\n\t"".endif\n\t""jnz 1b\n\t"".if %c[fuse]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm0, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm0\n\t""vaddpd %%ymm0, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm0, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm1, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm1\n\t""vaddpd %%ymm1, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm1, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm2, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm2\n\t""vaddpd %%ymm2, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm2, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm3, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm3\n\t""vaddpd %%ymm3, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm3, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm4, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm4\n\t""vaddpd %%ymm4, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm4, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm5, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm5\n\t""vaddpd %%ymm5, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm5, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm6, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm6\n\t""vaddpd %%ymm6, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm6, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm7, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm7\n\t""vaddpd %%ymm7, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm7, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm8, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm8\n\t""vaddpd %%ymm8, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm8, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm9, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm9\n\t""vaddpd %%ymm9, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm9, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm10, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm10\n\t""vaddpd %%ymm10, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm10, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm11, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm11\n\t""vaddpd %%ymm11, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm11, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t"".else\n\t""test %[nt], %[nt]\n\t""jz 2f\n\t""vmovntpd %%ymm0, (%[out])\n\t""vmovntpd %%ymm1, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm2, (%[out])\n\t""vmovntpd %%ymm3, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm4, (%[out])\n\t""vmovntpd %%ymm5, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm6, (%[out])\n\t""vmovntpd %%ymm7, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm8, (%[out])\n\t""vmovntpd %%ymm9, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm10, (%[out])\n\t""vmovntpd %%ymm11, 32(%[out])\n\t""jmp 3f\n\t""2:\n\t""vmovapd %%ymm0, (%[out])\n\t""vmovapd %%ymm1, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovapd %%ymm2, (%[out])\n\t""vmovapd %%ymm3, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovapd %%ymm4, (%[out])\n\t""vmovapd %%ymm5, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovapd %%ymm6, (%[out])\n\t""vmovapd %%ymm7, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovapd %%ymm8, (%[out])\n\t""vmovapd %%ymm9, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovapd %%ymm10, (%[out])\n\t""vmovapd %%ymm11, 32(%[out])\n\t""3:\n\t"".endif\n\t":[cm]"+&r"(cm),[a]"+&r"(a),[b]"+&r"(b),[out]"+&r"(out),[k]"+&r"(k),[pks]"+&r"(pks),[pky]"+&r"(pky),[pkd]"+&r"(pkd):[fuse]"i"(FUSE),[delta]"r"(delta),[stride]"r"(stride),[nt]"r"(int(n>=NTMIN&&rows==MR)),[sg]"i"(SG),[o0]"i"(0*(BASE+24)*8),[o1]"i"(1*(BASE+24)*8),[o2]"i"(2*(BASE+24)*8),[o3]"i"(3*(BASE+24)*8),[o4]"i"(4*(BASE+24)*8),[o5]"i"(5*(BASE+24)*8):"memory","cc","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15");if(rows!=MR)for(int i=0;i<rows;i++)for(int j=0;j<8;j+=4)_mm256_store_pd(c+(size_t)i*ldc+j,_mm256_load_pd(tmp+i*8+j));}template<bool FUSE=false>static __attribute__((always_inline))inline void kernel_plain(int n,const double*a,const double*b,double*c,int ldc,int rows,const double*cm=nullptr){alignas(32)double tmp[MR*8];double*out=rows==MR?c:tmp;ptrdiff_t stride=rows==MR?ldc*8:64;int k=n;ptrdiff_t delta=(ptrdiff_t)BASE*ldc*8;asm volatile("vxorpd %%ymm0, %%ymm0, %%ymm0\n\t""vxorpd %%ymm1, %%ymm1, %%ymm1\n\t""vxorpd %%ymm2, %%ymm2, %%ymm2\n\t""vxorpd %%ymm3, %%ymm3, %%ymm3\n\t""vxorpd %%ymm4, %%ymm4, %%ymm4\n\t""vxorpd %%ymm5, %%ymm5, %%ymm5\n\t""vxorpd %%ymm6, %%ymm6, %%ymm6\n\t""vxorpd %%ymm7, %%ymm7, %%ymm7\n\t""vxorpd %%ymm8, %%ymm8, %%ymm8\n\t""vxorpd %%ymm9, %%ymm9, %%ymm9\n\t""vxorpd %%ymm10, %%ymm10, %%ymm10\n\t""vxorpd %%ymm11, %%ymm11, %%ymm11\n\t"".p2align 6\n\t""1:\n\t""prefetcht0 512(%[b])\n\t""prefetcht0 576(%[b])\n\t""prefetcht0 640(%[b])\n\t""prefetcht0 704(%[b])\n\t""vmovapd 0(%[b]), %%ymm12\n\t""vmovapd 32(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 64(%[b]), %%ymm12\n\t""vmovapd 96(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 128(%[b]), %%ymm12\n\t""vmovapd 160(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 192(%[b]), %%ymm12\n\t""vmovapd 224(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""prefetcht0 768(%[b])\n\t""prefetcht0 832(%[b])\n\t""prefetcht0 896(%[b])\n\t""prefetcht0 960(%[b])\n\t""vmovapd 256(%[b]), %%ymm12\n\t""vmovapd 288(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+32(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 320(%[b]), %%ymm12\n\t""vmovapd 352(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+40(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 384(%[b]), %%ymm12\n\t""vmovapd 416(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+48(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""vmovapd 448(%[b]), %%ymm12\n\t""vmovapd 480(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vbroadcastsd %c[o2]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm4\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm5\n\t""vbroadcastsd %c[o3]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm6\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm7\n\t""vbroadcastsd %c[o4]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm8\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm9\n\t""vbroadcastsd %c[o5]+56(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm10\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm11\n\t""add $64, %[a]\n\t""add $512, %[b]\n\t""sub $8, %[k]\n\t""jnz 1b\n\t"".if %c[fuse]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm0, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm0\n\t""vaddpd %%ymm0, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm0, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm1, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm1\n\t""vaddpd %%ymm1, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm1, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm2, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm2\n\t""vaddpd %%ymm2, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm2, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm3, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm3\n\t""vaddpd %%ymm3, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm3, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm4, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm4\n\t""vaddpd %%ymm4, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm4, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm5, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm5\n\t""vaddpd %%ymm5, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm5, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm6, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm6\n\t""vaddpd %%ymm6, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm6, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm7, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm7\n\t""vaddpd %%ymm7, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm7, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm8, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm8\n\t""vaddpd %%ymm8, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm8, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm9, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm9\n\t""vaddpd %%ymm9, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm9, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm10, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm10\n\t""vaddpd %%ymm10, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm10, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm11, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm11\n\t""vaddpd %%ymm11, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm11, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t"".else\n\t""test %[nt], %[nt]\n\t""jz 2f\n\t""vmovntpd %%ymm0, (%[out])\n\t""vmovntpd %%ymm1, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm2, (%[out])\n\t""vmovntpd %%ymm3, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm4, (%[out])\n\t""vmovntpd %%ymm5, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm6, (%[out])\n\t""vmovntpd %%ymm7, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm8, (%[out])\n\t""vmovntpd %%ymm9, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm10, (%[out])\n\t""vmovntpd %%ymm11, 32(%[out])\n\t""jmp 3f\n\t""2:\n\t""vmovapd %%ymm0, (%[out])\n\t""vmovapd %%ymm1, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovapd %%ymm2, (%[out])\n\t""vmovapd %%ymm3, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovapd %%ymm4, (%[out])\n\t""vmovapd %%ymm5, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovapd %%ymm6, (%[out])\n\t""vmovapd %%ymm7, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovapd %%ymm8, (%[out])\n\t""vmovapd %%ymm9, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovapd %%ymm10, (%[out])\n\t""vmovapd %%ymm11, 32(%[out])\n\t""3:\n\t"".endif\n\t":[cm]"+&r"(cm),[a]"+&r"(a),[b]"+&r"(b),[out]"+&r"(out),[k]"+&r"(k):[fuse]"i"(FUSE),[delta]"r"(delta),[stride]"r"(stride),[nt]"r"(int(n>=NTMIN&&rows==MR)),[o0]"i"(0*(BASE+24)*8),[o1]"i"(1*(BASE+24)*8),[o2]"i"(2*(BASE+24)*8),[o3]"i"(3*(BASE+24)*8),[o4]"i"(4*(BASE+24)*8),[o5]"i"(5*(BASE+24)*8):"memory","cc","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15");if(rows!=MR)for(int i=0;i<rows;i++)for(int j=0;j<8;j+=4)_mm256_store_pd(c+(size_t)i*ldc+j,_mm256_load_pd(tmp+i*8+j));}struct Pend2{double*c;int ldc;const double*m;int h;int pz;int pos;bool active;};static Pend2 pend2={0,0,0,0,0,0,false};static constexpr int STEP2=16;template<int H=1024>static inline void pend2_prefetch(void){if(!pend2.active)return;int i=pend2.pos>>__builtin_ctz((unsigned)H),j=pend2.pos&(H-1);const double*c=pend2.c+(size_t)i*pend2.ldc+j;const double*m=pend2.m+pend2.pos;for(int t=0;t<STEP2;t+=8){_mm_prefetch((const char*)(c+t),_MM_HINT_T1);_mm_prefetch((const char*)(c+H+t),_MM_HINT_T1);_mm_prefetch((const char*)(c+(size_t)H*pend2.ldc+t),_MM_HINT_T1);_mm_prefetch((const char*)(c+(size_t)H*pend2.ldc+H+t),_MM_HINT_T1);_mm_prefetch((const char*)(m+t),_MM_HINT_T1);_mm_prefetch((const char*)(m+(H*H+UPPER_PAGE_PAD)+t),_MM_HINT_T1);_mm_prefetch((const char*)(m+2*(H*H+UPPER_PAGE_PAD)+t),_MM_HINT_T1);}}template<int H=1024>static inline void pend2_step(void){int i=pend2.pos>>__builtin_ctz((unsigned)H),j=pend2.pos&(H-1);for(int t=0;t<STEP2;t+=4){double*c11=pend2.c+(size_t)i*pend2.ldc+j+t,*c12=c11+H,*c21=c11+(size_t)H*pend2.ldc,*c22=c21+H;const double*m=pend2.m+pend2.pos+t;__m256d m1=_mm256_load_pd(c11),m2=_mm256_load_pd(c21),m3=_mm256_load_pd(c12),m4=_mm256_load_pd(c22),m5=_mm256_load_pd(m),m6=_mm256_load_pd(m+(H*H+UPPER_PAGE_PAD)),m7=_mm256_load_pd(m+2*(H*H+UPPER_PAGE_PAD));__m256d u2=_mm256_add_pd(m1,m6),u3=_mm256_add_pd(u2,m7),u4=_mm256_add_pd(u2,m5);__m256d r11=_mm256_add_pd(m1,m2),r12=_mm256_add_pd(u4,m3),r21=_mm256_sub_pd(u3,m4),r22=_mm256_add_pd(u3,m5);_mm256_store_pd(c11,r11);_mm256_store_pd(c12,r12);_mm256_store_pd(c21,r21);_mm256_store_pd(c22,r22);}pend2.pos+=STEP2;if(pend2.pos>=H*H)pend2.active=false;}static void pend2_finish(void){if(pend2.h==1024){while(pend2.active)pend2_step<1024>();}else{while(pend2.active)pend2_step<2048>();}}struct Pending{double*c;int ldc;const double*m;int pos=0;bool active=false;int rowlim=0;bool prog=false;};static Pending pending;alignas(4096)static double pending_work[3*(BASE*BASE+LEAF_PAD)+LEAF_PAD];static unsigned pending_tick=0;static constexpr int STEP=32,PERIOD=1;static inline void pending_prefetch(){if(!pending.active)return;if(pending.prog&&pending.pos/BASE>=pending.rowlim)return;int i=pending.pos/BASE,j=pending.pos%BASE;const double*c=pending.c+(size_t)i*pending.ldc+j;const double*m=pending.m+pending.pos;for(int t=0;t<STEP;t+=8){_mm_prefetch((const char*)(c+t),_MM_HINT_T1);_mm_prefetch((const char*)(c+BASE+t),_MM_HINT_T1);_mm_prefetch((const char*)(c+(size_t)BASE*pending.ldc+t),_MM_HINT_T1);_mm_prefetch((const char*)(c+(size_t)BASE*pending.ldc+BASE+t),_MM_HINT_T1);_mm_prefetch((const char*)(m+t),_MM_HINT_T1);_mm_prefetch((const char*)(m+(BASE*BASE+LEAF_PAD)+t),_MM_HINT_T1);_mm_prefetch((const char*)(m+2*(BASE*BASE+LEAF_PAD)+t),_MM_HINT_T1);}}static inline void pending_step(){if(pending.prog&&pending.pos/BASE>=pending.rowlim)return;int i=pending.pos/BASE,j=pending.pos%BASE;for(int t=0;t<STEP;t+=4){double*c11=pending.c+(size_t)i*pending.ldc+j+t,*c12=c11+BASE,*c21=c11+(size_t)BASE*pending.ldc,*c22=c21+BASE;const double*m=pending.m+pending.pos+t;__m256d m1=_mm256_load_pd(c11),m2=_mm256_load_pd(c21),m3=_mm256_load_pd(c12),m4=_mm256_load_pd(c22),m5=_mm256_load_pd(m),m6=_mm256_load_pd(m+(BASE*BASE+LEAF_PAD)),m7=_mm256_load_pd(m+2*(BASE*BASE+LEAF_PAD));__m256d r11=_mm256_add_pd(_mm256_sub_pd(_mm256_add_pd(m1,m4),m5),m7),r12=_mm256_add_pd(m3,m5),r21=_mm256_add_pd(m2,m4),r22=_mm256_add_pd(_mm256_add_pd(_mm256_sub_pd(m1,m2),m3),m6);_mm256_store_pd(c11,r11);_mm256_store_pd(c12,r12);_mm256_store_pd(c21,r21);_mm256_store_pd(c22,r22);}pending.pos+=STEP;if(pending.pos==BASE*BASE)pending.active=false;}static void pending_finish(){pending.prog=false;while(pending.active)pending_step();}static __attribute__((always_inline))inline void kernel2(int n,const double*a,const double*b,double*c,int ldc){int k=n;double*out=c;ptrdiff_t stride=(ptrdiff_t)ldc*8;asm volatile("vxorpd %%ymm0, %%ymm0, %%ymm0\n\t""vxorpd %%ymm1, %%ymm1, %%ymm1\n\t""vxorpd %%ymm2, %%ymm2, %%ymm2\n\t""vxorpd %%ymm3, %%ymm3, %%ymm3\n\t"".p2align 6\n\t""1:\n\t""prefetcht0 512(%[b])\n\t""prefetcht0 576(%[b])\n\t""prefetcht0 640(%[b])\n\t""prefetcht0 704(%[b])\n\t""vmovapd 0(%[b]), %%ymm12\n\t""vmovapd 32(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+0(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vmovapd 64(%[b]), %%ymm12\n\t""vmovapd 96(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+8(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vmovapd 128(%[b]), %%ymm12\n\t""vmovapd 160(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+16(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""vmovapd 192(%[b]), %%ymm12\n\t""vmovapd 224(%[b]), %%ymm13\n\t""vbroadcastsd %c[o0]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm0\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm1\n\t""vbroadcastsd %c[o1]+24(%[a]), %%ymm14\n\t""vfmadd231pd %%ymm12, %%ymm14, %%ymm2\n\t""vfmadd231pd %%ymm13, %%ymm14, %%ymm3\n\t""add $32, %[a]\n\t""add $256, %[b]\n\t""sub $4, %[k]\n\t""jnz 1b\n\t""vmovntpd %%ymm0, (%[out])\n\t""vmovntpd %%ymm1, 32(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm2, (%[out])\n\t""vmovntpd %%ymm3, 32(%[out])\n\t":[a]"+&r"(a),[b]"+&r"(b),[out]"+&r"(out),[k]"+&r"(k):[stride]"r"(stride),[o0]"i"(0*(BASE+24)*8),[o1]"i"(1*(BASE+24)*8):"memory","cc","ymm0","ymm1","ymm2","ymm3","ymm12","ymm13","ymm14");}template<bool FUSE=false>static __attribute__((noinline))void kernel_tail16(int n,const double*a,const double*b,double*out,int ldc,const double*cm=nullptr){const double*b1=b+8*(n+1);ptrdiff_t stride=(ptrdiff_t)ldc*8;int kk=n;ptrdiff_t delta=(ptrdiff_t)BASE*ldc*8;asm volatile("vxorpd %%ymm0, %%ymm0, %%ymm0\n\t""vxorpd %%ymm1, %%ymm1, %%ymm1\n\t""vxorpd %%ymm2, %%ymm2, %%ymm2\n\t""vxorpd %%ymm3, %%ymm3, %%ymm3\n\t""vxorpd %%ymm4, %%ymm4, %%ymm4\n\t""vxorpd %%ymm5, %%ymm5, %%ymm5\n\t""vxorpd %%ymm6, %%ymm6, %%ymm6\n\t""vxorpd %%ymm7, %%ymm7, %%ymm7\n\t"".p2align 6\n\t""1:\n\t""prefetcht0 512(%[b])\n\t""prefetcht0 512(%[b1])\n\t""vbroadcastsd %c[o0]+0(%[a]), %%ymm12\n\t""vbroadcastsd %c[o1]+0(%[a]), %%ymm13\n\t""vmovapd 0(%[b]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm0\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm4\n\t""vmovapd 32(%[b]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm1\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm5\n\t""vmovapd 0(%[b1]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm2\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm6\n\t""vmovapd 32(%[b1]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm3\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm7\n\t""prefetcht0 576(%[b])\n\t""prefetcht0 576(%[b1])\n\t""vbroadcastsd %c[o0]+8(%[a]), %%ymm12\n\t""vbroadcastsd %c[o1]+8(%[a]), %%ymm13\n\t""vmovapd 64(%[b]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm0\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm4\n\t""vmovapd 96(%[b]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm1\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm5\n\t""vmovapd 64(%[b1]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm2\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm6\n\t""vmovapd 96(%[b1]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm3\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm7\n\t""prefetcht0 640(%[b])\n\t""prefetcht0 640(%[b1])\n\t""vbroadcastsd %c[o0]+16(%[a]), %%ymm12\n\t""vbroadcastsd %c[o1]+16(%[a]), %%ymm13\n\t""vmovapd 128(%[b]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm0\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm4\n\t""vmovapd 160(%[b]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm1\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm5\n\t""vmovapd 128(%[b1]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm2\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm6\n\t""vmovapd 160(%[b1]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm3\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm7\n\t""prefetcht0 704(%[b])\n\t""prefetcht0 704(%[b1])\n\t""vbroadcastsd %c[o0]+24(%[a]), %%ymm12\n\t""vbroadcastsd %c[o1]+24(%[a]), %%ymm13\n\t""vmovapd 192(%[b]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm0\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm4\n\t""vmovapd 224(%[b]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm1\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm5\n\t""vmovapd 192(%[b1]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm2\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm6\n\t""vmovapd 224(%[b1]), %%ymm14\n\t""vfmadd231pd %%ymm14, %%ymm12, %%ymm3\n\t""vfmadd231pd %%ymm14, %%ymm13, %%ymm7\n\t""add $32, %[a]\n\t""add $256, %[b]\n\t""add $256, %[b1]\n\t""sub $4, %[kk]\n\t""jnz 1b\n\t"".if %c[fuse]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm0, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm0\n\t""vaddpd %%ymm0, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm0, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm1, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm1\n\t""vaddpd %%ymm1, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm1, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""vmovapd 64(%[out]), %%ymm12\n\t""vmovapd 4160(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 64(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm2, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 64(%[out])\n\t""vmovapd 4160(%[out]), %%ymm2\n\t""vaddpd %%ymm2, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4160(%[out])\n\t""vmovapd 64(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 64(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm2, %%ymm12, %%ymm12\n\t""vaddpd 2106432(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4160(%[out],%[delta],1)\n\t""vmovapd 96(%[out]), %%ymm12\n\t""vmovapd 4192(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 96(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm3, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 96(%[out])\n\t""vmovapd 4192(%[out]), %%ymm3\n\t""vaddpd %%ymm3, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4192(%[out])\n\t""vmovapd 96(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 96(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm3, %%ymm12, %%ymm12\n\t""vaddpd 2106464(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4192(%[out],%[delta],1)\n\t""add %[stride], %[out]\n\t""add $4096, %[cm]\n\t""vmovapd 0(%[out]), %%ymm12\n\t""vmovapd 4096(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 0(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm4, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out])\n\t""vmovapd 4096(%[out]), %%ymm4\n\t""vaddpd %%ymm4, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4096(%[out])\n\t""vmovapd 0(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 0(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm4, %%ymm12, %%ymm12\n\t""vaddpd 2106368(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4096(%[out],%[delta],1)\n\t""vmovapd 32(%[out]), %%ymm12\n\t""vmovapd 4128(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 32(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm5, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out])\n\t""vmovapd 4128(%[out]), %%ymm5\n\t""vaddpd %%ymm5, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4128(%[out])\n\t""vmovapd 32(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 32(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm5, %%ymm12, %%ymm12\n\t""vaddpd 2106400(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4128(%[out],%[delta],1)\n\t""vmovapd 64(%[out]), %%ymm12\n\t""vmovapd 4160(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 64(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm6, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 64(%[out])\n\t""vmovapd 4160(%[out]), %%ymm6\n\t""vaddpd %%ymm6, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4160(%[out])\n\t""vmovapd 64(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 64(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm6, %%ymm12, %%ymm12\n\t""vaddpd 2106432(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4160(%[out],%[delta],1)\n\t""vmovapd 96(%[out]), %%ymm12\n\t""vmovapd 4192(%[out],%[delta],1), %%ymm13\n\t""vaddpd %%ymm13, %%ymm12, %%ymm14\n\t""vmovapd 96(%[cm]), %%ymm15\n\t""vsubpd %%ymm15, %%ymm14, %%ymm14\n\t""vaddpd %%ymm7, %%ymm14, %%ymm14\n\t""vmovapd %%ymm14, 96(%[out])\n\t""vmovapd 4192(%[out]), %%ymm7\n\t""vaddpd %%ymm7, %%ymm15, %%ymm14\n\t""vmovapd %%ymm14, 4192(%[out])\n\t""vmovapd 96(%[out],%[delta],1), %%ymm15\n\t""vaddpd %%ymm15, %%ymm13, %%ymm14\n\t""vmovapd %%ymm14, 96(%[out],%[delta],1)\n\t""vsubpd %%ymm15, %%ymm12, %%ymm12\n\t""vaddpd %%ymm7, %%ymm12, %%ymm12\n\t""vaddpd 2106464(%[cm]), %%ymm12, %%ymm12\n\t""vmovapd %%ymm12, 4192(%[out],%[delta],1)\n\t"".else\n\t""vmovntpd %%ymm0, 0(%[out])\n\t""vmovntpd %%ymm1, 32(%[out])\n\t""vmovntpd %%ymm2, 64(%[out])\n\t""vmovntpd %%ymm3, 96(%[out])\n\t""add %[stride], %[out]\n\t""vmovntpd %%ymm4, 0(%[out])\n\t""vmovntpd %%ymm5, 32(%[out])\n\t""vmovntpd %%ymm6, 64(%[out])\n\t""vmovntpd %%ymm7, 96(%[out])\n\t"".endif\n\t":[cm]"+&r"(cm),[a]"+&r"(a),[b]"+&r"(b),[b1]"+&r"(b1),[out]"+&r"(out),[kk]"+&r"(kk):[fuse]"i"(FUSE),[delta]"r"(delta),[stride]"r"(stride),[o0]"i"(0),[o1]"i"((BASE+24)*8):"memory","cc","ymm0","ymm1","ymm2","ymm3","ymm4","ymm5","ymm6","ymm7","ymm8","ymm9","ymm10","ymm11","ymm12","ymm13","ymm14","ymm15");}static inline void fusion_prefetch(double*c,int ldc,const double*m,int rows,int cols){for(int r=0;r<rows;r++)for(int j=0;j<cols;j+=8){double*p=c+(size_t)r*ldc+j;const double*q=m+(size_t)r*BASE+j;_mm_prefetch((const char*)p,_MM_HINT_T1);_mm_prefetch((const char*)(p+BASE),_MM_HINT_T1);_mm_prefetch((const char*)(p+(size_t)BASE*ldc),_MM_HINT_T1);_mm_prefetch((const char*)(p+(size_t)BASE*ldc+BASE),_MM_HINT_T1);_mm_prefetch((const char*)q,_MM_HINT_T1);_mm_prefetch((const char*)(q+BASE*BASE+LEAF_PAD),_MM_HINT_T1);}}template<bool FUSE=false>static void leaf(int n,Mat a,Mat b,double*c,int ldc,const double*cm=nullptr){for(int k=0;k<n;k+=2)for(int j=0;j<n;j+=8){double*p=bp+(size_t)j*(n+1)+k*8;_mm256_store_pd(p+0,ld(b,k+0,j));_mm256_store_pd(p+4,ld(b,k+0,j+4));_mm256_store_pd(p+8,ld(b,k+1,j));_mm256_store_pd(p+12,ld(b,k+1,j+4));}pending_prefetch();double*pkb[2]={ap,ap2};int cur=0;{int mc0=n<MC?n:MC;for(int r=0;r<mc0;r+=MR){int rows=mc0-r<MR?mc0-r:MR;for(int i=0;i<rows;i++)for(int k=0;k<n;k+=4)_mm256_store_pd(pkb[0]+(size_t)(r+i)*(BASE+24)+k,ld(a,r+i,k));for(int i=rows;i<MR;i++)for(int k=0;k<n;k+=4)_mm256_store_pd(pkb[0]+(size_t)(r+i)*(BASE+24)+k,_mm256_setzero_pd());}}for(int ic=0;ic<n;){int mc=n-ic<=MC+MR?n-ic:MC;if(mc==MC+2){for(int j=0;j<n;j+=16){for(int jj=j;jj<j+16;jj+=NR)for(int r=0;r<MC;r+=MR){const double*mp=FUSE?cm+(size_t)(ic+r)*BASE+jj:nullptr;if(FUSE)fusion_prefetch(c+(size_t)(ic+r)*ldc+jj,ldc,mp,MR,8);kernel_plain<FUSE>(n,pkb[cur]+(size_t)r*(BASE+24),bp+(size_t)jj*(n+1),c+(size_t)(ic+r)*ldc+jj,ldc,MR,mp);if(pending.active&&(++pending_tick%PERIOD)==0){pending_step();if(pending.prog)pending_step();pending_prefetch();}if(pend2.active){pend2_step();pend2_prefetch();}}const double*mp=FUSE?cm+(size_t)(ic+MC)*BASE+j:nullptr;if(FUSE)fusion_prefetch(c+(size_t)(ic+MC)*ldc+j,ldc,mp,2,16);kernel_tail16<FUSE>(n,pkb[cur]+(size_t)MC*(BASE+24),bp+(size_t)j*(n+1),c+(size_t)(ic+MC)*ldc+j,ldc,mp);}ic+=mc;if(pending.prog)pending.rowlim=ic;continue;}const int ic2=ic+mc;const int nxt=(ic2<n)?(n-ic2<=MC+MR?n-ic2:MC):0;const int rows2=((nxt+MR-1)/MR)*MR;const int nch=rows2*2;const int qd=n/2;int ck=0;int skip=(n/NR)*((mc+MR-1)/MR)-nch;for(int j=0;j<n;j+=NR)for(int r=0;r<mc;r+=MR){if(mc-r==2){kernel2(n,pkb[cur]+(size_t)r*(BASE+24),bp+(size_t)j*(n+1),c+(size_t)(ic+r)*ldc+j,ldc);continue;}const double*pks=pk_zero,*pky=pk_zero,*pkm=pk_mp;double*pkd=pk_zero;int pkc=0;if(ck<nch&&--skip<0){const int row=ck/2,off=(ck%2)*qd;if(row<nxt){pks=a.x+(size_t)(ic2+row)*a.ld+(size_t)off;pky=a.sign?(a.y+(size_t)(ic2+row)*a.ly+(size_t)off):pk_zero;pkm=a.sign>0?pk_mp:pk_mn;}pkd=pkb[cur^1]+(size_t)row*(BASE+24)+(size_t)off;pkc=8;ck++;}const double*mp=FUSE?cm+(size_t)(ic+r)*BASE+j:nullptr;if(FUSE)fusion_prefetch(c+(size_t)(ic+r)*ldc+j,ldc,mp,MR,8);if(!pkc)kernel_plain<FUSE>(n,pkb[cur]+(size_t)r*(BASE+24),bp+(size_t)j*(n+1),c+(size_t)(ic+r)*ldc+j,ldc,mc-r<MR?mc-r:MR,mp);else if(a.sign==0)kernel<2,FUSE>(n,pkb[cur]+(size_t)r*(BASE+24),bp+(size_t)j*(n+1),c+(size_t)(ic+r)*ldc+j,ldc,mc-r<MR?mc-r:MR,pks,pky,pkd,mp);else if(a.sign>0)kernel<1,FUSE>(n,pkb[cur]+(size_t)r*(BASE+24),bp+(size_t)j*(n+1),c+(size_t)(ic+r)*ldc+j,ldc,mc-r<MR?mc-r:MR,pks,pky,pkd,mp);else kernel<-1,FUSE>(n,pkb[cur]+(size_t)r*(BASE+24),bp+(size_t)j*(n+1),c+(size_t)(ic+r)*ldc+j,ldc,mc-r<MR?mc-r:MR,pks,pky,pkd,mp);if(pending.active&&(++pending_tick%PERIOD)==0){pending_step();if(pending.prog)pending_step();pending_prefetch();}if(pend2.active){pend2_step();pend2_prefetch();}}for(int cc=ck;cc<nch;cc++){const int row=cc/2,off=(cc%2)*qd;if(row<nxt){for(int k=off;k<off+qd;k+=4)_mm256_store_pd(pkb[cur^1]+(size_t)row*(BASE+24)+k,ld(a,ic2+row,k));}else{for(int k=off;k<off+qd;k+=4)_mm256_store_pd(pkb[cur^1]+(size_t)row*(BASE+24)+k,_mm256_setzero_pd());}}cur^=1;ic+=mc;if(pending.prog)pending.rowlim=ic;}if(n>=NTMIN)_mm_sfence();}static void sum(int h,double*dst,Mat a,Mat b,int sign){for(int i=0;i<h;i++)for(int j=0;j<h;j+=8){__m256d v=sign>0?_mm256_add_pd(ld(a,i,j),ld(b,i,j)):_mm256_sub_pd(ld(a,i,j),ld(b,i,j));__m256d w=sign>0?_mm256_add_pd(ld(a,i,j+4),ld(b,i,j+4)):_mm256_sub_pd(ld(a,i,j+4),ld(b,i,j+4));asm volatile("":"+x"(v),"+x"(w): :"memory");if(h>=SUMNT){_mm256_stream_pd(dst+(size_t)i*h+j,v);_mm256_stream_pd(dst+(size_t)i*h+j+4,w);}else{_mm256_store_pd(dst+(size_t)i*h+j,v);_mm256_store_pd(dst+(size_t)i*h+j+4,w);}}if(h>=SUMNT)_mm_sfence();}static void combine(int h,double*c,int ldc,const double*m,bool wino=false){size_t z=(size_t)h*h+(wino?UPPER_PAGE_PAD:0);for(int i=0;i<h;i++)for(int j=0;j<h;j+=4){size_t t=(size_t)i*h+j;double*c11=c+(size_t)i*ldc+j,*c12=c11+h,*c21=c+(size_t)(i+h)*ldc+j,*c22=c21+h;__m256d m1=_mm256_load_pd(c11),m2=_mm256_load_pd(c21),m3=_mm256_load_pd(c12),m4=_mm256_load_pd(c22),m5=_mm256_load_pd(m+t),m6=_mm256_load_pd(m+z+t),m7=_mm256_load_pd(m+2*z+t);__m256d r11,r12,r21,r22;if(wino){__m256d u2=_mm256_add_pd(m1,m6),u3=_mm256_add_pd(u2,m7),u4=_mm256_add_pd(u2,m5);r11=_mm256_add_pd(m1,m2);r12=_mm256_add_pd(u4,m3);r21=_mm256_sub_pd(u3,m4);r22=_mm256_add_pd(u3,m5);}else{r11=_mm256_add_pd(_mm256_sub_pd(_mm256_add_pd(m1,m4),m5),m7);r12=_mm256_add_pd(m3,m5);r21=_mm256_add_pd(m2,m4);r22=_mm256_add_pd(_mm256_add_pd(_mm256_sub_pd(m1,m2),m3),m6);}_mm256_store_pd(c11,r11);_mm256_store_pd(c12,r12);_mm256_store_pd(c21,r21);_mm256_store_pd(c22,r22);}}static void pre(int h,Mat a,double*s1,double*s2,double*s4,double*s3,int ld3,bool right){for(int i=0;i<h;i++)for(int j=0;j<h;j+=8){__m256d a0=ld(a,i,j),a1=ld(a,i,j+4),b0=ld(a,i,j+h),b1=ld(a,i,j+h+4),c0=ld(a,i+h,j),c1=ld(a,i+h,j+4),d0=ld(a,i+h,j+h),d1=ld(a,i+h,j+h+4);__m256d p0,p1,q0,q1,r0,r1,t0,t1;if(!right){p0=_mm256_add_pd(c0,d0);p1=_mm256_add_pd(c1,d1);q0=_mm256_sub_pd(p0,a0);q1=_mm256_sub_pd(p1,a1);r0=_mm256_sub_pd(b0,q0);r1=_mm256_sub_pd(b1,q1);t0=_mm256_sub_pd(a0,c0);t1=_mm256_sub_pd(a1,c1);}else{p0=_mm256_sub_pd(b0,a0);p1=_mm256_sub_pd(b1,a1);q0=_mm256_sub_pd(d0,p0);q1=_mm256_sub_pd(d1,p1);r0=_mm256_sub_pd(q0,c0);r1=_mm256_sub_pd(q1,c1);t0=_mm256_sub_pd(d0,b0);t1=_mm256_sub_pd(d1,b1);}size_t off=(size_t)i*h+j,off3=(size_t)i*ld3+j;_mm256_stream_pd(s1+off,p0);_mm256_stream_pd(s1+off+4,p1);_mm256_stream_pd(s2+off,q0);_mm256_stream_pd(s2+off+4,q1);_mm256_stream_pd(s4+off,r0);_mm256_stream_pd(s4+off+4,r1);_mm256_stream_pd(s3+off3,t0);_mm256_stream_pd(s3+off3+4,t1);}_mm_sfence();}static const int lperm[7]={3,1,2,0,5,4,6};static void rec(int n,Mat a,Mat b,double*c,int ldc,double*work){if(n<=BASE){leaf(n,a,b,c,ldc);return;}int h=n/2;size_t z=(size_t)h*h;Mat aa[4]={{a.x,a.ld},{a.x+h,a.ld},{a.x+(size_t)h*a.ld,a.ld},{a.x+(size_t)h*a.ld+h,a.ld}};Mat bb[4]={{b.x,b.ld},{b.x+h,b.ld},{b.x+(size_t)h*b.ld,b.ld},{b.x+(size_t)h*b.ld+h,b.ld}};double*m=n==2*BASE?work+LEAF_PAD:work;if(n==2*BASE&&pending.active&&pending.m==m)m=pending_work+LEAF_PAD;size_t pz=z+(n==2*BASE?LEAF_PAD:UPPER_PAGE_PAD);double*p=work+3*z,*q=p+z,*sub=q+z;double*out[7]={c,c+(size_t)h*ldc,c+h,c+(size_t)h*ldc+h,m,m+pz,m+2*pz};if(n>BASE*2){pending_finish();double*pa0=m+3*pz,*pa1=pa0+pz,*pa2=pa1+pz,*pb0=pa2+pz,*pb1=pb0+pz,*pb2=pb1+pz;sub=pb2+pz;pre(h,a,pa0,pa1,pa2,c,ldc,false);pre(h,b,pb0,pb1,pb2,c+h,ldc,true);rec(h,aa[1],bb[2],out[1],ldc,sub);rec(h,aa[3],{pb2,h},out[3],ldc,sub);
#if !B2M_NO_BARA
pend2_finish();
#endif
rec(h,{pa0,h},{pb0,h},out[4],h,sub);rec(h,{pa1,h},{pb1,h},out[5],h,sub);rec(h,{c,ldc},{c+h,ldc},out[6],h,sub);rec(h,aa[0],bb[0],out[0],ldc,sub);rec(h,{pa2,h},bb[3],out[2],ldc,sub);pending_finish();pend2_finish();pend2={c,ldc,m,h,(int)pz,0,true};return;}const int ai[7]={0,2,0,3,0,2,1},aj[7]={3,3,-1,-1,1,0,3},as[7]={1,1,0,0,1,-1,-1};const int bi[7]={0,0,1,2,3,0,2},bj[7]={3,-1,3,0,-1,1,3},bs[7]={1,0,-1,-1,0,1,1};for(int r=0;r<7;r++){Mat x=aa[ai[r]],y=bb[bi[r]];if(h<=BASE){const int q=lperm[r];Mat x=aa[ai[q]],y=bb[bi[q]];if(as[q]){x.y=aa[aj[q]].x;x.ly=a.ld;x.sign=as[q];}if(bs[q]){y.y=bb[bj[q]].x;y.ly=b.ld;y.sign=bs[q];}if(r==6){pending_finish();leaf<true>(h,x,y,c,ldc,m);}else leaf(h,x,y,out[q],q<4?ldc:h);}else{if(as[r]){sum(h,p,x,aa[aj[r]],as[r]);x={p,h};}if(bs[r]){sum(h,q,y,bb[bj[r]],bs[r]);y={q,h};}rec(h,x,y,out[r],r<4?ldc:h,sub);}}if(h==BASE){}else{pending_finish();combine(h,c,ldc,m);}}void matrix_multiply(int n,const double*a,const double*b,double*c){madvise((void*)workspace,sizeof(workspace),MADV_HUGEPAGE);madvise((void*)pending_work,sizeof(pending_work),MADV_HUGEPAGE);rec(n,{a,n},{b,n},c,n,workspace);pending_finish();pend2_finish();_mm_sfence();}

评测结果