代码

/*
Credit: saffah_cc_v41_agg1 #117247, https://duck.ac/submission/117247:
byte-shuffle truncation for the micro24 eight-column tail; memory masks;
compiler scheduling options, three-row scheduling in micro16, unroll-4 combine.
pdoom word-ring experiment: VPMULLW and VPADDW keep the exact result
modulo 65536 directly in 16-bit lanes. Main tile is 4x48; the 16-column
tail uses 12 rows (and a 4-row remainder) with the same 4-row A packing.
No precision approximation, masks, saturating packs, or output permutations.
These extend our #112312 and retain its exact arithmetic and root dataflow.
*/
/*
Credit: pdoom #86352, https://duck.ac/submission/86352 : original exact
mod-65536 Strassen-Winograd engine, recursive packing and AVX2 pair-dot kernel.
saffah_cc_v41_agg1 #112006, https://duck.ac/submission/112006 : baseline code,
peeled first k pair, cache offsets, and packing that eliminates output shuffles.
saffah_codex_6s_agg2 #110387, https://duck.ac/submission/110387 : fused root
reconstruction/unpacking and constant-size recursion; #102935 : instruction order.
saffah_cc_v41_260924 #96771, https://duck.ac/submission/96771 : cache phase idea.
pdoom changes in this version:
- Fuse the root's eight input transforms with 64x64 packing. Stream the six
  retained raw quadrants and eight transformed quadrants to their packed slots.
- Reuse six retired root input-form slots for products: only 15 root slots.
- Write two adjacent 256-bit vectors per destination to complete each 64-byte line.
- Put 64 uint16_t padding elements after each 64x64 tile, and 32 elements
  between root slots; all index arithmetic uses the padded recursive footprint.
- Traverse B packing by k-pair before column panel, reusing adjacent input lines.
Inherited improvements from our #112187, https://duck.ac/submission/112187:
- Vectorized A 4x8-dword transpose and 256-bit B interleave while packing.
- Eight k-pairs per loop body; first k-pair initializes the accumulators.
- 1024 uint16_t elements between workspace slots, separating cache phases.
- Omit the first loop-exit check: every supported leaf has n=64 and 32 k-pairs.
All products, sums and differences are exact modulo 65536. Specialized to n=4096.
*/
#define ROOT_PAD 32
#pragma GCC optimize("O3,unroll-loops,web,rename-registers")
#pragma GCC target("avx2")
#include <immintrin.h>
#include <stdint.h>
#include <string.h>

#ifndef TILE_PAD
#define TILE_PAD 64
#endif
#ifndef BASE
#define BASE (1 << 6)
#endif
using U = uint16_t;
static constexpr int MAXN = 4096;
alignas(4096) static U pa_[MAXN * MAXN + (MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192], pb_[MAXN * MAXN + (MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192];
alignas(4096) static U pc_[MAXN * MAXN + (MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192], ws_[MAXN * MAXN + (MAXN/BASE)*(MAXN/BASE)*TILE_PAD + 8192];

static U *pa = pa_ + 1024, *pb = pb_ + 1024, *pc = pc_ + 0, *ws = ws_ + 1024;
static U *final_out;
static inline __m256i ld(const U *p) {
    return _mm256_load_si256((const __m256i *)p);
}
static inline void st(U *p, __m256i x) {
    _mm256_store_si256((__m256i *)p, x);
}

// The recursive quadrant layout makes every Strassen addition contiguous.
static void pack(U *d, const U *s, int n, int stride, bool right) {
    if (n > BASE) {
        int h = n / 2, q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
        pack(d, s, h, stride, right);
        pack(d + q, s + h, h, stride, right);
        pack(d + 2*q, s + h*stride, h, stride, right);
        pack(d + 3*q, s + h*stride + h, h, stride, right);
    } else if (!right) {
        for(int i=0;i<n;i+=4) for(int k=0;k<n;k+=16) {
            __m256i a=ld(s+i*stride+k), b=ld(s+(i+1)*stride+k);
            __m256i c=ld(s+(i+2)*stride+k), e=ld(s+(i+3)*stride+k);
            __m256i t0=_mm256_unpacklo_epi32(a,b),t1=_mm256_unpacklo_epi32(c,e);
            __m256i t2=_mm256_unpackhi_epi32(a,b),t3=_mm256_unpackhi_epi32(c,e);
            __m256i u0=_mm256_unpacklo_epi64(t0,t1),u1=_mm256_unpackhi_epi64(t0,t1);
            __m256i u2=_mm256_unpacklo_epi64(t2,t3),u3=_mm256_unpackhi_epi64(t2,t3);
            st(d,_mm256_permute2x128_si256(u0,u1,0x20));
            st(d+16,_mm256_permute2x128_si256(u2,u3,0x20));
            st(d+32,_mm256_permute2x128_si256(u0,u1,0x31));
            st(d+48,_mm256_permute2x128_si256(u2,u3,0x31));d+=64;
        }
    } else {
        for(int k=0;k<n;++k) {
            for(int j=0;j<48;j+=16) st(d+k*48+j,ld(s+k*stride+j));
            st(d+48*n+k*16,ld(s+k*stride+48));
        }
    }
}

static void unpack(U *d, const U *s, int n, int stride) {
    if (n > BASE) {
        int h = n/2, q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
        unpack(d,s,h,stride);
        unpack(d+h,s+q,h,stride);
        unpack(d+h*stride,s+2*q,h,stride);
        unpack(d+h*stride+h,s+3*q,h,stride);
    } else {
        for (int i=0;i<n;++i) {
            U *dp = d + (size_t)i*stride; const U *sp = s + (size_t)i*n;
            int k=0;
            for (;k+32<=n;k+=32) {
                _mm256_stream_si256((__m256i*)(dp+k),    _mm256_load_si256((const __m256i*)(sp+k)));
                _mm256_stream_si256((__m256i*)(dp+k+16), _mm256_load_si256((const __m256i*)(sp+k+16)));
            }
            for (;k<n;k+=16) _mm256_storeu_si256((__m256i*)(dp+k), _mm256_loadu_si256((const __m256i*)(sp+k)));
        }
    }
}

alignas(32) unsigned short s4aj_m16[16] = {0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0,0xFFFF,0};
alignas(32) unsigned char s4ak_sh16[32] = {0,1,4,5,8,9,12,13, 0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0,1,4,5,8,9,12,13, 0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80};
static inline void word4_48(const U *a,const U *b,U *c) {
    const U *end=a+256;
    asm volatile(
        "vmovdqa 0(%[b]), %%ymm12\n\t"
        "vpbroadcastw 0(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm0\n\t"
        "vpmullw 32(%[b]), %%ymm14, %%ymm1\n\t"
        "vpmullw 64(%[b]), %%ymm14, %%ymm2\n\t"
        "vpbroadcastw 4(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm3\n\t"
        "vpmullw 32(%[b]), %%ymm14, %%ymm4\n\t"
        "vpmullw 64(%[b]), %%ymm14, %%ymm5\n\t"
        "vpbroadcastw 8(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm6\n\t"
        "vpmullw 32(%[b]), %%ymm14, %%ymm7\n\t"
        "vpmullw 64(%[b]), %%ymm14, %%ymm8\n\t"
        "vpbroadcastw 12(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm9\n\t"
        "vpmullw 32(%[b]), %%ymm14, %%ymm10\n\t"
        "vpmullw 64(%[b]), %%ymm14, %%ymm11\n\t"
        "vmovdqa 96(%[b]), %%ymm12\n\t"
        "vpbroadcastw 2(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 128(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 160(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 6(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 128(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 160(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 10(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 128(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 160(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 14(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 128(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 160(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 192(%[b]), %%ymm12\n\t"
        "vpbroadcastw 16(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 224(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 256(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 20(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 224(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 256(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 24(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 224(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 256(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 28(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 224(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 256(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 288(%[b]), %%ymm12\n\t"
        "vpbroadcastw 18(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 320(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 352(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 22(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 320(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 352(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 26(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 320(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 352(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 30(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 320(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 352(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 384(%[b]), %%ymm12\n\t"
        "vpbroadcastw 32(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 416(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 448(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 36(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 416(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 448(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 40(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 416(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 448(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 44(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 416(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 448(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 480(%[b]), %%ymm12\n\t"
        "vpbroadcastw 34(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 512(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 544(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 38(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 512(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 544(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 42(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 512(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 544(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 46(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 512(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 544(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 576(%[b]), %%ymm12\n\t"
        "vpbroadcastw 48(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 608(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 640(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 52(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 608(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 640(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 56(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 608(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 640(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 60(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 608(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 640(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 672(%[b]), %%ymm12\n\t"
        "vpbroadcastw 50(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 704(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 736(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 54(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 704(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 736(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 58(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 704(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 736(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 62(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 704(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 736(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "add $64, %[a]\n\t"
        "add $768, %[b]\n\t"
        "1:\n\t"
        "vmovdqa 0(%[b]), %%ymm12\n\t"
        "vpbroadcastw 0(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 32(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 64(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 4(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 32(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 64(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 8(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 32(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 64(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 12(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 32(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 64(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 96(%[b]), %%ymm12\n\t"
        "vpbroadcastw 2(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 128(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 160(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 6(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 128(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 160(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 10(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 128(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 160(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 14(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 128(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 160(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 192(%[b]), %%ymm12\n\t"
        "vpbroadcastw 16(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 224(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 256(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 20(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 224(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 256(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 24(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 224(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 256(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 28(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 224(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 256(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 288(%[b]), %%ymm12\n\t"
        "vpbroadcastw 18(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 320(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 352(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 22(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 320(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 352(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 26(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 320(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 352(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 30(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 320(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 352(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 384(%[b]), %%ymm12\n\t"
        "vpbroadcastw 32(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 416(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 448(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 36(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 416(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 448(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 40(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 416(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 448(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 44(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 416(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 448(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 480(%[b]), %%ymm12\n\t"
        "vpbroadcastw 34(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 512(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 544(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 38(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 512(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 544(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 42(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 512(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 544(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 46(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 512(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 544(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 576(%[b]), %%ymm12\n\t"
        "vpbroadcastw 48(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 608(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 640(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 52(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 608(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 640(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 56(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 608(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 640(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 60(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 608(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 640(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 672(%[b]), %%ymm12\n\t"
        "vpbroadcastw 50(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpmullw 704(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpmullw 736(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 54(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpmullw 704(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpmullw 736(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 58(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpmullw 704(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpmullw 736(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 62(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpmullw 704(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpmullw 736(%[b]), %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "add $64, %[a]\n\t"
        "add $768, %[b]\n\t"
        "cmp %[end], %[a]\n\t"
        "jb 1b\n\t"
        "vmovdqa %%ymm0, 0(%[c])\n\t"
        "vmovdqa %%ymm1, 32(%[c])\n\t"
        "vmovdqa %%ymm2, 64(%[c])\n\t"
        "vmovdqa %%ymm3, 128(%[c])\n\t"
        "vmovdqa %%ymm4, 160(%[c])\n\t"
        "vmovdqa %%ymm5, 192(%[c])\n\t"
        "vmovdqa %%ymm6, 256(%[c])\n\t"
        "vmovdqa %%ymm7, 288(%[c])\n\t"
        "vmovdqa %%ymm8, 320(%[c])\n\t"
        "vmovdqa %%ymm9, 384(%[c])\n\t"
        "vmovdqa %%ymm10, 416(%[c])\n\t"
        "vmovdqa %%ymm11, 448(%[c])\n\t"
        : [a] "+&r"(a), [b] "+&r"(b)
        : [c] "r"(c), [end] "r"(end)
        : "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
static inline void word12_16(const U *a,const U *b,U *c) {
    const U *end=a+256;
    asm volatile(
        "vmovdqa 0(%[b]), %%ymm12\n\t"
        "vpbroadcastw 0(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm0\n\t"
        "vpbroadcastw 4(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm1\n\t"
        "vpbroadcastw 8(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm2\n\t"
        "vpbroadcastw 12(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm3\n\t"
        "vpbroadcastw 512(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm4\n\t"
        "vpbroadcastw 516(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm5\n\t"
        "vpbroadcastw 520(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm6\n\t"
        "vpbroadcastw 524(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm7\n\t"
        "vpbroadcastw 1024(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm8\n\t"
        "vpbroadcastw 1028(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm9\n\t"
        "vpbroadcastw 1032(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm10\n\t"
        "vpbroadcastw 1036(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm11\n\t"
        "vmovdqa 32(%[b]), %%ymm12\n\t"
        "vpbroadcastw 2(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 6(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 10(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 14(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 514(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 518(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 522(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 526(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1026(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1030(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1034(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1038(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 64(%[b]), %%ymm12\n\t"
        "vpbroadcastw 16(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 20(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 24(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 28(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 528(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 532(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 536(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 540(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1040(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1044(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1048(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1052(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 96(%[b]), %%ymm12\n\t"
        "vpbroadcastw 18(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 22(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 26(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 30(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 530(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 534(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 538(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 542(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1042(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1046(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1050(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1054(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 128(%[b]), %%ymm12\n\t"
        "vpbroadcastw 32(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 36(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 40(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 44(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 544(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 548(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 552(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 556(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1056(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1060(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1064(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1068(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 160(%[b]), %%ymm12\n\t"
        "vpbroadcastw 34(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 38(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 42(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 46(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 546(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 550(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 554(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 558(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1058(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1062(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1066(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1070(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 192(%[b]), %%ymm12\n\t"
        "vpbroadcastw 48(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 52(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 56(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 60(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 560(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 564(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 568(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 572(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1072(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1076(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1080(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1084(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 224(%[b]), %%ymm12\n\t"
        "vpbroadcastw 50(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 54(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 58(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 62(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 562(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 566(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 570(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 574(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1074(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1078(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1082(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1086(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "add $64, %[a]\n\t"
        "add $256, %[b]\n\t"
        "1:\n\t"
        "vmovdqa 0(%[b]), %%ymm12\n\t"
        "vpbroadcastw 0(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 4(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 8(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 12(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 512(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 516(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 520(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 524(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1024(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1028(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1032(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1036(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 32(%[b]), %%ymm12\n\t"
        "vpbroadcastw 2(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 6(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 10(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 14(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 514(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 518(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 522(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 526(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1026(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1030(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1034(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1038(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 64(%[b]), %%ymm12\n\t"
        "vpbroadcastw 16(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 20(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 24(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 28(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 528(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 532(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 536(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 540(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1040(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1044(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1048(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1052(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 96(%[b]), %%ymm12\n\t"
        "vpbroadcastw 18(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 22(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 26(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 30(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 530(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 534(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 538(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 542(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1042(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1046(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1050(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1054(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 128(%[b]), %%ymm12\n\t"
        "vpbroadcastw 32(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 36(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 40(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 44(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 544(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 548(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 552(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 556(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1056(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1060(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1064(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1068(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 160(%[b]), %%ymm12\n\t"
        "vpbroadcastw 34(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 38(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 42(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 46(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 546(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 550(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 554(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 558(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1058(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1062(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1066(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1070(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 192(%[b]), %%ymm12\n\t"
        "vpbroadcastw 48(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 52(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 56(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 60(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 560(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 564(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 568(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 572(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1072(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1076(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1080(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1084(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "vmovdqa 224(%[b]), %%ymm12\n\t"
        "vpbroadcastw 50(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 54(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 58(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 62(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vpbroadcastw 562(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm4, %%ymm4\n\t"
        "vpbroadcastw 566(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm5, %%ymm5\n\t"
        "vpbroadcastw 570(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm6, %%ymm6\n\t"
        "vpbroadcastw 574(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm7, %%ymm7\n\t"
        "vpbroadcastw 1074(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm8, %%ymm8\n\t"
        "vpbroadcastw 1078(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm9, %%ymm9\n\t"
        "vpbroadcastw 1082(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm10, %%ymm10\n\t"
        "vpbroadcastw 1086(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm11, %%ymm11\n\t"
        "add $64, %[a]\n\t"
        "add $256, %[b]\n\t"
        "cmp %[end], %[a]\n\t"
        "jb 1b\n\t"
        "vmovdqa %%ymm0, 0(%[c])\n\t"
        "vmovdqa %%ymm1, 128(%[c])\n\t"
        "vmovdqa %%ymm2, 256(%[c])\n\t"
        "vmovdqa %%ymm3, 384(%[c])\n\t"
        "vmovdqa %%ymm4, 512(%[c])\n\t"
        "vmovdqa %%ymm5, 640(%[c])\n\t"
        "vmovdqa %%ymm6, 768(%[c])\n\t"
        "vmovdqa %%ymm7, 896(%[c])\n\t"
        "vmovdqa %%ymm8, 1024(%[c])\n\t"
        "vmovdqa %%ymm9, 1152(%[c])\n\t"
        "vmovdqa %%ymm10, 1280(%[c])\n\t"
        "vmovdqa %%ymm11, 1408(%[c])\n\t"
        : [a] "+&r"(a), [b] "+&r"(b)
        : [c] "r"(c), [end] "r"(end)
        : "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
static inline void word4_16(const U *a,const U *b,U *c) {
    const U *end=a+256;
    asm volatile(
        "vmovdqa 0(%[b]), %%ymm12\n\t"
        "vpbroadcastw 0(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm0\n\t"
        "vpbroadcastw 4(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm1\n\t"
        "vpbroadcastw 8(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm2\n\t"
        "vpbroadcastw 12(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm3\n\t"
        "vmovdqa 32(%[b]), %%ymm12\n\t"
        "vpbroadcastw 2(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 6(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 10(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 14(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 64(%[b]), %%ymm12\n\t"
        "vpbroadcastw 16(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 20(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 24(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 28(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 96(%[b]), %%ymm12\n\t"
        "vpbroadcastw 18(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 22(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 26(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 30(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 128(%[b]), %%ymm12\n\t"
        "vpbroadcastw 32(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 36(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 40(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 44(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 160(%[b]), %%ymm12\n\t"
        "vpbroadcastw 34(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 38(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 42(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 46(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 192(%[b]), %%ymm12\n\t"
        "vpbroadcastw 48(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 52(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 56(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 60(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 224(%[b]), %%ymm12\n\t"
        "vpbroadcastw 50(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 54(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 58(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 62(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "add $64, %[a]\n\t"
        "add $256, %[b]\n\t"
        "1:\n\t"
        "vmovdqa 0(%[b]), %%ymm12\n\t"
        "vpbroadcastw 0(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 4(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 8(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 12(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 32(%[b]), %%ymm12\n\t"
        "vpbroadcastw 2(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 6(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 10(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 14(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 64(%[b]), %%ymm12\n\t"
        "vpbroadcastw 16(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 20(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 24(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 28(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 96(%[b]), %%ymm12\n\t"
        "vpbroadcastw 18(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 22(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 26(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 30(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 128(%[b]), %%ymm12\n\t"
        "vpbroadcastw 32(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 36(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 40(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 44(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 160(%[b]), %%ymm12\n\t"
        "vpbroadcastw 34(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 38(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 42(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 46(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 192(%[b]), %%ymm12\n\t"
        "vpbroadcastw 48(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 52(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 56(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 60(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "vmovdqa 224(%[b]), %%ymm12\n\t"
        "vpbroadcastw 50(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm0, %%ymm0\n\t"
        "vpbroadcastw 54(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm1, %%ymm1\n\t"
        "vpbroadcastw 58(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm2, %%ymm2\n\t"
        "vpbroadcastw 62(%[a]), %%ymm14\n\t"
        "vpmullw %%ymm12, %%ymm14, %%ymm15\n\t"
        "vpaddw %%ymm15, %%ymm3, %%ymm3\n\t"
        "add $64, %[a]\n\t"
        "add $256, %[b]\n\t"
        "cmp %[end], %[a]\n\t"
        "jb 1b\n\t"
        "vmovdqa %%ymm0, 0(%[c])\n\t"
        "vmovdqa %%ymm1, 128(%[c])\n\t"
        "vmovdqa %%ymm2, 256(%[c])\n\t"
        "vmovdqa %%ymm3, 384(%[c])\n\t"
        : [a] "+&r"(a), [b] "+&r"(b)
        : [c] "r"(c), [end] "r"(end)
        : "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
static void base(const U *a,const U *b,U *c,int n) {
    for(int i=0;i<64;i+=4) word4_48(a+i*64,b,c+i*64);
    for(int i=0;i<60;i+=12) word12_16(a+i*64,b+48*64,c+i*64+48);
    word4_16(a+60*64,b+48*64,c+60*64+48);
}

template<bool Sub>
static inline void combine(U *d,const U *a,const U *b,int len) {
    #pragma GCC unroll 4
    for(int i=0;i<len;i+=16) {
        __m256i x=ld(a+i),y=ld(b+i);
        st(d+i,Sub?_mm256_sub_epi16(x,y):_mm256_add_epi16(x,y));
    }
}

// At the root, the three completed packed quadrants can be sent straight
// to their final row-major destinations rather than materialized in pc.
template<int N>
static __attribute__((noinline)) void combine3_unpack(
    U *d12,U *d21,U *d22,
    const U *p6,const U *p7,const U *p5,
    const U *p1,const U *p4,const U *p3,int stride) {
    if constexpr (N > BASE) {
        constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
        combine3_unpack<h>(d12,d21,d22,p6,p7,p5,p1,p4,p3,stride);
        combine3_unpack<h>(d12+h,d21+h,d22+h,p6+q,p7+q,p5+q,p1+q,p4+q,p3+q,stride);
        combine3_unpack<h>(d12+h*stride,d21+h*stride,d22+h*stride,
                           p6+2*q,p7+2*q,p5+2*q,p1+2*q,p4+2*q,p3+2*q,stride);
        combine3_unpack<h>(d12+h*stride+h,d21+h*stride+h,d22+h*stride+h,
                           p6+3*q,p7+3*q,p5+3*q,p1+3*q,p4+3*q,p3+3*q,stride);
    } else {
        for(int row=0;row<N;++row) {
            const int off=row*N;
            U *o12=d12+(size_t)row*stride;
            U *o21=d21+(size_t)row*stride;
            U *o22=d22+(size_t)row*stride;
            for(int col=0;col<N;col+=16) {
                int idx=off+col;
                __m256i u2=_mm256_add_epi16(ld(p1+idx),ld(p6+idx));
                __m256i u3=_mm256_add_epi16(u2,ld(p7+idx));
                __m256i v5=ld(p5+idx);
                __m256i c12=_mm256_add_epi16(_mm256_add_epi16(u2,v5),ld(p3+idx));
                __m256i c21=_mm256_sub_epi16(u3,ld(p4+idx));
                __m256i c22=_mm256_add_epi16(u3,v5);
                _mm256_stream_si256((__m256i*)(o12+col),c12);
                _mm256_stream_si256((__m256i*)(o21+col),c21);
                _mm256_stream_si256((__m256i*)(o22+col),c22);
            }
        }
    }
}

// Add the surviving P1 into the root C11 product while mapping its packed
// quadrant layout to the final row-major C11 cells.
template<int N>
static __attribute__((noinline)) void unpack_add(U *d,const U *p2,const U *p1,int stride) {
    if constexpr(N>BASE) {
        constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
        unpack_add<h>(d,p2,p1,stride);
        unpack_add<h>(d+h,p2+q,p1+q,stride);
        unpack_add<h>(d+h*stride,p2+2*q,p1+2*q,stride);
        unpack_add<h>(d+h*stride+h,p2+3*q,p1+3*q,stride);
    } else {
        for(int row=0;row<N;++row) {
            U *o=d+(size_t)row*stride;
            int off=row*N;
            for(int col=0;col<N;col+=16)
                _mm256_stream_si256((__m256i*)(o+col),
                    _mm256_add_epi16(ld(p2+off+col),ld(p1+off+col)));
        }
    }
}

// Strassen-Winograd: seven products, fifteen additions, two temporaries.
// All operations are in Z/(2^16); truncation never changes the answer.
template<int N>
static __attribute__((noinline)) void mul(const U *a,const U *b,U *c,U *work) {
    if constexpr (N<=BASE) {
        base(a,b,c,N);
    } else {
    constexpr int h=N/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
    const U *a11=a,*a12=a+q,*a21=a+2*q,*a22=a+3*q;
    const U *b11=b,*b12=b+q,*b21=b+2*q,*b22=b+3*q;
    U *c11=c,*c12=c+q,*c21=c+2*q,*c22=c+3*q;
    U *x=work,*y=work+q+512,*next=work+2*q+1024;
    combine<true>(x,a11,a21,q);
    combine<true>(y,b22,b12,q);
    mul<h>(x,y,c21,next);                       // P7
    combine<false>(x,a21,a22,q);
    combine<true>(y,b12,b11,q);
    mul<h>(x,y,c22,next);                       // P5
    combine<true>(x,x,a11,q);
    combine<true>(y,b22,y,q);
    mul<h>(x,y,c12,next);                       // P6
    combine<true>(x,a12,x,q);
    mul<h>(x,b22,c11,next);                     // P3
    combine<true>(y,y,b21,q);
    mul<h>(a22,y,x,next);                       // P4
    mul<h>(a11,b11,y,next);                     // P1
    if constexpr (N == 4096) {
        combine3_unpack<2048>(final_out+2048,
            final_out+(size_t)2048*4096,
            final_out+(size_t)2048*4096+2048,
            c12,c21,c22,y,x,c11,4096);
    } else {
        for(int i=0;i<q;i+=16) {
            __m256i u2=_mm256_add_epi16(ld(y+i),ld(c12+i));
            __m256i u3=_mm256_add_epi16(u2,ld(c21+i));
            __m256i p5=ld(c22+i);
            st(c12+i,_mm256_add_epi16(_mm256_add_epi16(u2,p5),ld(c11+i)));
            st(c21+i,_mm256_sub_epi16(u3,ld(x+i)));
            st(c22+i,_mm256_add_epi16(u3,p5));
        }
    }
    mul<h>(a12,b21,c11,next);                   // P2
    if constexpr (N == 4096) unpack_add<2048>(final_out,c11,y,4096);
    else combine<false>(c11,c11,y,q);
    }
}


static constexpr int RQ=2048*2048+32*32*TILE_PAD,RS=RQ+ROOT_PAD;
alignas(4096) static U root_mem[15*RS+4096];
alignas(4096) static U root_tile[4*4096];
template<bool Right>
static void pack_root(const U*s,int n,int offset,U *dest) {
 if(n>64){int h=n/2,q=h*h+(h/BASE)*(h/BASE)*TILE_PAD;
  pack_root<Right>(s,h,offset,dest);
  pack_root<Right>(s+h,h,offset+q,dest);
  pack_root<Right>(s+h*4096,h,offset+2*q,dest);
  pack_root<Right>(s+h*4096+h,h,offset+3*q,dest);
 }else{
  pack(root_tile,s,64,4096,Right);
  pack(root_tile+4096,s+2048,64,4096,Right);
  pack(root_tile+8192,s+2048*4096,64,4096,Right);
  pack(root_tile+12288,s+2048*4096+2048,64,4096,Right);
  U *r0=dest+offset,*r1=r0+RS,*r2=r1+RS,*v7=r2+RS,*v5=v7+RS,*v6=v5+RS,*v34=v6+RS;
  for(int i=0;i<4096;i+=32){
   __m256i a=ld(root_tile+i),b=ld(root_tile+4096+i),c=ld(root_tile+8192+i),d=ld(root_tile+12288+i);
   __m256i aa=ld(root_tile+i+16),bb=ld(root_tile+4096+i+16),cc=ld(root_tile+8192+i+16),dd=ld(root_tile+12288+i+16);
   _mm256_stream_si256((__m256i*)(r0+i),a);_mm256_stream_si256((__m256i*)(r0+i+16),aa);
   _mm256_stream_si256((__m256i*)(r1+i),Right?c:b);_mm256_stream_si256((__m256i*)(r1+i+16),Right?cc:bb);
   _mm256_stream_si256((__m256i*)(r2+i),d);_mm256_stream_si256((__m256i*)(r2+i+16),dd);
   __m256i x7=Right?_mm256_sub_epi16(d,b):_mm256_sub_epi16(a,c);
   __m256i xx7=Right?_mm256_sub_epi16(dd,bb):_mm256_sub_epi16(aa,cc);
   _mm256_stream_si256((__m256i*)(v7+i),x7);_mm256_stream_si256((__m256i*)(v7+i+16),xx7);
   __m256i x5=Right?_mm256_sub_epi16(b,a):_mm256_add_epi16(c,d);
   __m256i xx5=Right?_mm256_sub_epi16(bb,aa):_mm256_add_epi16(cc,dd);
   _mm256_stream_si256((__m256i*)(v5+i),x5);_mm256_stream_si256((__m256i*)(v5+i+16),xx5);
   __m256i x6=Right?_mm256_sub_epi16(d,x5):_mm256_sub_epi16(x5,a);
   __m256i xx6=Right?_mm256_sub_epi16(dd,xx5):_mm256_sub_epi16(xx5,aa);
   _mm256_stream_si256((__m256i*)(v6+i),x6);_mm256_stream_si256((__m256i*)(v6+i+16),xx6);
   __m256i x34=Right?_mm256_sub_epi16(x6,c):_mm256_sub_epi16(b,x6);
   __m256i xx34=Right?_mm256_sub_epi16(xx6,cc):_mm256_sub_epi16(bb,xx6);
   _mm256_stream_si256((__m256i*)(v34+i),x34);_mm256_stream_si256((__m256i*)(v34+i+16),xx34);
  }
 }
}
void matrix_multiply(int n,const short*A,const short*B,short*C){
 U *ar=root_mem,*br=ar+7*RS,*pr=br+7*RS;
 pack_root<false>((const U*)A,2048,0,ar);
 pack_root<true>((const U*)B,2048,0,br);
 _mm_mfence();
 const U *a11=ar,*a12=ar+RS,*a22=ar+2*RS,*s7=ar+3*RS,*s5=ar+4*RS,*s6=ar+5*RS,*s3=ar+6*RS;
 const U *b11=br,*b21=br+RS,*b22=br+2*RS,*t7=br+3*RS,*t5=br+4*RS,*t6=br+5*RS,*t4=br+6*RS;
 U *p1=ar+5*RS,*p2=br+5*RS,*p3=ar+4*RS,*p4=br+4*RS,*p5=ar+3*RS,*p6=br+3*RS,*p7=pr;
 mul<2048>(s7,t7,p7,ws);mul<2048>(s5,t5,p5,ws);mul<2048>(s6,t6,p6,ws);
 mul<2048>(s3,b22,p3,ws);mul<2048>(a22,t4,p4,ws);mul<2048>(a11,b11,p1,ws);mul<2048>(a12,b21,p2,ws);
 U *out=(U*)C;
 combine3_unpack<2048>(out+2048,out+2048*4096,out+2048*4096+2048,p6,p7,p5,p1,p4,p3,4096);
 unpack_add<2048>(out,p2,p1,4096);_mm_sfence();
}

评测结果