/* int8 6-bit quantised GEMM for mmmf4k. generated by gen_i8.py */
#pragma GCC optimize("O3")
#pragma GCC target("avx2,bmi,bmi2,popcnt,lzcnt")
#include <immintrin.h>
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#define NN 4096
#define NG 4
#define KC 256
#define NP (NN/2)
#define APAD 1024
#define NCH (KC/16)
#define GSTR 81984
#define NGRP (NN/16)
#define JBN (NN/(16*NG))
#define NKB (NP/KC)
#define RB 32
#define INV961 (1.0f/484.0f)
struct MS { int16_t e[16]; int16_t o[16]; };
static const struct MS MASKV __attribute__((aligned(32))) =
{{1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0},{0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1}};
#define MASKS (&MASKV)
static uint32_t *g_A = 0;
static uint8_t *g_B = 0;
static int32_t *g_corr = 0;
static int32_t *g_SC = 0;
static int32_t *g_cs = 0;
static uint8_t *g_qa = 0, *g_qb = 0;
__attribute__((target("avx2"))) static void mikro(const uint32_t *ap, const uint8_t *bx, int nchunks, int32_t *acc)
{
asm volatile(
"vmovdqu 0(%[acc]), %%ymm6\n\t"
"vmovdqu 32(%[acc]), %%ymm10\n\t"
"vmovdqu 64(%[acc]), %%ymm7\n\t"
"vmovdqu 96(%[acc]), %%ymm11\n\t"
"vmovdqu 128(%[acc]), %%ymm8\n\t"
"vmovdqu 160(%[acc]), %%ymm12\n\t"
"vmovdqu 192(%[acc]), %%ymm9\n\t"
"vmovdqu 224(%[acc]), %%ymm13\n\t"
"movl %[nch], %%r10d\n\t"
".p2align 5\n\t"
"1:\n\t"
"vpxor %%xmm2, %%xmm2, %%xmm2\n\t"
"vpxor %%xmm3, %%xmm3, %%xmm3\n\t"
"vpxor %%xmm4, %%xmm4, %%xmm4\n\t"
"vpxor %%xmm5, %%xmm5, %%xmm5\n\t"
"vpbroadcastd 128(%[ap]), %%ymm0\n\t"
"vpbroadcastd 132(%[ap]), %%ymm15\n\t"
"vpmaddubsw 128(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 160(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82112(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82144(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164096(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164128(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246080(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246112(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 136(%[ap]), %%ymm0\n\t"
"vpbroadcastd 140(%[ap]), %%ymm15\n\t"
"vpmaddubsw 192(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 224(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82176(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82208(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164160(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164192(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246144(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246176(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 144(%[ap]), %%ymm0\n\t"
"vpbroadcastd 148(%[ap]), %%ymm15\n\t"
"vpmaddubsw 256(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 288(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82240(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82272(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164224(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164256(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246208(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246240(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 152(%[ap]), %%ymm0\n\t"
"vpbroadcastd 156(%[ap]), %%ymm15\n\t"
"vpmaddubsw 320(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 352(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82304(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82336(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164288(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164320(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246272(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246304(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 160(%[ap]), %%ymm0\n\t"
"vpbroadcastd 164(%[ap]), %%ymm15\n\t"
"vpmaddubsw 384(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 416(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82368(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82400(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164352(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164384(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246336(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246368(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 168(%[ap]), %%ymm0\n\t"
"vpbroadcastd 172(%[ap]), %%ymm15\n\t"
"vpmaddubsw 448(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 480(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82432(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82464(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164416(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164448(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246400(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246432(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 176(%[ap]), %%ymm0\n\t"
"vpbroadcastd 180(%[ap]), %%ymm15\n\t"
"vpmaddubsw 512(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 544(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82496(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82528(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164480(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164512(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246464(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246496(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 184(%[ap]), %%ymm0\n\t"
"vpbroadcastd 188(%[ap]), %%ymm15\n\t"
"vpmaddubsw 576(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 608(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82560(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82592(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164544(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164576(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246528(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246560(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 192(%[ap]), %%ymm0\n\t"
"vpbroadcastd 196(%[ap]), %%ymm15\n\t"
"vpmaddubsw 640(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 672(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82624(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82656(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164608(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164640(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246592(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246624(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 200(%[ap]), %%ymm0\n\t"
"vpbroadcastd 204(%[ap]), %%ymm15\n\t"
"vpmaddubsw 704(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 736(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82688(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82720(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164672(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164704(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246656(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246688(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 208(%[ap]), %%ymm0\n\t"
"vpbroadcastd 212(%[ap]), %%ymm15\n\t"
"vpmaddubsw 768(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 800(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82752(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82784(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164736(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164768(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246720(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246752(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 216(%[ap]), %%ymm0\n\t"
"vpbroadcastd 220(%[ap]), %%ymm15\n\t"
"vpmaddubsw 832(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 864(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82816(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82848(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164800(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164832(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246784(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246816(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 224(%[ap]), %%ymm0\n\t"
"vpbroadcastd 228(%[ap]), %%ymm15\n\t"
"vpmaddubsw 896(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 928(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82880(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82912(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164864(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164896(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246848(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246880(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 232(%[ap]), %%ymm0\n\t"
"vpbroadcastd 236(%[ap]), %%ymm15\n\t"
"vpmaddubsw 960(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 992(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 82944(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 82976(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164928(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 164960(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246912(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 246944(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 240(%[ap]), %%ymm0\n\t"
"vpbroadcastd 244(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1024(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1056(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83008(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83040(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 164992(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165024(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 246976(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247008(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 248(%[ap]), %%ymm0\n\t"
"vpbroadcastd 252(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1088(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1120(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83072(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83104(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165056(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165088(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247040(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247072(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 256(%[ap]), %%ymm0\n\t"
"vpbroadcastd 260(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1152(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1184(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83136(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83168(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165120(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165152(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247104(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247136(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 264(%[ap]), %%ymm0\n\t"
"vpbroadcastd 268(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1216(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1248(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83200(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83232(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165184(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165216(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247168(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247200(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 272(%[ap]), %%ymm0\n\t"
"vpbroadcastd 276(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1280(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1312(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83264(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83296(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165248(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165280(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247232(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247264(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 280(%[ap]), %%ymm0\n\t"
"vpbroadcastd 284(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1344(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1376(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83328(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83360(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165312(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165344(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247296(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247328(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 288(%[ap]), %%ymm0\n\t"
"vpbroadcastd 292(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1408(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1440(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83392(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83424(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165376(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165408(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247360(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247392(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 296(%[ap]), %%ymm0\n\t"
"vpbroadcastd 300(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1472(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1504(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83456(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83488(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165440(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165472(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247424(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247456(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 304(%[ap]), %%ymm0\n\t"
"vpbroadcastd 308(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1536(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1568(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83520(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83552(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165504(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165536(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247488(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247520(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 312(%[ap]), %%ymm0\n\t"
"vpbroadcastd 316(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1600(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1632(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83584(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83616(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165568(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165600(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247552(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247584(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 320(%[ap]), %%ymm0\n\t"
"vpbroadcastd 324(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1664(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1696(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83648(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83680(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165632(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165664(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247616(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247648(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 328(%[ap]), %%ymm0\n\t"
"vpbroadcastd 332(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1728(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1760(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83712(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83744(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165696(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165728(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247680(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247712(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 336(%[ap]), %%ymm0\n\t"
"vpbroadcastd 340(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1792(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1824(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83776(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83808(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165760(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165792(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247744(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247776(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 344(%[ap]), %%ymm0\n\t"
"vpbroadcastd 348(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1856(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1888(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83840(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83872(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165824(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165856(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247808(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247840(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 352(%[ap]), %%ymm0\n\t"
"vpbroadcastd 356(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1920(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 1952(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83904(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 83936(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165888(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165920(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247872(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247904(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 360(%[ap]), %%ymm0\n\t"
"vpbroadcastd 364(%[ap]), %%ymm15\n\t"
"vpmaddubsw 1984(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 2016(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 83968(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 84000(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 165952(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 165984(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 247936(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 247968(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 368(%[ap]), %%ymm0\n\t"
"vpbroadcastd 372(%[ap]), %%ymm15\n\t"
"vpmaddubsw 2048(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 2080(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 84032(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 84064(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 166016(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 166048(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 248000(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 248032(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpbroadcastd 376(%[ap]), %%ymm0\n\t"
"vpbroadcastd 380(%[ap]), %%ymm15\n\t"
"vpmaddubsw 2112(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 2144(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm2, %%ymm2\n\t"
"vpmaddubsw 84096(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 84128(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm3, %%ymm3\n\t"
"vpmaddubsw 166080(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 166112(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm4, %%ymm4\n\t"
"vpmaddubsw 248064(%[bx]), %%ymm0, %%ymm1\n\t"
"vpmaddubsw 248096(%[bx]), %%ymm15, %%ymm14\n\t"
"vpaddw %%ymm14, %%ymm1, %%ymm1\n\t"
"vpaddw %%ymm1, %%ymm5, %%ymm5\n\t"
"vpmaddwd 0(%[me]), %%ymm2, %%ymm14\n\t"
"vpmaddwd 32(%[me]), %%ymm2, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm6, %%ymm6\n\t"
"vpaddd %%ymm15, %%ymm10, %%ymm10\n\t"
"vpmaddwd 0(%[me]), %%ymm3, %%ymm14\n\t"
"vpmaddwd 32(%[me]), %%ymm3, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm7, %%ymm7\n\t"
"vpaddd %%ymm15, %%ymm11, %%ymm11\n\t"
"vpmaddwd 0(%[me]), %%ymm4, %%ymm14\n\t"
"vpmaddwd 32(%[me]), %%ymm4, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm8, %%ymm8\n\t"
"vpaddd %%ymm15, %%ymm12, %%ymm12\n\t"
"vpmaddwd 0(%[me]), %%ymm5, %%ymm14\n\t"
"vpmaddwd 32(%[me]), %%ymm5, %%ymm15\n\t"
"vpaddd %%ymm14, %%ymm9, %%ymm9\n\t"
"vpaddd %%ymm15, %%ymm13, %%ymm13\n\t"
"addq $256, %[ap]\n\t"
"addq $2048, %[bx]\n\t"
"decl %%r10d\n\t"
"jnz 1b\n\t"
"vmovdqu %%ymm6, 0(%[acc])\n\t"
"vmovdqu %%ymm10, 32(%[acc])\n\t"
"vmovdqu %%ymm7, 64(%[acc])\n\t"
"vmovdqu %%ymm11, 96(%[acc])\n\t"
"vmovdqu %%ymm8, 128(%[acc])\n\t"
"vmovdqu %%ymm12, 160(%[acc])\n\t"
"vmovdqu %%ymm9, 192(%[acc])\n\t"
"vmovdqu %%ymm13, 224(%[acc])\n\t"
: [ap]"+r"(ap), [bx]"+r"(bx)
: [acc]"r"(acc), [nch]"r"(nchunks), [me]"r"(MASKS)
: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15", "r10", "cc", "memory");
}
__attribute__((target("avx2"))) static inline __m256i qf8(const float *r)
{
return _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(r), _mm256_set1_ps(22.0f)));
}
/* 16 floats -> 16 bytes of A_q = round(x*31)+32 clamped to [1,63] */
__attribute__((target("avx2"))) static inline void qA16(const float *r, uint8_t *o)
{
__m256i v0 = qf8(r), v1 = qf8(r + 8);
v0 = _mm256_min_epi32(_mm256_max_epi32(_mm256_add_epi32(v0, _mm256_set1_epi32(22)),
_mm256_set1_epi32(0)), _mm256_set1_epi32(44));
v1 = _mm256_min_epi32(_mm256_max_epi32(_mm256_add_epi32(v1, _mm256_set1_epi32(22)),
_mm256_set1_epi32(0)), _mm256_set1_epi32(44));
_mm_storeu_si128((__m128i *)o, _mm_packus_epi16(
_mm_packus_epi32(_mm256_castsi256_si128(v0), _mm256_extracti128_si256(v0, 1)),
_mm_packus_epi32(_mm256_castsi256_si128(v1), _mm256_extracti128_si256(v1, 1))));
}
/* 16 floats -> 16 bytes of B_q = round(x*31) clamped to [-31,31] */
__attribute__((target("avx2"))) static inline void qB16(const float *r, uint8_t *o)
{
__m256i v0 = qf8(r), v1 = qf8(r + 8);
v0 = _mm256_min_epi32(_mm256_max_epi32(v0, _mm256_set1_epi32(-22)), _mm256_set1_epi32(22));
v1 = _mm256_min_epi32(_mm256_max_epi32(v1, _mm256_set1_epi32(-22)), _mm256_set1_epi32(22));
_mm_storeu_si128((__m128i *)o, _mm_packs_epi16(
_mm_packs_epi32(_mm256_castsi256_si128(v0), _mm256_extracti128_si256(v0, 1)),
_mm_packs_epi32(_mm256_castsi256_si128(v1), _mm256_extracti128_si256(v1, 1))));
}
/* A panel: row i, pair p -> dword [a0,a1,a0,a1] broadcast by vpbroadcastd */
__attribute__((target("avx2"))) static void packA(int n, const float *A)
{
for (int i = 0; i < n; i++) {
const float *r = A + (size_t)i * n;
uint32_t *o = g_A + (size_t)i * (n / 2 + APAD);
for (int k = 0; k < n; k += 16) {
uint8_t q[16];
qA16(r + k, q);
__m256i w32 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)q));
_mm256_storeu_si256((__m256i *)(o + (k >> 1)),
_mm256_mullo_epi32(w32, _mm256_set1_epi32(65537)));
}
}
}
/* B panel: record (g,p) = 32 B, i16 lane m = (Bq[2p][16g+m], Bq[2p+1][16g+m]) */
__attribute__((target("avx2"))) static void packB(int n, const float *B, int32_t *cs)
{
for (int p = 0; p < n / 2; p++) {
const float *r0 = B + (size_t)(2 * p) * n, *r1 = r0 + n;
for (int k = 0; k < n; k += 16) {
qB16(r0 + k, g_qa + k);
qB16(r1 + k, g_qb + k);
__m256i a0 = _mm256_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(g_qa + k)));
__m256i a1 = _mm256_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(g_qa + k + 8)));
__m256i b0 = _mm256_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(g_qb + k)));
__m256i b1 = _mm256_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(g_qb + k + 8)));
_mm256_storeu_si256((__m256i *)(cs + k), _mm256_add_epi32(
_mm256_loadu_si256((const __m256i *)(cs + k)), _mm256_add_epi32(a0, b0)));
_mm256_storeu_si256((__m256i *)(cs + k + 8), _mm256_add_epi32(
_mm256_loadu_si256((const __m256i *)(cs + k + 8)), _mm256_add_epi32(a1, b1)));
__m128i va = _mm_loadu_si128((const __m128i *)(g_qa + k));
__m128i vb = _mm_loadu_si128((const __m128i *)(g_qb + k));
__m256i rec = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_unpacklo_epi8(va, vb)), _mm_unpackhi_epi8(va, vb), 1);
_mm256_storeu_si256((__m256i *)(g_B + (size_t)(k / 16) * GSTR + (size_t)p * 32), rec);
}
}
}
/* SC layout per 16-column group: 8 i32 of EVEN columns then 8 i32 of ODD columns */
__attribute__((target("avx2"))) static void decode(int n, float *C)
{
const __m256 inv = _mm256_set1_ps(INV961);
for (int i = 0; i < n; i++) {
const int32_t *s = g_SC + (size_t)i * n;
float *c = C + (size_t)i * n;
for (int g = 0; g < n / 16; g++) {
const int32_t *q = s + g * 16;
__m256i E = _mm256_loadu_si256((const __m256i *)q);
__m256i O = _mm256_loadu_si256((const __m256i *)(q + 8));
__m256i lo = _mm256_unpacklo_epi32(E, O);
__m256i hi = _mm256_unpackhi_epi32(E, O);
__m256i r0 = _mm256_sub_epi32(_mm256_permute2x128_si256(lo, hi, 0x20),
_mm256_loadu_si256((const __m256i *)(g_corr + g * 16)));
__m256i r1 = _mm256_sub_epi32(_mm256_permute2x128_si256(lo, hi, 0x31),
_mm256_loadu_si256((const __m256i *)(g_corr + g * 16 + 8)));
_mm256_storeu_ps(c + g * 16, _mm256_mul_ps(_mm256_cvtepi32_ps(r0), inv));
_mm256_storeu_ps(c + g * 16 + 8, _mm256_mul_ps(_mm256_cvtepi32_ps(r1), inv));
}
}
}
static void simple_mm(int n, const float *A, const float *B, float *C)
{
for (int i = 0; i < n; i++)
for (int j = 0; j < n; j++) {
float s = 0.f;
for (int k = 0; k < n; k++) s += A[(size_t)i * n + k] * B[(size_t)k * n + j];
C[(size_t)i * n + j] = s;
}
}
void matrix_multiply(int n, const float *A, const float *B, float *C)
{
if (n != NN) { simple_mm(n, A, B, C); return; }
if (!g_A) {
if (posix_memalign((void **)&g_A, 64, (size_t)NN * (NP + APAD) * 4)) return;
if (posix_memalign((void **)&g_B, 64, (size_t)NGRP * GSTR)) return;
if (posix_memalign((void **)&g_corr, 64, (size_t)NN * 4)) return;
if (posix_memalign((void **)&g_SC, 64, (size_t)NN * NN * 4)) return;
if (posix_memalign((void **)&g_cs, 64, (size_t)NN * 4)) return;
if (posix_memalign((void **)&g_qa, 64, (size_t)NN + 64)) return;
if (posix_memalign((void **)&g_qb, 64, (size_t)NN + 64)) return;
}
memset(g_cs, 0, (size_t)NN * 4);
packA(n, A);
packB(n, B, g_cs);
for (int j = 0; j < NN; j++) g_corr[j] = g_cs[j] * 22;
memset(g_SC, 0, (size_t)NN * NN * 4);
for (int kb = 0; kb < NKB; kb++) {
for (int ib = 0; ib < NN; ib += RB) {
for (int jb = 0; jb < JBN; jb++) {
const uint8_t *b0 = g_B + (size_t)(jb * NG) * GSTR + (size_t)kb * KC * 32 - 128;
for (int i = ib; i < ib + RB; i++)
mikro(g_A + (size_t)i * (NP + APAD) + (size_t)kb * KC - 32, b0, NCH / 4,
g_SC + (size_t)i * NN + jb * (16 * NG));
}
}
}
decode(n, C);
}
| Compilation | N/A | N/A | Compile OK | Score: N/A | 显示更多 |
| Testcase #1 | 599.694 ms | 193 MB + 36 KB | Accepted | Score: 100 | 显示更多 |