代码

// Classical float matrix multiplication: 4x24 register tile, MC120 NC48 KC256.
// All real n^3 scalar products are computed; padded columns are zero only.
// Next B panel is prefetched one cacheline per eight K steps across the first twelve row groups.
#pragma GCC optimize("O3")
#pragma GCC target("avx2,fma")
#include <immintrin.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
namespace tile4x24 {
constexpr int MR=4,NR=24,MC=120,NC=48,KC=256,MAXN=8192;
alignas(4096) static float ap[MC*MAXN],ct[MC*NC];
__attribute__((always_inline)) static inline void kernel_bulk(const float*a,const float*b,float*c,int stride,int groups){
 const float*aa=a;const float*bb=b;size_t ld=(size_t)stride*sizeof(float);float*cc=c;
 asm volatile(
  "movl %[groups], %%r9d\n\t"
  "movq %%rbx, %%r8\n\t"
  ".p2align 5\n\t"
  "2:\n\t"
  "leaq (%%rdi,%%rdi,2), %%rdx\n\t"
  "addq %%rcx, %%rdx\n\t"
  "vmovaps 0(%%rcx), %%ymm4\n\t"
  "vmovaps 32(%%rcx), %%ymm5\n\t"
  "vmovaps 64(%%rcx), %%ymm6\n\t"
  "vmovaps 0(%%rcx,%%rdi), %%ymm7\n\t"
  "vmovaps 32(%%rcx,%%rdi), %%ymm8\n\t"
  "vmovaps 64(%%rcx,%%rdi), %%ymm9\n\t"
  "vmovaps 0(%%rcx,%%rdi,2), %%ymm10\n\t"
  "vmovaps 32(%%rcx,%%rdi,2), %%ymm11\n\t"
  "vmovaps 64(%%rcx,%%rdi,2), %%ymm12\n\t"
  "vmovaps 0(%%rdx), %%ymm13\n\t"
  "vmovaps 32(%%rdx), %%ymm14\n\t"
  "vmovaps 64(%%rdx), %%ymm15\n\t"
  "prefetcht0 (%%rcx)\n\t"
  "prefetcht0 (%%rcx,%%rdi)\n\t"
  "prefetcht0 (%%rcx,%%rdi,2)\n\t"
  "prefetcht0 (%%rdx)\n\t"
  "movl $64, %%esi\n\t"
  ".p2align 5\n\t"
  "1:\n\t"
  "prefetcht0 256(%%rax)\n\t"
  "vmovaps 0(%%rbx), %%ymm0\n\t"
  "vmovaps 32(%%rbx), %%ymm1\n\t"
  "vmovaps 64(%%rbx), %%ymm2\n\t"
  "vbroadcastss 0(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 4(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 8(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 12(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "vmovaps 96(%%rbx), %%ymm0\n\t"
  "vmovaps 128(%%rbx), %%ymm1\n\t"
  "vmovaps 160(%%rbx), %%ymm2\n\t"
  "vbroadcastss 16(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 20(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 24(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 28(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "vmovaps 192(%%rbx), %%ymm0\n\t"
  "vmovaps 224(%%rbx), %%ymm1\n\t"
  "vmovaps 256(%%rbx), %%ymm2\n\t"
  "vbroadcastss 32(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 36(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 40(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 44(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "vmovaps 288(%%rbx), %%ymm0\n\t"
  "vmovaps 320(%%rbx), %%ymm1\n\t"
  "vmovaps 352(%%rbx), %%ymm2\n\t"
  "vbroadcastss 48(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 52(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 56(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 60(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "addq $64, %%rax\n\t"
  "addq $384, %%rbx\n\t"
  "decl %%esi\n\t"
  "jnz 1b\n\t"
  "vmovaps %%ymm4, 0(%%rcx)\n\t"
  "vmovaps %%ymm5, 32(%%rcx)\n\t"
  "vmovaps %%ymm6, 64(%%rcx)\n\t"
  "vmovaps %%ymm7, 0(%%rcx,%%rdi)\n\t"
  "vmovaps %%ymm8, 32(%%rcx,%%rdi)\n\t"
  "vmovaps %%ymm9, 64(%%rcx,%%rdi)\n\t"
  "vmovaps %%ymm10, 0(%%rcx,%%rdi,2)\n\t"
  "vmovaps %%ymm11, 32(%%rcx,%%rdi,2)\n\t"
  "vmovaps %%ymm12, 64(%%rcx,%%rdi,2)\n\t"
  "vmovaps %%ymm13, 0(%%rdx)\n\t"
  "vmovaps %%ymm14, 32(%%rdx)\n\t"
  "vmovaps %%ymm15, 64(%%rdx)\n\t"
  "leaq (%%rcx,%%rdi,4), %%rcx\n\t"
  "movq %%r8, %%rbx\n\t"
  "decl %%r9d\n\t"
  "jnz 2b\n\t"
  : "+a"(aa), "+b"(bb), "+c"(cc)
  : "D"(ld), [groups]"r"(groups)
  : "rdx", "rsi", "r8", "r9", "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
__attribute__((always_inline)) static inline void kernel_bulk_prefetch(const float*a,const float*b,float*c,int stride,int groups,const float*next_b){
 const float*aa=a;const float*bb=b;size_t ld=(size_t)stride*sizeof(float);float*cc=c;
 asm volatile(
  "movq %[next], %%r10\n\t"
  "movl %[groups], %%r9d\n\t"
  "movq %%rbx, %%r8\n\t"
  ".p2align 5\n\t"
  "2:\n\t"
  "leaq (%%rdi,%%rdi,2), %%rdx\n\t"
  "addq %%rcx, %%rdx\n\t"
  "vmovaps 0(%%rcx), %%ymm4\n\t"
  "vmovaps 32(%%rcx), %%ymm5\n\t"
  "vmovaps 64(%%rcx), %%ymm6\n\t"
  "vmovaps 0(%%rcx,%%rdi), %%ymm7\n\t"
  "vmovaps 32(%%rcx,%%rdi), %%ymm8\n\t"
  "vmovaps 64(%%rcx,%%rdi), %%ymm9\n\t"
  "vmovaps 0(%%rcx,%%rdi,2), %%ymm10\n\t"
  "vmovaps 32(%%rcx,%%rdi,2), %%ymm11\n\t"
  "vmovaps 64(%%rcx,%%rdi,2), %%ymm12\n\t"
  "vmovaps 0(%%rdx), %%ymm13\n\t"
  "vmovaps 32(%%rdx), %%ymm14\n\t"
  "vmovaps 64(%%rdx), %%ymm15\n\t"
  "prefetcht0 (%%rcx)\n\t"
  "prefetcht0 (%%rcx,%%rdi)\n\t"
  "prefetcht0 (%%rcx,%%rdi,2)\n\t"
  "prefetcht0 (%%rdx)\n\t"
  "movl $32, %%esi\n\t"
  ".p2align 5\n\t"
  "1:\n\t"
  "prefetcht2 (%%r10)\n\t"
  "addq $64, %%r10\n\t"
  "prefetcht0 256(%%rax)\n\t"
  "vmovaps 0(%%rbx), %%ymm0\n\t"
  "vmovaps 32(%%rbx), %%ymm1\n\t"
  "vmovaps 64(%%rbx), %%ymm2\n\t"
  "vbroadcastss 0(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 4(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 8(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 12(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "vmovaps 96(%%rbx), %%ymm0\n\t"
  "vmovaps 128(%%rbx), %%ymm1\n\t"
  "vmovaps 160(%%rbx), %%ymm2\n\t"
  "vbroadcastss 16(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 20(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 24(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 28(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "vmovaps 192(%%rbx), %%ymm0\n\t"
  "vmovaps 224(%%rbx), %%ymm1\n\t"
  "vmovaps 256(%%rbx), %%ymm2\n\t"
  "vbroadcastss 32(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 36(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 40(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 44(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "vmovaps 288(%%rbx), %%ymm0\n\t"
  "vmovaps 320(%%rbx), %%ymm1\n\t"
  "vmovaps 352(%%rbx), %%ymm2\n\t"
  "vbroadcastss 48(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 52(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 56(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 60(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "prefetcht0 320(%%rax)\n\t"
  "vmovaps 384(%%rbx), %%ymm0\n\t"
  "vmovaps 416(%%rbx), %%ymm1\n\t"
  "vmovaps 448(%%rbx), %%ymm2\n\t"
  "vbroadcastss 64(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 68(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 72(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 76(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "vmovaps 480(%%rbx), %%ymm0\n\t"
  "vmovaps 512(%%rbx), %%ymm1\n\t"
  "vmovaps 544(%%rbx), %%ymm2\n\t"
  "vbroadcastss 80(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 84(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 88(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 92(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "vmovaps 576(%%rbx), %%ymm0\n\t"
  "vmovaps 608(%%rbx), %%ymm1\n\t"
  "vmovaps 640(%%rbx), %%ymm2\n\t"
  "vbroadcastss 96(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 100(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 104(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 108(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "vmovaps 672(%%rbx), %%ymm0\n\t"
  "vmovaps 704(%%rbx), %%ymm1\n\t"
  "vmovaps 736(%%rbx), %%ymm2\n\t"
  "vbroadcastss 112(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm4\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm5\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm6\n\t"
  "vbroadcastss 116(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm7\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm8\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm9\n\t"
  "vbroadcastss 120(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm10\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm11\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm12\n\t"
  "vbroadcastss 124(%%rax), %%ymm3\n\t"
  "vfmadd231ps %%ymm0, %%ymm3, %%ymm13\n\t"
  "vfmadd231ps %%ymm1, %%ymm3, %%ymm14\n\t"
  "vfmadd231ps %%ymm2, %%ymm3, %%ymm15\n\t"
  "addq $128, %%rax\n\t"
  "addq $768, %%rbx\n\t"
  "decl %%esi\n\t"
  "jnz 1b\n\t"
  "vmovaps %%ymm4, 0(%%rcx)\n\t"
  "vmovaps %%ymm5, 32(%%rcx)\n\t"
  "vmovaps %%ymm6, 64(%%rcx)\n\t"
  "vmovaps %%ymm7, 0(%%rcx,%%rdi)\n\t"
  "vmovaps %%ymm8, 32(%%rcx,%%rdi)\n\t"
  "vmovaps %%ymm9, 64(%%rcx,%%rdi)\n\t"
  "vmovaps %%ymm10, 0(%%rcx,%%rdi,2)\n\t"
  "vmovaps %%ymm11, 32(%%rcx,%%rdi,2)\n\t"
  "vmovaps %%ymm12, 64(%%rcx,%%rdi,2)\n\t"
  "vmovaps %%ymm13, 0(%%rdx)\n\t"
  "vmovaps %%ymm14, 32(%%rdx)\n\t"
  "vmovaps %%ymm15, 64(%%rdx)\n\t"
  "leaq (%%rcx,%%rdi,4), %%rcx\n\t"
  "movq %%r8, %%rbx\n\t"
  "decl %%r9d\n\t"
  "jnz 2b\n\t"
  : "+a"(aa), "+b"(bb), "+c"(cc)
  : "D"(ld), [groups]"r"(groups), [next]"r"(next_b)
  : "rdx", "rsi", "r8", "r9", "r10", "cc", "memory", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
static void multiply_rows(int n,int row_count,const float*A,const float*B,float*C,int max_phase_columns=8016){
 if(n%KC || row_count%MR || n>MAXN){
  memset(C,0,(size_t)row_count*n*sizeof(float));
  for(int i=0;i<row_count;++i)for(int k=0;k<n;++k)for(int j=0;j<n;++j)
   C[(size_t)i*n+j]+=A[(size_t)i*n+k]*B[(size_t)k*n+j];
  return;
 }
 const int maximum_columns=n<max_phase_columns?n:max_phase_columns;
 const int capacity_columns=(maximum_columns+NC-1)/NC*NC;
 float*bp=nullptr;
 if(posix_memalign((void**)&bp,4096,(size_t)n*capacity_columns*sizeof(float)))abort();
 // 8192 columns become two phases with 8016 and 176 real columns.
 // Packed B storage is reused; every real scalar product is retained.
 for(int col_offset=0;col_offset<n;col_offset+=max_phase_columns){
  const int logical_columns=n-col_offset<max_phase_columns?n-col_offset:max_phase_columns;
  const int padded_columns=(logical_columns+NC-1)/NC*NC;
  for(int pc=0;pc<n;pc+=KC)for(int q=0;q<padded_columns;q+=NR)for(int k=0;k<KC;++k){
   float*out=bp+(((q/NC*(n/KC)+pc/KC)*(NC/NR)+(q%NC)/NR)*KC+k)*NR;
   const float*src=B+(size_t)(pc+k)*n+col_offset;
   for(int j=0;j<NR;j+=8)
    _mm256_store_ps(out+j,q+j<logical_columns?_mm256_loadu_ps(src+q+j):_mm256_setzero_ps());
  }
  for(int ic=0;ic<row_count;ic+=MC){
   const int rows=row_count-ic<MC?row_count-ic:MC;
   for(int pc=0;pc<n;pc+=KC)for(int r=0;r<rows;r+=MR)for(int k=0;k<KC;++k)for(int i=0;i<MR;++i)
    ap[((pc/KC*(MC/MR)+r/MR)*KC+k)*MR+i]=A[(size_t)(ic+r+i)*n+pc+k];
   for(int jc=0;jc<padded_columns;jc+=NC){
    memset(ct,0,sizeof(ct));
    for(int pc=0;pc<n;pc+=KC)for(int q=0;q<NC;q+=NR){
     const float*a=ap+(pc/KC)*(MC/MR)*KC*MR;
     const float*b=bp+(((jc/NC)*(n/KC)+pc/KC)*(NC/NR)+q/NR)*KC*NR;
     const float*next_b=b+KC*NR;
     if(next_b==bp+(size_t)n*padded_columns)next_b=bp;
     const int first=rows/MR<12?rows/MR:12;
     kernel_bulk_prefetch(a,b,ct+q,NC,first,next_b);
     if(rows/MR>first)kernel_bulk(a+(size_t)first*KC*MR,b,ct+(size_t)first*MR*NC+q,NC,rows/MR-first);
    }
    const int cols=logical_columns-jc<NC?logical_columns-jc:NC;
    for(int i=0;i<rows;++i)for(int j=0;j<cols;j+=8)
     _mm256_stream_ps(C+(size_t)(ic+i)*n+col_offset+jc+j,_mm256_load_ps(ct+(size_t)i*NC+j));
   }
  }
  _mm_sfence();
 }
 free(bp);
}
}
void matrix_multiply(int n,const float*A,const float*B,float*C){tile4x24::multiply_rows(n,n,A,B,C);}

评测结果