|
|
|
|
########################################################################
|
|
|
|
|
# Implement fast SHA-256 with AVX1 instructions. (x86_64)
|
|
|
|
|
#
|
|
|
|
|
# Copyright (C) 2013 Intel Corporation.
|
|
|
|
|
#
|
|
|
|
|
# Authors:
|
|
|
|
|
# James Guilford <james.guilford@intel.com>
|
|
|
|
|
# Kirk Yap <kirk.s.yap@intel.com>
|
|
|
|
|
# Tim Chen <tim.c.chen@linux.intel.com>
|
|
|
|
|
#
|
|
|
|
|
# This software is available to you under a choice of one of two
|
|
|
|
|
# licenses. You may choose to be licensed under the terms of the GNU
|
|
|
|
|
# General Public License (GPL) Version 2, available from the file
|
|
|
|
|
# COPYING in the main directory of this source tree, or the
|
|
|
|
|
# OpenIB.org BSD license below:
|
|
|
|
|
#
|
|
|
|
|
# Redistribution and use in source and binary forms, with or
|
|
|
|
|
# without modification, are permitted provided that the following
|
|
|
|
|
# conditions are met:
|
|
|
|
|
#
|
|
|
|
|
# - Redistributions of source code must retain the above
|
|
|
|
|
# copyright notice, this list of conditions and the following
|
|
|
|
|
# disclaimer.
|
|
|
|
|
#
|
|
|
|
|
# - Redistributions in binary form must reproduce the above
|
|
|
|
|
# copyright notice, this list of conditions and the following
|
|
|
|
|
# disclaimer in the documentation and/or other materials
|
|
|
|
|
# provided with the distribution.
|
|
|
|
|
#
|
|
|
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
|
|
|
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
|
|
|
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
|
|
|
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
|
|
|
|
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
|
|
|
|
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
|
|
|
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
|
|
|
# SOFTWARE.
|
|
|
|
|
########################################################################
|
|
|
|
|
#
|
|
|
|
|
# This code is described in an Intel White-Paper:
|
|
|
|
|
# "Fast SHA-256 Implementations on Intel Architecture Processors"
|
|
|
|
|
#
|
|
|
|
|
# To find it, surf to http://www.intel.com/p/en_US/embedded
|
|
|
|
|
# and search for that title.
|
|
|
|
|
#
|
|
|
|
|
########################################################################
|
|
|
|
|
# This code schedules 1 block at a time, with 4 lanes per block
|
|
|
|
|
########################################################################
|
|
|
|
|
|
|
|
|
|
#ifdef HAS_AVX
|
|
|
|
|
#ifndef ENTRY
|
|
|
|
|
#define ENTRY(name) \
|
|
|
|
|
.globl name ; \
|
|
|
|
|
.align 4,0x90 ; \
|
|
|
|
|
name:
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifndef END
|
|
|
|
|
#define END(name) \
|
|
|
|
|
.size name, .-name
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifndef ENDPROC
|
|
|
|
|
#define ENDPROC(name) \
|
|
|
|
|
.type name, @function ; \
|
|
|
|
|
END(name)
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#define NUM_INVALID 100
|
|
|
|
|
|
|
|
|
|
#define TYPE_R32 0
|
|
|
|
|
#define TYPE_R64 1
|
|
|
|
|
#define TYPE_XMM 2
|
|
|
|
|
#define TYPE_INVALID 100
|
|
|
|
|
|
|
|
|
|
.macro R32_NUM opd r32
|
|
|
|
|
\opd = NUM_INVALID
|
|
|
|
|
.ifc \r32,%eax
|
|
|
|
|
\opd = 0
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%ecx
|
|
|
|
|
\opd = 1
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%edx
|
|
|
|
|
\opd = 2
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%ebx
|
|
|
|
|
\opd = 3
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%esp
|
|
|
|
|
\opd = 4
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%ebp
|
|
|
|
|
\opd = 5
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%esi
|
|
|
|
|
\opd = 6
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%edi
|
|
|
|
|
\opd = 7
|
|
|
|
|
.endif
|
|
|
|
|
#ifdef X86_64
|
|
|
|
|
.ifc \r32,%r8d
|
|
|
|
|
\opd = 8
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%r9d
|
|
|
|
|
\opd = 9
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%r10d
|
|
|
|
|
\opd = 10
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%r11d
|
|
|
|
|
\opd = 11
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%r12d
|
|
|
|
|
\opd = 12
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%r13d
|
|
|
|
|
\opd = 13
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%r14d
|
|
|
|
|
\opd = 14
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r32,%r15d
|
|
|
|
|
\opd = 15
|
|
|
|
|
.endif
|
|
|
|
|
#endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro R64_NUM opd r64
|
|
|
|
|
\opd = NUM_INVALID
|
|
|
|
|
#ifdef X86_64
|
|
|
|
|
.ifc \r64,%rax
|
|
|
|
|
\opd = 0
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%rcx
|
|
|
|
|
\opd = 1
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%rdx
|
|
|
|
|
\opd = 2
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%rbx
|
|
|
|
|
\opd = 3
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%rsp
|
|
|
|
|
\opd = 4
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%rbp
|
|
|
|
|
\opd = 5
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%rsi
|
|
|
|
|
\opd = 6
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%rdi
|
|
|
|
|
\opd = 7
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%r8
|
|
|
|
|
\opd = 8
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%r9
|
|
|
|
|
\opd = 9
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%r10
|
|
|
|
|
\opd = 10
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%r11
|
|
|
|
|
\opd = 11
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%r12
|
|
|
|
|
\opd = 12
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%r13
|
|
|
|
|
\opd = 13
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%r14
|
|
|
|
|
\opd = 14
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \r64,%r15
|
|
|
|
|
\opd = 15
|
|
|
|
|
.endif
|
|
|
|
|
#endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro XMM_NUM opd xmm
|
|
|
|
|
\opd = NUM_INVALID
|
|
|
|
|
.ifc \xmm,%xmm0
|
|
|
|
|
\opd = 0
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm1
|
|
|
|
|
\opd = 1
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm2
|
|
|
|
|
\opd = 2
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm3
|
|
|
|
|
\opd = 3
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm4
|
|
|
|
|
\opd = 4
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm5
|
|
|
|
|
\opd = 5
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm6
|
|
|
|
|
\opd = 6
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm7
|
|
|
|
|
\opd = 7
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm8
|
|
|
|
|
\opd = 8
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm9
|
|
|
|
|
\opd = 9
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm10
|
|
|
|
|
\opd = 10
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm11
|
|
|
|
|
\opd = 11
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm12
|
|
|
|
|
\opd = 12
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm13
|
|
|
|
|
\opd = 13
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm14
|
|
|
|
|
\opd = 14
|
|
|
|
|
.endif
|
|
|
|
|
.ifc \xmm,%xmm15
|
|
|
|
|
\opd = 15
|
|
|
|
|
.endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro TYPE type reg
|
|
|
|
|
R32_NUM reg_type_r32 \reg
|
|
|
|
|
R64_NUM reg_type_r64 \reg
|
|
|
|
|
XMM_NUM reg_type_xmm \reg
|
|
|
|
|
.if reg_type_r64 <> NUM_INVALID
|
|
|
|
|
\type = TYPE_R64
|
|
|
|
|
.elseif reg_type_r32 <> NUM_INVALID
|
|
|
|
|
\type = TYPE_R32
|
|
|
|
|
.elseif reg_type_xmm <> NUM_INVALID
|
|
|
|
|
\type = TYPE_XMM
|
|
|
|
|
.else
|
|
|
|
|
\type = TYPE_INVALID
|
|
|
|
|
.endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro PFX_OPD_SIZE
|
|
|
|
|
.byte 0x66
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro PFX_REX opd1 opd2 W=0
|
|
|
|
|
.if ((\opd1 | \opd2) & 8) || \W
|
|
|
|
|
.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
|
|
|
|
|
.endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro MODRM mod opd1 opd2
|
|
|
|
|
.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro PSHUFB_XMM xmm1 xmm2
|
|
|
|
|
XMM_NUM pshufb_opd1 \xmm1
|
|
|
|
|
XMM_NUM pshufb_opd2 \xmm2
|
|
|
|
|
PFX_OPD_SIZE
|
|
|
|
|
PFX_REX pshufb_opd1 pshufb_opd2
|
|
|
|
|
.byte 0x0f, 0x38, 0x00
|
|
|
|
|
MODRM 0xc0 pshufb_opd1 pshufb_opd2
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro PCLMULQDQ imm8 xmm1 xmm2
|
|
|
|
|
XMM_NUM clmul_opd1 \xmm1
|
|
|
|
|
XMM_NUM clmul_opd2 \xmm2
|
|
|
|
|
PFX_OPD_SIZE
|
|
|
|
|
PFX_REX clmul_opd1 clmul_opd2
|
|
|
|
|
.byte 0x0f, 0x3a, 0x44
|
|
|
|
|
MODRM 0xc0 clmul_opd1 clmul_opd2
|
|
|
|
|
.byte \imm8
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro PEXTRD imm8 xmm gpr
|
|
|
|
|
R32_NUM extrd_opd1 \gpr
|
|
|
|
|
XMM_NUM extrd_opd2 \xmm
|
|
|
|
|
PFX_OPD_SIZE
|
|
|
|
|
PFX_REX extrd_opd1 extrd_opd2
|
|
|
|
|
.byte 0x0f, 0x3a, 0x16
|
|
|
|
|
MODRM 0xc0 extrd_opd1 extrd_opd2
|
|
|
|
|
.byte \imm8
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro MOVQ_R64_XMM opd1 opd2
|
|
|
|
|
TYPE movq_r64_xmm_opd1_type \opd1
|
|
|
|
|
.if movq_r64_xmm_opd1_type == TYPE_XMM
|
|
|
|
|
XMM_NUM movq_r64_xmm_opd1 \opd1
|
|
|
|
|
R64_NUM movq_r64_xmm_opd2 \opd2
|
|
|
|
|
.else
|
|
|
|
|
R64_NUM movq_r64_xmm_opd1 \opd1
|
|
|
|
|
XMM_NUM movq_r64_xmm_opd2 \opd2
|
|
|
|
|
.endif
|
|
|
|
|
PFX_OPD_SIZE
|
|
|
|
|
PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
|
|
|
|
|
.if movq_r64_xmm_opd1_type == TYPE_XMM
|
|
|
|
|
.byte 0x0f, 0x7e
|
|
|
|
|
.else
|
|
|
|
|
.byte 0x0f, 0x6e
|
|
|
|
|
.endif
|
|
|
|
|
MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
## assume buffers not aligned
|
|
|
|
|
#define VMOVDQ vmovdqu
|
|
|
|
|
|
|
|
|
|
################################ Define Macros
|
|
|
|
|
|
|
|
|
|
# addm [mem], reg
|
|
|
|
|
# Add reg to mem using reg-mem add and store
|
|
|
|
|
.macro addm p1 p2
|
|
|
|
|
add \p1, \p2
|
|
|
|
|
mov \p2, \p1
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.macro MY_ROR p1 p2
|
|
|
|
|
shld $(32-(\p1)), \p2, \p2
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
################################
|
|
|
|
|
|
|
|
|
|
# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask
|
|
|
|
|
# Load xmm with mem and byte swap each dword
|
|
|
|
|
.macro COPY_XMM_AND_BSWAP p1 p2 p3
|
|
|
|
|
VMOVDQ \p2, \p1
|
|
|
|
|
vpshufb \p3, \p1, \p1
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
################################
|
|
|
|
|
|
|
|
|
|
X0 = %xmm4
|
|
|
|
|
X1 = %xmm5
|
|
|
|
|
X2 = %xmm6
|
|
|
|
|
X3 = %xmm7
|
|
|
|
|
|
|
|
|
|
XTMP0 = %xmm0
|
|
|
|
|
XTMP1 = %xmm1
|
|
|
|
|
XTMP2 = %xmm2
|
|
|
|
|
XTMP3 = %xmm3
|
|
|
|
|
XTMP4 = %xmm8
|
|
|
|
|
XFER = %xmm9
|
|
|
|
|
XTMP5 = %xmm11
|
|
|
|
|
|
|
|
|
|
SHUF_00BA = %xmm10 # shuffle xBxA -> 00BA
|
|
|
|
|
SHUF_DC00 = %xmm12 # shuffle xDxC -> DC00
|
|
|
|
|
BYTE_FLIP_MASK = %xmm13
|
|
|
|
|
|
|
|
|
|
NUM_BLKS = %rdx # 3rd arg
|
|
|
|
|
CTX = %rsi # 2nd arg
|
|
|
|
|
INP = %rdi # 1st arg
|
|
|
|
|
|
|
|
|
|
SRND = %rdi # clobbers INP
|
|
|
|
|
c = %ecx
|
|
|
|
|
d = %r8d
|
|
|
|
|
e = %edx
|
|
|
|
|
TBL = %rbp
|
|
|
|
|
a = %eax
|
|
|
|
|
b = %ebx
|
|
|
|
|
|
|
|
|
|
f = %r9d
|
|
|
|
|
g = %r10d
|
|
|
|
|
h = %r11d
|
|
|
|
|
|
|
|
|
|
y0 = %r13d
|
|
|
|
|
y1 = %r14d
|
|
|
|
|
y2 = %r15d
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
_INP_END_SIZE = 8
|
|
|
|
|
_INP_SIZE = 8
|
|
|
|
|
_XFER_SIZE = 16
|
|
|
|
|
_XMM_SAVE_SIZE = 0
|
|
|
|
|
|
|
|
|
|
_INP_END = 0
|
|
|
|
|
_INP = _INP_END + _INP_END_SIZE
|
|
|
|
|
_XFER = _INP + _INP_SIZE
|
|
|
|
|
_XMM_SAVE = _XFER + _XFER_SIZE
|
|
|
|
|
STACK_SIZE = _XMM_SAVE + _XMM_SAVE_SIZE
|
|
|
|
|
|
|
|
|
|
# rotate_Xs
|
|
|
|
|
# Rotate values of symbols X0...X3
|
|
|
|
|
.macro rotate_Xs
|
|
|
|
|
X_ = X0
|
|
|
|
|
X0 = X1
|
|
|
|
|
X1 = X2
|
|
|
|
|
X2 = X3
|
|
|
|
|
X3 = X_
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
# ROTATE_ARGS
|
|
|
|
|
# Rotate values of symbols a...h
|
|
|
|
|
.macro ROTATE_ARGS
|
|
|
|
|
TMP_ = h
|
|
|
|
|
h = g
|
|
|
|
|
g = f
|
|
|
|
|
f = e
|
|
|
|
|
e = d
|
|
|
|
|
d = c
|
|
|
|
|
c = b
|
|
|
|
|
b = a
|
|
|
|
|
a = TMP_
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro FOUR_ROUNDS_AND_SCHED
|
|
|
|
|
## compute s0 four at a time and s1 two at a time
|
|
|
|
|
## compute W[-16] + W[-7] 4 at a time
|
|
|
|
|
|
|
|
|
|
mov e, y0 # y0 = e
|
|
|
|
|
MY_ROR (25-11), y0 # y0 = e >> (25-11)
|
|
|
|
|
mov a, y1 # y1 = a
|
|
|
|
|
vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7]
|
|
|
|
|
MY_ROR (22-13), y1 # y1 = a >> (22-13)
|
|
|
|
|
xor e, y0 # y0 = e ^ (e >> (25-11))
|
|
|
|
|
mov f, y2 # y2 = f
|
|
|
|
|
MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6))
|
|
|
|
|
xor a, y1 # y1 = a ^ (a >> (22-13)
|
|
|
|
|
xor g, y2 # y2 = f^g
|
|
|
|
|
vpaddd X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16]
|
|
|
|
|
xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
|
|
|
|
|
and e, y2 # y2 = (f^g)&e
|
|
|
|
|
MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2))
|
|
|
|
|
## compute s0
|
|
|
|
|
vpalignr $4, X0, X1, XTMP1 # XTMP1 = W[-15]
|
|
|
|
|
xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
|
|
|
|
|
MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
|
|
|
|
|
xor g, y2 # y2 = CH = ((f^g)&e)^g
|
|
|
|
|
MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
|
|
|
|
|
add y0, y2 # y2 = S1 + CH
|
|
|
|
|
add _XFER(%rsp), y2 # y2 = k + w + S1 + CH
|
|
|
|
|
mov a, y0 # y0 = a
|
|
|
|
|
add y2, h # h = h + S1 + CH + k + w
|
|
|
|
|
mov a, y2 # y2 = a
|
|
|
|
|
vpsrld $7, XTMP1, XTMP2
|
|
|
|
|
or c, y0 # y0 = a|c
|
|
|
|
|
add h, d # d = d + h + S1 + CH + k + w
|
|
|
|
|
and c, y2 # y2 = a&c
|
|
|
|
|
vpslld $(32-7), XTMP1, XTMP3
|
|
|
|
|
and b, y0 # y0 = (a|c)&b
|
|
|
|
|
add y1, h # h = h + S1 + CH + k + w + S0
|
|
|
|
|
vpor XTMP2, XTMP3, XTMP3 # XTMP1 = W[-15] MY_ROR 7
|
|
|
|
|
or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c)
|
|
|
|
|
add y0, h # h = h + S1 + CH + k + w + S0 + MAJ
|
|
|
|
|
ROTATE_ARGS
|
|
|
|
|
mov a, y1 # y1 = a
|
|
|
|
|
MY_ROR (25-11), y0 # y0 = e >> (25-11)
|
|
|
|
|
xor e, y0 # y0 = e ^ (e >> (25-11))
|
|
|
|
|
mov f, y2 # y2 = f
|
|
|
|
|
MY_ROR (22-13), y1 # y1 = a >> (22-13)
|
|
|
|
|
vpsrld $18, XTMP1, XTMP2 #
|
|
|
|
|
xor a, y1 # y1 = a ^ (a >> (22-13)
|
|
|
|
|
MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6))
|
|
|
|
|
xor g, y2 # y2 = f^g
|
|
|
|
|
vpsrld $3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3
|
|
|
|
|
MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2))
|
|
|
|
|
xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
|
|
|
|
|
and e, y2 # y2 = (f^g)&e
|
|
|
|
|
MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
|
|
|
|
|
vpslld $(32-18), XTMP1, XTMP1
|
|
|
|
|
xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
|
|
|
|
|
xor g, y2 # y2 = CH = ((f^g)&e)^g
|
|
|
|
|
vpxor XTMP1, XTMP3, XTMP3 #
|
|
|
|
|
add y0, y2 # y2 = S1 + CH
|
|
|
|
|
add (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
|
|
|
|
|
MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
|
|
|
|
|
vpxor XTMP2, XTMP3, XTMP3 # XTMP1 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR
|
|
|
|
|
mov a, y0 # y0 = a
|
|
|
|
|
add y2, h # h = h + S1 + CH + k + w
|
|
|
|
|
mov a, y2 # y2 = a
|
|
|
|
|
vpxor XTMP4, XTMP3, XTMP1 # XTMP1 = s0
|
|
|
|
|
or c, y0 # y0 = a|c
|
|
|
|
|
add h, d # d = d + h + S1 + CH + k + w
|
|
|
|
|
and c, y2 # y2 = a&c
|
|
|
|
|
## compute low s1
|
|
|
|
|
vpshufd $0b11111010, X3, XTMP2 # XTMP2 = W[-2] {BBAA}
|
|
|
|
|
and b, y0 # y0 = (a|c)&b
|
|
|
|
|
add y1, h # h = h + S1 + CH + k + w + S0
|
|
|
|
|
vpaddd XTMP1, XTMP0, XTMP0 # XTMP0 = W[-16] + W[-7] + s0
|
|
|
|
|
or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c)
|
|
|
|
|
add y0, h # h = h + S1 + CH + k + w + S0 + MAJ
|
|
|
|
|
ROTATE_ARGS
|
|
|
|
|
mov e, y0 # y0 = e
|
|
|
|
|
mov a, y1 # y1 = a
|
|
|
|
|
MY_ROR (25-11), y0 # y0 = e >> (25-11)
|
|
|
|
|
xor e, y0 # y0 = e ^ (e >> (25-11))
|
|
|
|
|
MY_ROR (22-13), y1 # y1 = a >> (22-13)
|
|
|
|
|
mov f, y2 # y2 = f
|
|
|
|
|
xor a, y1 # y1 = a ^ (a >> (22-13)
|
|
|
|
|
MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6))
|
|
|
|
|
vpsrld $10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA}
|
|
|
|
|
xor g, y2 # y2 = f^g
|
|
|
|
|
vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] MY_ROR 19 {xBxA}
|
|
|
|
|
xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
|
|
|
|
|
and e, y2 # y2 = (f^g)&e
|
|
|
|
|
vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] MY_ROR 17 {xBxA}
|
|
|
|
|
MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2))
|
|
|
|
|
xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
|
|
|
|
|
xor g, y2 # y2 = CH = ((f^g)&e)^g
|
|
|
|
|
MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
|
|
|
|
|
vpxor XTMP3, XTMP2, XTMP2 #
|
|
|
|
|
add y0, y2 # y2 = S1 + CH
|
|
|
|
|
MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
|
|
|
|
|
add (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
|
|
|
|
|
vpxor XTMP2, XTMP4, XTMP4 # XTMP4 = s1 {xBxA}
|
|
|
|
|
mov a, y0 # y0 = a
|
|
|
|
|
add y2, h # h = h + S1 + CH + k + w
|
|
|
|
|
mov a, y2 # y2 = a
|
|
|
|
|
vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA}
|
|
|
|
|
or c, y0 # y0 = a|c
|
|
|
|
|
add h, d # d = d + h + S1 + CH + k + w
|
|
|
|
|
and c, y2 # y2 = a&c
|
|
|
|
|
vpaddd XTMP4, XTMP0, XTMP0 # XTMP0 = {..., ..., W[1], W[0]}
|
|
|
|
|
and b, y0 # y0 = (a|c)&b
|
|
|
|
|
add y1, h # h = h + S1 + CH + k + w + S0
|
|
|
|
|
## compute high s1
|
|
|
|
|
vpshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {DDCC}
|
|
|
|
|
or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c)
|
|
|
|
|
add y0, h # h = h + S1 + CH + k + w + S0 + MAJ
|
|
|
|
|
ROTATE_ARGS
|
|
|
|
|
mov e, y0 # y0 = e
|
|
|
|
|
MY_ROR (25-11), y0 # y0 = e >> (25-11)
|
|
|
|
|
mov a, y1 # y1 = a
|
|
|
|
|
MY_ROR (22-13), y1 # y1 = a >> (22-13)
|
|
|
|
|
xor e, y0 # y0 = e ^ (e >> (25-11))
|
|
|
|
|
mov f, y2 # y2 = f
|
|
|
|
|
MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6))
|
|
|
|
|
vpsrld $10, XTMP2, XTMP5 # XTMP5 = W[-2] >> 10 {DDCC}
|
|
|
|
|
xor a, y1 # y1 = a ^ (a >> (22-13)
|
|
|
|
|
xor g, y2 # y2 = f^g
|
|
|
|
|
vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] MY_ROR 19 {xDxC}
|
|
|
|
|
xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
|
|
|
|
|
and e, y2 # y2 = (f^g)&e
|
|
|
|
|
MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2))
|
|
|
|
|
vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] MY_ROR 17 {xDxC}
|
|
|
|
|
xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
|
|
|
|
|
MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
|
|
|
|
|
xor g, y2 # y2 = CH = ((f^g)&e)^g
|
|
|
|
|
vpxor XTMP3, XTMP2, XTMP2
|
|
|
|
|
MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
|
|
|
|
|
add y0, y2 # y2 = S1 + CH
|
|
|
|
|
add (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
|
|
|
|
|
vpxor XTMP2, XTMP5, XTMP5 # XTMP5 = s1 {xDxC}
|
|
|
|
|
mov a, y0 # y0 = a
|
|
|
|
|
add y2, h # h = h + S1 + CH + k + w
|
|
|
|
|
mov a, y2 # y2 = a
|
|
|
|
|
vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00}
|
|
|
|
|
or c, y0 # y0 = a|c
|
|
|
|
|
add h, d # d = d + h + S1 + CH + k + w
|
|
|
|
|
and c, y2 # y2 = a&c
|
|
|
|
|
vpaddd XTMP0, XTMP5, X0 # X0 = {W[3], W[2], W[1], W[0]}
|
|
|
|
|
and b, y0 # y0 = (a|c)&b
|
|
|
|
|
add y1, h # h = h + S1 + CH + k + w + S0
|
|
|
|
|
or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c)
|
|
|
|
|
add y0, h # h = h + S1 + CH + k + w + S0 + MAJ
|
|
|
|
|
ROTATE_ARGS
|
|
|
|
|
rotate_Xs
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
## input is [rsp + _XFER + %1 * 4]
|
|
|
|
|
.macro DO_ROUND round
|
|
|
|
|
mov e, y0 # y0 = e
|
|
|
|
|
MY_ROR (25-11), y0 # y0 = e >> (25-11)
|
|
|
|
|
mov a, y1 # y1 = a
|
|
|
|
|
xor e, y0 # y0 = e ^ (e >> (25-11))
|
|
|
|
|
MY_ROR (22-13), y1 # y1 = a >> (22-13)
|
|
|
|
|
mov f, y2 # y2 = f
|
|
|
|
|
xor a, y1 # y1 = a ^ (a >> (22-13)
|
|
|
|
|
MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6))
|
|
|
|
|
xor g, y2 # y2 = f^g
|
|
|
|
|
xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
|
|
|
|
|
MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2))
|
|
|
|
|
and e, y2 # y2 = (f^g)&e
|
|
|
|
|
xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
|
|
|
|
|
MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
|
|
|
|
|
xor g, y2 # y2 = CH = ((f^g)&e)^g
|
|
|
|
|
add y0, y2 # y2 = S1 + CH
|
|
|
|
|
MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
|
|
|
|
|
offset = \round * 4 + _XFER #
|
|
|
|
|
add offset(%rsp), y2 # y2 = k + w + S1 + CH
|
|
|
|
|
mov a, y0 # y0 = a
|
|
|
|
|
add y2, h # h = h + S1 + CH + k + w
|
|
|
|
|
mov a, y2 # y2 = a
|
|
|
|
|
or c, y0 # y0 = a|c
|
|
|
|
|
add h, d # d = d + h + S1 + CH + k + w
|
|
|
|
|
and c, y2 # y2 = a&c
|
|
|
|
|
and b, y0 # y0 = (a|c)&b
|
|
|
|
|
add y1, h # h = h + S1 + CH + k + w + S0
|
|
|
|
|
or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c)
|
|
|
|
|
add y0, h # h = h + S1 + CH + k + w + S0 + MAJ
|
|
|
|
|
ROTATE_ARGS
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
########################################################################
|
|
|
|
|
## void sha256_transform_avx(void *input_data, UINT32 digest[8], UINT64 num_blks)
|
|
|
|
|
## arg 1 : pointer to input data
|
|
|
|
|
## arg 2 : pointer to digest
|
|
|
|
|
## arg 3 : Num blocks
|
|
|
|
|
########################################################################
|
|
|
|
|
.text
|
|
|
|
|
ENTRY(sha256_transform_avx)
|
|
|
|
|
.align 32
|
|
|
|
|
pushq %rbx
|
|
|
|
|
pushq %rbp
|
|
|
|
|
pushq %r13
|
|
|
|
|
pushq %r14
|
|
|
|
|
pushq %r15
|
|
|
|
|
pushq %r12
|
|
|
|
|
|
|
|
|
|
mov %rsp, %r12
|
|
|
|
|
subq $STACK_SIZE, %rsp # allocate stack space
|
|
|
|
|
and $~15, %rsp # align stack pointer
|
|
|
|
|
|
|
|
|
|
shl $6, NUM_BLKS # convert to bytes
|
|
|
|
|
jz done_hash
|
|
|
|
|
add INP, NUM_BLKS # pointer to end of data
|
|
|
|
|
mov NUM_BLKS, _INP_END(%rsp)
|
|
|
|
|
|
|
|
|
|
## load initial digest
|
|
|
|
|
mov 4*0(CTX), a
|
|
|
|
|
mov 4*1(CTX), b
|
|
|
|
|
mov 4*2(CTX), c
|
|
|
|
|
mov 4*3(CTX), d
|
|
|
|
|
mov 4*4(CTX), e
|
|
|
|
|
mov 4*5(CTX), f
|
|
|
|
|
mov 4*6(CTX), g
|
|
|
|
|
mov 4*7(CTX), h
|
|
|
|
|
|
|
|
|
|
vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
|
|
|
|
|
vmovdqa _SHUF_00BA(%rip), SHUF_00BA
|
|
|
|
|
vmovdqa _SHUF_DC00(%rip), SHUF_DC00
|
|
|
|
|
loop0:
|
|
|
|
|
lea K256(%rip), TBL
|
|
|
|
|
|
|
|
|
|
## byte swap first 16 dwords
|
|
|
|
|
COPY_XMM_AND_BSWAP X0, 0*16(INP), BYTE_FLIP_MASK
|
|
|
|
|
COPY_XMM_AND_BSWAP X1, 1*16(INP), BYTE_FLIP_MASK
|
|
|
|
|
COPY_XMM_AND_BSWAP X2, 2*16(INP), BYTE_FLIP_MASK
|
|
|
|
|
COPY_XMM_AND_BSWAP X3, 3*16(INP), BYTE_FLIP_MASK
|
|
|
|
|
|
|
|
|
|
mov INP, _INP(%rsp)
|
|
|
|
|
|
|
|
|
|
## schedule 48 input dwords, by doing 3 rounds of 16 each
|
|
|
|
|
mov $3, SRND
|
|
|
|
|
.align 16
|
|
|
|
|
loop1:
|
|
|
|
|
vpaddd (TBL), X0, XFER
|
|
|
|
|
vmovdqa XFER, _XFER(%rsp)
|
|
|
|
|
FOUR_ROUNDS_AND_SCHED
|
|
|
|
|
|
|
|
|
|
vpaddd 1*16(TBL), X0, XFER
|
|
|
|
|
vmovdqa XFER, _XFER(%rsp)
|
|
|
|
|
FOUR_ROUNDS_AND_SCHED
|
|
|
|
|
|
|
|
|
|
vpaddd 2*16(TBL), X0, XFER
|
|
|
|
|
vmovdqa XFER, _XFER(%rsp)
|
|
|
|
|
FOUR_ROUNDS_AND_SCHED
|
|
|
|
|
|
|
|
|
|
vpaddd 3*16(TBL), X0, XFER
|
|
|
|
|
vmovdqa XFER, _XFER(%rsp)
|
|
|
|
|
add $4*16, TBL
|
|
|
|
|
FOUR_ROUNDS_AND_SCHED
|
|
|
|
|
|
|
|
|
|
sub $1, SRND
|
|
|
|
|
jne loop1
|
|
|
|
|
|
|
|
|
|
mov $2, SRND
|
|
|
|
|
loop2:
|
|
|
|
|
vpaddd (TBL), X0, XFER
|
|
|
|
|
vmovdqa XFER, _XFER(%rsp)
|
|
|
|
|
DO_ROUND 0
|
|
|
|
|
DO_ROUND 1
|
|
|
|
|
DO_ROUND 2
|
|
|
|
|
DO_ROUND 3
|
|
|
|
|
|
|
|
|
|
vpaddd 1*16(TBL), X1, XFER
|
|
|
|
|
vmovdqa XFER, _XFER(%rsp)
|
|
|
|
|
add $2*16, TBL
|
|
|
|
|
DO_ROUND 0
|
|
|
|
|
DO_ROUND 1
|
|
|
|
|
DO_ROUND 2
|
|
|
|
|
DO_ROUND 3
|
|
|
|
|
|
|
|
|
|
vmovdqa X2, X0
|
|
|
|
|
vmovdqa X3, X1
|
|
|
|
|
|
|
|
|
|
sub $1, SRND
|
|
|
|
|
jne loop2
|
|
|
|
|
|
|
|
|
|
addm (4*0)(CTX),a
|
|
|
|
|
addm (4*1)(CTX),b
|
|
|
|
|
addm (4*2)(CTX),c
|
|
|
|
|
addm (4*3)(CTX),d
|
|
|
|
|
addm (4*4)(CTX),e
|
|
|
|
|
addm (4*5)(CTX),f
|
|
|
|
|
addm (4*6)(CTX),g
|
|
|
|
|
addm (4*7)(CTX),h
|
|
|
|
|
|
|
|
|
|
mov _INP(%rsp), INP
|
|
|
|
|
add $64, INP
|
|
|
|
|
cmp _INP_END(%rsp), INP
|
|
|
|
|
jne loop0
|
|
|
|
|
|
|
|
|
|
done_hash:
|
|
|
|
|
|
|
|
|
|
mov %r12, %rsp
|
|
|
|
|
|
|
|
|
|
popq %r12
|
|
|
|
|
popq %r15
|
|
|
|
|
popq %r14
|
|
|
|
|
popq %r13
|
|
|
|
|
popq %rbp
|
|
|
|
|
popq %rbx
|
|
|
|
|
ret
|
|
|
|
|
ENDPROC(sha256_transform_avx)
|
|
|
|
|
|
|
|
|
|
.data
|
|
|
|
|
.align 64
|
|
|
|
|
K256:
|
|
|
|
|
.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
|
|
|
|
|
.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
|
|
|
|
|
.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
|
|
|
|
|
.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
|
|
|
|
|
.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
|
|
|
|
|
.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
|
|
|
|
|
.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
|
|
|
|
|
.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
|
|
|
|
|
.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
|
|
|
|
|
.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
|
|
|
|
|
.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
|
|
|
|
|
.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
|
|
|
|
|
.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
|
|
|
|
|
.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
|
|
|
|
|
.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
|
|
|
|
|
.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
|
|
|
|
|
|
|
|
|
|
PSHUFFLE_BYTE_FLIP_MASK:
|
|
|
|
|
.octa 0x0c0d0e0f08090a0b0405060700010203
|
|
|
|
|
|
|
|
|
|
# shuffle xBxA -> 00BA
|
|
|
|
|
_SHUF_00BA:
|
|
|
|
|
.octa 0xFFFFFFFFFFFFFFFF0b0a090803020100
|
|
|
|
|
|
|
|
|
|
# shuffle xDxC -> DC00
|
|
|
|
|
_SHUF_DC00:
|
|
|
|
|
.octa 0x0b0a090803020100FFFFFFFFFFFFFFFF
|
|
|
|
|
#endif
|
