public/openssl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
openssl/crypto/chacha/asm/chacha-s390x.pl
Richard Levitte 1aa89a7a3a Unify all assembler file generators 
They now generally conform to the following argument sequence:

    script.pl "$(PERLASM_SCHEME)" [ C preprocessor arguments ... ] \
              $(PROCESSOR) <output file>

However, in the spirit of being able to use these scripts manually,
they also allow for no argument, or for only the flavour, or for only
the output file.  This is done by only using the last argument as
output file if it's a file (it has an extension), and only using the
first argument as flavour if it isn't a file (it doesn't have an
extension).

While we're at it, we make all $xlate calls the same, i.e. the $output
argument is always quoted, and we always die on error when trying to
start $xlate.

There's a perl lesson in this, regarding operator priority...

This will always succeed, even when it fails:

    open FOO, "something" || die "ERR: $!";

The reason is that '||' has higher priority than list operators (a
function is essentially a list operator and gobbles up everything
following it that isn't lower priority), and since a non-empty string
is always true, so that ends up being exactly the same as:

    open FOO, "something";

This, however, will fail if "something" can't be opened:

    open FOO, "something" or die "ERR: $!";

The reason is that 'or' has lower priority that list operators,
i.e. it's performed after the 'open' call.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/9884)
2019-09-16 16:29:57 +02:00

1056 lines
26 KiB
Perl
Executable file
Raw Blame History

#! /usr/bin/env perl
# Copyright 2016-2019 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# December 2015
#
# ChaCha20 for s390x.
#
# 3 times faster than compiler-generated code.
#
# August 2018
#
# Add vx code path: 4x"vertical".
#
# Copyright IBM Corp. 2018
# Author: Patrick Steuer <patrick.steuer@de.ibm.com>
#
# February 2019
#
# Add 6x"horizontal" VX implementation. It's ~25% faster than IBM's
# 4x"vertical" submission [on z13] and >3 faster than scalar code.
# But to harness overheads revert to transliteration of VSX code path
# from chacha-ppc module, which is also 4x"vertical", to handle inputs
# not longer than 256 bytes.
use strict;
use FindBin qw($Bin);
use lib "$Bin/../..";
use perlasm::s390x qw(:DEFAULT :VX :LD AUTOLOAD LABEL INCLUDE);
# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
my ($z,$SIZE_T);
if ($flavour =~ /3[12]/) {
$z=0; # S/390 ABI
$SIZE_T=4;
} else {
$z=1; # zSeries ABI
$SIZE_T=8;
}
my $sp="%r15";
my $stdframe=16*$SIZE_T+4*8;
sub ROUND {
my @x=map("%r$_",(0..7,"x","x","x","x",(10..13)));
my @t=map("%r$_",(8,9));
my ($a0,$b0,$c0,$d0)=@_;
my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
my ($xc,$xc_)=map("$_",@t);
# Consider order in which variables are addressed by their
# index:
#
# a b c d
#
# 0 4 8 12 < even round
# 1 5 9 13
# 2 6 10 14
# 3 7 11 15
# 0 5 10 15 < odd round
# 1 6 11 12
# 2 7 8 13
# 3 4 9 14
#
# 'a', 'b' and 'd's are permanently allocated in registers,
# @x[0..7,12..15], while 'c's are maintained in memory. If
# you observe 'c' column, you'll notice that pair of 'c's is
# invariant between rounds. This means that we have to reload
# them once per round, in the middle. This is why you'll see
# 'c' stores and loads in the middle, but none in the beginning
# or end.
alr (@x[$a0],@x[$b0]); # Q1
alr (@x[$a1],@x[$b1]); # Q2
xr (@x[$d0],@x[$a0]);
xr (@x[$d1],@x[$a1]);
rll (@x[$d0],@x[$d0],16);
rll (@x[$d1],@x[$d1],16);
alr ($xc,@x[$d0]);
alr ($xc_,@x[$d1]);
xr (@x[$b0],$xc);
xr (@x[$b1],$xc_);
rll (@x[$b0],@x[$b0],12);
rll (@x[$b1],@x[$b1],12);
alr (@x[$a0],@x[$b0]);
alr (@x[$a1],@x[$b1]);
xr (@x[$d0],@x[$a0]);
xr (@x[$d1],@x[$a1]);
rll (@x[$d0],@x[$d0],8);
rll (@x[$d1],@x[$d1],8);
alr ($xc,@x[$d0]);
alr ($xc_,@x[$d1]);
xr (@x[$b0],$xc);
xr (@x[$b1],$xc_);
rll (@x[$b0],@x[$b0],7);
rll (@x[$b1],@x[$b1],7);
stm ($xc,$xc_,"$stdframe+4*8+4*$c0($sp)"); # reload pair of 'c's
lm ($xc,$xc_,"$stdframe+4*8+4*$c2($sp)");
alr (@x[$a2],@x[$b2]); # Q3
alr (@x[$a3],@x[$b3]); # Q4
xr (@x[$d2],@x[$a2]);
xr (@x[$d3],@x[$a3]);
rll (@x[$d2],@x[$d2],16);
rll (@x[$d3],@x[$d3],16);
alr ($xc,@x[$d2]);
alr ($xc_,@x[$d3]);
xr (@x[$b2],$xc);
xr (@x[$b3],$xc_);
rll (@x[$b2],@x[$b2],12);
rll (@x[$b3],@x[$b3],12);
alr (@x[$a2],@x[$b2]);
alr (@x[$a3],@x[$b3]);
xr (@x[$d2],@x[$a2]);
xr (@x[$d3],@x[$a3]);
rll (@x[$d2],@x[$d2],8);
rll (@x[$d3],@x[$d3],8);
alr ($xc,@x[$d2]);
alr ($xc_,@x[$d3]);
xr (@x[$b2],$xc);
xr (@x[$b3],$xc_);
rll (@x[$b2],@x[$b2],7);
rll (@x[$b3],@x[$b3],7);
}
sub VX_lane_ROUND {
my ($a0,$b0,$c0,$d0)=@_;
my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
my @x=map("%v$_",(0..15));
vaf (@x[$a0],@x[$a0],@x[$b0]); # Q1
vx (@x[$d0],@x[$d0],@x[$a0]);
verllf (@x[$d0],@x[$d0],16);
vaf (@x[$a1],@x[$a1],@x[$b1]); # Q2
vx (@x[$d1],@x[$d1],@x[$a1]);
verllf (@x[$d1],@x[$d1],16);
vaf (@x[$a2],@x[$a2],@x[$b2]); # Q3
vx (@x[$d2],@x[$d2],@x[$a2]);
verllf (@x[$d2],@x[$d2],16);
vaf (@x[$a3],@x[$a3],@x[$b3]); # Q4
vx (@x[$d3],@x[$d3],@x[$a3]);
verllf (@x[$d3],@x[$d3],16);
vaf (@x[$c0],@x[$c0],@x[$d0]);
vx (@x[$b0],@x[$b0],@x[$c0]);
verllf (@x[$b0],@x[$b0],12);
vaf (@x[$c1],@x[$c1],@x[$d1]);
vx (@x[$b1],@x[$b1],@x[$c1]);
verllf (@x[$b1],@x[$b1],12);
vaf (@x[$c2],@x[$c2],@x[$d2]);
vx (@x[$b2],@x[$b2],@x[$c2]);
verllf (@x[$b2],@x[$b2],12);
vaf (@x[$c3],@x[$c3],@x[$d3]);
vx (@x[$b3],@x[$b3],@x[$c3]);
verllf (@x[$b3],@x[$b3],12);
vaf (@x[$a0],@x[$a0],@x[$b0]);
vx (@x[$d0],@x[$d0],@x[$a0]);
verllf (@x[$d0],@x[$d0],8);
vaf (@x[$a1],@x[$a1],@x[$b1]);
vx (@x[$d1],@x[$d1],@x[$a1]);
verllf (@x[$d1],@x[$d1],8);
vaf (@x[$a2],@x[$a2],@x[$b2]);
vx (@x[$d2],@x[$d2],@x[$a2]);
verllf (@x[$d2],@x[$d2],8);
vaf (@x[$a3],@x[$a3],@x[$b3]);
vx (@x[$d3],@x[$d3],@x[$a3]);
verllf (@x[$d3],@x[$d3],8);
vaf (@x[$c0],@x[$c0],@x[$d0]);
vx (@x[$b0],@x[$b0],@x[$c0]);
verllf (@x[$b0],@x[$b0],7);
vaf (@x[$c1],@x[$c1],@x[$d1]);
vx (@x[$b1],@x[$b1],@x[$c1]);
verllf (@x[$b1],@x[$b1],7);
vaf (@x[$c2],@x[$c2],@x[$d2]);
vx (@x[$b2],@x[$b2],@x[$c2]);
verllf (@x[$b2],@x[$b2],7);
vaf (@x[$c3],@x[$c3],@x[$d3]);
vx (@x[$b3],@x[$b3],@x[$c3]);
verllf (@x[$b3],@x[$b3],7);
}
sub VX_ROUND {
my @a=@_[0..5];
my @b=@_[6..11];
my @c=@_[12..17];
my @d=@_[18..23];
my $odd=@_[24];
vaf (@a[$_],@a[$_],@b[$_]) for (0..5);
vx (@d[$_],@d[$_],@a[$_]) for (0..5);
verllf (@d[$_],@d[$_],16) for (0..5);
vaf (@c[$_],@c[$_],@d[$_]) for (0..5);
vx (@b[$_],@b[$_],@c[$_]) for (0..5);
verllf (@b[$_],@b[$_],12) for (0..5);
vaf (@a[$_],@a[$_],@b[$_]) for (0..5);
vx (@d[$_],@d[$_],@a[$_]) for (0..5);
verllf (@d[$_],@d[$_],8) for (0..5);
vaf (@c[$_],@c[$_],@d[$_]) for (0..5);
vx (@b[$_],@b[$_],@c[$_]) for (0..5);
verllf (@b[$_],@b[$_],7) for (0..5);
vsldb (@c[$_],@c[$_],@c[$_],8) for (0..5);
vsldb (@b[$_],@b[$_],@b[$_],$odd?12:4) for (0..5);
vsldb (@d[$_],@d[$_],@d[$_],$odd?4:12) for (0..5);
}
PERLASM_BEGIN($output);
INCLUDE ("s390x_arch.h");
TEXT ();
################
# void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len,
# const unsigned int key[8], const unsigned int counter[4])
my ($out,$inp,$len,$key,$counter)=map("%r$_",(2..6));
{
my $frame=$stdframe+4*20;
my @x=map("%r$_",(0..7,"x","x","x","x",(10..13)));
my @t=map("%r$_",(8,9));
GLOBL ("ChaCha20_ctr32");
TYPE ("ChaCha20_ctr32","\@function");
ALIGN (32);
LABEL ("ChaCha20_ctr32");
larl ("%r1","OPENSSL_s390xcap_P");
lghi ("%r0",64);
&{$z? \&ltgr:\&ltr} ($len,$len); # len==0?
bzr ("%r14");
lg ("%r1","S390X_STFLE+16(%r1)");
&{$z? \&clgr:\&clr} ($len,"%r0");
jle (".Lshort");
tmhh ("%r1",0x4000); # check for vx bit
jnz (".LChaCha20_ctr32_vx");
LABEL (".Lshort");
&{$z? \&aghi:\&ahi} ($len,-64);
&{$z? \&lghi:\&lhi} ("%r1",-$frame);
&{$z? \&stmg:\&stm} ("%r6","%r15","6*$SIZE_T($sp)");
&{$z? \&slgr:\&slr} ($out,$inp); # difference
la ($len,"0($inp,$len)"); # end of input minus 64
larl ("%r7",".Lsigma");
lgr ("%r0",$sp);
la ($sp,"0(%r1,$sp)");
&{$z? \&stg:\&st} ("%r0","0($sp)");
lmg ("%r8","%r11","0($key)"); # load key
lmg ("%r12","%r13","0($counter)"); # load counter
lmg ("%r6","%r7","0(%r7)"); # load sigma constant
la ("%r14","0($inp)");
&{$z? \&stg:\&st} ($out,"$frame+3*$SIZE_T($sp)");
&{$z? \&stg:\&st} ($len,"$frame+4*$SIZE_T($sp)");
stmg ("%r6","%r13","$stdframe($sp)");# copy key schedule to stack
srlg (@x[12],"%r12",32); # 32-bit counter value
j (".Loop_outer");
ALIGN (16);
LABEL (".Loop_outer");
lm (@x[0],@x[7],"$stdframe+4*0($sp)"); # load x[0]-x[7]
lm (@t[0],@t[1],"$stdframe+4*10($sp)"); # load x[10]-x[11]
lm (@x[13],@x[15],"$stdframe+4*13($sp)"); # load x[13]-x[15]
stm (@t[0],@t[1],"$stdframe+4*8+4*10($sp)");# offload x[10]-x[11]
lm (@t[0],@t[1],"$stdframe+4*8($sp)"); # load x[8]-x[9]
st (@x[12],"$stdframe+4*12($sp)"); # save counter
&{$z? \&stg:\&st} ("%r14","$frame+2*$SIZE_T($sp)");# save input pointer
lhi ("%r14",10);
j (".Loop");
ALIGN (4);
LABEL (".Loop");
ROUND (0, 4, 8,12);
ROUND (0, 5,10,15);
brct ("%r14",".Loop");
&{$z? \&lg:\&l} ("%r14","$frame+2*$SIZE_T($sp)");# pull input pointer
stm (@t[0],@t[1],"$stdframe+4*8+4*8($sp)"); # offload x[8]-x[9]
&{$z? \&lmg:\&lm} (@t[0],@t[1],"$frame+3*$SIZE_T($sp)");
al (@x[0],"$stdframe+4*0($sp)"); # accumulate key schedule
al (@x[1],"$stdframe+4*1($sp)");
al (@x[2],"$stdframe+4*2($sp)");
al (@x[3],"$stdframe+4*3($sp)");
al (@x[4],"$stdframe+4*4($sp)");
al (@x[5],"$stdframe+4*5($sp)");
al (@x[6],"$stdframe+4*6($sp)");
al (@x[7],"$stdframe+4*7($sp)");
lrvr (@x[0],@x[0]);
lrvr (@x[1],@x[1]);
lrvr (@x[2],@x[2]);
lrvr (@x[3],@x[3]);
lrvr (@x[4],@x[4]);
lrvr (@x[5],@x[5]);
lrvr (@x[6],@x[6]);
lrvr (@x[7],@x[7]);
al (@x[12],"$stdframe+4*12($sp)");
al (@x[13],"$stdframe+4*13($sp)");
al (@x[14],"$stdframe+4*14($sp)");
al (@x[15],"$stdframe+4*15($sp)");
lrvr (@x[12],@x[12]);
lrvr (@x[13],@x[13]);
lrvr (@x[14],@x[14]);
lrvr (@x[15],@x[15]);
la (@t[0],"0(@t[0],%r14)"); # reconstruct output pointer
&{$z? \&clgr:\&clr} ("%r14",@t[1]);
jh (".Ltail");
x (@x[0],"4*0(%r14)"); # xor with input
x (@x[1],"4*1(%r14)");
st (@x[0],"4*0(@t[0])"); # store output
x (@x[2],"4*2(%r14)");
st (@x[1],"4*1(@t[0])");
x (@x[3],"4*3(%r14)");
st (@x[2],"4*2(@t[0])");
x (@x[4],"4*4(%r14)");
st (@x[3],"4*3(@t[0])");
lm (@x[0],@x[3],"$stdframe+4*8+4*8($sp)"); # load x[8]-x[11]
x (@x[5],"4*5(%r14)");
st (@x[4],"4*4(@t[0])");
x (@x[6],"4*6(%r14)");
al (@x[0],"$stdframe+4*8($sp)");
st (@x[5],"4*5(@t[0])");
x (@x[7],"4*7(%r14)");
al (@x[1],"$stdframe+4*9($sp)");
st (@x[6],"4*6(@t[0])");
x (@x[12],"4*12(%r14)");
al (@x[2],"$stdframe+4*10($sp)");
st (@x[7],"4*7(@t[0])");
x (@x[13],"4*13(%r14)");
al (@x[3],"$stdframe+4*11($sp)");
st (@x[12],"4*12(@t[0])");
x (@x[14],"4*14(%r14)");
st (@x[13],"4*13(@t[0])");
x (@x[15],"4*15(%r14)");
st (@x[14],"4*14(@t[0])");
lrvr (@x[0],@x[0]);
st (@x[15],"4*15(@t[0])");
lrvr (@x[1],@x[1]);
lrvr (@x[2],@x[2]);
lrvr (@x[3],@x[3]);
lhi (@x[12],1);
x (@x[0],"4*8(%r14)");
al (@x[12],"$stdframe+4*12($sp)"); # increment counter
x (@x[1],"4*9(%r14)");
st (@x[0],"4*8(@t[0])");
x (@x[2],"4*10(%r14)");
st (@x[1],"4*9(@t[0])");
x (@x[3],"4*11(%r14)");
st (@x[2],"4*10(@t[0])");
st (@x[3],"4*11(@t[0])");
&{$z? \&clgr:\&clr} ("%r14",@t[1]); # done yet?
la ("%r14","64(%r14)");
jl (".Loop_outer");
LABEL (".Ldone");
xgr ("%r0","%r0");
xgr ("%r1","%r1");
xgr ("%r2","%r2");
xgr ("%r3","%r3");
stmg ("%r0","%r3","$stdframe+4*4($sp)"); # wipe key copy
stmg ("%r0","%r3","$stdframe+4*12($sp)");
&{$z? \&lmg:\&lm} ("%r6","%r15","$frame+6*$SIZE_T($sp)");
br ("%r14");
ALIGN (16);
LABEL (".Ltail");
la (@t[1],"64($t[1])");
stm (@x[0],@x[7],"$stdframe+4*0($sp)");
&{$z? \&slgr:\&slr} (@t[1],"%r14");
lm (@x[0],@x[3],"$stdframe+4*8+4*8($sp)");
&{$z? \&lghi:\&lhi} (@x[6],0);
stm (@x[12],@x[15],"$stdframe+4*12($sp)");
al (@x[0],"$stdframe+4*8($sp)");
al (@x[1],"$stdframe+4*9($sp)");
al (@x[2],"$stdframe+4*10($sp)");
al (@x[3],"$stdframe+4*11($sp)");
lrvr (@x[0],@x[0]);
lrvr (@x[1],@x[1]);
lrvr (@x[2],@x[2]);
lrvr (@x[3],@x[3]);
stm (@x[0],@x[3],"$stdframe+4*8($sp)");
LABEL (".Loop_tail");
llgc (@x[4],"0(@x[6],%r14)");
llgc (@x[5],"$stdframe(@x[6],$sp)");
xr (@x[5],@x[4]);
stc (@x[5],"0(@x[6],@t[0])");
la (@x[6],"1(@x[6])");
brct (@t[1],".Loop_tail");
j (".Ldone");
SIZE ("ChaCha20_ctr32",".-ChaCha20_ctr32");
}
########################################################################
# 4x"vertical" layout minimizes amount of instructions, but pipeline
# runs underutilized [because of vector instructions' high latency].
# On the other hand minimum amount of data it takes to fully utilize
# the pipeline is higher, so that effectively, short inputs would be
# processed slower. Hence this code path targeting <=256 bytes lengths.
#
{
my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
$xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%v$_",(0..15));
my @K=map("%v$_",(16..19));
my $CTR="%v26";
my ($xt0,$xt1,$xt2,$xt3)=map("%v$_",(27..30));
my $beperm="%v31";
my ($x00,$x10,$x20,$x30)=(0,map("r$_",(8..10)));
my $FRAME=$stdframe+4*16;
ALIGN (32);
LABEL ("ChaCha20_ctr32_4x");
LABEL (".LChaCha20_ctr32_4x");
&{$z? \&stmg:\&stm} ("%r6","%r7","6*$SIZE_T($sp)");
if (!$z) {
std ("%f4","16*$SIZE_T+2*8($sp)");
std ("%f6","16*$SIZE_T+3*8($sp)");
}
&{$z? \&lghi:\&lhi} ("%r1",-$FRAME);
lgr ("%r0",$sp);
la ($sp,"0(%r1,$sp)");
&{$z? \&stg:\&st} ("%r0","0($sp)"); # back-chain
if ($z) {
std ("%f8","$stdframe+8*0($sp)");
std ("%f9","$stdframe+8*1($sp)");
std ("%f10","$stdframe+8*2($sp)");
std ("%f11","$stdframe+8*3($sp)");
std ("%f12","$stdframe+8*4($sp)");
std ("%f13","$stdframe+8*5($sp)");
std ("%f14","$stdframe+8*6($sp)");
std ("%f15","$stdframe+8*7($sp)");
}
larl ("%r7",".Lsigma");
lhi ("%r0",10);
lhi ("%r1",0);
vl (@K[0],"0(%r7)"); # load sigma
vl (@K[1],"0($key)"); # load key
vl (@K[2],"16($key)");
vl (@K[3],"0($counter)"); # load counter
vl ($beperm,"0x40(%r7)");
vl ($xt1,"0x50(%r7)");
vrepf ($CTR,@K[3],0);
vlvgf (@K[3],"%r1",0); # clear @K[3].word[0]
vaf ($CTR,$CTR,$xt1);
#LABEL (".Loop_outer_4x");
vlm ($xa0,$xa3,"0x60(%r7)"); # load [smashed] sigma
vrepf ($xb0,@K[1],0); # smash the key
vrepf ($xb1,@K[1],1);
vrepf ($xb2,@K[1],2);
vrepf ($xb3,@K[1],3);
vrepf ($xc0,@K[2],0);
vrepf ($xc1,@K[2],1);
vrepf ($xc2,@K[2],2);
vrepf ($xc3,@K[2],3);
vlr ($xd0,$CTR);
vrepf ($xd1,@K[3],1);
vrepf ($xd2,@K[3],2);
vrepf ($xd3,@K[3],3);
LABEL (".Loop_4x");
VX_lane_ROUND(0, 4, 8,12);
VX_lane_ROUND(0, 5,10,15);
brct ("%r0",".Loop_4x");
vaf ($xd0,$xd0,$CTR);
vmrhf ($xt0,$xa0,$xa1); # transpose data
vmrhf ($xt1,$xa2,$xa3);
vmrlf ($xt2,$xa0,$xa1);
vmrlf ($xt3,$xa2,$xa3);
vpdi ($xa0,$xt0,$xt1,0b0000);
vpdi ($xa1,$xt0,$xt1,0b0101);
vpdi ($xa2,$xt2,$xt3,0b0000);
vpdi ($xa3,$xt2,$xt3,0b0101);
vmrhf ($xt0,$xb0,$xb1);
vmrhf ($xt1,$xb2,$xb3);
vmrlf ($xt2,$xb0,$xb1);
vmrlf ($xt3,$xb2,$xb3);
vpdi ($xb0,$xt0,$xt1,0b0000);
vpdi ($xb1,$xt0,$xt1,0b0101);
vpdi ($xb2,$xt2,$xt3,0b0000);
vpdi ($xb3,$xt2,$xt3,0b0101);
vmrhf ($xt0,$xc0,$xc1);
vmrhf ($xt1,$xc2,$xc3);
vmrlf ($xt2,$xc0,$xc1);
vmrlf ($xt3,$xc2,$xc3);
vpdi ($xc0,$xt0,$xt1,0b0000);
vpdi ($xc1,$xt0,$xt1,0b0101);
vpdi ($xc2,$xt2,$xt3,0b0000);
vpdi ($xc3,$xt2,$xt3,0b0101);
vmrhf ($xt0,$xd0,$xd1);
vmrhf ($xt1,$xd2,$xd3);
vmrlf ($xt2,$xd0,$xd1);
vmrlf ($xt3,$xd2,$xd3);
vpdi ($xd0,$xt0,$xt1,0b0000);
vpdi ($xd1,$xt0,$xt1,0b0101);
vpdi ($xd2,$xt2,$xt3,0b0000);
vpdi ($xd3,$xt2,$xt3,0b0101);
#vrepif ($xt0,4);
#vaf ($CTR,$CTR,$xt0); # next counter value
vaf ($xa0,$xa0,@K[0]);
vaf ($xb0,$xb0,@K[1]);
vaf ($xc0,$xc0,@K[2]);
vaf ($xd0,$xd0,@K[3]);
vperm ($xa0,$xa0,$xa0,$beperm);
vperm ($xb0,$xb0,$xb0,$beperm);
vperm ($xc0,$xc0,$xc0,$beperm);
vperm ($xd0,$xd0,$xd0,$beperm);
#&{$z? \&clgfi:\&clfi} ($len,0x40);
#jl (".Ltail_4x");
vlm ($xt0,$xt3,"0($inp)");
vx ($xt0,$xt0,$xa0);
vx ($xt1,$xt1,$xb0);
vx ($xt2,$xt2,$xc0);
vx ($xt3,$xt3,$xd0);
vstm ($xt0,$xt3,"0($out)");
la ($inp,"0x40($inp)");
la ($out,"0x40($out)");
&{$z? \&aghi:\&ahi} ($len,-0x40);
#je (".Ldone_4x");
vaf ($xa0,$xa1,@K[0]);
vaf ($xb0,$xb1,@K[1]);
vaf ($xc0,$xc1,@K[2]);
vaf ($xd0,$xd1,@K[3]);
vperm ($xa0,$xa0,$xa0,$beperm);
vperm ($xb0,$xb0,$xb0,$beperm);
vperm ($xc0,$xc0,$xc0,$beperm);
vperm ($xd0,$xd0,$xd0,$beperm);
&{$z? \&clgfi:\&clfi} ($len,0x40);
jl (".Ltail_4x");
vlm ($xt0,$xt3,"0($inp)");
vx ($xt0,$xt0,$xa0);
vx ($xt1,$xt1,$xb0);
vx ($xt2,$xt2,$xc0);
vx ($xt3,$xt3,$xd0);
vstm ($xt0,$xt3,"0($out)");
la ($inp,"0x40($inp)");
la ($out,"0x40($out)");
&{$z? \&aghi:\&ahi} ($len,-0x40);
je (".Ldone_4x");
vaf ($xa0,$xa2,@K[0]);
vaf ($xb0,$xb2,@K[1]);
vaf ($xc0,$xc2,@K[2]);
vaf ($xd0,$xd2,@K[3]);
vperm ($xa0,$xa0,$xa0,$beperm);
vperm ($xb0,$xb0,$xb0,$beperm);
vperm ($xc0,$xc0,$xc0,$beperm);
vperm ($xd0,$xd0,$xd0,$beperm);
&{$z? \&clgfi:\&clfi} ($len,0x40);
jl (".Ltail_4x");
vlm ($xt0,$xt3,"0($inp)");
vx ($xt0,$xt0,$xa0);
vx ($xt1,$xt1,$xb0);
vx ($xt2,$xt2,$xc0);
vx ($xt3,$xt3,$xd0);
vstm ($xt0,$xt3,"0($out)");
la ($inp,"0x40($inp)");
la ($out,"0x40($out)");
&{$z? \&aghi:\&ahi} ($len,-0x40);
je (".Ldone_4x");
vaf ($xa0,$xa3,@K[0]);
vaf ($xb0,$xb3,@K[1]);
vaf ($xc0,$xc3,@K[2]);
vaf ($xd0,$xd3,@K[3]);
vperm ($xa0,$xa0,$xa0,$beperm);
vperm ($xb0,$xb0,$xb0,$beperm);
vperm ($xc0,$xc0,$xc0,$beperm);
vperm ($xd0,$xd0,$xd0,$beperm);
&{$z? \&clgfi:\&clfi} ($len,0x40);
jl (".Ltail_4x");
vlm ($xt0,$xt3,"0($inp)");
vx ($xt0,$xt0,$xa0);
vx ($xt1,$xt1,$xb0);
vx ($xt2,$xt2,$xc0);
vx ($xt3,$xt3,$xd0);
vstm ($xt0,$xt3,"0($out)");
#la $inp,0x40($inp));
#la $out,0x40($out));
#lhi %r0,10);
#&{$z? \&aghi:\&ahi} $len,-0x40);
#jne .Loop_outer_4x);
LABEL (".Ldone_4x");
if (!$z) {
ld ("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
ld ("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
} else {
ld ("%f8","$stdframe+8*0($sp)");
ld ("%f9","$stdframe+8*1($sp)");
ld ("%f10","$stdframe+8*2($sp)");
ld ("%f11","$stdframe+8*3($sp)");
ld ("%f12","$stdframe+8*4($sp)");
ld ("%f13","$stdframe+8*5($sp)");
ld ("%f14","$stdframe+8*6($sp)");
ld ("%f15","$stdframe+8*7($sp)");
}
&{$z? \&lmg:\&lm} ("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
la ($sp,"$FRAME($sp)");
br ("%r14");
ALIGN (16);
LABEL (".Ltail_4x");
if (!$z) {
vlr ($xt0,$xb0);
ld ("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
ld ("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
vst ($xa0,"$stdframe+0x00($sp)");
vst ($xt0,"$stdframe+0x10($sp)");
vst ($xc0,"$stdframe+0x20($sp)");
vst ($xd0,"$stdframe+0x30($sp)");
} else {
vlr ($xt0,$xc0);
ld ("%f8","$stdframe+8*0($sp)");
ld ("%f9","$stdframe+8*1($sp)");
ld ("%f10","$stdframe+8*2($sp)");
ld ("%f11","$stdframe+8*3($sp)");
vlr ($xt1,$xd0);
ld ("%f12","$stdframe+8*4($sp)");
ld ("%f13","$stdframe+8*5($sp)");
ld ("%f14","$stdframe+8*6($sp)");
ld ("%f15","$stdframe+8*7($sp)");
vst ($xa0,"$stdframe+0x00($sp)");
vst ($xb0,"$stdframe+0x10($sp)");
vst ($xt0,"$stdframe+0x20($sp)");
vst ($xt1,"$stdframe+0x30($sp)");
}
lghi ("%r1",0);
LABEL (".Loop_tail_4x");
llgc ("%r5","0(%r1,$inp)");
llgc ("%r6","$stdframe(%r1,$sp)");
xr ("%r6","%r5");
stc ("%r6","0(%r1,$out)");
la ("%r1","1(%r1)");
brct ($len,".Loop_tail_4x");
&{$z? \&lmg:\&lm} ("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
la ($sp,"$FRAME($sp)");
br ("%r14");
SIZE ("ChaCha20_ctr32_4x",".-ChaCha20_ctr32_4x");
}
########################################################################
# 6x"horizontal" layout is optimal fit for the platform in its current
# shape, more specifically for given vector instructions' latency. Well,
# computational part of 8x"vertical" would be faster, but it consumes
# all registers and dealing with that will diminish the return...
#
{
my ($a0,$b0,$c0,$d0, $a1,$b1,$c1,$d1,
$a2,$b2,$c2,$d2, $a3,$b3,$c3,$d3,
$a4,$b4,$c4,$d4, $a5,$b5,$c5,$d5)=map("%v$_",(0..23));
my @K=map("%v$_",(27,24..26));
my ($t0,$t1,$t2,$t3)=map("%v$_",27..30);
my $beperm="%v31";
my $FRAME=$stdframe + 4*16;
GLOBL ("ChaCha20_ctr32_vx");
ALIGN (32);
LABEL ("ChaCha20_ctr32_vx");
LABEL (".LChaCha20_ctr32_vx");
&{$z? \&clgfi:\&clfi} ($len,256);
jle (".LChaCha20_ctr32_4x");
&{$z? \&stmg:\&stm} ("%r6","%r7","6*$SIZE_T($sp)");
if (!$z) {
std ("%f4","16*$SIZE_T+2*8($sp)");
std ("%f6","16*$SIZE_T+3*8($sp)");
}
&{$z? \&lghi:\&lhi} ("%r1",-$FRAME);
lgr ("%r0",$sp);
la ($sp,"0(%r1,$sp)");
&{$z? \&stg:\&st} ("%r0","0($sp)"); # back-chain
if ($z) {
std ("%f8","$FRAME-8*8($sp)");
std ("%f9","$FRAME-8*7($sp)");
std ("%f10","$FRAME-8*6($sp)");
std ("%f11","$FRAME-8*5($sp)");
std ("%f12","$FRAME-8*4($sp)");
std ("%f13","$FRAME-8*3($sp)");
std ("%f14","$FRAME-8*2($sp)");
std ("%f15","$FRAME-8*1($sp)");
}
larl ("%r7",".Lsigma");
lhi ("%r0",10);
vlm (@K[1],@K[2],"0($key)"); # load key
vl (@K[3],"0($counter)"); # load counter
vlm (@K[0],"$beperm","0(%r7)"); # load sigma, increments, ...
LABEL (".Loop_outer_vx");
vlr ($a0,@K[0]);
vlr ($b0,@K[1]);
vlr ($a1,@K[0]);
vlr ($b1,@K[1]);
vlr ($a2,@K[0]);
vlr ($b2,@K[1]);
vlr ($a3,@K[0]);
vlr ($b3,@K[1]);
vlr ($a4,@K[0]);
vlr ($b4,@K[1]);
vlr ($a5,@K[0]);
vlr ($b5,@K[1]);
vlr ($d0,@K[3]);
vaf ($d1,@K[3],$t1); # K[3]+1
vaf ($d2,@K[3],$t2); # K[3]+2
vaf ($d3,@K[3],$t3); # K[3]+3
vaf ($d4,$d2,$t2); # K[3]+4
vaf ($d5,$d2,$t3); # K[3]+5
vlr ($c0,@K[2]);
vlr ($c1,@K[2]);
vlr ($c2,@K[2]);
vlr ($c3,@K[2]);
vlr ($c4,@K[2]);
vlr ($c5,@K[2]);
vlr ($t1,$d1);
vlr ($t2,$d2);
vlr ($t3,$d3);
ALIGN (4);
LABEL (".Loop_vx");
VX_ROUND($a0,$a1,$a2,$a3,$a4,$a5,
$b0,$b1,$b2,$b3,$b4,$b5,
$c0,$c1,$c2,$c3,$c4,$c5,
$d0,$d1,$d2,$d3,$d4,$d5,
0);
VX_ROUND($a0,$a1,$a2,$a3,$a4,$a5,
$b0,$b1,$b2,$b3,$b4,$b5,
$c0,$c1,$c2,$c3,$c4,$c5,
$d0,$d1,$d2,$d3,$d4,$d5,
1);
brct ("%r0",".Loop_vx");
vaf ($a0,$a0,@K[0]);
vaf ($b0,$b0,@K[1]);
vaf ($c0,$c0,@K[2]);
vaf ($d0,$d0,@K[3]);
vaf ($a1,$a1,@K[0]);
vaf ($d1,$d1,$t1); # +K[3]+1
vperm ($a0,$a0,$a0,$beperm);
vperm ($b0,$b0,$b0,$beperm);
vperm ($c0,$c0,$c0,$beperm);
vperm ($d0,$d0,$d0,$beperm);
&{$z? \&clgfi:\&clfi} ($len,0x40);
jl (".Ltail_vx");
vaf ($d2,$d2,$t2); # +K[3]+2
vaf ($d3,$d3,$t3); # +K[3]+3
vlm ($t0,$t3,"0($inp)");
vx ($a0,$a0,$t0);
vx ($b0,$b0,$t1);
vx ($c0,$c0,$t2);
vx ($d0,$d0,$t3);
vlm (@K[0],$t3,"0(%r7)"); # re-load sigma and increments
vstm ($a0,$d0,"0($out)");
la ($inp,"0x40($inp)");
la ($out,"0x40($out)");
&{$z? \&aghi:\&ahi} ($len,-0x40);
je (".Ldone_vx");
vaf ($b1,$b1,@K[1]);
vaf ($c1,$c1,@K[2]);
vperm ($a0,$a1,$a1,$beperm);
vperm ($b0,$b1,$b1,$beperm);
vperm ($c0,$c1,$c1,$beperm);
vperm ($d0,$d1,$d1,$beperm);
&{$z? \&clgfi:\&clfi} ($len,0x40);
jl (".Ltail_vx");
vlm ($a1,$d1,"0($inp)");
vx ($a0,$a0,$a1);
vx ($b0,$b0,$b1);
vx ($c0,$c0,$c1);
vx ($d0,$d0,$d1);
vstm ($a0,$d0,"0($out)");
la ($inp,"0x40($inp)");
la ($out,"0x40($out)");
&{$z? \&aghi:\&ahi} ($len,-0x40);
je (".Ldone_vx");
vaf ($a2,$a2,@K[0]);
vaf ($b2,$b2,@K[1]);
vaf ($c2,$c2,@K[2]);
vperm ($a0,$a2,$a2,$beperm);
vperm ($b0,$b2,$b2,$beperm);
vperm ($c0,$c2,$c2,$beperm);
vperm ($d0,$d2,$d2,$beperm);
&{$z? \&clgfi:\&clfi} ($len,0x40);
jl (".Ltail_vx");
vlm ($a1,$d1,"0($inp)");
vx ($a0,$a0,$a1);
vx ($b0,$b0,$b1);
vx ($c0,$c0,$c1);
vx ($d0,$d0,$d1);
vstm ($a0,$d0,"0($out)");
la ($inp,"0x40($inp)");
la ($out,"0x40($out)");
&{$z? \&aghi:\&ahi} ($len,-0x40);
je (".Ldone_vx");
vaf ($a3,$a3,@K[0]);
vaf ($b3,$b3,@K[1]);
vaf ($c3,$c3,@K[2]);
vaf ($d2,@K[3],$t3); # K[3]+3
vperm ($a0,$a3,$a3,$beperm);
vperm ($b0,$b3,$b3,$beperm);
vperm ($c0,$c3,$c3,$beperm);
vperm ($d0,$d3,$d3,$beperm);
&{$z? \&clgfi:\&clfi} ($len,0x40);
jl (".Ltail_vx");
vaf ($d3,$d2,$t1); # K[3]+4
vlm ($a1,$d1,"0($inp)");
vx ($a0,$a0,$a1);
vx ($b0,$b0,$b1);
vx ($c0,$c0,$c1);
vx ($d0,$d0,$d1);
vstm ($a0,$d0,"0($out)");
la ($inp,"0x40($inp)");
la ($out,"0x40($out)");
&{$z? \&aghi:\&ahi} ($len,-0x40);
je (".Ldone_vx");
vaf ($a4,$a4,@K[0]);
vaf ($b4,$b4,@K[1]);
vaf ($c4,$c4,@K[2]);
vaf ($d4,$d4,$d3); # +K[3]+4
vaf ($d3,$d3,$t1); # K[3]+5
vaf (@K[3],$d2,$t3); # K[3]+=6
vperm ($a0,$a4,$a4,$beperm);
vperm ($b0,$b4,$b4,$beperm);
vperm ($c0,$c4,$c4,$beperm);
vperm ($d0,$d4,$d4,$beperm);
&{$z? \&clgfi:\&clfi} ($len,0x40);
jl (".Ltail_vx");
vlm ($a1,$d1,"0($inp)");
vx ($a0,$a0,$a1);
vx ($b0,$b0,$b1);
vx ($c0,$c0,$c1);
vx ($d0,$d0,$d1);
vstm ($a0,$d0,"0($out)");
la ($inp,"0x40($inp)");
la ($out,"0x40($out)");
&{$z? \&aghi:\&ahi} ($len,-0x40);
je (".Ldone_vx");
vaf ($a5,$a5,@K[0]);
vaf ($b5,$b5,@K[1]);
vaf ($c5,$c5,@K[2]);
vaf ($d5,$d5,$d3); # +K[3]+5
vperm ($a0,$a5,$a5,$beperm);
vperm ($b0,$b5,$b5,$beperm);
vperm ($c0,$c5,$c5,$beperm);
vperm ($d0,$d5,$d5,$beperm);
&{$z? \&clgfi:\&clfi} ($len,0x40);
jl (".Ltail_vx");
vlm ($a1,$d1,"0($inp)");
vx ($a0,$a0,$a1);
vx ($b0,$b0,$b1);
vx ($c0,$c0,$c1);
vx ($d0,$d0,$d1);
vstm ($a0,$d0,"0($out)");
la ($inp,"0x40($inp)");
la ($out,"0x40($out)");
lhi ("%r0",10);
&{$z? \&aghi:\&ahi} ($len,-0x40);
jne (".Loop_outer_vx");
LABEL (".Ldone_vx");
if (!$z) {
ld ("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
ld ("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
} else {
ld ("%f8","$FRAME-8*8($sp)");
ld ("%f9","$FRAME-8*7($sp)");
ld ("%f10","$FRAME-8*6($sp)");
ld ("%f11","$FRAME-8*5($sp)");
ld ("%f12","$FRAME-8*4($sp)");
ld ("%f13","$FRAME-8*3($sp)");
ld ("%f14","$FRAME-8*2($sp)");
ld ("%f15","$FRAME-8*1($sp)");
}
&{$z? \&lmg:\&lm} ("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
la ($sp,"$FRAME($sp)");
br ("%r14");
ALIGN (16);
LABEL (".Ltail_vx");
if (!$z) {
ld ("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
ld ("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
} else {
ld ("%f8","$FRAME-8*8($sp)");
ld ("%f9","$FRAME-8*7($sp)");
ld ("%f10","$FRAME-8*6($sp)");
ld ("%f11","$FRAME-8*5($sp)");
ld ("%f12","$FRAME-8*4($sp)");
ld ("%f13","$FRAME-8*3($sp)");
ld ("%f14","$FRAME-8*2($sp)");
ld ("%f15","$FRAME-8*1($sp)");
}
vstm ($a0,$d0,"$stdframe($sp)");
lghi ("%r1",0);
LABEL (".Loop_tail_vx");
llgc ("%r5","0(%r1,$inp)");
llgc ("%r6","$stdframe(%r1,$sp)");
xr ("%r6","%r5");
stc ("%r6","0(%r1,$out)");
la ("%r1","1(%r1)");
brct ($len,".Loop_tail_vx");
&{$z? \&lmg:\&lm} ("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
la ($sp,"$FRAME($sp)");
br ("%r14");
SIZE ("ChaCha20_ctr32_vx",".-ChaCha20_ctr32_vx");
}
################
ALIGN (32);
LABEL (".Lsigma");
LONG (0x61707865,0x3320646e,0x79622d32,0x6b206574); # endian-neutral sigma
LONG (1,0,0,0);
LONG (2,0,0,0);
LONG (3,0,0,0);
LONG (0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c); # byte swap
LONG (0,1,2,3);
LONG (0x61707865,0x61707865,0x61707865,0x61707865); # smashed sigma
LONG (0x3320646e,0x3320646e,0x3320646e,0x3320646e);
LONG (0x79622d32,0x79622d32,0x79622d32,0x79622d32);
LONG (0x6b206574,0x6b206574,0x6b206574,0x6b206574);
ASCIZ ("\"ChaCha20 for s390x, CRYPTOGAMS by <appro\@openssl.org>\"");
ALIGN (4);
PERLASM_END();
Reference in a new issue View git blame Copy permalink