[Vendor] mssqldb: 2019-11-28 -> 2020-04-28 (#11364)

update go-mssqldb 2019-11-28 (1d7a30a10f73) -> 2020-04-28 (06a60b6afbbc)

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
-	github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73
+	github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/dustin/go-humanize v1.0.0
 	github.com/editorconfig/editorconfig-core-go/v2 v2.1.1
@@ -102,7 +102,7 @@ require (
 	github.com/yohcop/openid-go v1.0.0
 	github.com/yuin/goldmark v1.1.25
 	github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60
-	golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
+	golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
 	golang.org/x/net v0.0.0-20200506145744-7e3656a0809f
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f

go.sum

@@ -147,8 +147,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM=
 github.com/denisenkom/go-mssqldb v0.0.0-20190924004331-208c0a498538 h1:bpWCJ5MddHsv4Xtl3azkK89mZzd/vvut32mvAnKbyUA=
 github.com/denisenkom/go-mssqldb v0.0.0-20190924004331-208c0a498538/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
-github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73 h1:OGNva6WhsKst5OZf7eZOklDztV3hwtTHovdrLHV+MsA=
+github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzqk8QCaRC4os14xoKDdbHqqlJtJA0oc1ZAjg=
-github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
@@ -683,6 +683,8 @@ golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=

vendor/github.com/denisenkom/go-mssqldb/README.md

@@ -18,7 +18,7 @@ Other supported formats are listed below.
 
 ### Common parameters:
 
-* `user id` - enter the SQL Server Authentication user id or the Windows Authentication user id in the DOMAIN\User format. On Windows, if user id is empty or missing Single-Sign-On is used.
+* `user id` - enter the SQL Server Authentication user id or the Windows Authentication user id in the DOMAIN\User format. On Windows, if user id is empty or missing Single-Sign-On is used. The user domain sensitive to the case which is defined in the connection string.
 * `password`
 * `database`
 * `connection timeout` - in seconds (default is 0 for no timeout), set to 0 for no timeout. Recommended to set to 0 and use context to manage query and connection timeouts.
@@ -106,6 +106,26 @@ Other supported formats are listed below.
   * `odbc:server=localhost;user id=sa;password={foo{bar}` // Literal `{`, password is "foo{bar"
   * `odbc:server=localhost;user id=sa;password={foo}}bar}` // Escaped `} with `}}`, password is "foo}bar"
 
+### Azure Active Directory authentication - preview
+
+The configuration of functionality might change in the future.
+
+Azure Active Directory (AAD) access tokens are relatively short lived and need to be 
+valid when a new connection is made. Authentication is supported using a callback func that
+provides a fresh and valid token using a connector:
+``` golang
+conn, err := mssql.NewAccessTokenConnector(
+  "Server=test.database.windows.net;Database=testdb",
+  tokenProvider)
+if err != nil {
+	// handle errors in DSN
+}
+db := sql.OpenDB(conn)
+```
+Where `tokenProvider` is a function that returns a fresh access token or an error. None of these statements
+actually trigger the retrieval of a token, this happens when the first statment is issued and a connection
+is created.
+
 ## Executing Stored Procedures
 
 To run a stored procedure, set the query text to the procedure name:

vendor/github.com/denisenkom/go-mssqldb/accesstokenconnector.go

@@ -0,0 +1,51 @@
+// +build go1.10
+
+package mssql
+
+import (
+	"context"
+	"database/sql/driver"
+	"errors"
+	"fmt"
+)
+
+var _ driver.Connector = &accessTokenConnector{}
+
+// accessTokenConnector wraps Connector and injects a
+// fresh access token when connecting to the database
+type accessTokenConnector struct {
+	Connector
+
+	accessTokenProvider func() (string, error)
+}
+
+// NewAccessTokenConnector creates a new connector from a DSN and a token provider.
+// The token provider func will be called when a new connection is requested and should return a valid access token.
+// The returned connector may be used with sql.OpenDB.
+func NewAccessTokenConnector(dsn string, tokenProvider func() (string, error)) (driver.Connector, error) {
+	if tokenProvider == nil {
+		return nil, errors.New("mssql: tokenProvider cannot be nil")
+	}
+
+	conn, err := NewConnector(dsn)
+	if err != nil {
+		return nil, err
+	}
+
+	c := &accessTokenConnector{
+		Connector:           *conn,
+		accessTokenProvider: tokenProvider,
+	}
+	return c, nil
+}
+
+// Connect returns a new database connection
+func (c *accessTokenConnector) Connect(ctx context.Context) (driver.Conn, error) {
+	var err error
+	c.Connector.params.fedAuthAccessToken, err = c.accessTokenProvider()
+	if err != nil {
+		return nil, fmt.Errorf("mssql: error retrieving access token: %+v", err)
+	}
+
+	return c.Connector.Connect(ctx)
+}

vendor/github.com/denisenkom/go-mssqldb/conn_str.go

@@ -37,6 +37,7 @@ type connectParams struct {
 	failOverPartner           string
 	failOverPort              uint64
 	packetSize                uint16
+	fedAuthAccessToken        string
 }
 
 func parseConnectParams(dsn string) (connectParams, error) {

vendor/github.com/denisenkom/go-mssqldb/mssql.go

@@ -397,7 +397,10 @@ func (s *Stmt) Close() error {
 }
 
 func (s *Stmt) SetQueryNotification(id, options string, timeout time.Duration) {
-	to := uint32(timeout / time.Second)
+	// 2.2.5.3.1 Query Notifications Header
+	// https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-tds/e168d373-a7b7-41aa-b6ca-25985466a7e0
+	// Timeout in milliseconds in TDS protocol.
+	to := uint32(timeout / time.Millisecond)
 	if to < 1 {
 		to = 1
 	}

vendor/github.com/denisenkom/go-mssqldb/ntlm.go

@@ -4,11 +4,14 @@ package mssql
 
 import (
 	"crypto/des"
+	"crypto/hmac"
 	"crypto/md5"
 	"crypto/rand"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"strings"
+	"time"
 	"unicode/utf16"
 
 	"golang.org/x/crypto/md4"
@@ -198,86 +201,204 @@ func ntlmSessionResponse(clientNonce [8]byte, serverChallenge [8]byte, password
 	return response(hash, passwordHash)
 }
 
-func (auth *ntlmAuth) NextBytes(bytes []byte) ([]byte, error) {
+func ntlmHashNoPadding(val string) []byte {
-	if string(bytes[0:8]) != "NTLMSSP\x00" {
+	hash := make([]byte, 16)
-		return nil, errorNTLM
+	h := md4.New()
-	}
+	h.Write(utf16le(val))
-	if binary.LittleEndian.Uint32(bytes[8:12]) != _CHALLENGE_MESSAGE {
+	h.Sum(hash[:0])
-		return nil, errorNTLM
-	}
-	flags := binary.LittleEndian.Uint32(bytes[20:24])
-	var challenge [8]byte
-	copy(challenge[:], bytes[24:32])
 
-	var lm, nt []byte
+	return hash
-	if (flags & _NEGOTIATE_EXTENDED_SESSIONSECURITY) != 0 {
+}
+
+func hmacMD5(passwordHash, data []byte) []byte {
+	hmacEntity := hmac.New(md5.New, passwordHash)
+	hmacEntity.Write(data)
+
+	return hmacEntity.Sum(nil)
+}
+
+func getNTLMv2AndLMv2ResponsePayloads(userDomain, username, password string, challenge, nonce [8]byte, targetInfoFields []byte, timestamp time.Time) (ntlmV2Payload, lmV2Payload []byte) {
+	// NTLMv2 response payload: http://davenport.sourceforge.net/ntlm.html#theNtlmv2Response
+
+	ntlmHash := ntlmHashNoPadding(password)
+	usernameAndTargetBytes := utf16le(strings.ToUpper(username) + userDomain)
+	ntlmV2Hash := hmacMD5(ntlmHash, usernameAndTargetBytes)
+	targetInfoLength := len(targetInfoFields)
+	blob := make([]byte, 32+targetInfoLength)
+	binary.BigEndian.PutUint32(blob[:4], 0x01010000)
+	binary.BigEndian.PutUint32(blob[4:8], 0x00000000)
+	binary.BigEndian.PutUint64(blob[8:16], uint64(timestamp.UnixNano()))
+	copy(blob[16:24], nonce[:])
+	binary.BigEndian.PutUint32(blob[24:28], 0x00000000)
+	copy(blob[28:], targetInfoFields)
+	binary.BigEndian.PutUint32(blob[28+targetInfoLength:], 0x00000000)
+	challengeLength := len(challenge)
+	blobLength := len(blob)
+	challengeAndBlob := make([]byte, challengeLength+blobLength)
+	copy(challengeAndBlob[:challengeLength], challenge[:])
+	copy(challengeAndBlob[challengeLength:], blob)
+	hashedChallenge := hmacMD5(ntlmV2Hash, challengeAndBlob)
+	ntlmV2Payload = append(hashedChallenge, blob...)
+
+	// LMv2 response payload: http://davenport.sourceforge.net/ntlm.html#theLmv2Response
+	ntlmV2hash := hmacMD5(ntlmHash, usernameAndTargetBytes)
+	challengeAndNonce := make([]byte, 16)
+	copy(challengeAndNonce[:8], challenge[:])
+	copy(challengeAndNonce[8:], nonce[:])
+	hashedChallenge = hmacMD5(ntlmV2hash, challengeAndNonce)
+	lmV2Payload = append(hashedChallenge, nonce[:]...)
+
+	return
+}
+
+func negotiateExtendedSessionSecurity(flags uint32, message []byte, challenge [8]byte, username, password, userDom string) (lm, nt []byte, err error) {
 	nonce := clientChallenge()
+
+	// Official specification: https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-nlmp/b38c36ed-2804-4868-a9ff-8dd3182128e4
+	// Unofficial walk through referenced by https://www.freetds.org/userguide/domains.htm: http://davenport.sourceforge.net/ntlm.html
+	if (flags & _NEGOTIATE_TARGET_INFO) != 0 {
+		targetInfoFields, err := getNTLMv2TargetInfoFields(message)
+		if err != nil {
+			return lm, nt, err
+		}
+
+		nt, lm = getNTLMv2AndLMv2ResponsePayloads(userDom, username, password, challenge, nonce, targetInfoFields, time.Now())
+
+		return lm, nt, nil
+	}
+
 	var lm_bytes [24]byte
 	copy(lm_bytes[:8], nonce[:])
 	lm = lm_bytes[:]
-		nt_bytes := ntlmSessionResponse(nonce, challenge, auth.Password)
+	nt_bytes := ntlmSessionResponse(nonce, challenge, password)
-		nt = nt_bytes[:]
-	} else {
-		lm_bytes := lmResponse(challenge, auth.Password)
-		lm = lm_bytes[:]
-		nt_bytes := ntResponse(challenge, auth.Password)
 	nt = nt_bytes[:]
+
+	return lm, nt, nil
 }
+
+func getNTLMv2TargetInfoFields(type2Message []byte) (info []byte, err error) {
+	type2MessageError := "mssql: while parsing NTLMv2 type 2 message, length %d too small for offset %d"
+	type2MessageLength := len(type2Message)
+	if type2MessageLength < 20 {
+		return nil, fmt.Errorf(type2MessageError, type2MessageLength, 20)
+	}
+
+	targetNameAllocated := binary.LittleEndian.Uint16(type2Message[14:16])
+	targetNameOffset := binary.LittleEndian.Uint32(type2Message[16:20])
+	endOfOffset := int(targetNameOffset + uint32(targetNameAllocated))
+	if type2MessageLength < endOfOffset {
+		return nil, fmt.Errorf(type2MessageError, type2MessageLength, endOfOffset)
+	}
+
+	targetInformationAllocated := binary.LittleEndian.Uint16(type2Message[42:44])
+	targetInformationDataOffset := binary.LittleEndian.Uint32(type2Message[44:48])
+	endOfOffset = int(targetInformationDataOffset + uint32(targetInformationAllocated))
+	if type2MessageLength < endOfOffset {
+		return nil, fmt.Errorf(type2MessageError, type2MessageLength, endOfOffset)
+	}
+
+	targetInformationBytes := make([]byte, targetInformationAllocated)
+	copy(targetInformationBytes, type2Message[targetInformationDataOffset:targetInformationDataOffset+uint32(targetInformationAllocated)])
+
+	return targetInformationBytes, nil
+}
+
+func buildNTLMResponsePayload(lm, nt []byte, flags uint32, domain, workstation, username string) ([]byte, error) {
 	lm_len := len(lm)
 	nt_len := len(nt)
-
+	domain16 := utf16le(domain)
-	domain16 := utf16le(auth.Domain)
 	domain_len := len(domain16)
-	user16 := utf16le(auth.UserName)
+	user16 := utf16le(username)
 	user_len := len(user16)
-	workstation16 := utf16le(auth.Workstation)
+	workstation16 := utf16le(workstation)
 	workstation_len := len(workstation16)
-
 	msg := make([]byte, 88+lm_len+nt_len+domain_len+user_len+workstation_len)
 	copy(msg, []byte("NTLMSSP\x00"))
 	binary.LittleEndian.PutUint32(msg[8:], _AUTHENTICATE_MESSAGE)
+
 	// Lm Challenge Response Fields
 	binary.LittleEndian.PutUint16(msg[12:], uint16(lm_len))
 	binary.LittleEndian.PutUint16(msg[14:], uint16(lm_len))
 	binary.LittleEndian.PutUint32(msg[16:], 88)
+
 	// Nt Challenge Response Fields
 	binary.LittleEndian.PutUint16(msg[20:], uint16(nt_len))
 	binary.LittleEndian.PutUint16(msg[22:], uint16(nt_len))
 	binary.LittleEndian.PutUint32(msg[24:], uint32(88+lm_len))
+
 	// Domain Name Fields
 	binary.LittleEndian.PutUint16(msg[28:], uint16(domain_len))
 	binary.LittleEndian.PutUint16(msg[30:], uint16(domain_len))
 	binary.LittleEndian.PutUint32(msg[32:], uint32(88+lm_len+nt_len))
+
 	// User Name Fields
 	binary.LittleEndian.PutUint16(msg[36:], uint16(user_len))
 	binary.LittleEndian.PutUint16(msg[38:], uint16(user_len))
 	binary.LittleEndian.PutUint32(msg[40:], uint32(88+lm_len+nt_len+domain_len))
+
 	// Workstation Fields
 	binary.LittleEndian.PutUint16(msg[44:], uint16(workstation_len))
 	binary.LittleEndian.PutUint16(msg[46:], uint16(workstation_len))
 	binary.LittleEndian.PutUint32(msg[48:], uint32(88+lm_len+nt_len+domain_len+user_len))
+
 	// Encrypted Random Session Key Fields
 	binary.LittleEndian.PutUint16(msg[52:], 0)
 	binary.LittleEndian.PutUint16(msg[54:], 0)
 	binary.LittleEndian.PutUint32(msg[56:], uint32(88+lm_len+nt_len+domain_len+user_len+workstation_len))
+
 	// Negotiate Flags
 	binary.LittleEndian.PutUint32(msg[60:], flags)
+
 	// Version
 	binary.LittleEndian.PutUint32(msg[64:], 0)
 	binary.LittleEndian.PutUint32(msg[68:], 0)
+
 	// MIC
 	binary.LittleEndian.PutUint32(msg[72:], 0)
 	binary.LittleEndian.PutUint32(msg[76:], 0)
 	binary.LittleEndian.PutUint32(msg[88:], 0)
 	binary.LittleEndian.PutUint32(msg[84:], 0)
+
 	// Payload
 	copy(msg[88:], lm)
 	copy(msg[88+lm_len:], nt)
 	copy(msg[88+lm_len+nt_len:], domain16)
 	copy(msg[88+lm_len+nt_len+domain_len:], user16)
 	copy(msg[88+lm_len+nt_len+domain_len+user_len:], workstation16)
+
 	return msg, nil
 }
 
+func (auth *ntlmAuth) NextBytes(bytes []byte) ([]byte, error) {
+	signature := string(bytes[0:8])
+	if signature != "NTLMSSP\x00" {
+		return nil, errorNTLM
+	}
+
+	messageTypeIndicator := binary.LittleEndian.Uint32(bytes[8:12])
+	if messageTypeIndicator != _CHALLENGE_MESSAGE {
+		return nil, errorNTLM
+	}
+
+	var challenge [8]byte
+	copy(challenge[:], bytes[24:32])
+	flags := binary.LittleEndian.Uint32(bytes[20:24])
+	if (flags & _NEGOTIATE_EXTENDED_SESSIONSECURITY) != 0 {
+		lm, nt, err := negotiateExtendedSessionSecurity(flags, bytes, challenge, auth.UserName, auth.Password, auth.Domain)
+		if err != nil {
+			return nil, err
+		}
+
+		return buildNTLMResponsePayload(lm, nt, flags, auth.Domain, auth.Workstation, auth.UserName)
+	}
+
+	lm_bytes := lmResponse(challenge, auth.Password)
+	lm := lm_bytes[:]
+	nt_bytes := ntResponse(challenge, auth.Password)
+	nt := nt_bytes[:]
+
+	return buildNTLMResponsePayload(lm, nt, flags, auth.Domain, auth.Workstation, auth.UserName)
+}
+
 func (auth *ntlmAuth) Free() {
 }

vendor/github.com/denisenkom/go-mssqldb/tds.go

@@ -106,6 +106,8 @@ const (
 	preloginTHREADID        = 3
 	preloginMARS            = 4
 	preloginTRACEID         = 5
+	preloginFEDAUTHREQUIRED = 6
+	preloginNONCEOPT        = 7
 	preloginTERMINATOR      = 0xff
 )
 
@@ -245,6 +247,12 @@ const (
 	fReadOnlyIntent = 32
 )
 
+// OptionFlags3
+// https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-tds/773a62b6-ee89-4c02-9e5e-344882630aac
+const (
+	fExtension = 0x10
+)
+
 type login struct {
 	TDSVersion     uint32
 	PacketSize     uint32
@@ -269,6 +277,89 @@ type login struct {
 	SSPI           []byte
 	AtchDBFile     string
 	ChangePassword string
+	FeatureExt     featureExts
+}
+
+type featureExts struct {
+	features map[byte]featureExt
+}
+
+type featureExt interface {
+	featureID() byte
+	toBytes() []byte
+}
+
+func (e *featureExts) Add(f featureExt) error {
+	if f == nil {
+		return nil
+	}
+	id := f.featureID()
+	if _, exists := e.features[id]; exists {
+		f := "Login error: Feature with ID '%v' is already present in FeatureExt block."
+		return fmt.Errorf(f, id)
+	}
+	if e.features == nil {
+		e.features = make(map[byte]featureExt)
+	}
+	e.features[id] = f
+	return nil
+}
+
+func (e featureExts) toBytes() []byte {
+	if len(e.features) == 0 {
+		return nil
+	}
+	var d []byte
+	for featureID, f := range e.features {
+		featureData := f.toBytes()
+
+		hdr := make([]byte, 5)
+		hdr[0] = featureID                                               // FedAuth feature extension BYTE
+		binary.LittleEndian.PutUint32(hdr[1:], uint32(len(featureData))) // FeatureDataLen DWORD
+		d = append(d, hdr...)
+
+		d = append(d, featureData...) // FeatureData *BYTE
+	}
+	if d != nil {
+		d = append(d, 0xff) // Terminator
+	}
+	return d
+}
+
+type featureExtFedAuthSTS struct {
+	FedAuthEcho  bool
+	FedAuthToken string
+	Nonce        []byte
+}
+
+func (e *featureExtFedAuthSTS) featureID() byte {
+	return 0x02
+}
+
+func (e *featureExtFedAuthSTS) toBytes() []byte {
+	if e == nil {
+		return nil
+	}
+
+	options := byte(0x01) << 1 // 0x01 => STS bFedAuthLibrary 7BIT
+	if e.FedAuthEcho {
+		options |= 1 // fFedAuthEcho
+	}
+
+	d := make([]byte, 5)
+	d[0] = options
+
+	// looks like string in
+	// https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-tds/f88b63bb-b479-49e1-a87b-deda521da508
+	tokenBytes := str2ucs2(e.FedAuthToken)
+	binary.LittleEndian.PutUint32(d[1:], uint32(len(tokenBytes))) // Should be a signed int32, but since the length is relatively small, this should work
+	d = append(d, tokenBytes...)
+
+	if len(e.Nonce) == 32 {
+		d = append(d, e.Nonce...)
+	}
+
+	return d
 }
 
 type loginHeader struct {
@@ -295,7 +386,7 @@ type loginHeader struct {
 	ServerNameOffset     uint16
 	ServerNameLength     uint16
 	ExtensionOffset      uint16
-	ExtensionLenght      uint16
+	ExtensionLength      uint16
 	CtlIntNameOffset     uint16
 	CtlIntNameLength     uint16
 	LanguageOffset       uint16
@@ -357,6 +448,8 @@ func sendLogin(w *tdsBuffer, login login) error {
 	database := str2ucs2(login.Database)
 	atchdbfile := str2ucs2(login.AtchDBFile)
 	changepassword := str2ucs2(login.ChangePassword)
+	featureExt := login.FeatureExt.toBytes()
+
 	hdr := loginHeader{
 		TDSVersion:           login.TDSVersion,
 		PacketSize:           login.PacketSize,
@@ -405,7 +498,18 @@ func sendLogin(w *tdsBuffer, login login) error {
 	offset += uint16(len(atchdbfile))
 	hdr.ChangePasswordOffset = offset
 	offset += uint16(len(changepassword))
-	hdr.Length = uint32(offset)
+
+	featureExtOffset := uint32(0)
+	featureExtLen := len(featureExt)
+	if featureExtLen > 0 {
+		hdr.OptionFlags3 |= fExtension
+		hdr.ExtensionOffset = offset
+		hdr.ExtensionLength = 4
+		offset += hdr.ExtensionLength // DWORD
+		featureExtOffset = uint32(offset)
+	}
+	hdr.Length = uint32(offset) + uint32(featureExtLen)
+
 	var err error
 	err = binary.Write(w, binary.LittleEndian, &hdr)
 	if err != nil {
@@ -455,6 +559,16 @@ func sendLogin(w *tdsBuffer, login login) error {
 	if err != nil {
 		return err
 	}
+	if featureExtOffset > 0 {
+		err = binary.Write(w, binary.LittleEndian, featureExtOffset)
+		if err != nil {
+			return err
+		}
+		_, err = w.Write(featureExt)
+		if err != nil {
+			return err
+		}
+	}
 	return w.FinishPacket()
 }
 
@@ -844,15 +958,23 @@ initiate_connection:
 		AppName:      p.appname,
 		TypeFlags:    p.typeFlags,
 	}
-	auth, auth_ok := getAuth(p.user, p.password, p.serverSPN, p.workstation)
+	auth, authOk := getAuth(p.user, p.password, p.serverSPN, p.workstation)
-	if auth_ok {
+	switch {
+	case p.fedAuthAccessToken != "": // accesstoken ignores user/password
+		featurext := &featureExtFedAuthSTS{
+			FedAuthEcho:  len(fields[preloginFEDAUTHREQUIRED]) > 0 && fields[preloginFEDAUTHREQUIRED][0] == 1,
+			FedAuthToken: p.fedAuthAccessToken,
+			Nonce:        fields[preloginNONCEOPT],
+		}
+		login.FeatureExt.Add(featurext)
+	case authOk:
 		login.SSPI, err = auth.InitialBytes()
 		if err != nil {
 			return nil, err
 		}
 		login.OptionFlags2 |= fIntSecurity
 		defer auth.Free()
-	} else {
+	default:
 		login.UserName = p.user
 		login.Password = p.password
 	}

vendor/github.com/denisenkom/go-mssqldb/token.go

@@ -24,6 +24,7 @@ const (
 	tokenInfo          token = 171 // 0xAB
 	tokenReturnValue   token = 0xAC
 	tokenLoginAck      token = 173 // 0xad
+	tokenFeatureExtAck token = 174 // 0xae
 	tokenRow           token = 209 // 0xd1
 	tokenNbcRow        token = 210 // 0xd2
 	tokenEnvChange     token = 227 // 0xE3
@@ -447,6 +448,22 @@ func parseLoginAck(r *tdsBuffer) loginAckStruct {
 	return res
 }
 
+// https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-tds/2eb82f8e-11f0-46dc-b42d-27302fa4701a
+func parseFeatureExtAck(r *tdsBuffer) {
+	// at most 1 featureAck per feature in featureExt
+	// go-mssqldb will add at most 1 feature, the spec defines 7 different features
+	for i := 0; i < 8; i++ {
+		featureID := r.byte() // FeatureID
+		if featureID == 0xff {
+			return
+		}
+		size := r.uint32() // FeatureAckDataLen
+		d := make([]byte, size)
+		r.ReadFull(d)
+	}
+	panic("parsed more than 7 featureAck's, protocol implementation error?")
+}
+
 // http://msdn.microsoft.com/en-us/library/dd357363.aspx
 func parseColMetadata72(r *tdsBuffer) (columns []columnStruct) {
 	count := r.uint16()
@@ -577,6 +594,8 @@ func processSingleResponse(sess *tdsSession, ch chan tokenStruct, outs map[strin
 		case tokenLoginAck:
 			loginAck := parseLoginAck(sess.buf)
 			ch <- loginAck
+		case tokenFeatureExtAck:
+			parseFeatureExtAck(sess.buf)
 		case tokenOrder:
 			order := parseOrder(sess.buf)
 			ch <- order

vendor/golang.org/x/crypto/blake2b/blake2b.go

@@ -5,6 +5,8 @@
 // Package blake2b implements the BLAKE2b hash algorithm defined by RFC 7693
 // and the extendable output function (XOF) BLAKE2Xb.
 //
+// BLAKE2b is optimized for 64-bit platforms—including NEON-enabled ARMs—and
+// produces digests of any size between 1 and 64 bytes.
 // For a detailed specification of BLAKE2b see https://blake2.net/blake2.pdf
 // and for BLAKE2Xb see https://blake2.net/blake2x.pdf
 //

vendor/golang.org/x/crypto/chacha20/chacha_generic.go

@@ -42,10 +42,14 @@ type Cipher struct {
 
 	// The last len bytes of buf are leftover key stream bytes from the previous
 	// XORKeyStream invocation. The size of buf depends on how many blocks are
-	// computed at a time.
+	// computed at a time by xorKeyStreamBlocks.
 	buf [bufSize]byte
 	len int
 
+	// overflow is set when the counter overflowed, no more blocks can be
+	// generated, and the next XORKeyStream call should panic.
+	overflow bool
+
 	// The counter-independent results of the first round are cached after they
 	// are computed the first time.
 	precompDone      bool
@@ -89,6 +93,7 @@ func newUnauthenticatedCipher(c *Cipher, key, nonce []byte) (*Cipher, error) {
 		return nil, errors.New("chacha20: wrong nonce size")
 	}
 
+	key, nonce = key[:KeySize], nonce[:NonceSize] // bounds check elimination hint
 	c.key = [8]uint32{
 		binary.LittleEndian.Uint32(key[0:4]),
 		binary.LittleEndian.Uint32(key[4:8]),
@@ -139,15 +144,18 @@ func quarterRound(a, b, c, d uint32) (uint32, uint32, uint32, uint32) {
 // SetCounter sets the Cipher counter. The next invocation of XORKeyStream will
 // behave as if (64 * counter) bytes had been encrypted so far.
 //
-// To prevent accidental counter reuse, SetCounter panics if counter is
+// To prevent accidental counter reuse, SetCounter panics if counter is less
-// less than the current value.
+// than the current value.
+//
+// Note that the execution time of XORKeyStream is not independent of the
+// counter value.
 func (s *Cipher) SetCounter(counter uint32) {
 	// Internally, s may buffer multiple blocks, which complicates this
 	// implementation slightly. When checking whether the counter has rolled
 	// back, we must use both s.counter and s.len to determine how many blocks
 	// we have already output.
 	outputCounter := s.counter - uint32(s.len)/blockSize
-	if counter < outputCounter {
+	if s.overflow || counter < outputCounter {
 		panic("chacha20: SetCounter attempted to rollback counter")
 	}
 
@@ -196,34 +204,52 @@ func (s *Cipher) XORKeyStream(dst, src []byte) {
 			dst[i] = src[i] ^ b
 		}
 		s.len -= len(keyStream)
-		src = src[len(keyStream):]
+		dst, src = dst[len(keyStream):], src[len(keyStream):]
-		dst = dst[len(keyStream):]
+	}
+	if len(src) == 0 {
+		return
 	}
 
-	const blocksPerBuf = bufSize / blockSize
+	// If we'd need to let the counter overflow and keep generating output,
-	numBufs := (uint64(len(src)) + bufSize - 1) / bufSize
+	// panic immediately. If instead we'd only reach the last block, remember
-	if uint64(s.counter)+numBufs*blocksPerBuf >= 1<<32 {
+	// not to generate any more output after the buffer is drained.
+	numBlocks := (uint64(len(src)) + blockSize - 1) / blockSize
+	if s.overflow || uint64(s.counter)+numBlocks > 1<<32 {
 		panic("chacha20: counter overflow")
+	} else if uint64(s.counter)+numBlocks == 1<<32 {
+		s.overflow = true
 	}
 
 	// xorKeyStreamBlocks implementations expect input lengths that are a
 	// multiple of bufSize. Platform-specific ones process multiple blocks at a
 	// time, so have bufSizes that are a multiple of blockSize.
 
-	rem := len(src) % bufSize
+	full := len(src) - len(src)%bufSize
-	full := len(src) - rem
-
 	if full > 0 {
 		s.xorKeyStreamBlocks(dst[:full], src[:full])
 	}
+	dst, src = dst[full:], src[full:]
+
+	// If using a multi-block xorKeyStreamBlocks would overflow, use the generic
+	// one that does one block at a time.
+	const blocksPerBuf = bufSize / blockSize
+	if uint64(s.counter)+blocksPerBuf > 1<<32 {
+		s.buf = [bufSize]byte{}
+		numBlocks := (len(src) + blockSize - 1) / blockSize
+		buf := s.buf[bufSize-numBlocks*blockSize:]
+		copy(buf, src)
+		s.xorKeyStreamBlocksGeneric(buf, buf)
+		s.len = len(buf) - copy(dst, buf)
+		return
+	}
 
 	// If we have a partial (multi-)block, pad it for xorKeyStreamBlocks, and
 	// keep the leftover keystream for the next XORKeyStream invocation.
-	if rem > 0 {
+	if len(src) > 0 {
 		s.buf = [bufSize]byte{}
-		copy(s.buf[:], src[full:])
+		copy(s.buf[:], src)
 		s.xorKeyStreamBlocks(s.buf[:], s.buf[:])
-		s.len = bufSize - copy(dst[full:], s.buf[:])
+		s.len = bufSize - copy(dst, s.buf[:])
 	}
 }
 
@@ -260,7 +286,9 @@ func (s *Cipher) xorKeyStreamBlocksGeneric(dst, src []byte) {
 		s.precompDone = true
 	}
 
-	for i := 0; i < len(src); i += blockSize {
+	// A condition of len(src) > 0 would be sufficient, but this also
+	// acts as a bounds check elimination hint.
+	for len(src) >= 64 && len(dst) >= 64 {
 		// The remainder of the first column round.
 		fcr0, fcr4, fcr8, fcr12 := quarterRound(c0, c4, c8, s.counter)
 
@@ -285,49 +313,28 @@ func (s *Cipher) xorKeyStreamBlocksGeneric(dst, src []byte) {
 			x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14)
 		}
 
-		// Finally, add back the initial state to generate the key stream.
+		// Add back the initial state to generate the key stream, then
-		x0 += c0
+		// XOR the key stream with the source and write out the result.
-		x1 += c1
+		addXor(dst[0:4], src[0:4], x0, c0)
-		x2 += c2
+		addXor(dst[4:8], src[4:8], x1, c1)
-		x3 += c3
+		addXor(dst[8:12], src[8:12], x2, c2)
-		x4 += c4
+		addXor(dst[12:16], src[12:16], x3, c3)
-		x5 += c5
+		addXor(dst[16:20], src[16:20], x4, c4)
-		x6 += c6
+		addXor(dst[20:24], src[20:24], x5, c5)
-		x7 += c7
+		addXor(dst[24:28], src[24:28], x6, c6)
-		x8 += c8
+		addXor(dst[28:32], src[28:32], x7, c7)
-		x9 += c9
+		addXor(dst[32:36], src[32:36], x8, c8)
-		x10 += c10
+		addXor(dst[36:40], src[36:40], x9, c9)
-		x11 += c11
+		addXor(dst[40:44], src[40:44], x10, c10)
-		x12 += s.counter
+		addXor(dst[44:48], src[44:48], x11, c11)
-		x13 += c13
+		addXor(dst[48:52], src[48:52], x12, s.counter)
-		x14 += c14
+		addXor(dst[52:56], src[52:56], x13, c13)
-		x15 += c15
+		addXor(dst[56:60], src[56:60], x14, c14)
+		addXor(dst[60:64], src[60:64], x15, c15)
 
 		s.counter += 1
-		if s.counter == 0 {
-			panic("chacha20: internal error: counter overflow")
-		}
 
-		in, out := src[i:], dst[i:]
+		src, dst = src[blockSize:], dst[blockSize:]
-		in, out = in[:blockSize], out[:blockSize] // bounds check elimination hint
-
-		// XOR the key stream with the source and write out the result.
-		xor(out[0:], in[0:], x0)
-		xor(out[4:], in[4:], x1)
-		xor(out[8:], in[8:], x2)
-		xor(out[12:], in[12:], x3)
-		xor(out[16:], in[16:], x4)
-		xor(out[20:], in[20:], x5)
-		xor(out[24:], in[24:], x6)
-		xor(out[28:], in[28:], x7)
-		xor(out[32:], in[32:], x8)
-		xor(out[36:], in[36:], x9)
-		xor(out[40:], in[40:], x10)
-		xor(out[44:], in[44:], x11)
-		xor(out[48:], in[48:], x12)
-		xor(out[52:], in[52:], x13)
-		xor(out[56:], in[56:], x14)
-		xor(out[60:], in[60:], x15)
 	}
 }
 

vendor/golang.org/x/crypto/chacha20/xor.go

@@ -13,10 +13,10 @@ const unaligned = runtime.GOARCH == "386" ||
 	runtime.GOARCH == "ppc64le" ||
 	runtime.GOARCH == "s390x"
 
-// xor reads a little endian uint32 from src, XORs it with u and
+// addXor reads a little endian uint32 from src, XORs it with (a + b) and
 // places the result in little endian byte order in dst.
-func xor(dst, src []byte, u uint32) {
+func addXor(dst, src []byte, a, b uint32) {
-	_, _ = src[3], dst[3] // eliminate bounds checks
+	_, _ = src[3], dst[3] // bounds check elimination hint
 	if unaligned {
 		// The compiler should optimize this code into
 		// 32-bit unaligned little endian loads and stores.
@@ -27,15 +27,16 @@ func xor(dst, src []byte, u uint32) {
 		v |= uint32(src[1]) << 8
 		v |= uint32(src[2]) << 16
 		v |= uint32(src[3]) << 24
-		v ^= u
+		v ^= a + b
 		dst[0] = byte(v)
 		dst[1] = byte(v >> 8)
 		dst[2] = byte(v >> 16)
 		dst[3] = byte(v >> 24)
 	} else {
-		dst[0] = src[0] ^ byte(u)
+		a += b
-		dst[1] = src[1] ^ byte(u>>8)
+		dst[0] = src[0] ^ byte(a)
-		dst[2] = src[2] ^ byte(u>>16)
+		dst[1] = src[1] ^ byte(a>>8)
-		dst[3] = src[3] ^ byte(u>>24)
+		dst[2] = src[2] ^ byte(a>>16)
+		dst[3] = src[3] ^ byte(a>>24)
 	}
 }

vendor/golang.org/x/crypto/poly1305/mac_noasm.go

@@ -2,10 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !amd64,!ppc64le gccgo purego
+// +build !amd64,!ppc64le,!s390x gccgo purego
 
 package poly1305
 
 type mac struct{ macGeneric }
-
-func newMAC(key *[32]byte) mac { return mac{newMACGeneric(key)} }

vendor/golang.org/x/crypto/poly1305/poly1305.go

@@ -26,7 +26,9 @@ const TagSize = 16
 // 16-byte result into out. Authenticating two different messages with the same
 // key allows an attacker to forge messages at will.
 func Sum(out *[16]byte, m []byte, key *[32]byte) {
-	sum(out, m, key)
+	h := New(key)
+	h.Write(m)
+	h.Sum(out[:0])
 }
 
 // Verify returns true if mac is a valid authenticator for m with the given key.
@@ -46,10 +48,9 @@ func Verify(mac *[16]byte, m []byte, key *[32]byte) bool {
 // two different messages with the same key allows an attacker
 // to forge messages at will.
 func New(key *[32]byte) *MAC {
-	return &MAC{
+	m := &MAC{}
-		mac:       newMAC(key),
+	initialize(key, &m.macState)
-		finalized: false,
+	return m
-	}
 }
 
 // MAC is an io.Writer computing an authentication tag
@@ -58,7 +59,7 @@ func New(key *[32]byte) *MAC {
 // MAC cannot be used like common hash.Hash implementations,
 // because using a poly1305 key twice breaks its security.
 // Therefore writing data to a running MAC after calling
-// Sum causes it to panic.
+// Sum or Verify causes it to panic.
 type MAC struct {
 	mac // platform-dependent implementation
 
@@ -71,10 +72,10 @@ func (h *MAC) Size() int { return TagSize }
 // Write adds more data to the running message authentication code.
 // It never returns an error.
 //
-// It must not be called after the first call of Sum.
+// It must not be called after the first call of Sum or Verify.
 func (h *MAC) Write(p []byte) (n int, err error) {
 	if h.finalized {
-		panic("poly1305: write to MAC after Sum")
+		panic("poly1305: write to MAC after Sum or Verify")
 	}
 	return h.mac.Write(p)
 }
@@ -87,3 +88,12 @@ func (h *MAC) Sum(b []byte) []byte {
 	h.finalized = true
 	return append(b, mac[:]...)
 }
+
+// Verify returns whether the authenticator of all data written to
+// the message authentication code matches the expected value.
+func (h *MAC) Verify(expected []byte) bool {
+	var mac [TagSize]byte
+	h.mac.Sum(&mac)
+	h.finalized = true
+	return subtle.ConstantTimeCompare(expected, mac[:]) == 1
+}

vendor/golang.org/x/crypto/poly1305/sum_amd64.go

@@ -9,17 +9,6 @@ package poly1305
 //go:noescape
 func update(state *macState, msg []byte)
 
-func sum(out *[16]byte, m []byte, key *[32]byte) {
-	h := newMAC(key)
-	h.Write(m)
-	h.Sum(out)
-}
-
-func newMAC(key *[32]byte) (h mac) {
-	initialize(key, &h.r, &h.s)
-	return
-}
-
 // mac is a wrapper for macGeneric that redirects calls that would have gone to
 // updateGeneric to update.
 //

vendor/golang.org/x/crypto/poly1305/sum_generic.go

@@ -31,16 +31,18 @@ func sumGeneric(out *[TagSize]byte, msg []byte, key *[32]byte) {
 	h.Sum(out)
 }
 
-func newMACGeneric(key *[32]byte) (h macGeneric) {
+func newMACGeneric(key *[32]byte) macGeneric {
-	initialize(key, &h.r, &h.s)
+	m := macGeneric{}
-	return
+	initialize(key, &m.macState)
+	return m
 }
 
 // macState holds numbers in saturated 64-bit little-endian limbs. That is,
 // the value of [x0, x1, x2] is x[0] + x[1] * 2⁶⁴ + x[2] * 2¹²⁸.
 type macState struct {
 	// h is the main accumulator. It is to be interpreted modulo 2¹³⁰ - 5, but
-	// can grow larger during and after rounds.
+	// can grow larger during and after rounds. It must, however, remain below
+	// 2 * (2¹³⁰ - 5).
 	h [3]uint64
 	// r and s are the private key components.
 	r [2]uint64
@@ -97,11 +99,12 @@ const (
 	rMask1 = 0x0FFFFFFC0FFFFFFC
 )
 
-func initialize(key *[32]byte, r, s *[2]uint64) {
+// initialize loads the 256-bit key into the two 128-bit secret values r and s.
-	r[0] = binary.LittleEndian.Uint64(key[0:8]) & rMask0
+func initialize(key *[32]byte, m *macState) {
-	r[1] = binary.LittleEndian.Uint64(key[8:16]) & rMask1
+	m.r[0] = binary.LittleEndian.Uint64(key[0:8]) & rMask0
-	s[0] = binary.LittleEndian.Uint64(key[16:24])
+	m.r[1] = binary.LittleEndian.Uint64(key[8:16]) & rMask1
-	s[1] = binary.LittleEndian.Uint64(key[24:32])
+	m.s[0] = binary.LittleEndian.Uint64(key[16:24])
+	m.s[1] = binary.LittleEndian.Uint64(key[24:32])
 }
 
 // uint128 holds a 128-bit number as two 64-bit limbs, for use with the

vendor/golang.org/x/crypto/poly1305/sum_noasm.go

@@ -1,13 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build s390x,!go1.11 !amd64,!s390x,!ppc64le gccgo purego
-
-package poly1305
-
-func sum(out *[TagSize]byte, msg []byte, key *[32]byte) {
-	h := newMAC(key)
-	h.Write(msg)
-	h.Sum(out)
-}

vendor/golang.org/x/crypto/poly1305/sum_ppc64le.go

@@ -9,17 +9,6 @@ package poly1305
 //go:noescape
 func update(state *macState, msg []byte)
 
-func sum(out *[16]byte, m []byte, key *[32]byte) {
-	h := newMAC(key)
-	h.Write(m)
-	h.Sum(out)
-}
-
-func newMAC(key *[32]byte) (h mac) {
-	initialize(key, &h.r, &h.s)
-	return
-}
-
 // mac is a wrapper for macGeneric that redirects calls that would have gone to
 // updateGeneric to update.
 //

vendor/golang.org/x/crypto/poly1305/sum_s390x.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build go1.11,!gccgo,!purego
+// +build !gccgo,!purego
 
 package poly1305
 
@@ -10,30 +10,66 @@ import (
 	"golang.org/x/sys/cpu"
 )
 
-// poly1305vx is an assembly implementation of Poly1305 that uses vector
+// updateVX is an assembly implementation of Poly1305 that uses vector
 // instructions. It must only be called if the vector facility (vx) is
 // available.
 //go:noescape
-func poly1305vx(out *[16]byte, m *byte, mlen uint64, key *[32]byte)
+func updateVX(state *macState, msg []byte)
 
-// poly1305vmsl is an assembly implementation of Poly1305 that uses vector
+// mac is a replacement for macGeneric that uses a larger buffer and redirects
-// instructions, including VMSL. It must only be called if the vector facility (vx) is
+// calls that would have gone to updateGeneric to updateVX if the vector
-// available and if VMSL is supported.
+// facility is installed.
-//go:noescape
+//
-func poly1305vmsl(out *[16]byte, m *byte, mlen uint64, key *[32]byte)
+// A larger buffer is required for good performance because the vector
+// implementation has a higher fixed cost per call than the generic
+// implementation.
+type mac struct {
+	macState
 
-func sum(out *[16]byte, m []byte, key *[32]byte) {
+	buffer [16 * TagSize]byte // size must be a multiple of block size (16)
+	offset int
+}
+
+func (h *mac) Write(p []byte) (int, error) {
+	nn := len(p)
+	if h.offset > 0 {
+		n := copy(h.buffer[h.offset:], p)
+		if h.offset+n < len(h.buffer) {
+			h.offset += n
+			return nn, nil
+		}
+		p = p[n:]
+		h.offset = 0
 		if cpu.S390X.HasVX {
-		var mPtr *byte
+			updateVX(&h.macState, h.buffer[:])
-		if len(m) > 0 {
-			mPtr = &m[0]
-		}
-		if cpu.S390X.HasVXE && len(m) > 256 {
-			poly1305vmsl(out, mPtr, uint64(len(m)), key)
 		} else {
-			poly1305vx(out, mPtr, uint64(len(m)), key)
+			updateGeneric(&h.macState, h.buffer[:])
 		}
+	}
+
+	tail := len(p) % len(h.buffer) // number of bytes to copy into buffer
+	body := len(p) - tail          // number of bytes to process now
+	if body > 0 {
+		if cpu.S390X.HasVX {
+			updateVX(&h.macState, p[:body])
 		} else {
-		sumGeneric(out, m, key)
+			updateGeneric(&h.macState, p[:body])
 		}
 	}
+	h.offset = copy(h.buffer[:], p[body:]) // copy tail bytes - can be 0
+	return nn, nil
+}
+
+func (h *mac) Sum(out *[TagSize]byte) {
+	state := h.macState
+	remainder := h.buffer[:h.offset]
+
+	// Use the generic implementation if we have 2 or fewer blocks left
+	// to sum. The vector implementation has a higher startup time.
+	if cpu.S390X.HasVX && len(remainder) > 2*TagSize {
+		updateVX(&state, remainder)
+	} else if len(remainder) > 0 {
+		updateGeneric(&state, remainder)
+	}
+	finalize(out, &state.h, &state.s)
+}

vendor/golang.org/x/crypto/poly1305/sum_s390x.s

@@ -2,115 +2,187 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build go1.11,!gccgo,!purego
+// +build !gccgo,!purego
 
 #include "textflag.h"
 
-// Implementation of Poly1305 using the vector facility (vx).
+// This implementation of Poly1305 uses the vector facility (vx)
+// to process up to 2 blocks (32 bytes) per iteration using an
+// algorithm based on the one described in:
+//
+// NEON crypto, Daniel J. Bernstein & Peter Schwabe
+// https://cryptojedi.org/papers/neoncrypto-20120320.pdf
+//
+// This algorithm uses 5 26-bit limbs to represent a 130-bit
+// value. These limbs are, for the most part, zero extended and
+// placed into 64-bit vector register elements. Each vector
+// register is 128-bits wide and so holds 2 of these elements.
+// Using 26-bit limbs allows us plenty of headroom to accomodate
+// accumulations before and after multiplication without
+// overflowing either 32-bits (before multiplication) or 64-bits
+// (after multiplication).
+//
+// In order to parallelise the operations required to calculate
+// the sum we use two separate accumulators and then sum those
+// in an extra final step. For compatibility with the generic
+// implementation we perform this summation at the end of every
+// updateVX call.
+//
+// To use two accumulators we must multiply the message blocks
+// by r² rather than r. Only the final message block should be
+// multiplied by r.
+//
+// Example:
+//
+// We want to calculate the sum (h) for a 64 byte message (m):
+//
+//   h = m[0:16]r⁴ + m[16:32]r³ + m[32:48]r² + m[48:64]r
+//
+// To do this we split the calculation into the even indices
+// and odd indices of the message. These form our SIMD 'lanes':
+//
+//   h = m[ 0:16]r⁴ + m[32:48]r² +   <- lane 0
+//       m[16:32]r³ + m[48:64]r      <- lane 1
+//
+// To calculate this iteratively we refactor so that both lanes
+// are written in terms of r² and r:
+//
+//   h = (m[ 0:16]r² + m[32:48])r² + <- lane 0
+//       (m[16:32]r² + m[48:64])r    <- lane 1
+//                ^             ^
+//                |             coefficients for second iteration
+//                coefficients for first iteration
+//
+// So in this case we would have two iterations. In the first
+// both lanes are multiplied by r². In the second only the
+// first lane is multiplied by r² and the second lane is
+// instead multiplied by r. This gives use the odd and even
+// powers of r that we need from the original equation.
+//
+// Notation:
+//
+//   h - accumulator
+//   r - key
+//   m - message
+//
+//   [a, b]       - SIMD register holding two 64-bit values
+//   [a, b, c, d] - SIMD register holding four 32-bit values
+//   xᵢ[n]        - limb n of variable x with bit width i
+//
+// Limbs are expressed in little endian order, so for 26-bit
+// limbs x₂₆[4] will be the most significant limb and x₂₆[0]
+// will be the least significant limb.
 
-// constants
+// masking constants
-#define MOD26 V0
+#define MOD24 V0 // [0x0000000000ffffff, 0x0000000000ffffff] - mask low 24-bits
-#define EX0   V1
+#define MOD26 V1 // [0x0000000003ffffff, 0x0000000003ffffff] - mask low 26-bits
-#define EX1   V2
-#define EX2   V3
 
-// temporaries
+// expansion constants (see EXPAND macro)
-#define T_0 V4
+#define EX0 V2
-#define T_1 V5
+#define EX1 V3
-#define T_2 V6
+#define EX2 V4
-#define T_3 V7
-#define T_4 V8
 
-// key (r)
+// key (r², r or 1 depending on context)
-#define R_0  V9
+#define R_0 V5
-#define R_1  V10
+#define R_1 V6
-#define R_2  V11
+#define R_2 V7
-#define R_3  V12
+#define R_3 V8
-#define R_4  V13
+#define R_4 V9
-#define R5_1 V14
-#define R5_2 V15
-#define R5_3 V16
-#define R5_4 V17
-#define RSAVE_0 R5
-#define RSAVE_1 R6
-#define RSAVE_2 R7
-#define RSAVE_3 R8
-#define RSAVE_4 R9
-#define R5SAVE_1 V28
-#define R5SAVE_2 V29
-#define R5SAVE_3 V30
-#define R5SAVE_4 V31
 
-// message block
+// precalculated coefficients (5r², 5r or 0 depending on context)
-#define F_0 V18
+#define R5_1 V10
-#define F_1 V19
+#define R5_2 V11
-#define F_2 V20
+#define R5_3 V12
-#define F_3 V21
+#define R5_4 V13
-#define F_4 V22
 
-// accumulator
+// message block (m)
-#define H_0 V23
+#define M_0 V14
-#define H_1 V24
+#define M_1 V15
-#define H_2 V25
+#define M_2 V16
-#define H_3 V26
+#define M_3 V17
-#define H_4 V27
+#define M_4 V18
 
-GLOBL ·keyMask<>(SB), RODATA, $16
+// accumulator (h)
-DATA ·keyMask<>+0(SB)/8, $0xffffff0ffcffff0f
+#define H_0 V19
-DATA ·keyMask<>+8(SB)/8, $0xfcffff0ffcffff0f
+#define H_1 V20
+#define H_2 V21
+#define H_3 V22
+#define H_4 V23
 
-GLOBL ·bswapMask<>(SB), RODATA, $16
+// temporary registers (for short-lived values)
-DATA ·bswapMask<>+0(SB)/8, $0x0f0e0d0c0b0a0908
+#define T_0 V24
-DATA ·bswapMask<>+8(SB)/8, $0x0706050403020100
+#define T_1 V25
+#define T_2 V26
+#define T_3 V27
+#define T_4 V28
 
-GLOBL ·constants<>(SB), RODATA, $64
+GLOBL ·constants<>(SB), RODATA, $0x30
-// MOD26
-DATA ·constants<>+0(SB)/8, $0x3ffffff
-DATA ·constants<>+8(SB)/8, $0x3ffffff
 // EX0
-DATA ·constants<>+16(SB)/8, $0x0006050403020100
+DATA ·constants<>+0x00(SB)/8, $0x0006050403020100
-DATA ·constants<>+24(SB)/8, $0x1016151413121110
+DATA ·constants<>+0x08(SB)/8, $0x1016151413121110
 // EX1
-DATA ·constants<>+32(SB)/8, $0x060c0b0a09080706
+DATA ·constants<>+0x10(SB)/8, $0x060c0b0a09080706
-DATA ·constants<>+40(SB)/8, $0x161c1b1a19181716
+DATA ·constants<>+0x18(SB)/8, $0x161c1b1a19181716
 // EX2
-DATA ·constants<>+48(SB)/8, $0x0d0d0d0d0d0f0e0d
+DATA ·constants<>+0x20(SB)/8, $0x0d0d0d0d0d0f0e0d
-DATA ·constants<>+56(SB)/8, $0x1d1d1d1d1d1f1e1d
+DATA ·constants<>+0x28(SB)/8, $0x1d1d1d1d1d1f1e1d
 
-// h = (f*g) % (2**130-5) [partial reduction]
+// MULTIPLY multiplies each lane of f and g, partially reduced
+// modulo 2¹³⁰ - 5. The result, h, consists of partial products
+// in each lane that need to be reduced further to produce the
+// final result.
+//
+//   h₁₃₀ = (f₁₃₀g₁₃₀) % 2¹³⁰ + (5f₁₃₀g₁₃₀) / 2¹³⁰
+//
+// Note that the multiplication by 5 of the high bits is
+// achieved by precalculating the multiplication of four of the
+// g coefficients by 5. These are g51-g54.
 #define MULTIPLY(f0, f1, f2, f3, f4, g0, g1, g2, g3, g4, g51, g52, g53, g54, h0, h1, h2, h3, h4) \
 	VMLOF  f0, g0, h0        \
-	VMLOF  f0, g1, h1        \
-	VMLOF  f0, g2, h2        \
 	VMLOF  f0, g3, h3        \
+	VMLOF  f0, g1, h1        \
 	VMLOF  f0, g4, h4        \
+	VMLOF  f0, g2, h2        \
 	VMLOF  f1, g54, T_0      \
-	VMLOF  f1, g0, T_1       \
-	VMLOF  f1, g1, T_2       \
 	VMLOF  f1, g2, T_3       \
+	VMLOF  f1, g0, T_1       \
 	VMLOF  f1, g3, T_4       \
+	VMLOF  f1, g1, T_2       \
 	VMALOF f2, g53, h0, h0   \
-	VMALOF f2, g54, h1, h1   \
-	VMALOF f2, g0, h2, h2    \
 	VMALOF f2, g1, h3, h3    \
+	VMALOF f2, g54, h1, h1   \
 	VMALOF f2, g2, h4, h4    \
+	VMALOF f2, g0, h2, h2    \
 	VMALOF f3, g52, T_0, T_0 \
-	VMALOF f3, g53, T_1, T_1 \
-	VMALOF f3, g54, T_2, T_2 \
 	VMALOF f3, g0, T_3, T_3  \
+	VMALOF f3, g53, T_1, T_1 \
 	VMALOF f3, g1, T_4, T_4  \
+	VMALOF f3, g54, T_2, T_2 \
 	VMALOF f4, g51, h0, h0   \
-	VMALOF f4, g52, h1, h1   \
-	VMALOF f4, g53, h2, h2   \
 	VMALOF f4, g54, h3, h3   \
+	VMALOF f4, g52, h1, h1   \
 	VMALOF f4, g0, h4, h4    \
+	VMALOF f4, g53, h2, h2   \
 	VAG    T_0, h0, h0       \
-	VAG    T_1, h1, h1       \
-	VAG    T_2, h2, h2       \
 	VAG    T_3, h3, h3       \
-	VAG    T_4, h4, h4
+	VAG    T_1, h1, h1       \
+	VAG    T_4, h4, h4       \
+	VAG    T_2, h2, h2
 
-// carry h0->h1 h3->h4, h1->h2 h4->h0, h0->h1 h2->h3, h3->h4
+// REDUCE performs the following carry operations in four
+// stages, as specified in Bernstein & Schwabe:
+//
+//   1: h₂₆[0]->h₂₆[1] h₂₆[3]->h₂₆[4]
+//   2: h₂₆[1]->h₂₆[2] h₂₆[4]->h₂₆[0]
+//   3: h₂₆[0]->h₂₆[1] h₂₆[2]->h₂₆[3]
+//   4: h₂₆[3]->h₂₆[4]
+//
+// The result is that all of the limbs are limited to 26-bits
+// except for h₂₆[1] and h₂₆[4] which are limited to 27-bits.
+//
+// Note that although each limb is aligned at 26-bit intervals
+// they may contain values that exceed 2²⁶ - 1, hence the need
+// to carry the excess bits in each limb.
 #define REDUCE(h0, h1, h2, h3, h4) \
 	VESRLG $26, h0, T_0  \
 	VESRLG $26, h3, T_1  \
@@ -136,144 +208,155 @@ DATA ·constants<>+56(SB)/8, $0x1d1d1d1d1d1f1e1d
 	VN     MOD26, h3, h3 \
 	VAG    T_2, h4, h4
 
-// expand in0 into d[0] and in1 into d[1]
+// EXPAND splits the 128-bit little-endian values in0 and in1
+// into 26-bit big-endian limbs and places the results into
+// the first and second lane of d₂₆[0:4] respectively.
+//
+// The EX0, EX1 and EX2 constants are arrays of byte indices
+// for permutation. The permutation both reverses the bytes
+// in the input and ensures the bytes are copied into the
+// destination limb ready to be shifted into their final
+// position.
 #define EXPAND(in0, in1, d0, d1, d2, d3, d4) \
-	VGBM   $0x0707, d1       \ // d1=tmp
-	VPERM  in0, in1, EX2, d4 \
 	VPERM  in0, in1, EX0, d0 \
 	VPERM  in0, in1, EX1, d2 \
-	VN     d1, d4, d4        \
+	VPERM  in0, in1, EX2, d4 \
 	VESRLG $26, d0, d1       \
 	VESRLG $30, d2, d3       \
 	VESRLG $4, d2, d2        \
-	VN     MOD26, d0, d0     \
+	VN     MOD26, d0, d0     \ // [in0₂₆[0], in1₂₆[0]]
-	VN     MOD26, d1, d1     \
+	VN     MOD26, d3, d3     \ // [in0₂₆[3], in1₂₆[3]]
-	VN     MOD26, d2, d2     \
+	VN     MOD26, d1, d1     \ // [in0₂₆[1], in1₂₆[1]]
-	VN     MOD26, d3, d3
+	VN     MOD24, d4, d4     \ // [in0₂₆[4], in1₂₆[4]]
+	VN     MOD26, d2, d2     // [in0₂₆[2], in1₂₆[2]]
 
-// pack h4:h0 into h1:h0 (no carry)
+// func updateVX(state *macState, msg []byte)
-#define PACK(h0, h1, h2, h3, h4) \
+TEXT ·updateVX(SB), NOSPLIT, $0
-	VESLG $26, h1, h1  \
+	MOVD state+0(FP), R1
-	VESLG $26, h3, h3  \
+	LMG  msg+8(FP), R2, R3 // R2=msg_base, R3=msg_len
-	VO    h0, h1, h0   \
-	VO    h2, h3, h2   \
-	VESLG $4, h2, h2   \
-	VLEIB $7, $48, h1  \
-	VSLB  h1, h2, h2   \
-	VO    h0, h2, h0   \
-	VLEIB $7, $104, h1 \
-	VSLB  h1, h4, h3   \
-	VO    h3, h0, h0   \
-	VLEIB $7, $24, h1  \
-	VSRLB h1, h4, h1
 
-// if h > 2**130-5 then h -= 2**130-5
+	// load EX0, EX1 and EX2
-#define MOD(h0, h1, t0, t1, t2) \
-	VZERO t0          \
-	VLEIG $1, $5, t0  \
-	VACCQ h0, t0, t1  \
-	VAQ   h0, t0, t0  \
-	VONE  t2          \
-	VLEIG $1, $-4, t2 \
-	VAQ   t2, t1, t1  \
-	VACCQ h1, t1, t1  \
-	VONE  t2          \
-	VAQ   t2, t1, t1  \
-	VN    h0, t1, t2  \
-	VNC   t0, t1, t1  \
-	VO    t1, t2, h0
-
-// func poly1305vx(out *[16]byte, m *byte, mlen uint64, key *[32]key)
-TEXT ·poly1305vx(SB), $0-32
-	// This code processes up to 2 blocks (32 bytes) per iteration
-	// using the algorithm described in:
-	// NEON crypto, Daniel J. Bernstein & Peter Schwabe
-	// https://cryptojedi.org/papers/neoncrypto-20120320.pdf
-	LMG out+0(FP), R1, R4 // R1=out, R2=m, R3=mlen, R4=key
-
-	// load MOD26, EX0, EX1 and EX2
 	MOVD $·constants<>(SB), R5
-	VLM  (R5), MOD26, EX2
+	VLM  (R5), EX0, EX2
 
-	// setup r
+	// generate masks
-	VL   (R4), T_0
+	VGMG $(64-24), $63, MOD24 // [0x00ffffff, 0x00ffffff]
-	MOVD $·keyMask<>(SB), R6
+	VGMG $(64-26), $63, MOD26 // [0x03ffffff, 0x03ffffff]
-	VL   (R6), T_1
-	VN   T_0, T_1, T_0
-	EXPAND(T_0, T_0, R_0, R_1, R_2, R_3, R_4)
 
-	// setup r*5
+	// load h (accumulator) and r (key) from state
-	VLEIG $0, $5, T_0
+	VZERO T_1               // [0, 0]
-	VLEIG $1, $5, T_0
+	VL    0(R1), T_0        // [h₆₄[0], h₆₄[1]]
+	VLEG  $0, 16(R1), T_1   // [h₆₄[2], 0]
+	VL    24(R1), T_2       // [r₆₄[0], r₆₄[1]]
+	VPDI  $0, T_0, T_2, T_3 // [h₆₄[0], r₆₄[0]]
+	VPDI  $5, T_0, T_2, T_4 // [h₆₄[1], r₆₄[1]]
 
-	// store r (for final block)
+	// unpack h and r into 26-bit limbs
-	VMLOF T_0, R_1, R5SAVE_1
+	// note: h₆₄[2] may have the low 3 bits set, so h₂₆[4] is a 27-bit value
-	VMLOF T_0, R_2, R5SAVE_2
+	VN     MOD26, T_3, H_0            // [h₂₆[0], r₂₆[0]]
-	VMLOF T_0, R_3, R5SAVE_3
+	VZERO  H_1                        // [0, 0]
-	VMLOF T_0, R_4, R5SAVE_4
+	VZERO  H_3                        // [0, 0]
-	VLGVG $0, R_0, RSAVE_0
+	VGMG   $(64-12-14), $(63-12), T_0 // [0x03fff000, 0x03fff000] - 26-bit mask with low 12 bits masked out
-	VLGVG $0, R_1, RSAVE_1
+	VESLG  $24, T_1, T_1              // [h₆₄[2]<<24, 0]
-	VLGVG $0, R_2, RSAVE_2
+	VERIMG $-26&63, T_3, MOD26, H_1   // [h₂₆[1], r₂₆[1]]
-	VLGVG $0, R_3, RSAVE_3
+	VESRLG $+52&63, T_3, H_2          // [h₂₆[2], r₂₆[2]] - low 12 bits only
-	VLGVG $0, R_4, RSAVE_4
+	VERIMG $-14&63, T_4, MOD26, H_3   // [h₂₆[1], r₂₆[1]]
+	VESRLG $40, T_4, H_4              // [h₂₆[4], r₂₆[4]] - low 24 bits only
+	VERIMG $+12&63, T_4, T_0, H_2     // [h₂₆[2], r₂₆[2]] - complete
+	VO     T_1, H_4, H_4              // [h₂₆[4], r₂₆[4]] - complete
 
-	// skip r**2 calculation
+	// replicate r across all 4 vector elements
+	VREPF $3, H_0, R_0 // [r₂₆[0], r₂₆[0], r₂₆[0], r₂₆[0]]
+	VREPF $3, H_1, R_1 // [r₂₆[1], r₂₆[1], r₂₆[1], r₂₆[1]]
+	VREPF $3, H_2, R_2 // [r₂₆[2], r₂₆[2], r₂₆[2], r₂₆[2]]
+	VREPF $3, H_3, R_3 // [r₂₆[3], r₂₆[3], r₂₆[3], r₂₆[3]]
+	VREPF $3, H_4, R_4 // [r₂₆[4], r₂₆[4], r₂₆[4], r₂₆[4]]
+
+	// zero out lane 1 of h
+	VLEIG $1, $0, H_0 // [h₂₆[0], 0]
+	VLEIG $1, $0, H_1 // [h₂₆[1], 0]
+	VLEIG $1, $0, H_2 // [h₂₆[2], 0]
+	VLEIG $1, $0, H_3 // [h₂₆[3], 0]
+	VLEIG $1, $0, H_4 // [h₂₆[4], 0]
+
+	// calculate 5r (ignore least significant limb)
+	VREPIF $5, T_0
+	VMLF   T_0, R_1, R5_1 // [5r₂₆[1], 5r₂₆[1], 5r₂₆[1], 5r₂₆[1]]
+	VMLF   T_0, R_2, R5_2 // [5r₂₆[2], 5r₂₆[2], 5r₂₆[2], 5r₂₆[2]]
+	VMLF   T_0, R_3, R5_3 // [5r₂₆[3], 5r₂₆[3], 5r₂₆[3], 5r₂₆[3]]
+	VMLF   T_0, R_4, R5_4 // [5r₂₆[4], 5r₂₆[4], 5r₂₆[4], 5r₂₆[4]]
+
+	// skip r² calculation if we are only calculating one block
 	CMPBLE R3, $16, skip
 
-	// calculate r**2
+	// calculate r²
-	MULTIPLY(R_0, R_1, R_2, R_3, R_4, R_0, R_1, R_2, R_3, R_4, R5SAVE_1, R5SAVE_2, R5SAVE_3, R5SAVE_4, H_0, H_1, H_2, H_3, H_4)
+	MULTIPLY(R_0, R_1, R_2, R_3, R_4, R_0, R_1, R_2, R_3, R_4, R5_1, R5_2, R5_3, R5_4, M_0, M_1, M_2, M_3, M_4)
-	REDUCE(H_0, H_1, H_2, H_3, H_4)
+	REDUCE(M_0, M_1, M_2, M_3, M_4)
-	VLEIG $0, $5, T_0
+	VGBM   $0x0f0f, T_0
-	VLEIG $1, $5, T_0
+	VERIMG $0, M_0, T_0, R_0 // [r₂₆[0], r²₂₆[0], r₂₆[0], r²₂₆[0]]
-	VMLOF T_0, H_1, R5_1
+	VERIMG $0, M_1, T_0, R_1 // [r₂₆[1], r²₂₆[1], r₂₆[1], r²₂₆[1]]
-	VMLOF T_0, H_2, R5_2
+	VERIMG $0, M_2, T_0, R_2 // [r₂₆[2], r²₂₆[2], r₂₆[2], r²₂₆[2]]
-	VMLOF T_0, H_3, R5_3
+	VERIMG $0, M_3, T_0, R_3 // [r₂₆[3], r²₂₆[3], r₂₆[3], r²₂₆[3]]
-	VMLOF T_0, H_4, R5_4
+	VERIMG $0, M_4, T_0, R_4 // [r₂₆[4], r²₂₆[4], r₂₆[4], r²₂₆[4]]
-	VLR   H_0, R_0
-	VLR   H_1, R_1
-	VLR   H_2, R_2
-	VLR   H_3, R_3
-	VLR   H_4, R_4
 
-	// initialize h
+	// calculate 5r² (ignore least significant limb)
-	VZERO H_0
+	VREPIF $5, T_0
-	VZERO H_1
+	VMLF   T_0, R_1, R5_1 // [5r₂₆[1], 5r²₂₆[1], 5r₂₆[1], 5r²₂₆[1]]
-	VZERO H_2
+	VMLF   T_0, R_2, R5_2 // [5r₂₆[2], 5r²₂₆[2], 5r₂₆[2], 5r²₂₆[2]]
-	VZERO H_3
+	VMLF   T_0, R_3, R5_3 // [5r₂₆[3], 5r²₂₆[3], 5r₂₆[3], 5r²₂₆[3]]
-	VZERO H_4
+	VMLF   T_0, R_4, R5_4 // [5r₂₆[4], 5r²₂₆[4], 5r₂₆[4], 5r²₂₆[4]]
 
 loop:
-	CMPBLE R3, $32, b2
+	CMPBLE R3, $32, b2 // 2 or fewer blocks remaining, need to change key coefficients
+
+	// load next 2 blocks from message
 	VLM (R2), T_0, T_1
+
+	// update message slice
 	SUB  $32, R3
 	MOVD $32(R2), R2
-	EXPAND(T_0, T_1, F_0, F_1, F_2, F_3, F_4)
+
-	VLEIB  $4, $1, F_4
+	// unpack message blocks into 26-bit big-endian limbs
-	VLEIB  $12, $1, F_4
+	EXPAND(T_0, T_1, M_0, M_1, M_2, M_3, M_4)
+
+	// add 2¹²⁸ to each message block value
+	VLEIB $4, $1, M_4
+	VLEIB $12, $1, M_4
 
 multiply:
-	VAG    H_0, F_0, F_0
+	// accumulate the incoming message
-	VAG    H_1, F_1, F_1
+	VAG H_0, M_0, M_0
-	VAG    H_2, F_2, F_2
+	VAG H_3, M_3, M_3
-	VAG    H_3, F_3, F_3
+	VAG H_1, M_1, M_1
-	VAG    H_4, F_4, F_4
+	VAG H_4, M_4, M_4
-	MULTIPLY(F_0, F_1, F_2, F_3, F_4, R_0, R_1, R_2, R_3, R_4, R5_1, R5_2, R5_3, R5_4, H_0, H_1, H_2, H_3, H_4)
+	VAG H_2, M_2, M_2
+
+	// multiply the accumulator by the key coefficient
+	MULTIPLY(M_0, M_1, M_2, M_3, M_4, R_0, R_1, R_2, R_3, R_4, R5_1, R5_2, R5_3, R5_4, H_0, H_1, H_2, H_3, H_4)
+
+	// carry and partially reduce the partial products
 	REDUCE(H_0, H_1, H_2, H_3, H_4)
+
 	CMPBNE R3, $0, loop
 
 finish:
-	// sum vectors
+	// sum lane 0 and lane 1 and put the result in lane 1
 	VZERO  T_0
 	VSUMQG H_0, T_0, H_0
-	VSUMQG H_1, T_0, H_1
-	VSUMQG H_2, T_0, H_2
 	VSUMQG H_3, T_0, H_3
+	VSUMQG H_1, T_0, H_1
 	VSUMQG H_4, T_0, H_4
+	VSUMQG H_2, T_0, H_2
 
-	// h may be >= 2*(2**130-5) so we need to reduce it again
+	// reduce again after summation
+	// TODO(mundaym): there might be a more efficient way to do this
+	// now that we only have 1 active lane. For example, we could
+	// simultaneously pack the values as we reduce them.
 	REDUCE(H_0, H_1, H_2, H_3, H_4)
 
-	// carry h1->h4
+	// carry h[1] through to h[4] so that only h[4] can exceed 2²⁶ - 1
+	// TODO(mundaym): in testing this final carry was unnecessary.
+	// Needs a proof before it can be removed though.
 	VESRLG $26, H_1, T_1
 	VN     MOD26, H_1, H_1
 	VAQ    T_1, H_2, H_2
@@ -284,95 +367,137 @@ finish:
 	VN     MOD26, H_3, H_3
 	VAQ    T_3, H_4, H_4
 
-	// h is now < 2*(2**130-5)
+	// h is now < 2(2¹³⁰ - 5)
-	// pack h into h1 (hi) and h0 (lo)
+	// Pack each lane in h₂₆[0:4] into h₁₂₈[0:1].
-	PACK(H_0, H_1, H_2, H_3, H_4)
+	VESLG $26, H_1, H_1
-
+	VESLG $26, H_3, H_3
-	// if h > 2**130-5 then h -= 2**130-5
+	VO    H_0, H_1, H_0
-	MOD(H_0, H_1, T_0, T_1, T_2)
+	VO    H_2, H_3, H_2
-
+	VESLG $4, H_2, H_2
-	// h += s
+	VLEIB $7, $48, H_1
-	MOVD  $·bswapMask<>(SB), R5
+	VSLB  H_1, H_2, H_2
-	VL    (R5), T_1
+	VO    H_0, H_2, H_0
-	VL    16(R4), T_0
+	VLEIB $7, $104, H_1
-	VPERM T_0, T_0, T_1, T_0    // reverse bytes (to big)
+	VSLB  H_1, H_4, H_3
-	VAQ   T_0, H_0, H_0
+	VO    H_3, H_0, H_0
-	VPERM H_0, H_0, T_1, H_0    // reverse bytes (to little)
+	VLEIB $7, $24, H_1
-	VST   H_0, (R1)
+	VSRLB H_1, H_4, H_1
 
+	// update state
+	VSTEG $1, H_0, 0(R1)
+	VSTEG $0, H_0, 8(R1)
+	VSTEG $1, H_1, 16(R1)
 	RET
 
-b2:
+b2:  // 2 or fewer blocks remaining
 	CMPBLE R3, $16, b1
 
-	// 2 blocks remaining
+	// Load the 2 remaining blocks (17-32 bytes remaining).
-	SUB    $17, R3
+	MOVD $-17(R3), R0    // index of final byte to load modulo 16
-	VL     (R2), T_0
+	VL   (R2), T_0       // load full 16 byte block
-	VLL    R3, 16(R2), T_1
+	VLL  R0, 16(R2), T_1 // load final (possibly partial) block and pad with zeros to 16 bytes
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBEQ R3, $16, 2(PC)
-	VLVGB  R3, R0, T_1
-	EXPAND(T_0, T_1, F_0, F_1, F_2, F_3, F_4)
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $12, $1, F_4
-	VLEIB  $4, $1, F_4
 
-	// setup [r²,r]
+	// The Poly1305 algorithm requires that a 1 bit be appended to
-	VLVGG $1, RSAVE_0, R_0
+	// each message block. If the final block is less than 16 bytes
-	VLVGG $1, RSAVE_1, R_1
+	// long then it is easiest to insert the 1 before the message
-	VLVGG $1, RSAVE_2, R_2
+	// block is split into 26-bit limbs. If, on the other hand, the
-	VLVGG $1, RSAVE_3, R_3
+	// final message block is 16 bytes long then we append the 1 bit
-	VLVGG $1, RSAVE_4, R_4
+	// after expansion as normal.
-	VPDI  $0, R5_1, R5SAVE_1, R5_1
+	MOVBZ  $1, R0
-	VPDI  $0, R5_2, R5SAVE_2, R5_2
+	MOVD   $-16(R3), R3   // index of byte in last block to insert 1 at (could be 16)
-	VPDI  $0, R5_3, R5SAVE_3, R5_3
+	CMPBEQ R3, $16, 2(PC) // skip the insertion if the final block is 16 bytes long
-	VPDI  $0, R5_4, R5SAVE_4, R5_4
+	VLVGB  R3, R0, T_1    // insert 1 into the byte at index R3
+
+	// Split both blocks into 26-bit limbs in the appropriate lanes.
+	EXPAND(T_0, T_1, M_0, M_1, M_2, M_3, M_4)
+
+	// Append a 1 byte to the end of the second to last block.
+	VLEIB $4, $1, M_4
+
+	// Append a 1 byte to the end of the last block only if it is a
+	// full 16 byte block.
+	CMPBNE R3, $16, 2(PC)
+	VLEIB  $12, $1, M_4
+
+	// Finally, set up the coefficients for the final multiplication.
+	// We have previously saved r and 5r in the 32-bit even indexes
+	// of the R_[0-4] and R5_[1-4] coefficient registers.
+	//
+	// We want lane 0 to be multiplied by r² so that can be kept the
+	// same. We want lane 1 to be multiplied by r so we need to move
+	// the saved r value into the 32-bit odd index in lane 1 by
+	// rotating the 64-bit lane by 32.
+	VGBM   $0x00ff, T_0         // [0, 0xffffffffffffffff] - mask lane 1 only
+	VERIMG $32, R_0, T_0, R_0   // [_,  r²₂₆[0], _,  r₂₆[0]]
+	VERIMG $32, R_1, T_0, R_1   // [_,  r²₂₆[1], _,  r₂₆[1]]
+	VERIMG $32, R_2, T_0, R_2   // [_,  r²₂₆[2], _,  r₂₆[2]]
+	VERIMG $32, R_3, T_0, R_3   // [_,  r²₂₆[3], _,  r₂₆[3]]
+	VERIMG $32, R_4, T_0, R_4   // [_,  r²₂₆[4], _,  r₂₆[4]]
+	VERIMG $32, R5_1, T_0, R5_1 // [_, 5r²₂₆[1], _, 5r₂₆[1]]
+	VERIMG $32, R5_2, T_0, R5_2 // [_, 5r²₂₆[2], _, 5r₂₆[2]]
+	VERIMG $32, R5_3, T_0, R5_3 // [_, 5r²₂₆[3], _, 5r₂₆[3]]
+	VERIMG $32, R5_4, T_0, R5_4 // [_, 5r²₂₆[4], _, 5r₂₆[4]]
 
 	MOVD $0, R3
 	BR   multiply
 
 skip:
-	VZERO H_0
-	VZERO H_1
-	VZERO H_2
-	VZERO H_3
-	VZERO H_4
-
 	CMPBEQ R3, $0, finish
 
-b1:
+b1:  // 1 block remaining
-	// 1 block remaining
+
-	SUB    $1, R3
+	// Load the final block (1-16 bytes). This will be placed into
-	VLL    R3, (R2), T_0
+	// lane 0.
-	ADD    $1, R3
+	MOVD $-1(R3), R0
+	VLL  R0, (R2), T_0 // pad to 16 bytes with zeros
+
+	// The Poly1305 algorithm requires that a 1 bit be appended to
+	// each message block. If the final block is less than 16 bytes
+	// long then it is easiest to insert the 1 before the message
+	// block is split into 26-bit limbs. If, on the other hand, the
+	// final message block is 16 bytes long then we append the 1 bit
+	// after expansion as normal.
 	MOVBZ  $1, R0
 	CMPBEQ R3, $16, 2(PC)
 	VLVGB  R3, R0, T_0
-	VZERO  T_1
-	EXPAND(T_0, T_1, F_0, F_1, F_2, F_3, F_4)
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $4, $1, F_4
-	VLEIG  $1, $1, R_0
-	VZERO  R_1
-	VZERO  R_2
-	VZERO  R_3
-	VZERO  R_4
-	VZERO  R5_1
-	VZERO  R5_2
-	VZERO  R5_3
-	VZERO  R5_4
 
-	// setup [r, 1]
+	// Set the message block in lane 1 to the value 0 so that it
-	VLVGG $0, RSAVE_0, R_0
+	// can be accumulated without affecting the final result.
-	VLVGG $0, RSAVE_1, R_1
+	VZERO T_1
-	VLVGG $0, RSAVE_2, R_2
+
-	VLVGG $0, RSAVE_3, R_3
+	// Split the final message block into 26-bit limbs in lane 0.
-	VLVGG $0, RSAVE_4, R_4
+	// Lane 1 will be contain 0.
-	VPDI  $0, R5SAVE_1, R5_1, R5_1
+	EXPAND(T_0, T_1, M_0, M_1, M_2, M_3, M_4)
-	VPDI  $0, R5SAVE_2, R5_2, R5_2
+
-	VPDI  $0, R5SAVE_3, R5_3, R5_3
+	// Append a 1 byte to the end of the last block only if it is a
-	VPDI  $0, R5SAVE_4, R5_4, R5_4
+	// full 16 byte block.
+	CMPBNE R3, $16, 2(PC)
+	VLEIB  $4, $1, M_4
+
+	// We have previously saved r and 5r in the 32-bit even indexes
+	// of the R_[0-4] and R5_[1-4] coefficient registers.
+	//
+	// We want lane 0 to be multiplied by r so we need to move the
+	// saved r value into the 32-bit odd index in lane 0. We want
+	// lane 1 to be set to the value 1. This makes multiplication
+	// a no-op. We do this by setting lane 1 in every register to 0
+	// and then just setting the 32-bit index 3 in R_0 to 1.
+	VZERO T_0
+	MOVD  $0, R0
+	MOVD  $0x10111213, R12
+	VLVGP R12, R0, T_1         // [_, 0x10111213, _, 0x00000000]
+	VPERM T_0, R_0, T_1, R_0   // [_,  r₂₆[0], _, 0]
+	VPERM T_0, R_1, T_1, R_1   // [_,  r₂₆[1], _, 0]
+	VPERM T_0, R_2, T_1, R_2   // [_,  r₂₆[2], _, 0]
+	VPERM T_0, R_3, T_1, R_3   // [_,  r₂₆[3], _, 0]
+	VPERM T_0, R_4, T_1, R_4   // [_,  r₂₆[4], _, 0]
+	VPERM T_0, R5_1, T_1, R5_1 // [_, 5r₂₆[1], _, 0]
+	VPERM T_0, R5_2, T_1, R5_2 // [_, 5r₂₆[2], _, 0]
+	VPERM T_0, R5_3, T_1, R5_3 // [_, 5r₂₆[3], _, 0]
+	VPERM T_0, R5_4, T_1, R5_4 // [_, 5r₂₆[4], _, 0]
+
+	// Set the value of lane 1 to be 1.
+	VLEIF $3, $1, R_0 // [_,  r₂₆[0], _, 1]
 
 	MOVD $0, R3
 	BR   multiply

vendor/golang.org/x/crypto/poly1305/sum_vmsl_s390x.s

@@ -1,909 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build go1.11,!gccgo,!purego
-
-#include "textflag.h"
-
-// Implementation of Poly1305 using the vector facility (vx) and the VMSL instruction.
-
-// constants
-#define EX0   V1
-#define EX1   V2
-#define EX2   V3
-
-// temporaries
-#define T_0 V4
-#define T_1 V5
-#define T_2 V6
-#define T_3 V7
-#define T_4 V8
-#define T_5 V9
-#define T_6 V10
-#define T_7 V11
-#define T_8 V12
-#define T_9 V13
-#define T_10 V14
-
-// r**2 & r**4
-#define R_0  V15
-#define R_1  V16
-#define R_2  V17
-#define R5_1 V18
-#define R5_2 V19
-// key (r)
-#define RSAVE_0 R7
-#define RSAVE_1 R8
-#define RSAVE_2 R9
-#define R5SAVE_1 R10
-#define R5SAVE_2 R11
-
-// message block
-#define M0 V20
-#define M1 V21
-#define M2 V22
-#define M3 V23
-#define M4 V24
-#define M5 V25
-
-// accumulator
-#define H0_0 V26
-#define H1_0 V27
-#define H2_0 V28
-#define H0_1 V29
-#define H1_1 V30
-#define H2_1 V31
-
-GLOBL ·keyMask<>(SB), RODATA, $16
-DATA ·keyMask<>+0(SB)/8, $0xffffff0ffcffff0f
-DATA ·keyMask<>+8(SB)/8, $0xfcffff0ffcffff0f
-
-GLOBL ·bswapMask<>(SB), RODATA, $16
-DATA ·bswapMask<>+0(SB)/8, $0x0f0e0d0c0b0a0908
-DATA ·bswapMask<>+8(SB)/8, $0x0706050403020100
-
-GLOBL ·constants<>(SB), RODATA, $48
-// EX0
-DATA ·constants<>+0(SB)/8, $0x18191a1b1c1d1e1f
-DATA ·constants<>+8(SB)/8, $0x0000050403020100
-// EX1
-DATA ·constants<>+16(SB)/8, $0x18191a1b1c1d1e1f
-DATA ·constants<>+24(SB)/8, $0x00000a0908070605
-// EX2
-DATA ·constants<>+32(SB)/8, $0x18191a1b1c1d1e1f
-DATA ·constants<>+40(SB)/8, $0x0000000f0e0d0c0b
-
-GLOBL ·c<>(SB), RODATA, $48
-// EX0
-DATA ·c<>+0(SB)/8, $0x0000050403020100
-DATA ·c<>+8(SB)/8, $0x0000151413121110
-// EX1
-DATA ·c<>+16(SB)/8, $0x00000a0908070605
-DATA ·c<>+24(SB)/8, $0x00001a1918171615
-// EX2
-DATA ·c<>+32(SB)/8, $0x0000000f0e0d0c0b
-DATA ·c<>+40(SB)/8, $0x0000001f1e1d1c1b
-
-GLOBL ·reduce<>(SB), RODATA, $32
-// 44 bit
-DATA ·reduce<>+0(SB)/8, $0x0
-DATA ·reduce<>+8(SB)/8, $0xfffffffffff
-// 42 bit
-DATA ·reduce<>+16(SB)/8, $0x0
-DATA ·reduce<>+24(SB)/8, $0x3ffffffffff
-
-// h = (f*g) % (2**130-5) [partial reduction]
-// uses T_0...T_9 temporary registers
-// input: m02_0, m02_1, m02_2, m13_0, m13_1, m13_2, r_0, r_1, r_2, r5_1, r5_2, m4_0, m4_1, m4_2, m5_0, m5_1, m5_2
-// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8, t9
-// output: m02_0, m02_1, m02_2, m13_0, m13_1, m13_2
-#define MULTIPLY(m02_0, m02_1, m02_2, m13_0, m13_1, m13_2, r_0, r_1, r_2, r5_1, r5_2, m4_0, m4_1, m4_2, m5_0, m5_1, m5_2, t0, t1, t2, t3, t4, t5, t6, t7, t8, t9) \
-	\ // Eliminate the dependency for the last 2 VMSLs
-	VMSLG m02_0, r_2, m4_2, m4_2                       \
-	VMSLG m13_0, r_2, m5_2, m5_2                       \ // 8 VMSLs pipelined
-	VMSLG m02_0, r_0, m4_0, m4_0                       \
-	VMSLG m02_1, r5_2, V0, T_0                         \
-	VMSLG m02_0, r_1, m4_1, m4_1                       \
-	VMSLG m02_1, r_0, V0, T_1                          \
-	VMSLG m02_1, r_1, V0, T_2                          \
-	VMSLG m02_2, r5_1, V0, T_3                         \
-	VMSLG m02_2, r5_2, V0, T_4                         \
-	VMSLG m13_0, r_0, m5_0, m5_0                       \
-	VMSLG m13_1, r5_2, V0, T_5                         \
-	VMSLG m13_0, r_1, m5_1, m5_1                       \
-	VMSLG m13_1, r_0, V0, T_6                          \
-	VMSLG m13_1, r_1, V0, T_7                          \
-	VMSLG m13_2, r5_1, V0, T_8                         \
-	VMSLG m13_2, r5_2, V0, T_9                         \
-	VMSLG m02_2, r_0, m4_2, m4_2                       \
-	VMSLG m13_2, r_0, m5_2, m5_2                       \
-	VAQ   m4_0, T_0, m02_0                             \
-	VAQ   m4_1, T_1, m02_1                             \
-	VAQ   m5_0, T_5, m13_0                             \
-	VAQ   m5_1, T_6, m13_1                             \
-	VAQ   m02_0, T_3, m02_0                            \
-	VAQ   m02_1, T_4, m02_1                            \
-	VAQ   m13_0, T_8, m13_0                            \
-	VAQ   m13_1, T_9, m13_1                            \
-	VAQ   m4_2, T_2, m02_2                             \
-	VAQ   m5_2, T_7, m13_2                             \
-
-// SQUARE uses three limbs of r and r_2*5 to output square of r
-// uses T_1, T_5 and T_7 temporary registers
-// input: r_0, r_1, r_2, r5_2
-// temp: TEMP0, TEMP1, TEMP2
-// output: p0, p1, p2
-#define SQUARE(r_0, r_1, r_2, r5_2, p0, p1, p2, TEMP0, TEMP1, TEMP2) \
-	VMSLG r_0, r_0, p0, p0     \
-	VMSLG r_1, r5_2, V0, TEMP0 \
-	VMSLG r_2, r5_2, p1, p1    \
-	VMSLG r_0, r_1, V0, TEMP1  \
-	VMSLG r_1, r_1, p2, p2     \
-	VMSLG r_0, r_2, V0, TEMP2  \
-	VAQ   TEMP0, p0, p0        \
-	VAQ   TEMP1, p1, p1        \
-	VAQ   TEMP2, p2, p2        \
-	VAQ   TEMP0, p0, p0        \
-	VAQ   TEMP1, p1, p1        \
-	VAQ   TEMP2, p2, p2        \
-
-// carry h0->h1->h2->h0 || h3->h4->h5->h3
-// uses T_2, T_4, T_5, T_7, T_8, T_9
-//       t6,  t7,  t8,  t9, t10, t11
-// input: h0, h1, h2, h3, h4, h5
-// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11
-// output: h0, h1, h2, h3, h4, h5
-#define REDUCE(h0, h1, h2, h3, h4, h5, t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) \
-	VLM    (R12), t6, t7  \ // 44 and 42 bit clear mask
-	VLEIB  $7, $0x28, t10 \ // 5 byte shift mask
-	VREPIB $4, t8         \ // 4 bit shift mask
-	VREPIB $2, t11        \ // 2 bit shift mask
-	VSRLB  t10, h0, t0    \ // h0 byte shift
-	VSRLB  t10, h1, t1    \ // h1 byte shift
-	VSRLB  t10, h2, t2    \ // h2 byte shift
-	VSRLB  t10, h3, t3    \ // h3 byte shift
-	VSRLB  t10, h4, t4    \ // h4 byte shift
-	VSRLB  t10, h5, t5    \ // h5 byte shift
-	VSRL   t8, t0, t0     \ // h0 bit shift
-	VSRL   t8, t1, t1     \ // h2 bit shift
-	VSRL   t11, t2, t2    \ // h2 bit shift
-	VSRL   t8, t3, t3     \ // h3 bit shift
-	VSRL   t8, t4, t4     \ // h4 bit shift
-	VESLG  $2, t2, t9     \ // h2 carry x5
-	VSRL   t11, t5, t5    \ // h5 bit shift
-	VN     t6, h0, h0     \ // h0 clear carry
-	VAQ    t2, t9, t2     \ // h2 carry x5
-	VESLG  $2, t5, t9     \ // h5 carry x5
-	VN     t6, h1, h1     \ // h1 clear carry
-	VN     t7, h2, h2     \ // h2 clear carry
-	VAQ    t5, t9, t5     \ // h5 carry x5
-	VN     t6, h3, h3     \ // h3 clear carry
-	VN     t6, h4, h4     \ // h4 clear carry
-	VN     t7, h5, h5     \ // h5 clear carry
-	VAQ    t0, h1, h1     \ // h0->h1
-	VAQ    t3, h4, h4     \ // h3->h4
-	VAQ    t1, h2, h2     \ // h1->h2
-	VAQ    t4, h5, h5     \ // h4->h5
-	VAQ    t2, h0, h0     \ // h2->h0
-	VAQ    t5, h3, h3     \ // h5->h3
-	VREPG  $1, t6, t6     \ // 44 and 42 bit masks across both halves
-	VREPG  $1, t7, t7     \
-	VSLDB  $8, h0, h0, h0 \ // set up [h0/1/2, h3/4/5]
-	VSLDB  $8, h1, h1, h1 \
-	VSLDB  $8, h2, h2, h2 \
-	VO     h0, h3, h3     \
-	VO     h1, h4, h4     \
-	VO     h2, h5, h5     \
-	VESRLG $44, h3, t0    \ // 44 bit shift right
-	VESRLG $44, h4, t1    \
-	VESRLG $42, h5, t2    \
-	VN     t6, h3, h3     \ // clear carry bits
-	VN     t6, h4, h4     \
-	VN     t7, h5, h5     \
-	VESLG  $2, t2, t9     \ // multiply carry by 5
-	VAQ    t9, t2, t2     \
-	VAQ    t0, h4, h4     \
-	VAQ    t1, h5, h5     \
-	VAQ    t2, h3, h3     \
-
-// carry h0->h1->h2->h0
-// input: h0, h1, h2
-// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8
-// output: h0, h1, h2
-#define REDUCE2(h0, h1, h2, t0, t1, t2, t3, t4, t5, t6, t7, t8) \
-	VLEIB  $7, $0x28, t3 \ // 5 byte shift mask
-	VREPIB $4, t4        \ // 4 bit shift mask
-	VREPIB $2, t7        \ // 2 bit shift mask
-	VGBM   $0x003F, t5   \ // mask to clear carry bits
-	VSRLB  t3, h0, t0    \
-	VSRLB  t3, h1, t1    \
-	VSRLB  t3, h2, t2    \
-	VESRLG $4, t5, t5    \ // 44 bit clear mask
-	VSRL   t4, t0, t0    \
-	VSRL   t4, t1, t1    \
-	VSRL   t7, t2, t2    \
-	VESRLG $2, t5, t6    \ // 42 bit clear mask
-	VESLG  $2, t2, t8    \
-	VAQ    t8, t2, t2    \
-	VN     t5, h0, h0    \
-	VN     t5, h1, h1    \
-	VN     t6, h2, h2    \
-	VAQ    t0, h1, h1    \
-	VAQ    t1, h2, h2    \
-	VAQ    t2, h0, h0    \
-	VSRLB  t3, h0, t0    \
-	VSRLB  t3, h1, t1    \
-	VSRLB  t3, h2, t2    \
-	VSRL   t4, t0, t0    \
-	VSRL   t4, t1, t1    \
-	VSRL   t7, t2, t2    \
-	VN     t5, h0, h0    \
-	VN     t5, h1, h1    \
-	VESLG  $2, t2, t8    \
-	VN     t6, h2, h2    \
-	VAQ    t0, h1, h1    \
-	VAQ    t8, t2, t2    \
-	VAQ    t1, h2, h2    \
-	VAQ    t2, h0, h0    \
-
-// expands two message blocks into the lower halfs of the d registers
-// moves the contents of the d registers into upper halfs
-// input: in1, in2, d0, d1, d2, d3, d4, d5
-// temp: TEMP0, TEMP1, TEMP2, TEMP3
-// output: d0, d1, d2, d3, d4, d5
-#define EXPACC(in1, in2, d0, d1, d2, d3, d4, d5, TEMP0, TEMP1, TEMP2, TEMP3) \
-	VGBM   $0xff3f, TEMP0      \
-	VGBM   $0xff1f, TEMP1      \
-	VESLG  $4, d1, TEMP2       \
-	VESLG  $4, d4, TEMP3       \
-	VESRLG $4, TEMP0, TEMP0    \
-	VPERM  in1, d0, EX0, d0    \
-	VPERM  in2, d3, EX0, d3    \
-	VPERM  in1, d2, EX2, d2    \
-	VPERM  in2, d5, EX2, d5    \
-	VPERM  in1, TEMP2, EX1, d1 \
-	VPERM  in2, TEMP3, EX1, d4 \
-	VN     TEMP0, d0, d0       \
-	VN     TEMP0, d3, d3       \
-	VESRLG $4, d1, d1          \
-	VESRLG $4, d4, d4          \
-	VN     TEMP1, d2, d2       \
-	VN     TEMP1, d5, d5       \
-	VN     TEMP0, d1, d1       \
-	VN     TEMP0, d4, d4       \
-
-// expands one message block into the lower halfs of the d registers
-// moves the contents of the d registers into upper halfs
-// input: in, d0, d1, d2
-// temp: TEMP0, TEMP1, TEMP2
-// output: d0, d1, d2
-#define EXPACC2(in, d0, d1, d2, TEMP0, TEMP1, TEMP2) \
-	VGBM   $0xff3f, TEMP0     \
-	VESLG  $4, d1, TEMP2      \
-	VGBM   $0xff1f, TEMP1     \
-	VPERM  in, d0, EX0, d0    \
-	VESRLG $4, TEMP0, TEMP0   \
-	VPERM  in, d2, EX2, d2    \
-	VPERM  in, TEMP2, EX1, d1 \
-	VN     TEMP0, d0, d0      \
-	VN     TEMP1, d2, d2      \
-	VESRLG $4, d1, d1         \
-	VN     TEMP0, d1, d1      \
-
-// pack h2:h0 into h1:h0 (no carry)
-// input: h0, h1, h2
-// output: h0, h1, h2
-#define PACK(h0, h1, h2) \
-	VMRLG  h1, h2, h2  \ // copy h1 to upper half h2
-	VESLG  $44, h1, h1 \ // shift limb 1 44 bits, leaving 20
-	VO     h0, h1, h0  \ // combine h0 with 20 bits from limb 1
-	VESRLG $20, h2, h1 \ // put top 24 bits of limb 1 into h1
-	VLEIG  $1, $0, h1  \ // clear h2 stuff from lower half of h1
-	VO     h0, h1, h0  \ // h0 now has 88 bits (limb 0 and 1)
-	VLEIG  $0, $0, h2  \ // clear upper half of h2
-	VESRLG $40, h2, h1 \ // h1 now has upper two bits of result
-	VLEIB  $7, $88, h1 \ // for byte shift (11 bytes)
-	VSLB   h1, h2, h2  \ // shift h2 11 bytes to the left
-	VO     h0, h2, h0  \ // combine h0 with 20 bits from limb 1
-	VLEIG  $0, $0, h1  \ // clear upper half of h1
-
-// if h > 2**130-5 then h -= 2**130-5
-// input: h0, h1
-// temp: t0, t1, t2
-// output: h0
-#define MOD(h0, h1, t0, t1, t2) \
-	VZERO t0          \
-	VLEIG $1, $5, t0  \
-	VACCQ h0, t0, t1  \
-	VAQ   h0, t0, t0  \
-	VONE  t2          \
-	VLEIG $1, $-4, t2 \
-	VAQ   t2, t1, t1  \
-	VACCQ h1, t1, t1  \
-	VONE  t2          \
-	VAQ   t2, t1, t1  \
-	VN    h0, t1, t2  \
-	VNC   t0, t1, t1  \
-	VO    t1, t2, h0  \
-
-// func poly1305vmsl(out *[16]byte, m *byte, mlen uint64, key *[32]key)
-TEXT ·poly1305vmsl(SB), $0-32
-	// This code processes 6 + up to 4 blocks (32 bytes) per iteration
-	// using the algorithm described in:
-	// NEON crypto, Daniel J. Bernstein & Peter Schwabe
-	// https://cryptojedi.org/papers/neoncrypto-20120320.pdf
-	// And as moddified for VMSL as described in
-	// Accelerating Poly1305 Cryptographic Message Authentication on the z14
-	// O'Farrell et al, CASCON 2017, p48-55
-	// https://ibm.ent.box.com/s/jf9gedj0e9d2vjctfyh186shaztavnht
-
-	LMG   out+0(FP), R1, R4 // R1=out, R2=m, R3=mlen, R4=key
-	VZERO V0                // c
-
-	// load EX0, EX1 and EX2
-	MOVD $·constants<>(SB), R5
-	VLM  (R5), EX0, EX2        // c
-
-	// setup r
-	VL    (R4), T_0
-	MOVD  $·keyMask<>(SB), R6
-	VL    (R6), T_1
-	VN    T_0, T_1, T_0
-	VZERO T_2                 // limbs for r
-	VZERO T_3
-	VZERO T_4
-	EXPACC2(T_0, T_2, T_3, T_4, T_1, T_5, T_7)
-
-	// T_2, T_3, T_4: [0, r]
-
-	// setup r*20
-	VLEIG $0, $0, T_0
-	VLEIG $1, $20, T_0       // T_0: [0, 20]
-	VZERO T_5
-	VZERO T_6
-	VMSLG T_0, T_3, T_5, T_5
-	VMSLG T_0, T_4, T_6, T_6
-
-	// store r for final block in GR
-	VLGVG $1, T_2, RSAVE_0  // c
-	VLGVG $1, T_3, RSAVE_1  // c
-	VLGVG $1, T_4, RSAVE_2  // c
-	VLGVG $1, T_5, R5SAVE_1 // c
-	VLGVG $1, T_6, R5SAVE_2 // c
-
-	// initialize h
-	VZERO H0_0
-	VZERO H1_0
-	VZERO H2_0
-	VZERO H0_1
-	VZERO H1_1
-	VZERO H2_1
-
-	// initialize pointer for reduce constants
-	MOVD $·reduce<>(SB), R12
-
-	// calculate r**2 and 20*(r**2)
-	VZERO R_0
-	VZERO R_1
-	VZERO R_2
-	SQUARE(T_2, T_3, T_4, T_6, R_0, R_1, R_2, T_1, T_5, T_7)
-	REDUCE2(R_0, R_1, R_2, M0, M1, M2, M3, M4, R5_1, R5_2, M5, T_1)
-	VZERO R5_1
-	VZERO R5_2
-	VMSLG T_0, R_1, R5_1, R5_1
-	VMSLG T_0, R_2, R5_2, R5_2
-
-	// skip r**4 calculation if 3 blocks or less
-	CMPBLE R3, $48, b4
-
-	// calculate r**4 and 20*(r**4)
-	VZERO T_8
-	VZERO T_9
-	VZERO T_10
-	SQUARE(R_0, R_1, R_2, R5_2, T_8, T_9, T_10, T_1, T_5, T_7)
-	REDUCE2(T_8, T_9, T_10, M0, M1, M2, M3, M4, T_2, T_3, M5, T_1)
-	VZERO T_2
-	VZERO T_3
-	VMSLG T_0, T_9, T_2, T_2
-	VMSLG T_0, T_10, T_3, T_3
-
-	// put r**2 to the right and r**4 to the left of R_0, R_1, R_2
-	VSLDB $8, T_8, T_8, T_8
-	VSLDB $8, T_9, T_9, T_9
-	VSLDB $8, T_10, T_10, T_10
-	VSLDB $8, T_2, T_2, T_2
-	VSLDB $8, T_3, T_3, T_3
-
-	VO T_8, R_0, R_0
-	VO T_9, R_1, R_1
-	VO T_10, R_2, R_2
-	VO T_2, R5_1, R5_1
-	VO T_3, R5_2, R5_2
-
-	CMPBLE R3, $80, load // less than or equal to 5 blocks in message
-
-	// 6(or 5+1) blocks
-	SUB    $81, R3
-	VLM    (R2), M0, M4
-	VLL    R3, 80(R2), M5
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBGE R3, $16, 2(PC)
-	VLVGB  R3, R0, M5
-	MOVD   $96(R2), R2
-	EXPACC(M0, M1, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3)
-	EXPACC(M2, M3, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3)
-	VLEIB  $2, $1, H2_0
-	VLEIB  $2, $1, H2_1
-	VLEIB  $10, $1, H2_0
-	VLEIB  $10, $1, H2_1
-
-	VZERO  M0
-	VZERO  M1
-	VZERO  M2
-	VZERO  M3
-	VZERO  T_4
-	VZERO  T_10
-	EXPACC(M4, M5, M0, M1, M2, M3, T_4, T_10, T_0, T_1, T_2, T_3)
-	VLR    T_4, M4
-	VLEIB  $10, $1, M2
-	CMPBLT R3, $16, 2(PC)
-	VLEIB  $10, $1, T_10
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M2, M3, M4, T_4, T_5, T_2, T_7, T_8, T_9)
-	VMRHG  V0, H0_1, H0_0
-	VMRHG  V0, H1_1, H1_0
-	VMRHG  V0, H2_1, H2_0
-	VMRLG  V0, H0_1, H0_1
-	VMRLG  V0, H1_1, H1_1
-	VMRLG  V0, H2_1, H2_1
-
-	SUB    $16, R3
-	CMPBLE R3, $0, square
-
-load:
-	// load EX0, EX1 and EX2
-	MOVD $·c<>(SB), R5
-	VLM  (R5), EX0, EX2
-
-loop:
-	CMPBLE R3, $64, add // b4	// last 4 or less blocks left
-
-	// next 4 full blocks
-	VLM  (R2), M2, M5
-	SUB  $64, R3
-	MOVD $64(R2), R2
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, T_0, T_1, T_3, T_4, T_5, T_2, T_7, T_8, T_9)
-
-	// expacc in-lined to create [m2, m3] limbs
-	VGBM   $0x3f3f, T_0     // 44 bit clear mask
-	VGBM   $0x1f1f, T_1     // 40 bit clear mask
-	VPERM  M2, M3, EX0, T_3
-	VESRLG $4, T_0, T_0     // 44 bit clear mask ready
-	VPERM  M2, M3, EX1, T_4
-	VPERM  M2, M3, EX2, T_5
-	VN     T_0, T_3, T_3
-	VESRLG $4, T_4, T_4
-	VN     T_1, T_5, T_5
-	VN     T_0, T_4, T_4
-	VMRHG  H0_1, T_3, H0_0
-	VMRHG  H1_1, T_4, H1_0
-	VMRHG  H2_1, T_5, H2_0
-	VMRLG  H0_1, T_3, H0_1
-	VMRLG  H1_1, T_4, H1_1
-	VMRLG  H2_1, T_5, H2_1
-	VLEIB  $10, $1, H2_0
-	VLEIB  $10, $1, H2_1
-	VPERM  M4, M5, EX0, T_3
-	VPERM  M4, M5, EX1, T_4
-	VPERM  M4, M5, EX2, T_5
-	VN     T_0, T_3, T_3
-	VESRLG $4, T_4, T_4
-	VN     T_1, T_5, T_5
-	VN     T_0, T_4, T_4
-	VMRHG  V0, T_3, M0
-	VMRHG  V0, T_4, M1
-	VMRHG  V0, T_5, M2
-	VMRLG  V0, T_3, M3
-	VMRLG  V0, T_4, M4
-	VMRLG  V0, T_5, M5
-	VLEIB  $10, $1, M2
-	VLEIB  $10, $1, M5
-
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	CMPBNE R3, $0, loop
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9)
-	VMRHG  V0, H0_1, H0_0
-	VMRHG  V0, H1_1, H1_0
-	VMRHG  V0, H2_1, H2_0
-	VMRLG  V0, H0_1, H0_1
-	VMRLG  V0, H1_1, H1_1
-	VMRLG  V0, H2_1, H2_1
-
-	// load EX0, EX1, EX2
-	MOVD $·constants<>(SB), R5
-	VLM  (R5), EX0, EX2
-
-	// sum vectors
-	VAQ H0_0, H0_1, H0_0
-	VAQ H1_0, H1_1, H1_0
-	VAQ H2_0, H2_1, H2_0
-
-	// h may be >= 2*(2**130-5) so we need to reduce it again
-	// M0...M4 are used as temps here
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
-
-next:  // carry h1->h2
-	VLEIB  $7, $0x28, T_1
-	VREPIB $4, T_2
-	VGBM   $0x003F, T_3
-	VESRLG $4, T_3
-
-	// byte shift
-	VSRLB T_1, H1_0, T_4
-
-	// bit shift
-	VSRL T_2, T_4, T_4
-
-	// clear h1 carry bits
-	VN T_3, H1_0, H1_0
-
-	// add carry
-	VAQ T_4, H2_0, H2_0
-
-	// h is now < 2*(2**130-5)
-	// pack h into h1 (hi) and h0 (lo)
-	PACK(H0_0, H1_0, H2_0)
-
-	// if h > 2**130-5 then h -= 2**130-5
-	MOD(H0_0, H1_0, T_0, T_1, T_2)
-
-	// h += s
-	MOVD  $·bswapMask<>(SB), R5
-	VL    (R5), T_1
-	VL    16(R4), T_0
-	VPERM T_0, T_0, T_1, T_0    // reverse bytes (to big)
-	VAQ   T_0, H0_0, H0_0
-	VPERM H0_0, H0_0, T_1, H0_0 // reverse bytes (to little)
-	VST   H0_0, (R1)
-	RET
-
-add:
-	// load EX0, EX1, EX2
-	MOVD $·constants<>(SB), R5
-	VLM  (R5), EX0, EX2
-
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9)
-	VMRHG  V0, H0_1, H0_0
-	VMRHG  V0, H1_1, H1_0
-	VMRHG  V0, H2_1, H2_0
-	VMRLG  V0, H0_1, H0_1
-	VMRLG  V0, H1_1, H1_1
-	VMRLG  V0, H2_1, H2_1
-	CMPBLE R3, $64, b4
-
-b4:
-	CMPBLE R3, $48, b3 // 3 blocks or less
-
-	// 4(3+1) blocks remaining
-	SUB    $49, R3
-	VLM    (R2), M0, M2
-	VLL    R3, 48(R2), M3
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBEQ R3, $16, 2(PC)
-	VLVGB  R3, R0, M3
-	MOVD   $64(R2), R2
-	EXPACC(M0, M1, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3)
-	VLEIB  $10, $1, H2_0
-	VLEIB  $10, $1, H2_1
-	VZERO  M0
-	VZERO  M1
-	VZERO  M4
-	VZERO  M5
-	VZERO  T_4
-	VZERO  T_10
-	EXPACC(M2, M3, M0, M1, M4, M5, T_4, T_10, T_0, T_1, T_2, T_3)
-	VLR    T_4, M2
-	VLEIB  $10, $1, M4
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $10, $1, T_10
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M4, M5, M2, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9)
-	VMRHG  V0, H0_1, H0_0
-	VMRHG  V0, H1_1, H1_0
-	VMRHG  V0, H2_1, H2_0
-	VMRLG  V0, H0_1, H0_1
-	VMRLG  V0, H1_1, H1_1
-	VMRLG  V0, H2_1, H2_1
-	SUB    $16, R3
-	CMPBLE R3, $0, square // this condition must always hold true!
-
-b3:
-	CMPBLE R3, $32, b2
-
-	// 3 blocks remaining
-
-	// setup [r²,r]
-	VSLDB $8, R_0, R_0, R_0
-	VSLDB $8, R_1, R_1, R_1
-	VSLDB $8, R_2, R_2, R_2
-	VSLDB $8, R5_1, R5_1, R5_1
-	VSLDB $8, R5_2, R5_2, R5_2
-
-	VLVGG $1, RSAVE_0, R_0
-	VLVGG $1, RSAVE_1, R_1
-	VLVGG $1, RSAVE_2, R_2
-	VLVGG $1, R5SAVE_1, R5_1
-	VLVGG $1, R5SAVE_2, R5_2
-
-	// setup [h0, h1]
-	VSLDB $8, H0_0, H0_0, H0_0
-	VSLDB $8, H1_0, H1_0, H1_0
-	VSLDB $8, H2_0, H2_0, H2_0
-	VO    H0_1, H0_0, H0_0
-	VO    H1_1, H1_0, H1_0
-	VO    H2_1, H2_0, H2_0
-	VZERO H0_1
-	VZERO H1_1
-	VZERO H2_1
-
-	VZERO M0
-	VZERO M1
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-
-	// H*[r**2, r]
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, H0_1, H1_1, T_10, M5)
-
-	SUB    $33, R3
-	VLM    (R2), M0, M1
-	VLL    R3, 32(R2), M2
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBEQ R3, $16, 2(PC)
-	VLVGB  R3, R0, M2
-
-	// H += m0
-	VZERO T_1
-	VZERO T_2
-	VZERO T_3
-	EXPACC2(M0, T_1, T_2, T_3, T_4, T_5, T_6)
-	VLEIB $10, $1, T_3
-	VAG   H0_0, T_1, H0_0
-	VAG   H1_0, T_2, H1_0
-	VAG   H2_0, T_3, H2_0
-
-	VZERO M0
-	VZERO M3
-	VZERO M4
-	VZERO M5
-	VZERO T_10
-
-	// (H+m0)*r
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M3, M4, M5, V0, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M3, M4, M5, T_10, H0_1, H1_1, H2_1, T_9)
-
-	// H += m1
-	VZERO V0
-	VZERO T_1
-	VZERO T_2
-	VZERO T_3
-	EXPACC2(M1, T_1, T_2, T_3, T_4, T_5, T_6)
-	VLEIB $10, $1, T_3
-	VAQ   H0_0, T_1, H0_0
-	VAQ   H1_0, T_2, H1_0
-	VAQ   H2_0, T_3, H2_0
-	REDUCE2(H0_0, H1_0, H2_0, M0, M3, M4, M5, T_9, H0_1, H1_1, H2_1, T_10)
-
-	// [H, m2] * [r**2, r]
-	EXPACC2(M2, H0_0, H1_0, H2_0, T_1, T_2, T_3)
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $10, $1, H2_0
-	VZERO  M0
-	VZERO  M1
-	VZERO  M2
-	VZERO  M3
-	VZERO  M4
-	VZERO  M5
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, H0_1, H1_1, M5, T_10)
-	SUB    $16, R3
-	CMPBLE R3, $0, next   // this condition must always hold true!
-
-b2:
-	CMPBLE R3, $16, b1
-
-	// 2 blocks remaining
-
-	// setup [r²,r]
-	VSLDB $8, R_0, R_0, R_0
-	VSLDB $8, R_1, R_1, R_1
-	VSLDB $8, R_2, R_2, R_2
-	VSLDB $8, R5_1, R5_1, R5_1
-	VSLDB $8, R5_2, R5_2, R5_2
-
-	VLVGG $1, RSAVE_0, R_0
-	VLVGG $1, RSAVE_1, R_1
-	VLVGG $1, RSAVE_2, R_2
-	VLVGG $1, R5SAVE_1, R5_1
-	VLVGG $1, R5SAVE_2, R5_2
-
-	// setup [h0, h1]
-	VSLDB $8, H0_0, H0_0, H0_0
-	VSLDB $8, H1_0, H1_0, H1_0
-	VSLDB $8, H2_0, H2_0, H2_0
-	VO    H0_1, H0_0, H0_0
-	VO    H1_1, H1_0, H1_0
-	VO    H2_1, H2_0, H2_0
-	VZERO H0_1
-	VZERO H1_1
-	VZERO H2_1
-
-	VZERO M0
-	VZERO M1
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-
-	// H*[r**2, r]
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M2, M3, M4, T_4, T_5, T_2, T_7, T_8, T_9)
-	VMRHG V0, H0_1, H0_0
-	VMRHG V0, H1_1, H1_0
-	VMRHG V0, H2_1, H2_0
-	VMRLG V0, H0_1, H0_1
-	VMRLG V0, H1_1, H1_1
-	VMRLG V0, H2_1, H2_1
-
-	// move h to the left and 0s at the right
-	VSLDB $8, H0_0, H0_0, H0_0
-	VSLDB $8, H1_0, H1_0, H1_0
-	VSLDB $8, H2_0, H2_0, H2_0
-
-	// get message blocks and append 1 to start
-	SUB    $17, R3
-	VL     (R2), M0
-	VLL    R3, 16(R2), M1
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBEQ R3, $16, 2(PC)
-	VLVGB  R3, R0, M1
-	VZERO  T_6
-	VZERO  T_7
-	VZERO  T_8
-	EXPACC2(M0, T_6, T_7, T_8, T_1, T_2, T_3)
-	EXPACC2(M1, T_6, T_7, T_8, T_1, T_2, T_3)
-	VLEIB  $2, $1, T_8
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $10, $1, T_8
-
-	// add [m0, m1] to h
-	VAG H0_0, T_6, H0_0
-	VAG H1_0, T_7, H1_0
-	VAG H2_0, T_8, H2_0
-
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-	VZERO T_10
-	VZERO M0
-
-	// at this point R_0 .. R5_2 look like [r**2, r]
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M2, M3, M4, M5, T_10, M0, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M2, M3, M4, M5, T_9, H0_1, H1_1, H2_1, T_10)
-	SUB    $16, R3, R3
-	CMPBLE R3, $0, next
-
-b1:
-	CMPBLE R3, $0, next
-
-	// 1 block remaining
-
-	// setup [r²,r]
-	VSLDB $8, R_0, R_0, R_0
-	VSLDB $8, R_1, R_1, R_1
-	VSLDB $8, R_2, R_2, R_2
-	VSLDB $8, R5_1, R5_1, R5_1
-	VSLDB $8, R5_2, R5_2, R5_2
-
-	VLVGG $1, RSAVE_0, R_0
-	VLVGG $1, RSAVE_1, R_1
-	VLVGG $1, RSAVE_2, R_2
-	VLVGG $1, R5SAVE_1, R5_1
-	VLVGG $1, R5SAVE_2, R5_2
-
-	// setup [h0, h1]
-	VSLDB $8, H0_0, H0_0, H0_0
-	VSLDB $8, H1_0, H1_0, H1_0
-	VSLDB $8, H2_0, H2_0, H2_0
-	VO    H0_1, H0_0, H0_0
-	VO    H1_1, H1_0, H1_0
-	VO    H2_1, H2_0, H2_0
-	VZERO H0_1
-	VZERO H1_1
-	VZERO H2_1
-
-	VZERO M0
-	VZERO M1
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-
-	// H*[r**2, r]
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
-
-	// set up [0, m0] limbs
-	SUB    $1, R3
-	VLL    R3, (R2), M0
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBEQ R3, $16, 2(PC)
-	VLVGB  R3, R0, M0
-	VZERO  T_1
-	VZERO  T_2
-	VZERO  T_3
-	EXPACC2(M0, T_1, T_2, T_3, T_4, T_5, T_6)// limbs: [0, m]
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $10, $1, T_3
-
-	// h+m0
-	VAQ H0_0, T_1, H0_0
-	VAQ H1_0, T_2, H1_0
-	VAQ H2_0, T_3, H2_0
-
-	VZERO M0
-	VZERO M1
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
-
-	BR next
-
-square:
-	// setup [r²,r]
-	VSLDB $8, R_0, R_0, R_0
-	VSLDB $8, R_1, R_1, R_1
-	VSLDB $8, R_2, R_2, R_2
-	VSLDB $8, R5_1, R5_1, R5_1
-	VSLDB $8, R5_2, R5_2, R5_2
-
-	VLVGG $1, RSAVE_0, R_0
-	VLVGG $1, RSAVE_1, R_1
-	VLVGG $1, RSAVE_2, R_2
-	VLVGG $1, R5SAVE_1, R5_1
-	VLVGG $1, R5SAVE_2, R5_2
-
-	// setup [h0, h1]
-	VSLDB $8, H0_0, H0_0, H0_0
-	VSLDB $8, H1_0, H1_0, H1_0
-	VSLDB $8, H2_0, H2_0, H2_0
-	VO    H0_1, H0_0, H0_0
-	VO    H1_1, H1_0, H1_0
-	VO    H2_1, H2_0, H2_0
-	VZERO H0_1
-	VZERO H1_1
-	VZERO H2_1
-
-	VZERO M0
-	VZERO M1
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-
-	// (h0*r**2) + (h1*r)
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
-	BR next

vendor/golang.org/x/crypto/ssh/agent/client.go

@@ -102,8 +102,9 @@ type ConstraintExtension struct {
 
 // AddedKey describes an SSH key to be added to an Agent.
 type AddedKey struct {
-	// PrivateKey must be a *rsa.PrivateKey, *dsa.PrivateKey or
+	// PrivateKey must be a *rsa.PrivateKey, *dsa.PrivateKey,
-	// *ecdsa.PrivateKey, which will be inserted into the agent.
+	// ed25519.PrivateKey or *ecdsa.PrivateKey, which will be inserted into the
+	// agent.
 	PrivateKey interface{}
 	// Certificate, if not nil, is communicated to the agent and will be
 	// stored with the key.
@@ -566,6 +567,17 @@ func (c *client) insertKey(s interface{}, comment string, constraints []byte) er
 			Comments:    comment,
 			Constraints: constraints,
 		})
+	case ed25519.PrivateKey:
+		req = ssh.Marshal(ed25519KeyMsg{
+			Type:        ssh.KeyAlgoED25519,
+			Pub:         []byte(k)[32:],
+			Priv:        []byte(k),
+			Comments:    comment,
+			Constraints: constraints,
+		})
+	// This function originally supported only *ed25519.PrivateKey, however the
+	// general idiom is to pass ed25519.PrivateKey by value, not by pointer.
+	// We still support the pointer variant for backwards compatibility.
 	case *ed25519.PrivateKey:
 		req = ssh.Marshal(ed25519KeyMsg{
 			Type:        ssh.KeyAlgoED25519,
@@ -683,6 +695,18 @@ func (c *client) insertCert(s interface{}, cert *ssh.Certificate, comment string
 			Comments:    comment,
 			Constraints: constraints,
 		})
+	case ed25519.PrivateKey:
+		req = ssh.Marshal(ed25519CertMsg{
+			Type:        cert.Type(),
+			CertBytes:   cert.Marshal(),
+			Pub:         []byte(k)[32:],
+			Priv:        []byte(k),
+			Comments:    comment,
+			Constraints: constraints,
+		})
+	// This function originally supported only *ed25519.PrivateKey, however the
+	// general idiom is to pass ed25519.PrivateKey by value, not by pointer.
+	// We still support the pointer variant for backwards compatibility.
 	case *ed25519.PrivateKey:
 		req = ssh.Marshal(ed25519CertMsg{
 			Type:        cert.Type(),

vendor/golang.org/x/crypto/ssh/certs.go

@@ -414,8 +414,8 @@ func (c *CertChecker) CheckCert(principal string, cert *Certificate) error {
 	return nil
 }
 
-// SignCert sets c.SignatureKey to the authority's public key and stores a
+// SignCert signs the certificate with an authority, setting the Nonce,
-// Signature, by authority, in the certificate.
+// SignatureKey, and Signature fields.
 func (c *Certificate) SignCert(rand io.Reader, authority Signer) error {
 	c.Nonce = make([]byte, 32)
 	if _, err := io.ReadFull(rand, c.Nonce); err != nil {

vendor/golang.org/x/crypto/ssh/cipher.go

@@ -119,7 +119,7 @@ var cipherModes = map[string]*cipherMode{
 	chacha20Poly1305ID: {64, 0, newChaCha20Cipher},
 
 	// CBC mode is insecure and so is not included in the default config.
-	// (See http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf). If absolutely
+	// (See https://www.ieee-security.org/TC/SP2013/papers/4977a526.pdf). If absolutely
 	// needed, it's possible to specify a custom Config to enable it.
 	// You should expect that an active attacker can recover plaintext if
 	// you do.

vendor/golang.org/x/crypto/ssh/kex.go

@@ -572,7 +572,7 @@ func (gex *dhGEXSHA) diffieHellman(theirPublic, myPrivate *big.Int) (*big.Int, e
 	return new(big.Int).Exp(theirPublic, myPrivate, gex.p), nil
 }
 
-func (gex *dhGEXSHA) Client(c packetConn, randSource io.Reader, magics *handshakeMagics) (*kexResult, error) {
+func (gex dhGEXSHA) Client(c packetConn, randSource io.Reader, magics *handshakeMagics) (*kexResult, error) {
 	// Send GexRequest
 	kexDHGexRequest := kexDHGexRequestMsg{
 		MinBits:      dhGroupExchangeMinimumBits,
@@ -677,7 +677,7 @@ func (gex *dhGEXSHA) Client(c packetConn, randSource io.Reader, magics *handshak
 // Server half implementation of the Diffie Hellman Key Exchange with SHA1 and SHA256.
 //
 // This is a minimal implementation to satisfy the automated tests.
-func (gex *dhGEXSHA) Server(c packetConn, randSource io.Reader, magics *handshakeMagics, priv Signer) (result *kexResult, err error) {
+func (gex dhGEXSHA) Server(c packetConn, randSource io.Reader, magics *handshakeMagics, priv Signer) (result *kexResult, err error) {
 	// Receive GexRequest
 	packet, err := c.readPacket()
 	if err != nil {

vendor/golang.org/x/crypto/ssh/keys.go

@@ -1246,15 +1246,23 @@ func passphraseProtectedOpenSSHKey(passphrase []byte) openSSHDecryptFunc {
 		}
 		key, iv := k[:32], k[32:]
 
-		if cipherName != "aes256-ctr" {
-			return nil, fmt.Errorf("ssh: unknown cipher %q, only supports %q", cipherName, "aes256-ctr")
-		}
 		c, err := aes.NewCipher(key)
 		if err != nil {
 			return nil, err
 		}
+		switch cipherName {
+		case "aes256-ctr":
 			ctr := cipher.NewCTR(c, iv)
 			ctr.XORKeyStream(privKeyBlock, privKeyBlock)
+		case "aes256-cbc":
+			if len(privKeyBlock)%c.BlockSize() != 0 {
+				return nil, fmt.Errorf("ssh: invalid encrypted private key length, not a multiple of the block size")
+			}
+			cbc := cipher.NewCBCDecrypter(c, iv)
+			cbc.CryptBlocks(privKeyBlock, privKeyBlock)
+		default:
+			return nil, fmt.Errorf("ssh: unknown cipher %q, only supports %q or %q", cipherName, "aes256-ctr", "aes256-cbc")
+		}
 
 		return privKeyBlock, nil
 	}

vendor/modules.txt

@@ -158,7 +158,7 @@ github.com/couchbaselabs/go-couchbase
 ## explicit
 # github.com/davecgh/go-spew v1.1.1
 github.com/davecgh/go-spew/spew
-# github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73
+# github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc
 ## explicit
 github.com/denisenkom/go-mssqldb
 github.com/denisenkom/go-mssqldb/internal/cp
@@ -670,7 +670,7 @@ go.mongodb.org/mongo-driver/bson/bsonrw
 go.mongodb.org/mongo-driver/bson/bsontype
 go.mongodb.org/mongo-driver/bson/primitive
 go.mongodb.org/mongo-driver/x/bsonx/bsoncore
-# golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
+# golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
 ## explicit
 golang.org/x/crypto/acme
 golang.org/x/crypto/acme/autocert