OpenMathLib/OpenBLAS
1
0
Fork 0
mirror of https://github.com/OpenMathLib/OpenBLAS synced 2026-05-31 00:45:48 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #5124 from XiWeiGu/LoongArch64-LA264-lapack-fixed

Browse Source 
LoongArch64: Fixed lapack test for LA264
This commit is contained in:
Martin Kroeker
2025-02-12 14:58:30 +01:00
committed by GitHub
parent e8b11a126b 2c4a5cc6e6
commit 8d487ef6eb
9 changed files with 298 additions and 2054 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
kernel/loongarch64/amax_lsx.S
View File

@@ -56,17 +56,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
LDINT INCX, 0(INCX)
#endif
vxor.v VM0, VM0, VM0
bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
#ifdef DOUBLE
vldrepl.d VM0, X, 0
#else
vldrepl.w VM0, X, 0
#endif
VFSUB VM0, VM0, VM0
bne INCX, TEMP, .L20
srai.d I, N, 3

79
kernel/loongarch64/cnrm2_lsx.S
View File

@@ -47,6 +47,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define VX4 $vr21
#define res1 $vr19
#define res2 $vr20
#define RCP $f2
#define VALPHA $vr3
PROLOGUE
@@ -55,10 +57,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
LDINT INCX, 0(INCX)
#endif
vxor.v res1, res1, res1
vxor.v res2, res2, res2
bge $r0, N, .L999
beq $r0, INCX, .L999
addi.d $sp, $sp, -32
st.d $ra, $sp, 0
st.d N, $sp, 8
st.d X, $sp, 16
st.d INCX, $sp, 24
#ifdef DYNAMIC_ARCH
bl camax_k_LA264
#else
bl camax_k
#endif
ld.d $ra, $sp, 0
ld.d N, $sp, 8
ld.d X, $sp, 16
ld.d INCX, $sp, 24
addi.d $sp, $sp, 32
frecip.s RCP, $f0
vreplvei.w VALPHA, $vr2, 0
vxor.v res1, res1, res1
vxor.v res2, res2, res2
fcmp.ceq.s $fcc0, $f0, $f19
bcnez $fcc0, .L999
li.d TEMP, 1
slli.d TEMP, TEMP, ZBASE_SHIFT
slli.d INCX, INCX, ZBASE_SHIFT
@@ -69,16 +91,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.L10:
vld VX0, X, 0 * SIZE
vfcvtl.d.s VX1, VX0
vfcvth.d.s VX2, VX0
vfmadd.d res1, VX1, VX1, res1
vfmadd.d res2, VX2, VX2, res2
vld VX0, X, 4 * SIZE
vfcvtl.d.s VX3, VX0
vfcvth.d.s VX4, VX0
vfmadd.d res1, VX3, VX3, res1
vfmadd.d res2, VX4, VX4, res2
addi.d I, I, -1
vld VX0, X, 0 * SIZE
vld VX1, X, 4 * SIZE
vfmul.s VX0, VX0, VALPHA
vfmul.s VX1, VX1, VALPHA
vfmadd.s res1, VX0, VX0, res1
vfmadd.s res2, VX1, VX1, res2
addi.d X, X, 8 * SIZE
blt $r0, I, .L10
b .L996
@@ -99,10 +120,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vinsgr2vr.w VX0, t3, 2
vinsgr2vr.w VX0, t4, 3
add.d X, X, INCX
vfcvtl.d.s VX1, VX0
vfcvth.d.s VX2, VX0
vfmadd.d res1, VX1, VX1, res1
vfmadd.d res2, VX2, VX2, res2
vfmul.s VX0, VX0, VALPHA
vfmadd.s res1, VX0, VX0, res1
ld.w t1, X, 0 * SIZE
ld.w t2, X, 1 * SIZE
add.d X, X, INCX
@@ -113,19 +133,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vinsgr2vr.w VX0, t3, 2
vinsgr2vr.w VX0, t4, 3
add.d X, X, INCX
vfcvtl.d.s VX3, VX0
vfcvth.d.s VX4, VX0
vfmadd.d res1, VX3, VX3, res1
vfmadd.d res2, VX4, VX4, res2
vfmul.s VX0, VX0, VALPHA
vfmadd.s res2, VX0, VX0, res2
addi.d I, I, -1
blt $r0, I, .L21
b .L996
.align 3
.L996:
vfadd.d res1, res1, res2
vreplvei.d VX1, res1, 1
vfadd.d res1, VX1, res1
vfadd.s res1, res1, res2
vreplvei.w VX1, res1, 1
vreplvei.w VX2, res1, 2
vreplvei.w VX3, res1, 3
vfadd.s res1, VX1, res1
vfadd.s res1, VX2, res1
vfadd.s res1, VX3, res1
.align 3
.L997:
@@ -137,18 +160,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
fld.s a1, X, 0 * SIZE
fld.s a2, X, 1 * SIZE
addi.d I, I, -1
fcvt.d.s a1, a1
fcvt.d.s a2, a2
fmadd.d res, a1, a1, res
fmadd.d res, a2, a2, res
fmul.s a1, a1, RCP
fmul.s a2, a2, RCP
fmadd.s res, a1, a1, res
fmadd.s res, a2, a2, res
add.d X, X, INCX
blt $r0, I, .L998
.align 3
.L999:
fsqrt.d res, res
fsqrt.s res, res
fmul.s $f0, res, $f0
move $r4, $r17
fcvt.s.d $f0, $f19
jirl $r0, $r1, 0x0
.align 3

8
kernel/loongarch64/copy_lsx.S
View File

@@ -270,9 +270,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add.d Y, Y, INCY
ST a2, Y, 0
add.d Y, Y, INCY
ST a3, X, 0
ST a3, Y, 0
add.d Y, Y, INCY
ST a4, X, 0
ST a4, Y, 0
add.d Y, Y, INCY
LD a1, X, 0
add.d X, X, INCX
@@ -286,9 +286,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add.d Y, Y, INCY
ST a2, Y, 0
add.d Y, Y, INCY
ST a3, X, 0
ST a3, Y, 0
add.d Y, Y, INCY
ST a4, X, 0
ST a4, Y, 0
add.d Y, Y, INCY
addi.d I, I, -1
blt $r0, I, .L222

597
kernel/loongarch64/crot_lsx.S
View File

@@ -75,6 +75,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
slli.d TEMP, TEMP, ZBASE_SHIFT
slli.d INCX, INCX, ZBASE_SHIFT
slli.d INCY, INCY, ZBASE_SHIFT
move YY, Y
move XX, X
MTG t1, C
MTG t2, S
MTG t3, a1
@@ -89,25 +91,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vreplgr2vr.w VXZ, t3
srai.d I, N, 2
#endif
bge $r0, I, .L997
beq INCX, $r0, .L996
beq INCY, $r0, .L996
bne INCX, TEMP, .L22 // INCX!=1 or INCY!=1
bne INCY, TEMP, .L22
.L11:
bge $r0, I, .L997
CMPEQ $fcc0, C, a1
bcnez $fcc0, .L110
CMPEQ $fcc0, S, a1
bcnez $fcc0, .L112 // C!=0 S==0
b .L111 // C!=0 S!=0
.align 3
.L110:
CMPEQ $fcc0, S, a1
bcnez $fcc0, .L114 // C==0 S==0
b .L113 // C==0 S!=0
.align 3
bne INCX, TEMP, .L221 // INCX!=1 or INCY!=1
bne INCY, TEMP, .L221
.L111: // C!=0 S!=0
vld VX0, X, 0 * SIZE
@@ -168,151 +156,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
b .L997
.align 3
.L112: // C!=0 S==0
vld VX0, X, 0 * SIZE
vld VX2, Y, 0 * SIZE
#ifdef DOUBLE
vld VX1, X, 2 * SIZE
vld VX3, Y, 2 * SIZE
vpickev.d x1, VX1, VX0
vpickod.d x2, VX1, VX0
vpickev.d x3, VX3, VX2
vpickod.d x4, VX3, VX2
vfmul.d VX0, x1, VXC
vfmul.d VX1, x3, VXC
vfmul.d VX2, x2, VXC
vfmul.d VX3, x4, VXC
vilvl.d x1, VX2 ,VX0
vilvh.d x2, VX2, VX0
vilvl.d x3, VX3 ,VX1
vilvh.d x4, VX3, VX1
vst x1, X, 0 * SIZE
vst x3, Y, 0 * SIZE
vst x2, X, 2 * SIZE
vst x4, Y, 2 * SIZE
addi.d X, X, 4 * SIZE
addi.d Y, Y, 4 * SIZE
#else
vld VX1, X, 4 * SIZE
vld VX3, Y, 4 * SIZE
vpickev.w x1, VX1, VX0
vpickod.w x2, VX1, VX0
vpickev.w x3, VX3, VX2
vpickod.w x4, VX3, VX2
vfmul.s VX0, x1, VXC
vfmul.s VX1, x3, VXC
vfmul.s VX2, x2, VXC
vfmul.s VX3, x4, VXC
vilvl.w x1, VX2 ,VX0
vilvh.w x2, VX2, VX0
vilvl.w x3, VX3 ,VX1
vilvh.w x4, VX3, VX1
vst x1, X, 0 * SIZE
vst x3, Y, 0 * SIZE
vst x2, X, 4 * SIZE
vst x4, Y, 4 * SIZE
addi.d X, X, 8 * SIZE
addi.d Y, Y, 8 * SIZE
#endif
addi.d I, I, -1
blt $r0, I, .L112
b .L997
.align 3
.L113: // C==0 S!=0
vld VX0, X, 0 * SIZE
vld VX2, Y, 0 * SIZE
#ifdef DOUBLE
vld VX1, X, 2 * SIZE
vld VX3, Y, 2 * SIZE
vpickev.d x1, VX1, VX0
vpickod.d x2, VX1, VX0
vpickev.d x3, VX3, VX2
vpickod.d x4, VX3, VX2
vfmul.d VX0, x3, VXS
vfmul.d VX1, x1, VXS
vfsub.d VX1, VXZ, VX1
vfmul.d VX2, x4, VXS
vfmul.d VX3, x2, VXS
vfsub.d VX3, VXZ, VX3
vilvl.d x1, VX2 ,VX0
vilvh.d x2, VX2, VX0
vilvl.d x3, VX3 ,VX1
vilvh.d x4, VX3, VX1
vst x1, X, 0 * SIZE
vst x3, Y, 0 * SIZE
vst x2, X, 2 * SIZE
vst x4, Y, 2 * SIZE
addi.d X, X, 4 * SIZE
addi.d Y, Y, 4 * SIZE
#else
vld VX1, X, 4 * SIZE
vld VX3, Y, 4 * SIZE
vpickev.w x1, VX1, VX0
vpickod.w x2, VX1, VX0
vpickev.w x3, VX3, VX2
vpickod.w x4, VX3, VX2
vfmul.s VX0, x3, VXS
vfmul.s VX1, x1, VXS
vfsub.s VX1, VXZ, VX1
vfmul.s VX2, x4, VXS
vfmul.s VX3, x2, VXS
vfsub.s VX3, VXZ, VX3
vilvl.w x1, VX2 ,VX0
vilvh.w x2, VX2, VX0
vilvl.w x3, VX3 ,VX1
vilvh.w x4, VX3, VX1
vst x1, X, 0 * SIZE
vst x3, Y, 0 * SIZE
vst x2, X, 4 * SIZE
vst x4, Y, 4 * SIZE
addi.d X, X, 8 * SIZE
addi.d Y, Y, 8 * SIZE
#endif
addi.d I, I, -1
blt $r0, I, .L113
b .L997
.align 3
.L114: // C==0 S==0
vst VXZ, X, 0 * SIZE
vst VXZ, Y, 0 * SIZE
#ifdef DOUBLE
vst VXZ, X, 2 * SIZE
vst VXZ, Y, 2 * SIZE
addi.d X, X, 4 * SIZE
addi.d Y, Y, 4 * SIZE
#else
vst VXZ, X, 4 * SIZE
vst VXZ, Y, 4 * SIZE
addi.d X, X, 8 * SIZE
addi.d Y, Y, 8 * SIZE
#endif
addi.d I, I, -1
blt $r0, I, .L114
b .L997
.align 3
.L22:
#ifdef DOUBLE
srai.d I, N, 2
#endif
bge $r0, I, .L997
move YY, Y
move XX, X
CMPEQ $fcc0, C, a1
bcnez $fcc0, .L220
CMPEQ $fcc0, S, a1
bcnez $fcc0, .L222 // C!=0 S==0
b .L221 // C!=0 S!=0
.align 3
.L220:
CMPEQ $fcc0, S, a1
bcnez $fcc0, .L224 // C==0 S==0
b .L223 // C==0 S!=0
.align 3
.L221: // C!=0 S!=0
#ifdef DOUBLE
ld.d t1, X, 0 * SIZE
@@ -355,50 +198,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vstelm.d VX1, YY, 0, 1
vstelm.d VX3, YY, 1 * SIZE, 1
add.d YY, YY, INCY
ld.d t1, X, 0 * SIZE
ld.d t2, X, 1 * SIZE
add.d X, X, INCX
ld.d t3, X, 0 * SIZE
ld.d t4, X, 1 * SIZE
vinsgr2vr.d x1, t1, 0
vinsgr2vr.d x2, t2, 0
vinsgr2vr.d x1, t3, 1
vinsgr2vr.d x2, t4, 1
add.d X, X, INCX
ld.d t1, Y, 0 * SIZE
ld.d t2, Y, 1 * SIZE
add.d Y, Y, INCY
ld.d t3, Y, 0 * SIZE
ld.d t4, Y, 1 * SIZE
vinsgr2vr.d x3, t1, 0
vinsgr2vr.d x4, t2, 0
vinsgr2vr.d x3, t3, 1
vinsgr2vr.d x4, t4, 1
add.d Y, Y, INCY
vfmul.d VX0, x1, VXC
vfmadd.d VX0, x3, VXS, VX0
vfmul.d VX1, x1, VXS
vfmsub.d VX1, x3, VXC, VX1
vfmul.d VX2, x2, VXC
vfmadd.d VX2, x4, VXS, VX2
vfmul.d VX3, x2, VXS
vfmsub.d VX3, x4, VXC, VX3
vstelm.d VX0, XX, 0, 0
vstelm.d VX2, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.d VX0, XX, 0, 1
vstelm.d VX2, XX, 1 * SIZE, 1
add.d XX, XX, INCX
vstelm.d VX1, YY, 0, 0
vstelm.d VX3, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.d VX1, YY, 0, 1
vstelm.d VX3, YY, 1 * SIZE, 1
add.d YY, YY, INCY
addi.d I, I, -1
blt $r0, I, .L221
b .L995
#else
ld.w t1, X, 0 * SIZE
ld.w t2, X, 1 * SIZE
@@ -473,396 +272,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vstelm.w VX1, YY, 0, 3
vstelm.w VX3, YY, 1 * SIZE, 3
add.d YY, YY, INCY
#endif
addi.d I, I, -1
blt $r0, I, .L221
b .L997
#endif
.align 3
.L222: // C!=0 S==0
#ifdef DOUBLE
ld.d t1, X, 0 * SIZE
ld.d t2, X, 1 * SIZE
add.d X, X, INCX
ld.d t3, X, 0 * SIZE
ld.d t4, X, 1 * SIZE
add.d X, X, INCX
vinsgr2vr.d x1, t1, 0
vinsgr2vr.d x2, t2, 0
vinsgr2vr.d x1, t3, 1
vinsgr2vr.d x2, t4, 1
ld.d t1, Y, 0 * SIZE
ld.d t2, Y, 1 * SIZE
add.d Y, Y, INCY
ld.d t3, Y, 0 * SIZE
ld.d t4, Y, 1 * SIZE
vinsgr2vr.d x3, t1, 0
vinsgr2vr.d x4, t2, 0
vinsgr2vr.d x3, t3, 1
vinsgr2vr.d x4, t4, 1
add.d Y, Y, INCY
vfmul.d VX0, x1, VXC
vfmul.d VX1, x3, VXC
vfmul.d VX2, x2, VXC
vfmul.d VX3, x4, VXC
vstelm.d VX0, XX, 0, 0
vstelm.d VX2, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.d VX0, XX, 0, 1
vstelm.d VX2, XX, 1 * SIZE, 1
add.d XX, XX, INCX
vstelm.d VX1, YY, 0, 0
vstelm.d VX3, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.d VX1, YY, 0, 1
vstelm.d VX3, YY, 1 * SIZE, 1
add.d YY, YY, INCY
ld.d t1, X, 0 * SIZE
ld.d t2, X, 1 * SIZE
add.d X, X, INCX
ld.d t3, X, 0 * SIZE
ld.d t4, X, 1 * SIZE
vinsgr2vr.d x1, t1, 0
vinsgr2vr.d x2, t2, 0
vinsgr2vr.d x1, t3, 1
vinsgr2vr.d x2, t4, 1
add.d X, X, INCX
ld.d t1, Y, 0 * SIZE
ld.d t2, Y, 1 * SIZE
add.d Y, Y, INCY
ld.d t3, Y, 0 * SIZE
ld.d t4, Y, 1 * SIZE
vinsgr2vr.d x3, t1, 0
vinsgr2vr.d x4, t2, 0
vinsgr2vr.d x3, t3, 1
vinsgr2vr.d x4, t4, 1
add.d Y, Y, INCY
vfmul.d VX0, x1, VXC
vfmul.d VX1, x3, VXC
vfmul.d VX2, x2, VXC
vfmul.d VX3, x4, VXC
vstelm.d VX0, XX, 0, 0
vstelm.d VX2, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.d VX0, XX, 0, 1
vstelm.d VX2, XX, 1 * SIZE, 1
add.d XX, XX, INCX
vstelm.d VX1, YY, 0, 0
vstelm.d VX3, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.d VX1, YY, 0, 1
vstelm.d VX3, YY, 1 * SIZE, 1
add.d YY, YY, INCY
addi.d I, I, -1
blt $r0, I, .L222
b .L995
#else
ld.w t1, X, 0 * SIZE
ld.w t2, X, 1 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
ld.w t4, X, 1 * SIZE
add.d X, X, INCX
vinsgr2vr.w x1, t1, 0
vinsgr2vr.w x2, t2, 0
vinsgr2vr.w x1, t3, 1
vinsgr2vr.w x2, t4, 1
ld.w t1, Y, 0 * SIZE
ld.w t2, Y, 1 * SIZE
add.d Y, Y, INCY
ld.w t3, Y, 0 * SIZE
ld.w t4, Y, 1 * SIZE
add.d Y, Y, INCY
vinsgr2vr.w x3, t1, 0
vinsgr2vr.w x4, t2, 0
vinsgr2vr.w x3, t3, 1
vinsgr2vr.w x4, t4, 1
ld.w t1, X, 0 * SIZE
ld.w t2, X, 1 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
ld.w t4, X, 1 * SIZE
vinsgr2vr.w x1, t1, 2
vinsgr2vr.w x2, t2, 2
vinsgr2vr.w x1, t3, 3
vinsgr2vr.w x2, t4, 3
add.d X, X, INCX
ld.w t1, Y, 0 * SIZE
ld.w t2, Y, 1 * SIZE
add.d Y, Y, INCY
ld.w t3, Y, 0 * SIZE
ld.w t4, Y, 1 * SIZE
vinsgr2vr.w x3, t1, 2
vinsgr2vr.w x4, t2, 2
vinsgr2vr.w x3, t3, 3
vinsgr2vr.w x4, t4, 3
add.d Y, Y, INCY
vfmul.s VX0, x1, VXC
vfmul.s VX1, x3, VXC
vfmul.s VX2, x2, VXC
vfmul.s VX3, x4, VXC
vstelm.w VX0, XX, 0, 0
vstelm.w VX2, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.w VX0, XX, 0, 1
vstelm.w VX2, XX, 1 * SIZE, 1
add.d XX, XX, INCX
vstelm.w VX0, XX, 0, 2
vstelm.w VX2, XX, 1 * SIZE, 2
add.d XX, XX, INCX
vstelm.w VX0, XX, 0, 3
vstelm.w VX2, XX, 1 * SIZE, 3
add.d XX, XX, INCX
vstelm.w VX1, YY, 0, 0
vstelm.w VX3, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.w VX1, YY, 0, 1
vstelm.w VX3, YY, 1 * SIZE, 1
add.d YY, YY, INCY
vstelm.w VX1, YY, 0, 2
vstelm.w VX3, YY, 1 * SIZE, 2
add.d YY, YY, INCY
vstelm.w VX1, YY, 0, 3
vstelm.w VX3, YY, 1 * SIZE, 3
add.d YY, YY, INCY
addi.d I, I, -1
blt $r0, I, .L222
b .L997
#endif
.align 3
.L223: // C==0 S!=0
#ifdef DOUBLE
ld.d t1, X, 0 * SIZE
ld.d t2, X, 1 * SIZE
add.d X, X, INCX
ld.d t3, X, 0 * SIZE
ld.d t4, X, 1 * SIZE
add.d X, X, INCX
vinsgr2vr.d x1, t1, 0
vinsgr2vr.d x2, t2, 0
vinsgr2vr.d x1, t3, 1
vinsgr2vr.d x2, t4, 1
ld.d t1, Y, 0 * SIZE
ld.d t2, Y, 1 * SIZE
add.d Y, Y, INCY
ld.d t3, Y, 0 * SIZE
ld.d t4, Y, 1 * SIZE
vinsgr2vr.d x3, t1, 0
vinsgr2vr.d x4, t2, 0
vinsgr2vr.d x3, t3, 1
vinsgr2vr.d x4, t4, 1
add.d Y, Y, INCY
vfmul.d VX0, x3, VXS
vfmul.d VX1, x1, VXS
vfsub.d VX1, VXZ, VX1
vfmul.d VX2, x4, VXS
vfmul.d VX3, x2, VXS
vfsub.d VX3, VXZ, VX3
vstelm.d VX0, XX, 0, 0
vstelm.d VX2, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.d VX0, XX, 0, 1
vstelm.d VX2, XX, 1 * SIZE, 1
add.d XX, XX, INCX
vstelm.d VX1, YY, 0, 0
vstelm.d VX3, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.d VX1, YY, 0, 1
vstelm.d VX3, YY, 1 * SIZE, 1
add.d YY, YY, INCY
ld.d t1, X, 0 * SIZE
ld.d t2, X, 1 * SIZE
add.d X, X, INCX
ld.d t3, X, 0 * SIZE
ld.d t4, X, 1 * SIZE
vinsgr2vr.d x1, t1, 0
vinsgr2vr.d x2, t2, 0
vinsgr2vr.d x1, t3, 1
vinsgr2vr.d x2, t4, 1
add.d X, X, INCX
ld.d t1, Y, 0 * SIZE
ld.d t2, Y, 1 * SIZE
add.d Y, Y, INCY
ld.d t3, Y, 0 * SIZE
ld.d t4, Y, 1 * SIZE
vinsgr2vr.d x3, t1, 0
vinsgr2vr.d x4, t2, 0
vinsgr2vr.d x3, t3, 1
vinsgr2vr.d x4, t4, 1
add.d Y, Y, INCY
vfmul.d VX0, x3, VXS
vfmul.d VX1, x1, VXS
vfsub.d VX1, VXZ, VX1
vfmul.d VX2, x4, VXS
vfmul.d VX3, x2, VXS
vfsub.d VX3, VXZ, VX3
vstelm.d VX0, XX, 0, 0
vstelm.d VX2, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.d VX0, XX, 0, 1
vstelm.d VX2, XX, 1 * SIZE, 1
add.d XX, XX, INCX
vstelm.d VX1, YY, 0, 0
vstelm.d VX3, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.d VX1, YY, 0, 1
vstelm.d VX3, YY, 1 * SIZE, 1
add.d YY, YY, INCY
addi.d I, I, -1
blt $r0, I, .L223
b .L995
#else
ld.w t1, X, 0 * SIZE
ld.w t2, X, 1 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
ld.w t4, X, 1 * SIZE
add.d X, X, INCX
vinsgr2vr.w x1, t1, 0
vinsgr2vr.w x2, t2, 0
vinsgr2vr.w x1, t3, 1
vinsgr2vr.w x2, t4, 1
ld.w t1, Y, 0 * SIZE
ld.w t2, Y, 1 * SIZE
add.d Y, Y, INCY
ld.w t3, Y, 0 * SIZE
ld.w t4, Y, 1 * SIZE
add.d Y, Y, INCY
vinsgr2vr.w x3, t1, 0
vinsgr2vr.w x4, t2, 0
vinsgr2vr.w x3, t3, 1
vinsgr2vr.w x4, t4, 1
ld.w t1, X, 0 * SIZE
ld.w t2, X, 1 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
ld.w t4, X, 1 * SIZE
vinsgr2vr.w x1, t1, 2
vinsgr2vr.w x2, t2, 2
vinsgr2vr.w x1, t3, 3
vinsgr2vr.w x2, t4, 3
add.d X, X, INCX
ld.w t1, Y, 0 * SIZE
ld.w t2, Y, 1 * SIZE
add.d Y, Y, INCY
ld.w t3, Y, 0 * SIZE
ld.w t4, Y, 1 * SIZE
vinsgr2vr.w x3, t1, 2
vinsgr2vr.w x4, t2, 2
vinsgr2vr.w x3, t3, 3
vinsgr2vr.w x4, t4, 3
add.d Y, Y, INCY
vfmul.s VX0, x3, VXS
vfmul.s VX1, x1, VXS
vfsub.s VX1, VXZ, VX1
vfmul.s VX2, x4, VXS
vfmul.s VX3, x2, VXS
vfsub.s VX3, VXZ, VX3
vstelm.w VX0, XX, 0, 0
vstelm.w VX2, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.w VX0, XX, 0, 1
vstelm.w VX2, XX, 1 * SIZE, 1
add.d XX, XX, INCX
vstelm.w VX0, XX, 0, 2
vstelm.w VX2, XX, 1 * SIZE, 2
add.d XX, XX, INCX
vstelm.w VX0, XX, 0, 3
vstelm.w VX2, XX, 1 * SIZE, 3
add.d XX, XX, INCX
vstelm.w VX1, YY, 0, 0
vstelm.w VX3, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.w VX1, YY, 0, 1
vstelm.w VX3, YY, 1 * SIZE, 1
add.d YY, YY, INCY
vstelm.w VX1, YY, 0, 2
vstelm.w VX3, YY, 1 * SIZE, 2
add.d YY, YY, INCY
vstelm.w VX1, YY, 0, 3
vstelm.w VX3, YY, 1 * SIZE, 3
add.d YY, YY, INCY
addi.d I, I, -1
blt $r0, I, .L223
b .L997
#endif
.align 3
.L224: // C==0 S==0
#ifdef DOUBLE
vstelm.d VXZ, XX, 0, 0
vstelm.d VXZ, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.d VXZ, XX, 0, 0
vstelm.d VXZ, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.d VXZ, XX, 0, 0
vstelm.d VXZ, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.d VXZ, XX, 0, 0
vstelm.d VXZ, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.d VXZ, YY, 0, 0
vstelm.d VXZ, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.d VXZ, YY, 0, 0
vstelm.d VXZ, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.d VXZ, YY, 0, 0
vstelm.d VXZ, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.d VXZ, YY, 0, 0
vstelm.d VXZ, YY, 1 * SIZE, 0
add.d YY, YY, INCY
addi.d I, I, -1
blt $r0, I, .L224
move X, XX
move Y, YY
b .L995
#else
vstelm.w VXZ, XX, 0, 0
vstelm.w VXZ, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.w VXZ, XX, 0, 0
vstelm.w VXZ, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.w VXZ, XX, 0, 0
vstelm.w VXZ, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.w VXZ, XX, 0, 0
vstelm.w VXZ, XX, 1 * SIZE, 0
add.d XX, XX, INCX
vstelm.w VXZ, YY, 0, 0
vstelm.w VXZ, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.w VXZ, YY, 0, 0
vstelm.w VXZ, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.w VXZ, YY, 0, 0
vstelm.w VXZ, YY, 1 * SIZE, 0
add.d YY, YY, INCY
vstelm.w VXZ, YY, 0, 0
vstelm.w VXZ, YY, 1 * SIZE, 0
add.d YY, YY, INCY
addi.d I, I, -1
blt $r0, I, .L224
move X, XX
move Y, YY
b .L997
#endif
.align 3
#ifdef DOUBLE
.L995:
andi I, N, 3
bge $r0, I, .L999
b .L998
.align 3
#endif
.L996:
move I, N
b .L998

86
kernel/loongarch64/dot_lsx.S
View File

@@ -53,8 +53,8 @@ PROLOGUE
#endif
/* init $f8 and $f9 to zero */
SUB s1, s1, s1
SUB s2, s2, s2
vxor.v $vr8, $vr8, $vr8
vxor.v $vr9, $vr9, $vr9
slli.d INCX, INCX, BASE_SHIFT
li.d TEMP, SIZE
slli.d INCY, INCY, BASE_SHIFT
@@ -64,20 +64,6 @@ PROLOGUE
/* !((inc_x == 1) && (inc_y == 1)) */
/* init $vr8 and $vr9 to zero */
#ifdef DOUBLE
vldrepl.d $vr0, X, 0
#else
vldrepl.w $vr0, X, 0
#endif
#ifdef DSDOT
vfcvtl.d.s $vr0, $vr0
vfsub.d $vr8, $vr0, $vr0
vfsub.d $vr9, $vr0, $vr0
#else
VFSUB $vr8, $vr0, $vr0
VFSUB $vr9, $vr0, $vr0
#endif
#ifdef DOUBLE
srai.d I, N, 3
@@ -99,31 +85,31 @@ PROLOGUE
addi.w I, I, -1
addi.d X, X, 64
addi.d Y, Y, 64
#ifdef DSDOT
#ifndef DOUBLE
vfcvtl.d.s $vr10, $vr0
vfcvtl.d.s $vr11, $vr4
vfcvth.d.s $vr12, $vr0
vfcvth.d.s $vr13, $vr4
vfmadd.d $vr8, $vr10, $vr12, $vr8
vfmadd.d $vr9, $vr11, $vr13, $vr9
vfmadd.d $vr8, $vr10, $vr11, $vr8
vfmadd.d $vr9, $vr12, $vr13, $vr9
vfcvtl.d.s $vr10, $vr1
vfcvtl.d.s $vr11, $vr5
vfcvth.d.s $vr12, $vr1
vfcvth.d.s $vr13, $vr5
vfmadd.d $vr8, $vr10, $vr12, $vr8
vfmadd.d $vr9, $vr11, $vr13, $vr9
vfmadd.d $vr8, $vr10, $vr11, $vr8
vfmadd.d $vr9, $vr12, $vr13, $vr9
vfcvtl.d.s $vr10, $vr2
vfcvtl.d.s $vr11, $vr6
vfcvth.d.s $vr12, $vr2
vfcvth.d.s $vr13, $vr6
vfmadd.d $vr8, $vr10, $vr12, $vr8
vfmadd.d $vr9, $vr11, $vr13, $vr9
vfmadd.d $vr8, $vr10, $vr11, $vr8
vfmadd.d $vr9, $vr12, $vr13, $vr9
vfcvtl.d.s $vr10, $vr3
vfcvtl.d.s $vr11, $vr7
vfcvth.d.s $vr12, $vr3
vfcvth.d.s $vr13, $vr7
vfmadd.d $vr8, $vr10, $vr12, $vr8
vfmadd.d $vr9, $vr11, $vr13, $vr9
vfmadd.d $vr8, $vr10, $vr11, $vr8
vfmadd.d $vr9, $vr12, $vr13, $vr9
#else
VFMADD $vr8, $vr0, $vr4, $vr8
VFMADD $vr9, $vr1, $vr5, $vr9
@@ -149,13 +135,13 @@ PROLOGUE
addi.w I, I, -1
addi.d X, X, 16
addi.d Y, Y, 16
#ifdef DSDOT
#ifndef DOUBLE
vfcvtl.d.s $vr10, $vr0
vfcvtl.d.s $vr11, $vr4
vfcvth.d.s $vr12, $vr0
vfcvth.d.s $vr13, $vr4
vfmadd.d $vr8, $vr10, $vr12, $vr8
vfmadd.d $vr9, $vr11, $vr13, $vr9
vfmadd.d $vr8, $vr10, $vr11, $vr8
vfmadd.d $vr9, $vr12, $vr13, $vr9
#else
VFMADD $vr8, $vr0, $vr4, $vr8
#endif
@@ -163,23 +149,10 @@ PROLOGUE
.align 3
.L14:
/* store dot in s1 $f8 */
#ifdef DSDOT
vfadd.d $vr8, $vr8, $vr9
fsub.s s2, s2, s2 /* set s2 to 0.0 */
fsub.d s2, s2, s2 /* set s2 to 0.0 */
vpackod.d $vr0, $vr8, $vr8
vfadd.d $vr8, $vr8, $vr0
#else
VFADD $vr8, $vr8, $vr9
SUB s2, s2, s2 /* set s2 to 0.0 */
vpackod.d $vr0, $vr8, $vr8
#ifdef DOUBLE
VFADD $vr8, $vr8, $vr0
#else
VFADD $vr8, $vr8, $vr0
vpackod.w $vr0, $vr8, $vr8
VFADD $vr8, $vr8, $vr0
#endif /* defined DOUBLE */
#endif /* defined DSDOT */
.align 3
.L15:
#ifdef DOUBLE
@@ -193,7 +166,7 @@ PROLOGUE
/* DOUBLE: 1 ; FLOAT: 1~3 */
LD a1, X, 0
LD b1, Y, 0
#ifdef DSDOT
#ifndef DOUBLE
fcvt.d.s a1, a1
fcvt.d.s b1, b1
fmadd.d s1, b1, a1, s1
@@ -236,7 +209,7 @@ PROLOGUE
add.d X, X, INCX
LD b1, Y, 0 * SIZE
add.d Y, Y, INCY
#ifdef DSDOT
#ifndef DOUBLE
fcvt.d.s a1, a1
fcvt.d.s b1, b1
fmadd.d s1, b1, a1, s1
@@ -248,7 +221,7 @@ PROLOGUE
add.d X, X, INCX
LD b1, Y, 0 * SIZE
add.d Y, Y, INCY
#ifdef DSDOT
#ifndef DOUBLE
fcvt.d.s a1, a1
fcvt.d.s b1, b1
fmadd.d s2, b1, a1, s2
@@ -260,7 +233,7 @@ PROLOGUE
add.d X, X, INCX
LD b1, Y, 0 * SIZE
add.d Y, Y, INCY
#ifdef DSDOT
#ifndef DOUBLE
fcvt.d.s a1, a1
fcvt.d.s b1, b1
fmadd.d s1, b1, a1, s1
@@ -272,7 +245,7 @@ PROLOGUE
add.d X, X, INCX
LD b1, Y, 0 * SIZE
add.d Y, Y, INCY
#ifdef DSDOT
#ifndef DOUBLE
fcvt.d.s a1, a1
fcvt.d.s b1, b1
fmadd.d s2, b1, a1, s2
@@ -284,7 +257,7 @@ PROLOGUE
add.d X, X, INCX
LD b1, Y, 0 * SIZE
add.d Y, Y, INCY
#ifdef DSDOT
#ifndef DOUBLE
fcvt.d.s a1, a1
fcvt.d.s b1, b1
fmadd.d s1, b1, a1, s1
@@ -296,7 +269,7 @@ PROLOGUE
add.d X, X, INCX
LD b1, Y, 0 * SIZE
add.d Y, Y, INCY
#ifdef DSDOT
#ifndef DOUBLE
fcvt.d.s a1, a1
fcvt.d.s b1, b1
fmadd.d s2, b1, a1, s2
@@ -308,7 +281,7 @@ PROLOGUE
add.d X, X, INCX
LD b1, Y, 0 * SIZE
add.d Y, Y, INCY
#ifdef DSDOT
#ifndef DOUBLE
fcvt.d.s a1, a1
fcvt.d.s b1, b1
fmadd.d s1, b1, a1, s1
@@ -321,7 +294,7 @@ PROLOGUE
LD b1, Y, 0 * SIZE
add.d Y, Y, INCY
addi.d I, I, -1
#ifdef DSDOT
#ifndef DOUBLE
fcvt.d.s a1, a1
fcvt.d.s b1, b1
fmadd.d s2, b1, a1, s2
@@ -342,7 +315,7 @@ PROLOGUE
LD b1, Y, 0 * SIZE
add.d Y, Y, INCY
addi.d I, I, -1
#ifdef DSDOT
#ifndef DOUBLE
fcvt.d.s a1, a1
fcvt.d.s b1, b1
fmadd.d s1, b1, a1, s1
@@ -353,12 +326,13 @@ PROLOGUE
.align 3
.L999:
#ifdef DSDOT
fadd.d $f0, s1, s2
#else
ADD $f0, s1, s2
#endif
move $r4, $r17
#if defined(DOUBLE)
#elif defined(DSDOT)
#else
fcvt.s.d $f0, $f0
#endif
jirl $r0, $r1, 0x0
EPILOGUE

234
kernel/loongarch64/iamax_lsx.S
View File

@@ -56,19 +56,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define VI3 $vr8
#define VI4 $vr19
#define VT0 $vr23
#define VZE $vr3
#define VT1 $vr4
#define VT2 $vr5
#define VC0 $vr6
PROLOGUE
li.d i0, 0
bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
vldi VZE, 0
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
bne INCX, TEMP, .L20
vld VM0, X, 0
#ifdef DOUBLE
vfsub.d VT1, VZE, VM0
addi.d i0, i0, 1
srai.d I, N, 3
vfmaxa.d VM0, VM0, VT1
bge $r0, I, .L11
slli.d i0, i0, 1 //2
vreplgr2vr.d VINC2, i0
@@ -79,12 +86,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
addi.d i0, i0, 1
vinsgr2vr.d VI1, i0, 1
addi.d i0, i0, 3
vinsgr2vr.d VI0, i0, 0 //1
vinsgr2vr.d VI0, i0, 0 //initialize the index value for vectorization
addi.d i0, i0, 1
vinsgr2vr.d VI0, i0, 1 //2
vinsgr2vr.d VI0, i0, 1
#else
vfsub.s VT1, VZE, VM0
addi.w i0, i0, 1
srai.d I, N, 3
vfmaxa.s VM0, VM0, VT1
bge $r0, I, .L21
slli.w i0, i0, 2 //4
vreplgr2vr.w VINC2, i0
@@ -115,39 +124,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vadd.d VI1, VI1, VINC4
vld VX1, X, 2 * SIZE
vadd.d VI2, VI1, VINC2
vfmaxa.d x1, VX0, VX1
vfcmp.ceq.d VT0, VX0, x1
vbitsel.v x2, VI2, VI1, VT0
vfsub.d VT1, VZE, VX0
vfsub.d VT2, VZE, VX1
vfmaxa.d VX0, VX0, VT1
vfmaxa.d VX1, VX1, VT2
vfcmp.clt.d VT0, VX0, VX1 //abx(x0) < abs(x1)
vbitsel.v x1, VX0, VX1, VT0 //abs(maxf)
vbitsel.v x2, VI1, VI2, VT0 //i
vld VX0, X, 4 * SIZE
vadd.d VI1, VI2, VINC2
vld VX1, X, 6 * SIZE
vadd.d VI2, VI1, VINC2
vfmaxa.d x3, VX0, VX1
vfcmp.ceq.d VT0, VX0, x3
vbitsel.v x4, VI2, VI1, VT0
vfmaxa.d x3, x1, x3
vfcmp.ceq.d VT0, x1, x3
vbitsel.v x2, x4, x2, VT0
vfmaxa.d VM1, VM0, x3
vfcmp.ceq.d VT0, VM0, VM1
vbitsel.v VM0, VM1, VM0, VT0
vbitsel.v VI0, x2, VI0, VT0
vfsub.d VT1, VZE, VX0
vfsub.d VT2, VZE, VX1
vfmaxa.d VX0, VX0, VT1
vfmaxa.d VX1, VX1, VT2
vfcmp.clt.d VT0, VX0, VX1
vbitsel.v x3, VX0, VX1, VT0 //abs(maxf)
vbitsel.v x4, VI1, VI2, VT0 //i
vfcmp.clt.d VC0, x1, x3
vbitsel.v x1, x1, x3, VC0 //abs(maxf)
vbitsel.v x2, x2, x4, VC0 //i
vfcmp.clt.d VT0, VM0, x1
addi.d I, I, -1
addi.d X, X, 8 * SIZE
vbitsel.v VM0, VM0, x1, VT0
vbitsel.v VI0, VI0, x2, VT0
#else
vld VX0, X, 0 * SIZE
vadd.w VI1, VI1, VINC4
vld VX1, X, 4 * SIZE
vadd.w VI2, VI1, VINC2
vfmaxa.s VM1, VX0, VX1
vfcmp.ceq.s VT0, VX0, VM1
vfsub.s VT1, VZE, VX0
vfsub.s VT2, VZE, VX1
vfmaxa.s VX0, VX0, VT1
vfmaxa.s VX1, VX1, VT2
vfcmp.clt.s VT0, VX0, VX1
vbitsel.v x1, VX0, VX1, VT0 //abs(maxf)
vbitsel.v x2, VI1, VI2, VT0 //i
addi.d I, I, -1
vbitsel.v VI2, VI2, VI1, VT0
vfmaxa.s VM1, VM0, VM1
vfcmp.ceq.s VT0, VM0, VM1
vfcmp.clt.s VT0, VM0, x1
addi.d X, X, 8 * SIZE
vbitsel.v VM0, VM1, VM0, VT0
vbitsel.v VI0, VI2, VI0, VT0
vbitsel.v VM0, VM0, x1, VT0
vbitsel.v VI0, VI0, x2, VT0
#endif
blt $r0, I, .L10
.align 3
@@ -158,7 +179,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vreplvei.d VI2, VI0, 1
vreplvei.d x1, VM0, 0
vreplvei.d x2, VM0, 1
fcmp.ceq.d $fcc0, $f10, $f9
fcmp.ceq.d $fcc0, $f9, $f10
bceqz $fcc0, .L16
vfcmp.clt.d VT0, VI1, VI2
vbitsel.v VI0, VI2, VI1, VT0
@@ -172,28 +193,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vreplvei.w x2, VM0, 1
vreplvei.w x3, VM0, 2
vreplvei.w x4, VM0, 3
vfmaxa.s VM1, x1, x2
vfcmp.ceq.s VT0, VM1, x1
vbitsel.v VINC2, VI2, VI1, VT0
vfmaxa.s VM0, x3, x4
vfcmp.ceq.s VT0, x3, VM0
vbitsel.v VINC4, VI4, VI3, VT0
vfmaxa.s VM0, VM0, VM1
vfcmp.ceq.s VT0, VM0, VM1
vbitsel.v VI0, VINC4, VINC2, VT0
fcmp.ceq.d $fcc0, $f15, $f9
bceqz $fcc0, .L26
vfcmp.clt.s VT0, VI1, VI0
vbitsel.v VI0, VI0, VI1, VT0
b .L26
#endif
.align 3
#ifdef DOUBLE
.L16:
vfmaxa.d VM0, x1, x2
vfcmp.ceq.d VT0, x1, VM0
vbitsel.v VI0, VI2, VI1, VT0
vfcmp.clt.d VT0, x1, x2
vbitsel.v VI0, VI1, VI2, VT0
vbitsel.v VM0, x1, x2, VT0
.align 3
.L17:
@@ -212,10 +220,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.L13:
fld.d $f9, X, 0
vfmaxa.d VM1, x1, VM0
vfcmp.ceq.d VT0, VM0, VM1
vbitsel.v VM0, VM1, VM0, VT0
vbitsel.v VI0, VI1, VI0, VT0
fsub.d $f10, $f3, $f9
vfmaxa.d x1, x1, x2
vfcmp.clt.d VT0, VM0, x1
vbitsel.v VM0, VM0, x1, VT0
vbitsel.v VI0, VI0, VI1, VT0
addi.d I, I, -1
addi.d i1, i1, 1
addi.d X, X, SIZE
@@ -241,10 +250,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add.d TEMP, TEMP, INCX
vinsgr2vr.d VM0, t2, 1
slli.d i0, i0, 1 //2
vfsub.d VT1, VZE, VM0
vreplgr2vr.d VINC2, i0
slli.d i0, i0, 1 //4
vreplgr2vr.d VINC4, i0
addi.d i0, i0, -7
vfmaxa.d VM0, VM0, VT1
vinsgr2vr.d VI1, i0, 0 //initialize the index value for vectorization
addi.d i0, i0, 1
vinsgr2vr.d VI1, i0, 1
@@ -269,9 +280,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add.d X, X, INCX
vinsgr2vr.d VX1, t2, 1
vadd.d VI2, VI1, VINC2
vfmaxa.d x1, VX0, VX1
vfcmp.ceq.d VT0, VX0, x1
vbitsel.v x2, VI2, VI1, VT0
vfsub.d VT1, VZE, VX0
vfsub.d VT2, VZE, VX1
vfmaxa.d VX0, VX0, VT1
vfmaxa.d VX1, VX1, VT2
vfcmp.clt.d VT0, VX0, VX1
vbitsel.v x1, VX0, VX1, VT0
vbitsel.v x2, VI1, VI2, VT0
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.d VX0, t1, 0
@@ -286,16 +302,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add.d X, X, INCX
vinsgr2vr.d VX1, t2, 1
vadd.d VI2, VI1, VINC2
vfmaxa.d x3, VX0, VX1
vfcmp.ceq.d VT0, VX0, x3
vbitsel.v x4, VI2, VI1, VT0
vfmaxa.d x3, x1, x3
vfcmp.ceq.d VT0, x1, x3
vbitsel.v x2, x4, x2, VT0
vfmaxa.d VM1, VM0, x3
vbitsel.v VM0, VM1, VM0, VT0
vfcmp.ceq.d VT0, VM0, VM1
vbitsel.v VI0, x2, VI0, VT0
vfsub.d VT1, VZE, VX0
vfsub.d VT2, VZE, VX1
vfmaxa.d VX0, VX0, VT1
vfmaxa.d VX1, VX1, VT2
vfcmp.clt.d VT0, VX0, VX1
vbitsel.v x3, VX0, VX1, VT0
vbitsel.v x4, VI1, VI2, VT0
vfcmp.clt.d VC0, x1, x3
vbitsel.v x1, x1, x3, VC0
vbitsel.v x2, x2, x4, VC0
vfcmp.clt.d VT0, VM0, x1
vbitsel.v VM0, VM0, x1, VT0
vbitsel.v VI0, VI0, x2, VT0
addi.d I, I, -1
blt $r0, I, .L24
.align 3
@@ -313,9 +333,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.align 3
.L26:
vfmaxa.d VM0, x1, x2
vfcmp.ceq.d VT0, x1, VM0
vbitsel.v VI0, VI2, VI1, VT0
vfcmp.clt.d VT0, x1, x2
vbitsel.v VI0, VI1, VI2, VT0
vbitsel.v VM0, x1, x2, VT0
.align 3
.L27:
@@ -389,14 +409,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vinsgr2vr.w VX1, t3, 2
vinsgr2vr.w VX1, t4, 3
vadd.w VI2, VI1, VINC2
vfmaxa.s VM1, VX0, VX1
vfcmp.ceq.s VT0, VX0, VM1
vbitsel.v VI2, VI2, VI1, VT0
vfmaxa.s VM1, VM0, VM1
vfcmp.ceq.s VT0, VM0, VM1
vfsub.s VT1, VZE, VX0
vfsub.s VT2, VZE, VX1
vfmaxa.s VX0, VX0, VT1
vfmaxa.s VX1, VX1, VT2
vfcmp.clt.s VT0, VX0, VX1
vbitsel.v x1, VX0, VX1, VT0
vbitsel.v x2, VI1, VI2, VT0 //i
addi.d I, I, -1
vbitsel.v VM0, VM1, VM0, VT0
vbitsel.v VI0, VI2, VI0, VT0
vfcmp.clt.s VT0, VM0, x1
vbitsel.v VM0, VM0, x1, VT0
vbitsel.v VI0, VI0, x2, VT0
blt $r0, I, .L24
.align 3
@@ -409,42 +433,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vreplvei.w x2, VM0, 1
vreplvei.w x3, VM0, 2
vreplvei.w x4, VM0, 3
vfmaxa.s VM1, x1, x2
vfcmp.ceq.s VT0, VM1, x1
vbitsel.v VINC2, VI2, VI1, VT0
vfmaxa.s VM0, x3, x4
vfcmp.ceq.s VT0, x3, VM0
vbitsel.v VINC4, VI4, VI3, VT0
vfmaxa.s VM0, VM0, VM1
vfcmp.ceq.s VT0, VM0, VM1
vbitsel.v VI0, VINC4, VINC2, VT0
fcmp.ceq.d $fcc0, $f15, $f9
bceqz $fcc0, .L26
vfcmp.clt.s VT0, VI1, VI0
vbitsel.v VI0, VI0, VI1, VT0
.align 3
.L26:
fcmp.ceq.d $fcc0, $f15, $f10
bceqz $fcc0, .L27
vfcmp.clt.s VT0, VI2, VI0
vbitsel.v VI0, VI0, VI2, VT0
fcmp.ceq.s $fcc0, $f9, $f10
bceqz $fcc0, .L31
vfcmp.clt.s VT0, VI1, VI2
vbitsel.v VI1, VI2, VI1, VT0
b .L32
.align 3
.L27:
fcmp.ceq.d $fcc0, $f15, $f11
bceqz $fcc0, .L28
vfcmp.clt.s VT0, VI3, VI0
vbitsel.v VI0, VI0, VI3, VT0
.L31:
vfcmp.clt.s VT0, x1, x2
vbitsel.v VI1, VI1, VI2, VT0
vbitsel.v x1, x1, x2, VT0
.align 3
.L28:
fcmp.ceq.d $fcc0, $f15, $f12
bceqz $fcc0, .L29
vfcmp.clt.s VT0, VI4, VI0
vbitsel.v VI0, VI0, VI4, VT0
.L32:
fcmp.ceq.s $fcc0, $f11, $f12
bceqz $fcc0, .L33
vfcmp.clt.s VT1, VI3, VI4
vbitsel.v VI3, VI4, VI3, VT1
b .L34
.align 3
.L33:
vfcmp.clt.s VT1, x3, x4
vbitsel.v x3, x3, x4, VT1
vbitsel.v VI3, VI3, VI4, VT1
.align 3
.L34:
fcmp.ceq.s $fcc0, $f9, $f11
bceqz $fcc0, .L35
vfcmp.clt.s VT0, VI1, VI3
vbitsel.v VI0, VI3, VI1, VT0
vxor.v VM0, x1, VZE
b .L29
.align 3
.L35:
vfcmp.clt.s VT0, x1, x3
vbitsel.v VM0, x1, x3, VT0
vbitsel.v VI0, VI1, VI3, VT0
.align 3
.L29:
movfr2gr.s i0, $f20
.align 3
@@ -462,10 +489,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.L22:
LD $f9, X, 0
VFMAXA VM1, x1, VM0
VCMPEQ VT0, VM0, VM1
vbitsel.v VM0, VM1, VM0, VT0
vbitsel.v VI0, VI1, VI0, VT0
#ifdef DOUBLE
fsub.d $f10, $f3, $f9
vfmaxa.d x1, x1, x2
vfcmp.clt.d VT0, VM0, x1
#else
fsub.s $f10, $f3, $f9
vfmaxa.s x1, x1, x2
vfcmp.clt.s VT0, VM0, x1
#endif
vbitsel.v VM0, VM0, x1, VT0
vbitsel.v VI0, VI0, VI1, VT0
addi.d I, I, -1
addi.d i1, i1, 1
add.d X, X, INCX

1195
kernel/loongarch64/rot_lsx.S
View File

File diff suppressed because it is too large Load Diff

81
kernel/loongarch64/snrm2_lsx.S
View File

@@ -52,6 +52,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/* Don't change following FR unless you know the effects. */
#define res1 $vr19
#define res2 $vr20
#define RCP $f2
#define VALPHA $vr3
// The optimization for snrm2 cannot simply involve
// extending the data type from float to double and
// then summing the squares of the data. LAPACK tests
// have shown that this approach can still lead to data overflow.
// Instead, we need to find the maximum absolute value in the entire
// array and divide each data element by this maximum value before
// performing the calculation. This approach can avoid overflow (and does not require extending the data type).
PROLOGUE
@@ -59,10 +69,31 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
LDINT N, 0(N)
LDINT INCX, 0(INCX)
#endif
vxor.v res1, res1, res1
vxor.v res2, res2, res2
bge $r0, N, .L999
beq $r0, INCX, .L999
addi.d $sp, $sp, -32
st.d $ra, $sp, 0
st.d N, $sp, 8
st.d X, $sp, 16
st.d INCX, $sp, 24
#ifdef DYNAMIC_ARCH
bl samax_k_LA264
#else
bl samax_k
#endif
ld.d $ra, $sp, 0
ld.d N, $sp, 8
ld.d X, $sp, 16
ld.d INCX, $sp, 24
addi.d $sp, $sp, 32
frecip.s RCP, $f0
vreplvei.w VALPHA, $vr2, 0
vxor.v res1, res1, res1
vxor.v res2, res2, res2
fcmp.ceq.s $fcc0, $f0, $f19
bcnez $fcc0, .L999
li.d TEMP, SIZE
slli.d INCX, INCX, BASE_SHIFT
srai.d I, N, 3
@@ -75,14 +106,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vld VX5, X, 4 * SIZE
addi.d I, I, -1
addi.d X, X, 8 * SIZE
vfcvtl.d.s VX1, VX0
vfcvth.d.s VX2, VX0
vfcvtl.d.s VX3, VX5
vfcvth.d.s VX4, VX5
vfmadd.d res1, VX1, VX1, res1
vfmadd.d res2, VX2, VX2, res2
vfmadd.d res1, VX3, VX3, res1
vfmadd.d res2, VX4, VX4, res2
vfmul.s VX0, VX0, VALPHA
vfmul.s VX5, VX5, VALPHA
vfmadd.s res1, VX0, VX0, res1
vfmadd.s res2, VX5, VX5, res2
blt $r0, I, .L10
b .L996
.align 3
@@ -104,10 +133,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vinsgr2vr.w VX0, t2, 1
vinsgr2vr.w VX0, t3, 2
vinsgr2vr.w VX0, t4, 3
vfcvtl.d.s VX1, VX0
vfcvth.d.s VX2, VX0
vfmadd.d res1, VX1, VX1, res1
vfmadd.d res2, VX2, VX2, res2
vfmul.s VX0, VX0, VALPHA
vfmadd.s res1, VX0, VX0, res1
ld.w t1, X, 0
add.d X, X, INCX
ld.w t2, X, 0
@@ -120,19 +148,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vinsgr2vr.w VX0, t2, 1
vinsgr2vr.w VX0, t3, 2
vinsgr2vr.w VX0, t4, 3
vfcvtl.d.s VX3, VX0
vfcvth.d.s VX4, VX0
vfmadd.d res1, VX3, VX3, res1
vfmadd.d res2, VX4, VX4, res2
vfmul.s VX0, VX0, VALPHA
vfmadd.s res2, VX0, VX0, res2
addi.d I, I, -1
blt $r0, I, .L21
b .L996
.align 3
.L996:
vfadd.d res1, res1, res2
vreplvei.d VX1, res1, 1
vfadd.d res1, VX1, res1
vfadd.s res1, res1, res2
vreplvei.w VX1, res1, 1
vreplvei.w VX2, res1, 2
vreplvei.w VX3, res1, 3
vfadd.s res1, VX1, res1
vfadd.s res1, VX2, res1
vfadd.s res1, VX3, res1
.align 3
.L997:
@@ -143,16 +172,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.L998:
fld.s $f15, X, 0
addi.d I, I, -1
fcvt.d.s $f15, $f15
fmadd.d $f19, $f15, $f15, $f19
fmul.s $f15, $f15, RCP
fmadd.s $f19, $f15, $f15, $f19
add.d X, X, INCX
blt $r0, I, .L998
.align 3
.L999:
fsqrt.d $f19, $f19
fsqrt.s $f19, $f19
fmul.s $f0, $f19, $f0
move $r4, $r17
fcvt.s.d $f0, $f19
jirl $r0, $r1, 0x0
.align 3

65
kernel/loongarch64/swap_lsx.S
View File

@@ -348,62 +348,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
move XX, X
.L222:
LD a1, X, 0
add.d X, X, INCX
LD a2, X, 0
add.d X, X, INCX
LD a3, X, 0
add.d X, X, INCX
LD a4, X, 0
add.d X, X, INCX
LD b1, Y, 0
ST a1, Y, 0
add.d Y, Y, INCY
LD b2, Y, 0
ST a2, Y, 0
add.d Y, Y, INCY
LD b3, Y, 0
ST a3, Y, 0
add.d Y, Y, INCY
LD b4, Y, 0
ST a4, Y, 0
add.d Y, Y, INCY
LD a1, X, 0
add.d X, X, INCX
ST b1, XX, 0
add.d XX, XX, INCX
LD b1, Y, 0
ST a1, Y, 0
add.d Y, Y, INCY
LD a2, X, 0
add.d X, X, INCX
ST b2, XX, 0
add.d XX, XX, INCX
LD b2, Y, 0
ST a2, Y, 0
add.d Y, Y, INCY
LD a3, X, 0
add.d X, X, INCX
ST b3, XX, 0
add.d XX, XX, INCX
LD b3, Y, 0
ST a3, Y, 0
LD a4, X, 0
add.d X, X, INCX
ST b4, XX, 0
add.d XX, XX, INCX
LD b4, Y, 0
ST a4, Y, 0
add.d Y, Y, INCY
ST b1, XX, 0
add.d XX, XX, INCX
ST b2, XX, 0
add.d XX, XX, INCX
ST b3, XX, 0
add.d XX, XX, INCX
ST b4, XX, 0
add.d XX, XX, INCX
addi.d I, I, -1
.rept 8
LD $f12, X, 0
LD $f14, Y, 0
ST $f12, Y, 0
ST $f14, X, 0
add.d X, X, INCX
add.d Y, Y, INCY
.endr
addi.d I, I, -1
blt $r0, I, .L222
.align 3