OpenMathLib/OpenBLAS
1
0
Fork 0
mirror of https://github.com/OpenMathLib/OpenBLAS synced 2026-06-19 00:35:10 +08:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity

Fix 32-bit integer overflow in LAPACKE nancheck index arithmetic

Browse Source 
The optional NaN checks performed by LAPACKE front-ends computed array
offsets and lengths in 32-bit lapack_int arithmetic, which overflows for
large matrices (e.g. LAPACKE_dpotrf with n = lda >= 46341 overflows
j*lda in LAPACKE_dtr_nancheck, leading to out-of-bounds reads and
crashes). Several nancheck helpers (ge, gb, tp, tf) already cast to
size_t; this applies the same treatment to the ones that were missed:

- ?tr_nancheck: cast a[i+j*lda] index to size_t (also covers po/sy/he
  checks, i.e. the POTRF/Cholesky path)
- ?_nancheck (vector): iterate by element count instead of n*inc, with
  size_t indexing
- ?sp/?pp/?pf/?hp/?tp/?tf_nancheck: compute n*(n+1)/2 length in size_t
- ?tz_nancheck: widen the part offsets to int64_t (keeping the -1
  sentinel) and compute them with a widening cast

Verified with UBSan: LAPACKE_str_nancheck at n = lda = 47000 previously
reported signed integer overflow and crashed with SEGV at -O0; it now
passes and still detects the NaN. Old and new code produce identical
results across 1907 enumerated small-size cases (all layouts, uplo,
diag, shapes and NaN positions for tr/tz/vector/packed variants).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yue Chang
2026-06-11 23:33:02 -04:00
parent a033631ee8
commit ede136ee4e
34 changed files with 62 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lapack-netlib/LAPACKE/utils/lapacke_c_nancheck.c
View File

@@ -42,8 +42,8 @@ lapack_logical LAPACKE_c_nancheck( lapack_int n,
if( incx == 0 ) return (lapack_logical) LAPACK_CISNAN( x[0] );
inc = ( incx > 0 ) ? incx : -incx ;
for( i = 0; i < n*inc; i+=inc ) {
if( LAPACK_CISNAN( x[i] ) )
for( i = 0; i < n; i++ ) {
if( LAPACK_CISNAN( x[(size_t)i*inc] ) )
return (lapack_logical) 1;
}
return (lapack_logical) 0;

2
lapack-netlib/LAPACKE/utils/lapacke_chp_nancheck.c
View File

@@ -39,6 +39,6 @@
lapack_logical LAPACKE_chp_nancheck( lapack_int n,
const lapack_complex_float *ap )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_c_nancheck( len, ap, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_cpf_nancheck.c
View File

@@ -40,6 +40,6 @@
lapack_logical LAPACKE_cpf_nancheck( lapack_int n,
const lapack_complex_float *a )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_c_nancheck( len, a, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_cpp_nancheck.c
View File

@@ -39,6 +39,6 @@
lapack_logical LAPACKE_cpp_nancheck( lapack_int n,
const lapack_complex_float *ap )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_c_nancheck( len, ap, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_csp_nancheck.c
View File

@@ -39,6 +39,6 @@
lapack_logical LAPACKE_csp_nancheck( lapack_int n,
const lapack_complex_float *ap )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_c_nancheck( len, ap, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_ctf_nancheck.c
View File

@@ -150,7 +150,7 @@ lapack_logical LAPACKE_ctf_nancheck( int matrix_layout, char transr,
}
} else {
/* Non-unit case - just check whole array for NaNs. */
len = n*(n+1)/2;
len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_cge_nancheck( LAPACK_COL_MAJOR, len, 1, a, len );
}
}

2
lapack-netlib/LAPACKE/utils/lapacke_ctp_nancheck.c
View File

@@ -76,7 +76,7 @@ lapack_logical LAPACKE_ctp_nancheck( int matrix_layout, char uplo, char diag,
return (lapack_logical) 0;
} else {
/* Non-unit case - just check whole array for NaNs. */
len = n*(n+1)/2;
len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_c_nancheck( len, ap, 1 );
}
}

4
lapack-netlib/LAPACKE/utils/lapacke_ctr_nancheck.c
View File

@@ -68,14 +68,14 @@ lapack_logical LAPACKE_ctr_nancheck( int matrix_layout, char uplo, char diag,
if( ( colmaj || lower ) && !( colmaj && lower ) ) {
for( j = st; j < n; j++ ) {
for( i = 0; i < MIN( j+1-st, lda ); i++ ) {
if( LAPACK_CISNAN( a[i+j*lda] ) )
if( LAPACK_CISNAN( a[i+(size_t)j*lda] ) )
return (lapack_logical) 1;
}
}
} else {
for( j = 0; j < n-st; j++ ) {
for( i = j+st; i < MIN( n, lda ); i++ ) {
if( LAPACK_CISNAN( a[i+j*lda] ) )
if( LAPACK_CISNAN( a[i+(size_t)j*lda] ) )
return (lapack_logical) 1;
}
}

12
lapack-netlib/LAPACKE/utils/lapacke_ctz_nancheck.c
View File

@@ -103,27 +103,27 @@ lapack_logical LAPACKE_ctz_nancheck( int matrix_layout, char direct, char uplo,
}
/* Initial offsets and sizes of triangular and rectangular parts */
lapack_int tri_offset = 0;
int64_t tri_offset = 0;
lapack_int tri_n = MIN(m,n);
lapack_int rect_offset = -1;
int64_t rect_offset = -1;
lapack_int rect_m = ( m > n ) ? m - n : m;
lapack_int rect_n = ( n > m ) ? n - m : n;
/* Fix offsets depending on the shape of the matrix */
if( front ) {
if( lower && m > n ) {
rect_offset = tri_n * ( !colmaj ? lda : 1 );
rect_offset = (int64_t)tri_n * ( !colmaj ? lda : 1 );
} else if( !lower && n > m ) {
rect_offset = tri_n * ( colmaj ? lda : 1 );
rect_offset = (int64_t)tri_n * ( colmaj ? lda : 1 );
}
} else {
if( m > n ) {
tri_offset = rect_m * ( !colmaj ? lda : 1 );
tri_offset = (int64_t)rect_m * ( !colmaj ? lda : 1 );
if( !lower ) {
rect_offset = 0;
}
} else if( n > m ) {
tri_offset = rect_n * ( colmaj ? lda : 1 );
tri_offset = (int64_t)rect_n * ( colmaj ? lda : 1 );
if( lower ) {
rect_offset = 0;
}

4
lapack-netlib/LAPACKE/utils/lapacke_d_nancheck.c
View File

@@ -42,8 +42,8 @@ lapack_logical LAPACKE_d_nancheck( lapack_int n,
if( incx == 0 ) return (lapack_logical) LAPACK_DISNAN( x[0] );
inc = ( incx > 0 ) ? incx : -incx ;
for( i = 0; i < n*inc; i+=inc ) {
if( LAPACK_DISNAN( x[i] ) )
for( i = 0; i < n; i++ ) {
if( LAPACK_DISNAN( x[(size_t)i*inc] ) )
return (lapack_logical) 1;
}
return (lapack_logical) 0;

2
lapack-netlib/LAPACKE/utils/lapacke_dpf_nancheck.c
View File

@@ -40,6 +40,6 @@
lapack_logical LAPACKE_dpf_nancheck( lapack_int n,
const double *a )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_d_nancheck( len, a, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_dpp_nancheck.c
View File

@@ -39,6 +39,6 @@
lapack_logical LAPACKE_dpp_nancheck( lapack_int n,
const double *ap )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_d_nancheck( len, ap, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_dsp_nancheck.c
View File

@@ -39,6 +39,6 @@
lapack_logical LAPACKE_dsp_nancheck( lapack_int n,
const double *ap )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_d_nancheck( len, ap, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_dtf_nancheck.c
View File

@@ -150,7 +150,7 @@ lapack_logical LAPACKE_dtf_nancheck( int matrix_layout, char transr,
}
} else {
/* Non-unit case - just check whole array for NaNs. */
len = n*(n+1)/2;
len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_dge_nancheck( LAPACK_COL_MAJOR, len, 1, a, len );
}
}

2
lapack-netlib/LAPACKE/utils/lapacke_dtp_nancheck.c
View File

@@ -76,7 +76,7 @@ lapack_logical LAPACKE_dtp_nancheck( int matrix_layout, char uplo, char diag,
return (lapack_logical) 0;
} else {
/* Non-unit case - just check whole array for NaNs. */
len = n*(n+1)/2;
len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_d_nancheck( len, ap, 1 );
}
}

4
lapack-netlib/LAPACKE/utils/lapacke_dtr_nancheck.c
View File

@@ -68,14 +68,14 @@ lapack_logical LAPACKE_dtr_nancheck( int matrix_layout, char uplo, char diag,
if( ( colmaj || lower ) && !( colmaj && lower ) ) {
for( j = st; j < n; j++ ) {
for( i = 0; i < MIN( j+1-st, lda ); i++ ) {
if( LAPACK_DISNAN( a[i+j*lda] ) )
if( LAPACK_DISNAN( a[i+(size_t)j*lda] ) )
return (lapack_logical) 1;
}
}
} else {
for( j = 0; j < n-st; j++ ) {
for( i = j+st; i < MIN( n, lda ); i++ ) {
if( LAPACK_DISNAN( a[i+j*lda] ) )
if( LAPACK_DISNAN( a[i+(size_t)j*lda] ) )
return (lapack_logical) 1;
}
}

12
lapack-netlib/LAPACKE/utils/lapacke_dtz_nancheck.c
View File

@@ -102,27 +102,27 @@ lapack_logical LAPACKE_dtz_nancheck( int matrix_layout, char direct, char uplo,
}
/* Initial offsets and sizes of triangular and rectangular parts */
lapack_int tri_offset = 0;
int64_t tri_offset = 0;
lapack_int tri_n = MIN(m,n);
lapack_int rect_offset = -1;
int64_t rect_offset = -1;
lapack_int rect_m = ( m > n ) ? m - n : m;
lapack_int rect_n = ( n > m ) ? n - m : n;
/* Fix offsets depending on the shape of the matrix */
if( front ) {
if( lower && m > n ) {
rect_offset = tri_n * ( !colmaj ? lda : 1 );
rect_offset = (int64_t)tri_n * ( !colmaj ? lda : 1 );
} else if( !lower && n > m ) {
rect_offset = tri_n * ( colmaj ? lda : 1 );
rect_offset = (int64_t)tri_n * ( colmaj ? lda : 1 );
}
} else {
if( m > n ) {
tri_offset = rect_m * ( !colmaj ? lda : 1 );
tri_offset = (int64_t)rect_m * ( !colmaj ? lda : 1 );
if( !lower ) {
rect_offset = 0;
}
} else if( n > m ) {
tri_offset = rect_n * ( colmaj ? lda : 1 );
tri_offset = (int64_t)rect_n * ( colmaj ? lda : 1 );
if( lower ) {
rect_offset = 0;
}

4
lapack-netlib/LAPACKE/utils/lapacke_s_nancheck.c
View File

@@ -42,8 +42,8 @@ lapack_logical LAPACKE_s_nancheck( lapack_int n,
if( incx == 0 ) return (lapack_logical) LAPACK_SISNAN( x[0] );
inc = ( incx > 0 ) ? incx : -incx ;
for( i = 0; i < n*inc; i+=inc ) {
if( LAPACK_SISNAN( x[i] ) )
for( i = 0; i < n; i++ ) {
if( LAPACK_SISNAN( x[(size_t)i*inc] ) )
return (lapack_logical) 1;
}
return (lapack_logical) 0;

2
lapack-netlib/LAPACKE/utils/lapacke_spf_nancheck.c
View File

@@ -40,6 +40,6 @@
lapack_logical LAPACKE_spf_nancheck( lapack_int n,
const float *a )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_s_nancheck( len, a, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_spp_nancheck.c
View File

@@ -39,6 +39,6 @@
lapack_logical LAPACKE_spp_nancheck( lapack_int n,
const float *ap )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_s_nancheck( len, ap, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_ssp_nancheck.c
View File

@@ -39,6 +39,6 @@
lapack_logical LAPACKE_ssp_nancheck( lapack_int n,
const float *ap )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_s_nancheck( len, ap, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_stf_nancheck.c
View File

@@ -151,7 +151,7 @@ lapack_logical LAPACKE_stf_nancheck( int matrix_layout, char transr,
}
} else {
/* Non-unit case - just check whole array for NaNs. */
len = n*(n+1)/2;
len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_sge_nancheck( LAPACK_COL_MAJOR, len, 1, a, len );
}
}

2
lapack-netlib/LAPACKE/utils/lapacke_stp_nancheck.c
View File

@@ -76,7 +76,7 @@ lapack_logical LAPACKE_stp_nancheck( int matrix_layout, char uplo, char diag,
return (lapack_logical) 0;
} else {
/* Non-unit case - just check whole array for NaNs. */
len = n*(n+1)/2;
len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_s_nancheck( len, ap, 1 );
}
}

4
lapack-netlib/LAPACKE/utils/lapacke_str_nancheck.c
View File

@@ -68,14 +68,14 @@ lapack_logical LAPACKE_str_nancheck( int matrix_layout, char uplo, char diag,
if( ( colmaj || lower ) && !( colmaj && lower ) ) {
for( j = st; j < n; j++ ) {
for( i = 0; i < MIN( j+1-st, lda ); i++ ) {
if( LAPACK_SISNAN( a[i+j*lda] ) )
if( LAPACK_SISNAN( a[i+(size_t)j*lda] ) )
return (lapack_logical) 1;
}
}
} else {
for( j = 0; j < n-st; j++ ) {
for( i = j+st; i < MIN( n, lda ); i++ ) {
if( LAPACK_SISNAN( a[i+j*lda] ) )
if( LAPACK_SISNAN( a[i+(size_t)j*lda] ) )
return (lapack_logical) 1;
}
}

12
lapack-netlib/LAPACKE/utils/lapacke_stz_nancheck.c
View File

@@ -102,27 +102,27 @@ lapack_logical LAPACKE_stz_nancheck( int matrix_layout, char direct, char uplo,
}
/* Initial offsets and sizes of triangular and rectangular parts */
lapack_int tri_offset = 0;
int64_t tri_offset = 0;
lapack_int tri_n = MIN(m,n);
lapack_int rect_offset = -1;
int64_t rect_offset = -1;
lapack_int rect_m = ( m > n ) ? m - n : m;
lapack_int rect_n = ( n > m ) ? n - m : n;
/* Fix offsets depending on the shape of the matrix */
if( front ) {
if( lower && m > n ) {
rect_offset = tri_n * ( !colmaj ? lda : 1 );
rect_offset = (int64_t)tri_n * ( !colmaj ? lda : 1 );
} else if( !lower && n > m ) {
rect_offset = tri_n * ( colmaj ? lda : 1 );
rect_offset = (int64_t)tri_n * ( colmaj ? lda : 1 );
}
} else {
if( m > n ) {
tri_offset = rect_m * ( !colmaj ? lda : 1 );
tri_offset = (int64_t)rect_m * ( !colmaj ? lda : 1 );
if( !lower ) {
rect_offset = 0;
}
} else if( n > m ) {
tri_offset = rect_n * ( colmaj ? lda : 1 );
tri_offset = (int64_t)rect_n * ( colmaj ? lda : 1 );
if( lower ) {
rect_offset = 0;
}

4
lapack-netlib/LAPACKE/utils/lapacke_z_nancheck.c
View File

@@ -42,8 +42,8 @@ lapack_logical LAPACKE_z_nancheck( lapack_int n,
if( incx == 0 ) return (lapack_logical) LAPACK_ZISNAN( x[0] );
inc = ( incx > 0 ) ? incx : -incx ;
for( i = 0; i < n*inc; i+=inc ) {
if( LAPACK_ZISNAN( x[i] ) )
for( i = 0; i < n; i++ ) {
if( LAPACK_ZISNAN( x[(size_t)i*inc] ) )
return (lapack_logical) 1;
}
return (lapack_logical) 0;

2
lapack-netlib/LAPACKE/utils/lapacke_zhp_nancheck.c
View File

@@ -39,6 +39,6 @@
lapack_logical LAPACKE_zhp_nancheck( lapack_int n,
const lapack_complex_double *ap )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_z_nancheck( len, ap, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_zpf_nancheck.c
View File

@@ -40,6 +40,6 @@
lapack_logical LAPACKE_zpf_nancheck( lapack_int n,
const lapack_complex_double *a )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_z_nancheck( len, a, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_zpp_nancheck.c
View File

@@ -39,6 +39,6 @@
lapack_logical LAPACKE_zpp_nancheck( lapack_int n,
const lapack_complex_double *ap )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_z_nancheck( len, ap, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_zsp_nancheck.c
View File

@@ -39,6 +39,6 @@
lapack_logical LAPACKE_zsp_nancheck( lapack_int n,
const lapack_complex_double *ap )
{
lapack_int len = n*(n+1)/2;
lapack_int len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_z_nancheck( len, ap, 1 );
}

2
lapack-netlib/LAPACKE/utils/lapacke_ztf_nancheck.c
View File

@@ -151,7 +151,7 @@ lapack_logical LAPACKE_ztf_nancheck( int matrix_layout, char transr,
}
} else {
/* Non-unit case - just check whole array for NaNs. */
len = n*(n+1)/2;
len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_zge_nancheck( LAPACK_COL_MAJOR, len, 1, a, len );
}
}

2
lapack-netlib/LAPACKE/utils/lapacke_ztp_nancheck.c
View File

@@ -76,7 +76,7 @@ lapack_logical LAPACKE_ztp_nancheck( int matrix_layout, char uplo, char diag,
return (lapack_logical) 0;
} else {
/* Non-unit case - just check whole array for NaNs. */
len = n*(n+1)/2;
len = (lapack_int)(((size_t)n*(n+1))/2);
return LAPACKE_z_nancheck( len, ap, 1 );
}
}

4
lapack-netlib/LAPACKE/utils/lapacke_ztr_nancheck.c
View File

@@ -68,14 +68,14 @@ lapack_logical LAPACKE_ztr_nancheck( int matrix_layout, char uplo, char diag,
if( ( colmaj || lower ) && !( colmaj && lower ) ) {
for( j = st; j < n; j++ ) {
for( i = 0; i < MIN( j+1-st, lda ); i++ ) {
if( LAPACK_ZISNAN( a[i+j*lda] ) )
if( LAPACK_ZISNAN( a[i+(size_t)j*lda] ) )
return (lapack_logical) 1;
}
}
} else {
for( j = 0; j < n-st; j++ ) {
for( i = j+st; i < MIN( n, lda ); i++ ) {
if( LAPACK_ZISNAN( a[i+j*lda] ) )
if( LAPACK_ZISNAN( a[i+(size_t)j*lda] ) )
return (lapack_logical) 1;
}
}

12
lapack-netlib/LAPACKE/utils/lapacke_ztz_nancheck.c
View File

@@ -103,27 +103,27 @@ lapack_logical LAPACKE_ztz_nancheck( int matrix_layout, char direct, char uplo,
}
/* Initial offsets and sizes of triangular and rectangular parts */
lapack_int tri_offset = 0;
int64_t tri_offset = 0;
lapack_int tri_n = MIN(m,n);
lapack_int rect_offset = -1;
int64_t rect_offset = -1;
lapack_int rect_m = ( m > n ) ? m - n : m;
lapack_int rect_n = ( n > m ) ? n - m : n;
/* Fix offsets depending on the shape of the matrix */
if( front ) {
if( lower && m > n ) {
rect_offset = tri_n * ( !colmaj ? lda : 1 );
rect_offset = (int64_t)tri_n * ( !colmaj ? lda : 1 );
} else if( !lower && n > m ) {
rect_offset = tri_n * ( colmaj ? lda : 1 );
rect_offset = (int64_t)tri_n * ( colmaj ? lda : 1 );
}
} else {
if( m > n ) {
tri_offset = rect_m * ( !colmaj ? lda : 1 );
tri_offset = (int64_t)rect_m * ( !colmaj ? lda : 1 );
if( !lower ) {
rect_offset = 0;
}
} else if( n > m ) {
tri_offset = rect_n * ( colmaj ? lda : 1 );
tri_offset = (int64_t)rect_n * ( colmaj ? lda : 1 );
if( lower ) {
rect_offset = 0;
}