From 724f02cffadca0f63828537d018247f48ee115c0 Mon Sep 17 00:00:00 2001 From: Paul Eggert Date: Sat, 24 Apr 2021 18:13:40 -0700 Subject: [PATCH] xmalloca, etc.: avoid unlikely trap * lib/xmalloca.h (xnmalloca): Use size_t, not ptrdiff_t, to avoid implementation-defined behavior (which could include a trap) if N is 0 and S exceeds PTRDIFF_MAX. * lib/xalloc-oversized.h: Adjust comment to match. --- ChangeLog | 8 +++++++- lib/xalloc-oversized.h | 4 ++-- lib/xmalloca.h | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 63471166c4..f91f2e3df6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,7 +1,13 @@ 2021-04-24 Paul Eggert - xmalloca, etc.: fix some xalloc-oversized issues + xmalloca, etc.: avoid unlikely trap * lib/malloca.h (nmalloca): + * lib/xmalloca.h (xnmalloca): + Use size_t, not ptrdiff_t, to avoid implementation-defined + behavior (which could include a trap) if N is 0 and S exceeds + PTRDIFF_MAX. + * lib/xalloc-oversized.h: Adjust comment to match. + * lib/xmalloca.h (xnmalloca): Convert S to ptrdiff_t to avoid arithmetic overflow if N and S are both narrower than ptrdiff_t. * lib/xalloc-oversized.h (xalloc_oversized): diff --git a/lib/xalloc-oversized.h b/lib/xalloc-oversized.h index 62f1ae94b8..2619a2aab4 100644 --- a/lib/xalloc-oversized.h +++ b/lib/xalloc-oversized.h @@ -38,8 +38,8 @@ Warning: (xalloc_oversized (N, S) ? NULL : malloc (N * S)) can misbehave if N and S are both narrower than ptrdiff_t and size_t, - and can be rewritten as (xalloc_oversized (N, S) ? NULL : malloc - (N * (ptrdiff_t) S)) or similarly with size_t. + and can be rewritten as (xalloc_oversized (N, S) ? NULL + : malloc (N * (size_t) S)). This is a macro, not a function, so that it works even if an argument exceeds MAX (PTRDIFF_MAX, SIZE_MAX). */ diff --git a/lib/xmalloca.h b/lib/xmalloca.h index c7815f6171..dbc648697e 100644 --- a/lib/xmalloca.h +++ b/lib/xmalloca.h @@ -54,7 +54,7 @@ extern void * xmmalloca (size_t n); #if HAVE_ALLOCA /* Rely on xmalloca (SIZE_MAX) calling xalloc_die (). */ # define xnmalloca(n, s) \ - xmalloca (xalloc_oversized (n, s) ? (size_t) (-1) : (n) * (ptrdiff_t) (s)) + xmalloca (xalloc_oversized (n, s) ? (size_t) (-1) : (n) * (size_t) (s)) #else # define xnmalloca(n, s) \ xnmalloc (n, s) -- 2.39.5