From e7eb4c68af1c567a690fd985567a2acb0d07e5af Mon Sep 17 00:00:00 2001 From: Bruno Haible Date: Tue, 4 Jul 2023 21:04:00 +0200 Subject: [PATCH] mbiter, mbfile, mbmemcasecoll: Improve handling of mbrtoc32 result. * lib/mbiter.h (mbi_avail): If cur.ptr has reached the limit but in_shift is true, call mbiter_multi_next. (mbiter_multi_next): Set in_shift to false after an incomplete multibyte character. * lib/mbfile.h (mbfile_multi_getc): Pass the input bytes incrementally into mbrtoc32. When mbf->state is not in the initial state, call mbrtoc32 again. * lib/mbmemcasecoll.c (apply_c32tolower): When the state is not in the initial state, call mbrtoc32 again. --- ChangeLog | 13 +++++++ lib/mbfile.h | 83 ++++++++++++++++++++-------------------- lib/mbiter.h | 9 +++-- lib/mbmemcasecoll.c | 92 +++++++++++++++++++++++++-------------------- 4 files changed, 110 insertions(+), 87 deletions(-) diff --git a/ChangeLog b/ChangeLog index 83f0075f3b..8aa54f38d8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,16 @@ +2023-07-04 Bruno Haible + + mbiter, mbfile, mbmemcasecoll: Improve handling of mbrtoc32 result. + * lib/mbiter.h (mbi_avail): If cur.ptr has reached the limit but + in_shift is true, call mbiter_multi_next. + (mbiter_multi_next): Set in_shift to false after an incomplete multibyte + character. + * lib/mbfile.h (mbfile_multi_getc): Pass the input bytes incrementally + into mbrtoc32. When mbf->state is not in the initial state, call + mbrtoc32 again. + * lib/mbmemcasecoll.c (apply_c32tolower): When the state is not in the + initial state, call mbrtoc32 again. + 2023-07-04 Bruno Haible mbiter, mbuiter, mbfile: Improve state handling after invalid input. diff --git a/lib/mbfile.h b/lib/mbfile.h index 7670dabfbb..ee1dd5a4b9 100644 --- a/lib/mbfile.h +++ b/lib/mbfile.h @@ -77,6 +77,7 @@ struct mbfile_multi { MBFILE_INLINE void mbfile_multi_getc (struct mbchar *mbc, struct mbfile_multi *mbf) { + unsigned int new_bufcount; size_t bytes; /* If EOF has already been seen, don't use getc. This matters if @@ -92,54 +93,51 @@ mbfile_multi_getc (struct mbchar *mbc, struct mbfile_multi *mbf) return; } - /* Before using mbrtowc, we need at least one byte. */ - if (mbf->bufcount == 0) + new_bufcount = mbf->bufcount; + + /* If mbf->state is not in the initial state, some more 32-bit wide character + may be hiding in the state. We need to call mbrtoc32 again. */ + if (mbsinit (&mbf->state)) { - int c = getc (mbf->fp); - if (c == EOF) + /* Before using mbrtoc32, we need at least one byte. */ + if (new_bufcount == 0) { - mbf->eof_seen = true; - goto eof; + int c = getc (mbf->fp); + if (c == EOF) + { + mbf->eof_seen = true; + goto eof; + } + mbf->buf[0] = (unsigned char) c; + new_bufcount++; } - mbf->buf[0] = (unsigned char) c; - mbf->bufcount++; - } - /* Handle most ASCII characters quickly, without calling mbrtowc(). */ - if (mbf->bufcount == 1 && mbsinit (&mbf->state) && is_basic (mbf->buf[0])) - { - /* These characters are part of the POSIX portable character set. - For most of them, namely those in the ISO C basic character set, - ISO C 99 guarantees that their wide character code is identical to - their char code. For the few other ones, this is the case as well, - in all locale encodings that are in use. The 32-bit wide character - code is the same as well. */ - mbc->wc = mbc->buf[0] = mbf->buf[0]; - mbc->wc_valid = true; - mbc->ptr = &mbc->buf[0]; - mbc->bytes = 1; - mbf->bufcount = 0; - return; + /* Handle most ASCII characters quickly, without calling mbrtoc32(). */ + if (new_bufcount == 1 && is_basic (mbf->buf[0])) + { + /* These characters are part of the POSIX portable character set. + For most of them, namely those in the ISO C basic character set, + ISO C 99 guarantees that their wide character code is identical to + their char code. For the few other ones, this is the case as well, + in all locale encodings that are in use. The 32-bit wide character + code is the same as well. */ + mbc->wc = mbc->buf[0] = mbf->buf[0]; + mbc->wc_valid = true; + mbc->ptr = &mbc->buf[0]; + mbc->bytes = 1; + mbf->bufcount = 0; + return; + } } - /* Use mbrtowc on an increasing number of bytes. Read only as many bytes + /* Use mbrtoc32 on an increasing number of bytes. Read only as many bytes from mbf->fp as needed. This is needed to give reasonable interactive behaviour when mbf->fp is connected to an interactive tty. */ for (;;) { - /* We don't know whether the 'mbrtowc' function updates the state when - it returns -2, - this is the ISO C 99 and glibc-2.2 behaviour - or - not - amended ANSI C, glibc-2.1 and Solaris 2.7 behaviour. We - don't have an autoconf test for this, yet. - The new behaviour would allow us to feed the bytes one by one into - mbrtowc. But the old behaviour forces us to feed all bytes since - the end of the last character into mbrtowc. Since we want to retry - with more bytes when mbrtowc returns -2, we must backup the state - before calling mbrtowc, because implementations with the new - behaviour will clobber it. */ - mbstate_t backup_state = mbf->state; - - bytes = mbrtoc32 (&mbc->wc, &mbf->buf[0], mbf->bufcount, &mbf->state); + /* Feed the bytes one by one into mbrtoc32. */ + bytes = mbrtoc32 (&mbc->wc, &mbf->buf[mbf->bufcount], new_bufcount - mbf->bufcount, &mbf->state); + mbf->bufcount = new_bufcount; if (bytes == (size_t) -1) { @@ -154,7 +152,6 @@ mbfile_multi_getc (struct mbchar *mbc, struct mbfile_multi *mbf) else if (bytes == (size_t) -2) { /* An incomplete multibyte character. */ - mbf->state = backup_state; if (mbf->bufcount == MBCHAR_BUF_SIZE) { /* An overlong incomplete multibyte sequence was encountered. */ @@ -165,18 +162,18 @@ mbfile_multi_getc (struct mbchar *mbc, struct mbfile_multi *mbf) } else { - /* Read one more byte and retry mbrtowc. */ + /* Read one more byte and retry mbrtoc32. */ int c = getc (mbf->fp); if (c == EOF) { /* An incomplete multibyte character at the end. */ mbf->eof_seen = true; - bytes = mbf->bufcount; + bytes = new_bufcount; mbc->wc_valid = false; break; } - mbf->buf[mbf->bufcount] = (unsigned char) c; - mbf->bufcount++; + mbf->buf[new_bufcount] = (unsigned char) c; + new_bufcount++; } } else diff --git a/lib/mbiter.h b/lib/mbiter.h index e4963010f3..8bd83d7262 100644 --- a/lib/mbiter.h +++ b/lib/mbiter.h @@ -155,8 +155,10 @@ mbiter_multi_next (struct mbiter_multi *iter) /* An incomplete multibyte character at the end. */ iter->cur.bytes = iter->limit - iter->cur.ptr; iter->cur.wc_valid = false; - /* Whether to set iter->in_shift = false and reset iter->state - or not is not important; the string end is reached anyway. */ + /* Cause the next mbi_avail invocation to return false. */ + iter->in_shift = false; + /* Whether to reset iter->state or not is not important; the + string end is reached anyway. */ } else { @@ -208,7 +210,8 @@ typedef struct mbiter_multi mbi_iterator_t; (iter).in_shift = false, memset (&(iter).state, '\0', sizeof (mbstate_t)), \ (iter).next_done = false) #define mbi_avail(iter) \ - ((iter).cur.ptr < (iter).limit && (mbiter_multi_next (&(iter)), true)) + (((iter).cur.ptr < (iter).limit || (iter).in_shift) \ + && (mbiter_multi_next (&(iter)), true)) #define mbi_advance(iter) \ ((iter).cur.ptr += (iter).cur.bytes, (iter).next_done = false) diff --git a/lib/mbmemcasecoll.c b/lib/mbmemcasecoll.c index de7390f6d1..f79f262dc0 100644 --- a/lib/mbmemcasecoll.c +++ b/lib/mbmemcasecoll.c @@ -51,59 +51,69 @@ apply_c32tolower (const char *inbuf, size_t inbufsize, remaining = inbufsize; while (remaining > 0) { - char32_t wc1; - size_t n1; mbstate_t state; memset (&state, '\0', sizeof (mbstate_t)); - n1 = mbrtoc32 (&wc1, inbuf, remaining, &state); - if (n1 == (size_t)(-2)) - break; - if (n1 != (size_t)(-1)) + do { - wint_t wc2; + char32_t wc1; + size_t n1; - if (n1 == 0) /* NUL character? */ - n1 = 1; - else if (n1 == (size_t)(-3)) - n1 = 0; + n1 = mbrtoc32 (&wc1, inbuf, remaining, &state); - wc2 = c32tolower (wc1); - if (wc2 != wc1) + if (n1 == (size_t)(-1)) { - size_t n2; - - memset (&state, '\0', sizeof (mbstate_t)); - n2 = c32rtomb (outbuf, wc2, &state); - if (n2 != (size_t)(-1)) + /* Invalid multibyte character on input. + Copy one byte without modification. */ + *outbuf++ = *inbuf++; + remaining -= 1; + break; + } + else if (n1 == (size_t)(-2)) + { + /* Incomplete multibyte sequence on input. + Pass it through unmodified. */ + while (remaining > 0) { - /* Store the translated multibyte character. */ - inbuf += n1; - remaining -= n1; - outbuf += n2; - continue; + *outbuf++ = *inbuf++; + remaining -= 1; } + break; } + else + { + wint_t wc2; - /* Nothing to translate. */ - memcpy (outbuf, inbuf, n1); - inbuf += n1; - remaining -= n1; - outbuf += n1; - continue; - } + if (n1 == 0) /* NUL character? */ + n1 = 1; + else if (n1 == (size_t)(-3)) + n1 = 0; - /* Invalid multibyte character on input. - Copy one byte without modification. */ - *outbuf++ = *inbuf++; - remaining -= 1; - } - /* Incomplete multibyte sequence on input. - Pass it through unmodified. */ - while (remaining > 0) - { - *outbuf++ = *inbuf++; - remaining -= 1; + wc2 = c32tolower (wc1); + if (wc2 != wc1) + { + mbstate_t state2; + size_t n2; + + memset (&state2, '\0', sizeof (mbstate_t)); + n2 = c32rtomb (outbuf, wc2, &state2); + if (n2 != (size_t)(-1)) + { + /* Store the translated multibyte character. */ + outbuf += n2; + goto done_storing; + } + } + + /* Nothing to translate. */ + memcpy (outbuf, inbuf, n1); + outbuf += n1; + done_storing: + inbuf += n1; + remaining -= n1; + } + } + while (! mbsinit (&state)); } /* Verify the output buffer was large enough. */ -- 2.39.5