Skip to content

log: Fix C-locale UTF-8 splitting and out-of-range codepoints #1166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
seks99x wants to merge 2 commits into
base: master
Choose a base branch
from
+114 −6
Open

log: Fix C-locale UTF-8 splitting and out-of-range codepoints #1166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e613028
Fix C-locale UTF-8 splitting and out-of-range codepoints in log.c
seks99x Nov 11, 2026
62b7e3f
Guard mbrtowc() Unicode bounds check with __STDC_ISO_10646__ in log.c
seks99x Nov 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 55 additions & 6 deletions log.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,17 +246,69 @@ void logfile_reopen(void)
}
}

/* Decode one UTF-8 character when running in the 7-bit C/POSIX locale
* (MB_CUR_MAX == 1) where mbrtowc() rejects all bytes >= 0x80. */
static size_t utf8_char_len(const char *buf, size_t len, int *is_control)
{
const uchar *s = (const uchar *)buf;
uint32 cp;
size_t need;

if (len < 2 || s[0] < 0xc2 || s[0] > 0xf4)
return 0;
if (s[0] < 0xe0) {
need = 2;
cp = s[0] & 0x1f;
} else if (s[0] < 0xf0) {
need = 3;
cp = s[0] & 0x0f;
} else {
need = 4;
cp = s[0] & 0x07;
}
if (len < need)
return 0;
for (size_t i = 1; i < need; i++) {
if ((s[i] & 0xc0) != 0x80)
return 0;
cp = (cp << 6) | (s[i] & 0x3f);
}
if ((need == 3 && cp < 0x800) || (need == 3 && cp >= 0xd800 && cp <= 0xdfff)
|| (need == 4 && (cp < 0x10000 || cp > 0x10ffff)))
return 0;

*is_control = (cp >= 0x7f && cp <= 0x9f);
return need;
}

/* Decode one character using the active locale. Invalid or incomplete input
* returns 0 so its bytes are filtered individually. */
static size_t locale_char_len(const char *buf, size_t len, filter_mbstate *state,
int *is_control)
{
#if defined HAVE_WCHAR_H && defined HAVE_MBRTOWC
wchar_t wc;
size_t char_len = mbrtowc(&wc, buf, len, state);
size_t char_len;

if (MB_CUR_MAX == 1) {
#if defined HAVE_SETLOCALE && defined LC_CTYPE
const char *ctype = setlocale(LC_CTYPE, NULL);
if (!ctype || strcmp(ctype, "C") == 0 || strcmp(ctype, "POSIX") == 0)
return utf8_char_len(buf, len, is_control);
#else
return utf8_char_len(buf, len, is_control);
#endif
}

if (char_len == (size_t)-1 || char_len == (size_t)-2) {
char_len = mbrtowc(&wc, buf, len, state);
if (char_len == (size_t)-1 || char_len == (size_t)-2
#ifdef __STDC_ISO_10646__
|| (wc >= 0xd800 && wc <= 0xdfff) || wc > 0x10ffff
#endif
) {
memset(state, 0, sizeof *state);
if (MB_CUR_MAX == 1)
return utf8_char_len(buf, len, is_control);
return 0;
}
if (char_len == 0)
Expand All @@ -265,11 +317,8 @@ static size_t locale_char_len(const char *buf, size_t len, filter_mbstate *state
&& (wc < L' ' || (wc >= 0x7f && wc <= 0x9f));
return char_len;
#else
(void)buf;
(void)len;
(void)state;
(void)is_control;
return 0;
return utf8_char_len(buf, len, is_control);
#endif
}

Expand Down
59 changes: 59 additions & 0 deletions testsuite/output-control-chars_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,13 @@ def check_locale_case(label, locale_names, name, expected, rejected):
'raw_csi': b'raw_\x9b_name',
'utf8_csi': b'utf8_\xc2\x9b_name',
'valid_utf8': b'valid_\xd8\x9b_name',
'sharp_s': b'sharp_\xc3\x9f_name',
'valid_4byte': b'four_\xf0\x9f\x92\xa9_name',
'inc_prefix': b'inc_\xe2\x82',
'bad_cont': b'badcont_\xe2\x28\xa1_name',
'overlong': b'overlong_\xc0\xaf_name',
'surrogate': b'surrogate_\xed\xa0\x80_name',
'above_max': b'abovemax_\xf4\x90\x80\x80_name',
'leading_cr': b'\rleading_cr_name',
'delete': b'delete_\x7f_name',
}
Expand Down Expand Up @@ -104,6 +111,22 @@ def check_locale_case(label, locale_names, name, expected, rejected):
if (utf8_locale is not None and 'valid_utf8' in created
and names['valid_utf8'] not in output):
test_fail("valid UTF-8 containing a C1-range continuation byte was changed")
if (utf8_locale is not None and 'sharp_s' in created
and names['sharp_s'] not in output):
test_fail("valid UTF-8 sharp s (0xc3 0x9f) was changed under UTF-8 locale")
if (utf8_locale is not None and 'valid_4byte' in created
and names['valid_4byte'] not in output):
test_fail("valid 4-byte UTF-8 (f0 9f 92 a9) was changed under UTF-8 locale")
if 'inc_prefix' in created and b'inc_\xe2\\#202\n' not in output:
test_fail(f"incomplete 3-byte prefix E2 82 did not escape 0x82: {output!r}")
if 'bad_cont' in created and names['bad_cont'] not in output:
test_fail(f"invalid continuation E2 28 A1 was not handled properly: {output!r}")
if 'overlong' in created and names['overlong'] not in output:
test_fail(f"overlong C0 AF was not handled properly: {output!r}")
if 'surrogate' in created and b'surrogate_\xed\xa0\\#200_name' not in output:
test_fail(f"surrogate ED A0 80 did not escape trailing 0x80: {output!r}")
if 'above_max' in created and b'abovemax_\xf4\\#220\\#200\\#200_name' not in output:
test_fail(f"above-U+10FFFF F4 90 80 80 did not escape C1 bytes: {output!r}")
if 'leading_cr' in created and names['leading_cr'] in output:
test_fail("leading carriage return reached terminal output")
if 'leading_cr' in created and b'\\#015leading_cr_name' not in output:
Expand All @@ -113,6 +136,42 @@ def check_locale_case(label, locale_names, name, expected, rejected):
if 'delete' in created and b'delete_\\#177_name' not in output:
test_fail("DEL was not escaped")

# Verify C/POSIX locale (or minimal container without locale database):
dst_c = base / 'dst-c'
dst_c.mkdir()
logfile_c = base / 'rsync-c.log'
env_c = os.environ.copy()
env_c['LC_ALL'] = 'C'
proc_c = subprocess.run(
rsync_argv('-av', '--8-bit-output', f'--log-file={logfile_c}',
'--log-file-format=%f', str(src) + '/', str(dst_c) + '/'),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env_c)
if proc_c.returncode != 0:
test_fail(f"rsync failed under C locale: {proc_c.stderr!r}")

c_output = proc_c.stdout + proc_c.stderr + logfile_c.read_bytes()
if 'sharp_s' in created and names['sharp_s'] not in c_output:
test_fail(f"C locale mangled valid UTF-8 sharp s: {c_output!r}")
if 'valid_utf8' in created and names['valid_utf8'] not in c_output:
test_fail(f"C locale mangled valid_utf8: {c_output!r}")
if 'valid_4byte' in created and names['valid_4byte'] not in c_output:
test_fail(f"C locale mangled valid 4-byte UTF-8: {c_output!r}")
if 'utf8_csi' in created:
if b'\xc2\x9b' in c_output:
test_fail("C locale let UTF-8-encoded CSI reach output")
if b'\\#302\\#233' not in c_output:
test_fail("C locale did not escape UTF-8-encoded CSI")
if 'inc_prefix' in created and b'inc_\xe2\\#202\n' not in c_output:
test_fail(f"C locale did not escape 0x82 in incomplete prefix E2 82: {c_output!r}")
if 'bad_cont' in created and names['bad_cont'] not in c_output:
test_fail(f"C locale mishandled E2 28 A1: {c_output!r}")
if 'overlong' in created and names['overlong'] not in c_output:
test_fail(f"C locale mishandled overlong C0 AF: {c_output!r}")
if 'surrogate' in created and b'surrogate_\xed\xa0\\#200_name' not in c_output:
test_fail(f"C locale did not escape trailing 0x80 in surrogate ED A0 80: {c_output!r}")
if 'above_max' in created and b'abovemax_\xf4\\#220\\#200\\#200_name' not in c_output:
test_fail(f"C locale did not escape C1 bytes in F4 90 80 80: {c_output!r}")

locale_cases = 0
locale_cases += check_locale_case(
'iso-8859-1', ('en_US.ISO-8859-1', 'no_NO.ISO-8859-1'),
Expand Down
Loading