From be664f4b273e10ab76ee55dd78998b3bf18207b7 Mon Sep 17 00:00:00 2001 From: blikblum <3047126+blikblum@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:18:06 +0000 Subject: [PATCH 1/2] Initial plan From 2203cd9a7d5f71302eda94b6649db13d636a2ee9 Mon Sep 17 00:00:00 2001 From: blikblum <3047126+blikblum@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:19:48 +0000 Subject: [PATCH 2/2] Preserve leading U+FEFF in decoded strings Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- src/DecodeStream.js | 2 +- test/DecodeStream.js | 14 ++++++++++++++ test/String.js | 10 ++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/DecodeStream.js b/src/DecodeStream.js index 226ffa1..abaf02e 100644 --- a/src/DecodeStream.js +++ b/src/DecodeStream.js @@ -18,7 +18,7 @@ export class DecodeStream { let buf = this.readBuffer(length); try { - let decoder = new TextDecoder(encoding); + let decoder = new TextDecoder(encoding, {ignoreBOM: true}); return decoder.decode(buf); } catch (err) { return buf; diff --git a/test/DecodeStream.js b/test/DecodeStream.js index 52f0b63..fd8924f 100644 --- a/test/DecodeStream.js +++ b/test/DecodeStream.js @@ -87,6 +87,20 @@ describe('DecodeStream', function() { assert.equal(stream.readString(buf.length, 'utf16be'), 'unicode! 👍'); }); + for (const [encoding, bytes] of [ + ['utf8', [0xef, 0xbb, 0xbf, 0x41]], + ['utf16le', [0xff, 0xfe, 0x41, 0x00]], + ['ucs2', [0xff, 0xfe, 0x41, 0x00]], + ['utf16be', [0xfe, 0xff, 0x00, 0x41]] + ]) { + it(`should preserve leading U+FEFF in each ${encoding} string`, function() { + const stream = new DecodeStream(new Uint8Array([...bytes, ...bytes])); + assert.equal(stream.readString(bytes.length, encoding), '\uFEFFA'); + assert.equal(stream.readString(bytes.length, encoding), '\uFEFFA'); + assert.equal(stream.pos, bytes.length * 2); + }); + } + it('should decode macroman', function() { const buf = new Uint8Array([0x8a, 0x63, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x20, 0x63, 0x68, 0x87, 0x72, 0x61, 0x63, 0x74, 0x65, 0x72, 0x73]); const stream = new DecodeStream(buf); diff --git a/test/String.js b/test/String.js index a5bbe14..07d924f 100644 --- a/test/String.js +++ b/test/String.js @@ -24,6 +24,16 @@ describe('String', function() { assert.equal(string.fromBuffer(Buffer.from('🍻')), '🍻'); }); + for (const encoding of ['utf8', 'utf16le', 'ucs2', 'utf16be']) { + it(`should round-trip leading U+FEFF in ${encoding} strings`, function() { + for (const length of [uint8, null]) { + const string = new StringT(length, encoding); + const value = '\uFEFFA\uFEFF'; + assert.equal(string.fromBuffer(string.toBuffer(value)), value); + } + }); + } + it('should decode encoding computed from function', function() { const string = new StringT(4, function() { return 'utf8'; }); assert.equal(string.fromBuffer(Buffer.from('🍻')), '🍻');