Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/DecodeStream.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export class DecodeStream {

let buf = this.readBuffer(length);
try {
let decoder = new TextDecoder(encoding);
let decoder = new TextDecoder(encoding, {ignoreBOM: true});
return decoder.decode(buf);
} catch (err) {
return buf;
Expand Down
14 changes: 14 additions & 0 deletions test/DecodeStream.js
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,20 @@ describe('DecodeStream', function() {
assert.equal(stream.readString(buf.length, 'utf16be'), 'unicode! 👍');
});

for (const [encoding, bytes] of [
['utf8', [0xef, 0xbb, 0xbf, 0x41]],
['utf16le', [0xff, 0xfe, 0x41, 0x00]],
['ucs2', [0xff, 0xfe, 0x41, 0x00]],
['utf16be', [0xfe, 0xff, 0x00, 0x41]]
]) {
it(`should preserve leading U+FEFF in each ${encoding} string`, function() {
const stream = new DecodeStream(new Uint8Array([...bytes, ...bytes]));
assert.equal(stream.readString(bytes.length, encoding), '\uFEFFA');
assert.equal(stream.readString(bytes.length, encoding), '\uFEFFA');
assert.equal(stream.pos, bytes.length * 2);
});
}

it('should decode macroman', function() {
const buf = new Uint8Array([0x8a, 0x63, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x20, 0x63, 0x68, 0x87, 0x72, 0x61, 0x63, 0x74, 0x65, 0x72, 0x73]);
const stream = new DecodeStream(buf);
Expand Down
10 changes: 10 additions & 0 deletions test/String.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,16 @@ describe('String', function() {
assert.equal(string.fromBuffer(Buffer.from('🍻')), '🍻');
});

for (const encoding of ['utf8', 'utf16le', 'ucs2', 'utf16be']) {
it(`should round-trip leading U+FEFF in ${encoding} strings`, function() {
for (const length of [uint8, null]) {
const string = new StringT(length, encoding);
const value = '\uFEFFA\uFEFF';
assert.equal(string.fromBuffer(string.toBuffer(value)), value);
}
});
}

it('should decode encoding computed from function', function() {
const string = new StringT(4, function() { return 'utf8'; });
assert.equal(string.fromBuffer(Buffer.from('🍻')), '🍻');
Expand Down
Loading