Skip to content

Commit

Permalink
fix: encoding utf8 strings in node 20
Browse files Browse the repository at this point in the history
The writeFileUtf8 binding used by node 20 was not correctly encoding
strings as utf8, and recording the file length as
String.prototype.length, which returns number of utf-16 code units.

If you wrote a string containing code points with a different number of
utf-8 code units than utf-16 code units, then read the file back the
returned string would be truncated by the difference in the number of
code units.
  • Loading branch information
Caleb ツ Everett committed Jan 3, 2025
1 parent 00cc7b5 commit f5e49e0
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 2 deletions.
4 changes: 2 additions & 2 deletions lib/binding.js
Original file line number Diff line number Diff line change
Expand Up @@ -722,7 +722,7 @@ Binding.prototype.writeString = function (
callback(err, written, returned && string);
};
}
return this.writeBuffer(fd, buffer, 0, string.length, position, wrapper, ctx);
return this.writeBuffer(fd, buffer, 0, buffer.length, position, wrapper, ctx);
};

/**
Expand Down Expand Up @@ -878,7 +878,7 @@ Binding.prototype.readFileUtf8 = function (name, flags) {
*/
Binding.prototype.writeFileUtf8 = function (filepath, data, flags, mode) {
const destFd = this.open(filepath, flags, mode);
this.writeBuffer(destFd, data, 0, data.length);
this.writeString(destFd, data, null, 'utf8');
};

/**
Expand Down
39 changes: 39 additions & 0 deletions test/lib/encoding.spec.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
const helper = require('../helper.js');
const assert = helper.assert;
const fs = require('fs');
const mock = require('../../lib/index.js');

const CHARS = [
// // 1 utf-16, 1 utf-8 byte
'A',

// 1 utf-16 code unit, 3 utf-8 bytes
'’',

// // 2 utf-16 code units, 4 utf-8 bytes
'😄',
];

const ENCODINGS = ['utf8', 'utf16le', 'latin1'];

for (const encoding of ENCODINGS) {
for (const char of CHARS) {
describe(`Encoding (${encoding} ${char})`, () => {
const buffer = Buffer.from(char, encoding);

beforeEach(() => mock());
afterEach(() => mock.restore());

beforeEach(() => fs.writeFileSync('file', char, {encoding}));

it(`writes ${buffer.length} bytes`, () => {
assert.strictEqual(fs.statSync('file').size, buffer.length);
});

it('reads the written value (buffer)', () => {
const out = fs.readFileSync('file');
assert.sameOrderedMembers(Array.from(out), Array.from(buffer));
});
});
}
}

0 comments on commit f5e49e0

Please sign in to comment.