Reading Files in NodeJS

posted in: Uncategorized | 0

When I first needed to read some files in node I just assumed they would fit in memory.  A few 1000 lines is easy:

const recordString = fs.readFileSync(filename, 'utf8');
const recordStrings = recordString.split("\n");

But then I needed to read in a 1.5GB file with 2.5 million lines and I was no longer able to just do that.  On my development machine it would fit. But on a small docker container it won’t.  Luckily someone thought of this and there is the built in readline library. But the actual read of each line is an asynchronous event so we have a tiny challenge to make it behave as a synchronous function.  In case you find yourself needing to read a large file line by line.  I wrote this in an older style JavaScript to accommodate where I was using it.

const fs = require('fs');
const readline = require('readline');

function readFile() {
return new Promise( (resolve, reject) => {
const reader = readline.createInterface({
input: fs.createReadStream('mytextfile.txt'),
terminal: false
})
.on('line', (line) => {
console.log("Read line: " + line);
})
.on('close', () => {
console.log("Finished reading");
return resolve(true);
});
});
}

async function main() {
console.log("start");
let result = await readFile();
console.log("result = " + result);
}
main();

 

Leave a Reply

Your email address will not be published. Required fields are marked *