From 4b3e47bd23673e9a0a5428fa89afbfa0ff8257a0 Mon Sep 17 00:00:00 2001
From: Richard Hansen <rhansen@rhansen.org>
Date: Thu, 17 Jun 2021 19:41:44 -0400
Subject: [PATCH] bin/importSqlFile.js: Read the file one line at a time

This avoids running out of memory if the file is large.
---
 src/bin/importSqlFile.js | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/bin/importSqlFile.js b/src/bin/importSqlFile.js
index b7e3a16c..148503e8 100644
--- a/src/bin/importSqlFile.js
+++ b/src/bin/importSqlFile.js
@@ -48,6 +48,7 @@ const unescape = (val) => {
 (async () => {
   const fs = require('fs');
   const log4js = require('log4js');
+  const readline = require('readline');
   const settings = require('../node/utils/Settings');
   const ueberDB = require('ueberdb2');
 
@@ -71,14 +72,12 @@ const unescape = (val) => {
   await util.promisify(db.init.bind(db))();
   log('done');
 
-  log('open output file...');
-  const lines = fs.readFileSync(sqlFile, 'utf8').split('\n');
+  log(`Opening ${sqlFile}...`);
+  const stream = fs.createReadStream(sqlFile, {encoding: 'utf8'});
 
-  const count = lines.length;
+  log(`Reading ${sqlFile}...`);
   let keyNo = 0;
-
-  process.stdout.write(`Start importing ${count} keys...\n`);
-  lines.forEach((l) => {
+  for await (const l of readline.createInterface({input: stream, crlfDelay: Infinity})) {
     if (l.substr(0, 27) === 'REPLACE INTO store VALUES (') {
       const pos = l.indexOf("', '");
       const key = l.substr(28, pos - 28);
@@ -88,11 +87,9 @@ const unescape = (val) => {
       console.log(`unval: ${unescape(value)}`);
       db.set(key, unescape(value), null);
       keyNo++;
-      if (keyNo % 1000 === 0) {
-        process.stdout.write(` ${keyNo}/${count}\n`);
-      }
+      if (keyNo % 1000 === 0) log(` ${keyNo}`);
     }
-  });
+  }
   process.stdout.write('\n');
   process.stdout.write('done. waiting for db to finish transaction. ' +
                        'depended on dbms this may take some time..\n');