如何避免在 API 导入请求期间出现"Allocation failed - JavaScript heap out of memory"

Question

我有一个nodejs脚本，可以读取excels文件中的数据并将其发送到带有"request"的nodejs API。 此脚本适用于少量数据。但是当我使用包含 60.000 行的 Excel 进行测试时，它因错误而中断：致命错误：
CALL_AND_RETRY_LAST分配失败 - JavaScript 堆内存不足

function fillWithFile(file, path, header, fileConfig) { 
let workbook = XLSX.readFile(path + '/' + file); // read file
for (var sheetIterator = 0; sheetIterator < workbook.SheetNames.length; sheetIterator++) { // for each tab in excel file
let worksheetJson = XLSX.utils.sheet_to_json(workbook.Sheets[workbook.SheetNames[sheetIterator]]); // transforme one tab of excel to JSON
let config = fileConfig[workbook.SheetNames[sheetIterator]]; // get the config for the current tab
let datasPromises = getPromises(worksheetJson, config, header);
Promise.all(datasPromises).then(function (data) {
var dataString = '{"data":' + JSON.stringify(data) + ',"options":{"purgeBefore":false}}';
let options = {
url: API_URL + '/' + config.service + 's/import',
method: 'POST',
headers: header,
body: dataString
}
request.post(options, function (err, res, body) {
if (err) throw err;
console.log('Server responded with:', body);
});
});
}
}
function getPromises(worksheetJson, config, header) {
let datasPromises = [];
let promises = [];
for (let lineIterator = 0; lineIterator < worksheetJson.length; lineIterator++) { // for each line
datasPromises.push(new Promise(function (resolve, reject) {
for (let key in config.columns) { // for each column in config
if (config.columns[key].service !== undefined) { // if service exist we need to get id of the object in this service.
promises.push(new Promise(function (resolve, reject) {
findChildren(worksheetJson[lineIterator], config.columns[key], header)
.then(function (res) {
resolve({ key: key, data: res.id });
});
}));
}
else {
promises.push(new Promise(function (resolve, reject) {
resolve({ key: key, data: worksheetJson[lineIterator][config.columns[key]] });
}));
}
}
let tmpObj = {};
Promise.all(promises).then(function (values) {
for (var i = 0; i < values.length; i++) {
tmpObj[values[i].key] = values[i].data;
}
resolve(tmpObj);
});
}));
}
return datasPromises;
}
function findChildren(sheetData, config, header) { // get children with get request
let dataObj = {};
let searchParams;
for (let key in config.columns) {
dataObj[key] = sheetData[config.columns[key]];
if (searchParams === undefined) searchParams = key + '=' + sheetData[config.columns[key]];
else searchParams += '&' + key + '=' + sheetData[config.columns[key]];
}
var headers = {
'Authorization': header.Authorization,
'Accept': 'application/json, text/plain, */*',
};
var options = {
url: API_URL + '/' + config.service + 's?' + searchParams,
headers: headers
};
return new Promise(function (resolve, reject) {
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
try {
resolve(JSON.parse(body));
} catch (e) {
reject(e);
}
}
else {
return reject(error);
}
});
});
}

脚本使用了大量内存，他最终崩溃了...... 有人知道我该如何解决这个问题吗？ 我想，我必须找到一种方法来强制承诺立即解决，以避免内存溢出。 感谢您的任何帮助。

Answer 1

我认为问题是你不控制并行性，当一切都"同时"运行时 - 它会导致内存问题。

我尝试使用 async/await 重写代码 - 实际上所有操作现在都是串行的：

async function fillWithFile(file, path, header, fileConfig) {
let workbook = XLSX.readFile(path + '/' + file); // read file
for (var sheetIterator = 0; sheetIterator < workbook.SheetNames.length; sheetIterator++) { // for each tab in excel file
const worksheetJson = XLSX.utils.sheet_to_json(workbook.Sheets[workbook.SheetNames[sheetIterator]]); // transforme one tab of excel to JSON
const config = fileConfig[workbook.SheetNames[sheetIterator]]; // get the config for the current tab
const data = await getData(worksheetJson, config, header);
const dataString = '{"data":' + JSON.stringify(data) + ',"options":{"purgeBefore":false}}';
const options = {
url: API_URL + '/' + config.service + 's/import',
method: 'POST',
headers: header,
body: dataString
};
await new Promise((resolve, reject) => {
request.post(options, function (err, res, body) {
if (err) {
reject(err);
return;
}
console.log('Server responded with:', body);
resolve(body);
})
});
}
}
async function getData(worksheetJson, config, header) {
const data = [];
for (let lineIterator = 0; lineIterator < worksheetJson.length; lineIterator++) { // for each line
const values = {};
for (let key in config.columns) { // for each column in config
if (config.columns[key].service !== undefined) { // if service exist we need to get id of the object in this service.
const res = await findChildren(worksheetJson[lineIterator], config.columns[key], header);
values[key] = res.id;
}
else {
values[key] = worksheetJson[lineIterator][config.columns[key]];
}
}
data.push(values);    
}
return data;
}
function findChildren(sheetData, config, header) { // get children with get request
let dataObj = {};
let searchParams;
for (let key in config.columns) {
dataObj[key] = sheetData[config.columns[key]];
if (searchParams === undefined) searchParams = key + '=' + sheetData[config.columns[key]];
else searchParams += '&' + key + '=' + sheetData[config.columns[key]];
}
var headers = {
'Authorization': header.Authorization,
'Accept': 'application/json, text/plain, */*',
};
var options = {
url: API_URL + '/' + config.service + 's?' + searchParams,
headers: headers
};
return new Promise(function (resolve, reject) {
request(options, function (error, response, body) {
if (!error && response.statusCode === 200) {
try {
resolve(JSON.parse(body));
} catch (e) {
reject(e);
}
}
else {
return reject(error);
}
});
});
}

您现在像这样运行导入：

fillWithFile(..args).then(() => console.log('done'))