我正在使用 node js 從服務器下載大檔案（300MB）并將回應通過管道傳輸到檔案寫入流。據我了解 nodejs 中的管道，資料流由 node 管理，我不必考慮排水和其他事件。我面臨的問題是，運行我的應用程式的 docker 的記憶體使用量與正在下載的檔案的增加量相同（即檔案似乎正在保存在記憶體中）。即使我洗掉 docker 中的檔案，這種記憶體使用情況仍然存在。我附上了用于創建請求和管道的代碼，供參考。代碼運行良好，但會導致性能問題，例如巨大的記憶體/CPU 使用率和因 OOM 錯誤而崩潰。我無法理解我做錯了什么。
let req = request({
url: firmwareURL,
maxAttempts: 5,
retryDelay: 5000,
retryStrategy: request.RetryStrategies.HTTPOrNetworkError});
// 1. Perform server request
req.on('response', (res) => {
console.log(methodName, 'Download response statusCode:', res.statusCode);
if (res.statusCode === 200) {
abortOperation = false;
isStarted = "yes";
// 1.1 Create local file stream if the file is found on url and WaterMark paramter, for bigger chunk
// basepath basefirmware folder firmware name file extension
fileStoragePath = `${firmwareDirectory}/${ip}`;
console.log("filestoragepath is",fileStoragePath);
fileName = `${firmwareVersion}.${firmwareURL.split(".").pop()}`;
// temporary store the file
tempFile = `${fileStoragePath}/${fileName}`;
console.log("tempfile is",tempFile);
writestream = fs.createWriteStream(tempFile, {
highWaterMark: Math.pow(2,20 )
}); // for 1mb buffer,can be increased
writestream.on('error', function (err) {
// on error
console.log('Error while creating a file write stream' err);
abortOperation = true;
isStarted = "no";
_deleteProgressPointer(ip);
});
// 1.2 Get content length of the current response
size = parseInt(res.headers['content-length'], 10);
console.log(methodName, 'File size is:', size);
req.pipe(writestream);
} else {
// 1.3 Ignore next request events on failure
console.log(methodName, 'File not found on server. res.statusCode:', res.statusCode);
abortOperation = true;
isStarted = "no";
_deleteProgressPointer(ip);
}
});
// 3. In case of error ignore next request events
req.on('error', (error) => {
console.log(methodName, 'File not found on server:', error);
abortOperation = true;
isStarted = "no";
_deleteProgressPointer(ip);
});
// 4. After stream is received close the connection
req.on('end', () => {
if (!abortOperation) {
if (null !== writestream) {
writestream.end();
writestream.on('finish', function () {
console.log(methodName, `File successfully downloaded for device ${ip} of firmware version ${firmwareVersion}`);
try {
// file extraction/storage operation
// further check whether the file extension is valid or not
if (ALLOWED_EXTENSION.includes(firmwareURL.split(".").pop())) {
try {
//req.unpipe(writestream);
fileio.removeFile(tempFile); //deleting downloaded file to avoid storage issues
});
console.log("upgrade ended");
return upgradeOp;
} catch (error) {
console.log(`Error while renamining file: ${tempFile}`);
}
} else {
console.log(methodName, ` Not an valid file extension: ${tempFile}`);
fileio.removeFile(tempFile);
console.log(methodName, ` Invalid: ${tempFile} removed`);
}
// delete the progress pointer
_deleteProgressPointer(ip);
} catch (error) {
// delete the progress pointer
_deleteProgressPointer(ip);
console.log(methodName, `Error during read/write operation :${error}`);
}
});
}
uj5u.com熱心網友回復：
問題是您使用的是requestretry包，它并不真正支持流式傳輸。它總是request使用回呼進行呼叫，并將提供一個以完整回應解決的承諾。該請求庫將當這樣的讀取整個回應體callback設定，這的確沒有緩沖在存盤器中的完整的回應。這不是你想要的。
我沒有看到僅使用requestretry進行流式處理的方法，因此您應該直接使用請求包（或者，鑒于其棄用，它的后繼庫之一）并自己處理重試邏輯。
轉載請註明出處，本文鏈接：https://www.uj5u.com/net/353501.html
標籤：javascript 节点.js 表达 堆内存 nodejs-stream