Introduction
When streaming files from AWS S3 using s3.getObject().createReadStream() in Node.js, errors like missing keys, access denied, or network failures are not thrown as exceptions — they are emitted as events on the stream. If you do not listen for the error event, your application may crash with an unhandled error or silently fail.
The Problem
1const AWS = require('aws-sdk');
2const s3 = new AWS.S3();
3const fs = require('fs');
4
5// This creates the stream but does NOT handle errors
6const stream = s3.getObject({
7 Bucket: 'my-bucket',
8 Key: 'nonexistent-file.txt'
9}).createReadStream();
10
11// Piping without error handling — crashes the process!
12stream.pipe(fs.createWriteStream('output.txt'));
13// Error: NoSuchKey: The specified key does not exist.
Fix 1: Listen for the Error Event
1const stream = s3.getObject({
2 Bucket: 'my-bucket',
3 Key: 'my-file.txt'
4}).createReadStream();
5
6stream.on('error', (err) => {
7 console.error('S3 stream error:', err.code, err.message);
8 // Handle specific errors
9 if (err.code === 'NoSuchKey') {
10 console.error('File not found in S3');
11 } else if (err.code === 'AccessDenied') {
12 console.error('Permission denied');
13 }
14});
15
16stream.pipe(fs.createWriteStream('output.txt'));
Fix 2: Use the Request's httpResponse Event
Catch HTTP-level errors before the stream starts flowing:
1const request = s3.getObject({
2 Bucket: 'my-bucket',
3 Key: 'my-file.txt'
4});
5
6request.on('httpHeaders', (statusCode, headers) => {
7 if (statusCode >= 400) {
8 console.error(`S3 returned HTTP ${statusCode}`);
9 }
10});
11
12const stream = request.createReadStream();
13stream.on('error', (err) => {
14 console.error('Stream error:', err.message);
15});
16
17stream.pipe(fs.createWriteStream('output.txt'));
Fix 3: Promise-Based with Error Handling
Wrap the stream in a Promise for async/await:
1function downloadFromS3(bucket, key, destPath) {
2 return new Promise((resolve, reject) => {
3 const stream = s3.getObject({ Bucket: bucket, Key: key }).createReadStream();
4 const writeStream = fs.createWriteStream(destPath);
5
6 stream.on('error', (err) => {
7 // Clean up partial file
8 fs.unlink(destPath, () => {});
9 reject(err);
10 });
11
12 writeStream.on('error', (err) => {
13 reject(err);
14 });
15
16 writeStream.on('finish', () => {
17 resolve(destPath);
18 });
19
20 stream.pipe(writeStream);
21 });
22}
23
24// Usage
25try {
26 await downloadFromS3('my-bucket', 'data.csv', './data.csv');
27 console.log('Download complete');
28} catch (err) {
29 console.error('Download failed:', err.code);
30}
Fix 4: Using AWS SDK v3 (Recommended)
The AWS SDK v3 uses a different pattern:
1const { S3Client, GetObjectCommand } = require('@aws-sdk/client-s3');
2const { pipeline } = require('stream/promises');
3const fs = require('fs');
4
5const s3 = new S3Client({ region: 'us-east-1' });
6
7async function downloadFile(bucket, key, destPath) {
8 try {
9 const response = await s3.send(new GetObjectCommand({
10 Bucket: bucket,
11 Key: key
12 }));
13
14 // response.Body is a readable stream
15 await pipeline(
16 response.Body,
17 fs.createWriteStream(destPath)
18 );
19
20 console.log('Download complete');
21 } catch (err) {
22 if (err.name === 'NoSuchKey') {
23 console.error('File not found');
24 } else {
25 console.error('Error:', err.message);
26 }
27 }
28}
stream/promises.pipeline handles error propagation and cleanup automatically.
Piping to an Express Response
1app.get('/download/:key', async (req, res) => {
2 const params = { Bucket: 'my-bucket', Key: req.params.key };
3
4 try {
5 // Check if object exists first
6 await s3.headObject(params).promise();
7 } catch (err) {
8 return res.status(404).json({ error: 'File not found' });
9 }
10
11 const stream = s3.getObject(params).createReadStream();
12
13 stream.on('error', (err) => {
14 console.error('Stream error:', err);
15 if (!res.headersSent) {
16 res.status(500).json({ error: 'Download failed' });
17 }
18 });
19
20 res.setHeader('Content-Type', 'application/octet-stream');
21 stream.pipe(res);
22});
Common S3 Error Codes
| Error Code | HTTP Status | Meaning |
NoSuchKey | 404 | Object does not exist |
NoSuchBucket | 404 | Bucket does not exist |
AccessDenied | 403 | Missing permissions |
InvalidBucketName | 400 | Malformed bucket name |
RequestTimeout | 400 | Upload/download too slow |
SlowDown | 503 | Rate limit exceeded |
Common Pitfalls
Unhandled error event: In Node.js, an error event on a stream without a listener crashes the process. Always attach an error handler before piping.
Partial file cleanup: If an error occurs mid-download, the destination file contains partial data. Delete it in the error handler.
Concurrency and Throughput: When designing systems for high throughput, consider using multiple streams and instances for parallel processing.
Transform Streams: For real-time data manipulation, integrate transform streams to process the data on the fly as it streams from S3.
Retry Logic: Implement retry mechanisms for transient network issues. The AWS SDK v2 has built-in retry (maxRetries option); SDK v3 uses @aws-sdk/middleware-retry.
Memory pressure: Do not buffer the entire S3 object in memory with .promise() + Body for large files. Use streaming.
Summary
Always attach an error event listener to S3 createReadStream() before piping
Use stream/promises.pipeline (Node 15+) or Promises to handle errors in async code
AWS SDK v3 provides a cleaner async pattern with GetObjectCommand
Clean up partial files when download errors occur mid-stream
Check if the object exists with headObject before streaming to HTTP responses
