How to Download Files with HTTP Client

Table Of Contents

• Problem
• Solution
• Explanation

Problem

You need to download files of various sizes and types in Node.js, handle them efficiently without loading everything into memory, and provide progress feedback for large downloads.

Solution

const fs = require('fs');
const path = require('path');
const { pipeline } = require('stream/promises');

// 1. Basic File Download with fetch()
async function downloadFile(url, filename) {
  try {
    const response = await fetch(url);
    
    if (!response.ok) {
      throw new Error(`Download failed: ${response.status} ${response.statusText}`);
    }
    
    // Get file data as ArrayBuffer
    const arrayBuffer = await response.arrayBuffer();
    const buffer = Buffer.from(arrayBuffer);
    
    // Write to file
    await fs.promises.writeFile(filename, buffer);
    
    console.log(`Downloaded ${filename} (${buffer.length} bytes)`);
    return {
      filename,
      size: buffer.length,
      contentType: response.headers.get('Content-Type')
    };
  } catch (error) {
    console.error('Download error:', error.message);
    throw error;
  }
}

// 2. Streaming Download for Large Files
async function downloadFileStream(url, filename) {
  try {
    const response = await fetch(url);
    
    if (!response.ok) {
      throw new Error(`Download failed: ${response.status}`);
    }
    
    if (!response.body) {
      throw new Error('Response body is not readable');
    }
    
    // Create write stream
    const fileStream = fs.createWriteStream(filename);
    
    // Use pipeline for proper error handling and cleanup
    await pipeline(response.body, fileStream);
    
    const stats = await fs.promises.stat(filename);
    console.log(`Downloaded ${filename} (${stats.size} bytes)`);
    
    return {
      filename,
      size: stats.size,
      contentType: response.headers.get('Content-Type')
    };
  } catch (error) {
    // Clean up partial file on error
    try {
      await fs.promises.unlink(filename);
    } catch (unlinkError) {
      // Ignore if file doesn't exist
    }
    
    console.error('Stream download error:', error.message);
    throw error;
  }
}

// 3. Download with Progress Tracking
async function downloadWithProgress(url, filename, onProgress) {
  try {
    const response = await fetch(url);
    
    if (!response.ok) {
      throw new Error(`Download failed: ${response.status}`);
    }
    
    const contentLength = response.headers.get('Content-Length');
    const totalSize = contentLength ? parseInt(contentLength) : 0;
    
    if (!response.body) {
      throw new Error('Response body is not readable');
    }
    
    const reader = response.body.getReader();
    const fileStream = fs.createWriteStream(filename);
    
    let downloadedSize = 0;
    
    try {
      while (true) {
        const { done, value } = await reader.read();
        
        if (done) break;
        
        downloadedSize += value.length;
        fileStream.write(value);
        
        // Report progress
        if (onProgress && totalSize > 0) {
          const progress = (downloadedSize / totalSize) * 100;
          onProgress({
            downloadedSize,
            totalSize,
            progress: Math.round(progress * 100) / 100
          });
        } else if (onProgress) {
          onProgress({
            downloadedSize,
            totalSize: null,
            progress: null
          });
        }
      }
    } finally {
      reader.releaseLock();
      fileStream.end();
    }
    
    console.log(`Download completed: ${filename} (${downloadedSize} bytes)`);
    
    return {
      filename,
      size: downloadedSize,
      contentType: response.headers.get('Content-Type')
    };
  } catch (error) {
    // Clean up partial file
    try {
      await fs.promises.unlink(filename);
    } catch (unlinkError) {
      // Ignore cleanup errors
    }
    
    console.error('Progress download error:', error.message);
    throw error;
  }
}

// 4. Download with Resumable Support
async function downloadResumable(url, filename, options = {}) {
  const { maxRetries = 3, chunkSize = 1024 * 1024 } = options; // 1MB chunks
  
  let startByte = 0;
  let attempt = 0;
  
  // Check if partial file exists
  try {
    const stats = await fs.promises.stat(filename);
    startByte = stats.size;
    console.log(`Resuming download from byte ${startByte}`);
  } catch (error) {
    // File doesn't exist, start from beginning
    console.log('Starting new download');
  }
  
  while (attempt < maxRetries) {
    try {
      const headers = startByte > 0 ? { 'Range': `bytes=${startByte}-` } : {};
      
      const response = await fetch(url, { headers });
      
      if (!response.ok && response.status !== 206) {
        throw new Error(`Download failed: ${response.status}`);
      }
      
      const contentLength = response.headers.get('Content-Length');
      const contentRange = response.headers.get('Content-Range');
      
      let totalSize = null;
      if (contentRange) {
        const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
        totalSize = match ? parseInt(match[1]) : null;
      } else if (contentLength) {
        totalSize = parseInt(contentLength);
      }
      
      if (!response.body) {
        throw new Error('Response body is not readable');
      }
      
      const fileStream = fs.createWriteStream(filename, { 
        flags: startByte > 0 ? 'a' : 'w' // Append if resuming
      });
      
      const reader = response.body.getReader();
      let downloadedInThisSession = 0;
      
      try {
        while (true) {
          const { done, value } = await reader.read();
          
          if (done) break;
          
          fileStream.write(value);
          downloadedInThisSession += value.length;
          
          if (downloadedInThisSession % (chunkSize * 10) === 0) {
            console.log(`Downloaded ${startByte + downloadedInThisSession} bytes`);
          }
        }
      } finally {
        reader.releaseLock();
        fileStream.end();
      }
      
      console.log(`Download completed: ${filename}`);
      
      return {
        filename,
        size: startByte + downloadedInThisSession,
        resumed: startByte > 0,
        contentType: response.headers.get('Content-Type')
      };
    } catch (error) {
      attempt++;
      console.error(`Download attempt ${attempt} failed:`, error.message);
      
      if (attempt < maxRetries) {
        // Update start byte for resume
        try {
          const stats = await fs.promises.stat(filename);
          startByte = stats.size;
        } catch (statError) {
          startByte = 0;
        }
        
        const delay = Math.pow(2, attempt) * 1000; // Exponential backoff
        console.log(`Retrying in ${delay}ms...`);
        await new Promise(resolve => setTimeout(resolve, delay));
      }
    }
  }
  
  throw new Error(`Download failed after ${maxRetries} attempts`);
}

// 5. Multiple File Downloads
async function downloadMultipleFiles(downloads, options = {}) {
  const { concurrent = 3, retries = 3 } = options;
  const results = [];
  
  // Process downloads in batches
  for (let i = 0; i < downloads.length; i += concurrent) {
    const batch = downloads.slice(i, i + concurrent);
    
    const batchPromises = batch.map(async (download, index) => {
      try {
        console.log(`Starting download ${i + index + 1}/${downloads.length}: ${download.filename}`);
        
        const result = await downloadWithProgress(
          download.url,
          download.filename,
          (progress) => {
            if (progress.progress !== null) {
              console.log(`${download.filename}: ${progress.progress}%`);
            }
          }
        );
        
        return {
          ...download,
          success: true,
          result
        };
      } catch (error) {
        return {
          ...download,
          success: false,
          error: error.message
        };
      }
    });
    
    const batchResults = await Promise.allSettled(batchPromises);
    results.push(...batchResults.map(r => r.value));
  }
  
  return results;
}

// 6. Download File to Memory
async function downloadToBuffer(url, maxSize = 100 * 1024 * 1024) { // 100MB limit
  try {
    const response = await fetch(url);
    
    if (!response.ok) {
      throw new Error(`Download failed: ${response.status}`);
    }
    
    const contentLength = response.headers.get('Content-Length');
    if (contentLength && parseInt(contentLength) > maxSize) {
      throw new Error(`File too large: ${contentLength} bytes (max: ${maxSize})`);
    }
    
    const arrayBuffer = await response.arrayBuffer();
    const buffer = Buffer.from(arrayBuffer);
    
    if (buffer.length > maxSize) {
      throw new Error(`File too large: ${buffer.length} bytes (max: ${maxSize})`);
    }
    
    return {
      buffer,
      size: buffer.length,
      contentType: response.headers.get('Content-Type'),
      filename: getFilenameFromResponse(response) || 'download'
    };
  } catch (error) {
    console.error('Buffer download error:', error.message);
    throw error;
  }
}

// 7. Download with Authentication
async function downloadWithAuth(url, filename, token) {
  try {
    const response = await fetch(url, {
      headers: {
        'Authorization': `Bearer ${token}`,
        'Accept': '*/*'
      }
    });
    
    if (response.status === 401) {
      throw new Error('Authentication failed - invalid token');
    }
    
    if (!response.ok) {
      throw new Error(`Download failed: ${response.status}`);
    }
    
    return await downloadFileStream(url, filename);
  } catch (error) {
    console.error('Authenticated download error:', error.message);
    throw error;
  }
}

// 8. Utility Functions
function getFilenameFromResponse(response) {
  const contentDisposition = response.headers.get('Content-Disposition');
  if (contentDisposition) {
    const filenameMatch = contentDisposition.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/);
    if (filenameMatch && filenameMatch[1]) {
      return filenameMatch[1].replace(/['"]/g, '');
    }
  }
  
  // Extract from URL
  const url = new URL(response.url);
  const pathname = url.pathname;
  const filename = path.basename(pathname);
  
  return filename || 'download';
}

function formatFileSize(bytes) {
  const sizes = ['Bytes', 'KB', 'MB', 'GB'];
  if (bytes === 0) return '0 Bytes';
  
  const i = Math.floor(Math.log(bytes) / Math.log(1024));
  return Math.round(bytes / Math.pow(1024, i) * 100) / 100 + ' ' + sizes[i];
}

// Usage Examples
async function runDownloadExamples() {
  console.log('=== File Download Examples ===');
  
  try {
    // 1. Basic download
    console.log('1. Basic file download...');
    await downloadFile(
      'https://httpbin.org/json',
      './downloads/sample.json'
    );
    
    // 2. Streaming download
    console.log('2. Streaming download...');
    await downloadFileStream(
      'https://httpbin.org/json',
      './downloads/sample-stream.json'
    );
    
    // 3. Download with progress
    console.log('3. Download with progress...');
    await downloadWithProgress(
      'https://httpbin.org/json',
      './downloads/sample-progress.json',
      (progress) => {
        if (progress.progress !== null) {
          console.log(`Progress: ${progress.progress}%`);
        }
      }
    );
    
    // 4. Download to buffer
    console.log('4. Download to buffer...');
    const bufferResult = await downloadToBuffer('https://httpbin.org/json');
    console.log(`Downloaded to buffer: ${formatFileSize(bufferResult.size)}`);
    
    // 5. Multiple downloads
    console.log('5. Multiple downloads...');
    const downloads = [
      { url: 'https://httpbin.org/json', filename: './downloads/file1.json' },
      { url: 'https://httpbin.org/xml', filename: './downloads/file2.xml' }
    ];
    
    const results = await downloadMultipleFiles(downloads, { concurrent: 2 });
    console.log(`Downloaded ${results.filter(r => r.success).length}/${results.length} files`);
    
  } catch (error) {
    console.error('Download example error:', error.message);
  }
}

// Create downloads directory
async function ensureDownloadDir() {
  try {
    await fs.promises.mkdir('./downloads', { recursive: true });
  } catch (error) {
    // Directory already exists
  }
}

// Run examples if this file is executed directly
if (require.main === module) {
  ensureDownloadDir().then(() => runDownloadExamples());
}

module.exports = {
  downloadFile,
  downloadFileStream,
  downloadWithProgress,
  downloadResumable,
  downloadMultipleFiles,
  downloadToBuffer,
  downloadWithAuth,
  getFilenameFromResponse,
  formatFileSize
};

Explanation

For small files, use response.arrayBuffer() to get the complete file data. For large files, use streaming with response.body.getReader() or Node.js streams to avoid memory issues.

Implement progress tracking by reading the Content-Length header and tracking downloaded bytes. For resumable downloads, use the Range header to request specific byte ranges. Always handle errors properly and clean up partial files when downloads fail.