From dc13b36389418992b2b5d6bb1c28d8cc792d8f22 Mon Sep 17 00:00:00 2001 From: Michael Barajas Date: Sat, 20 Dec 2014 14:58:48 -0500 Subject: [PATCH 01/15] Fix 302 redirects, removing url parameters and removed unused bin folder --- .gitignore | 3 ++- README.md | 37 +++++++++++++++++++++------------ bin/nwget.js | 0 lib/wget.js | 58 +++++++++++++++++++++++++++++++++++----------------- package.json | 19 ++++++++++------- 5 files changed, 77 insertions(+), 40 deletions(-) delete mode 100644 bin/nwget.js diff --git a/.gitignore b/.gitignore index 91dfed8..0aa35d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store -node_modules \ No newline at end of file +node_modules +.idea diff --git a/README.md b/README.md index 3527914..a293a33 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,23 @@ -# node-wget +# wget-improved + +wget-improved simplifies retrieving files from any URL + +Improvements over old wget module + - Handles 302 redirects (including infinite redirects) + - Passes URL parameters + - Better error reporting -A download tool, now supporting http/https resource and http/https proxy, written in nodejs. # Installing ``` -npm install wget +npm install wget-improved --save ``` - -# Usage - - -## download(src, output, options) - ```js var wget = require('wget'); -var src = 'https://raw.github.com/Fyrd/caniuse/master/data.json'; -var output = '/tmp/data.json'; +var src = 'http://nodejs.org/images/logo.svg'; +var output = '/tmp/logo.svg'; var options = { - proxy: 'http://host:port' +//see options below }; var download = wget.download(src, output, options); download.on('error', function(err) { @@ -64,4 +64,15 @@ req.end(); req.on('error', function(err) { console.log(err); }); -``` \ No newline at end of file +``` +#download and request method options +```js + +options = {} + options.proxy = {}; + options.proxy.protocol = 'http'; + options.proxy.host = 'someproxy.org'; + options.proxy.port = 1337; + options.proxy.proxyAuth = '{basic auth}; + options.proxy.headers = {'User-Agent': 'Node'}; +``` diff --git a/bin/nwget.js b/bin/nwget.js deleted file mode 100644 index e69de29..0000000 diff --git a/lib/wget.js b/lib/wget.js index 1b55d3e..94890bb 100644 --- a/lib/wget.js +++ b/lib/wget.js @@ -8,13 +8,13 @@ var util = require('util'); var fs = require('fs'); var EventEmitter = require('events').EventEmitter; -function download(src, output, options) { +function download(src, output, options, redirects) { var downloader = new EventEmitter(), srcUrl, tunnelAgent, req; - if (options) { + if (options) { options = parseOptions('download', options); } srcUrl = url.parse(src); @@ -24,11 +24,20 @@ function download(src, output, options) { protocol: srcUrl.protocol, host: srcUrl.hostname, port: srcUrl.port, - path: srcUrl.pathname, + path: srcUrl.pathname + (srcUrl.search || ""), proxy: options?options.proxy:undefined, method: 'GET' }, function(res) { var fileSize, writeStream, downloadedSize; + // Handle 302 redirects + if(res.statusCode === 302) { + redirects++; + if(redirects >= 6) { + return downloader.emit('error', 'Infinite redirect loop detected'); + } + return download(res.headers.location, output, options, redirects); + } + if (res.statusCode === 200) { downloadedSize = 0; fileSize = res.headers['content-length']; @@ -48,8 +57,6 @@ function download(src, output, options) { }); res.on('end', function() { writeStream.end(); - }); - writeStream.on('close', function(){ downloader.emit('end', output); }); } else { @@ -66,38 +73,51 @@ function download(src, output, options) { } function request(options, callback) { + var newOptions = {}, newProxy = {}, key; options = parseOptions('request', options); if (options.protocol === 'http') { if (options.proxy) { + for (key in options.proxy) { + if (key !== 'protocol') { + newProxy[key] = options.proxy[key]; + } + } if (options.proxy.protocol === 'http') { - delete options.proxy.protocol; // delete self-defined arg - options.agent = tunnel.httpOverHttp({proxy: options.proxy}); + options.agent = tunnel.httpOverHttp({proxy: newProxy}); } else if (options.proxy.protocol === 'https') { - delete options.proxy.protocol; // delete self-defined arg - options.agent = tunnel.httpOverHttps({proxy: options.proxy}); + options.agent = tunnel.httpOverHttps({proxy: newProxy}); } else { throw options.proxy.protocol + ' proxy is not supported!'; } } - delete options.protocol; // delete self-defined arg - delete options.proxy; // delete self-defined arg - return http.request(options, callback); + for (key in options) { + if (key !== 'protocol' && key !== 'proxy') { + newOptions[key] = options[key]; + } + } + return http.request(newOptions, callback); } if (options.protocol === 'https') { if (options.proxy) { + for (key in options.proxy) { + if (key !== 'protocol') { + newProxy[key] = options.proxy[key]; + } + } if (options.proxy.protocol === 'http') { - delete options.proxy.protocol; // delete self-defined arg - options.agent = tunnel.httpsOverHttp({proxy: options.proxy}); + options.agent = tunnel.httpsOverHttp({proxy: newProxy}); } else if (options.proxy.protocol === 'https') { - delete options.proxy.protocol; // delete self-defined arg - options.agent = tunnel.httpsOverHttps({proxy: options.proxy}); + options.agent = tunnel.httpsOverHttps({proxy: newProxy}); } else { throw options.proxy.protocol + ' proxy is not supported!'; } } - delete options.protocol; // delete self-defined arg - delete options.proxy; // delete self-defined arg - return https.request(options, callback); + for (key in options) { + if (key !== 'protocol' && key !== 'proxy') { + newOptions[key] = options[key]; + } + } + return https.request(newOptions, callback); } throw 'only allow http or https request!'; } diff --git a/package.json b/package.json index 934883b..e8e5087 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,19 @@ { - "name": "wget", - "version": "0.0.1", - "description": "wget in nodejs.", - "keywords": ["download", "http", "https", "ftp", "proxy"], - "author": "Chengwei Wu ", + "name": "wget-improved", + "version": "0.9.0", + "description": "wget in nodejs, forked from wget to add improvements and help maintain the project", + "keywords": ["download", "http", "https", "ftp", "proxy", "wget"], + "author": "Michael Barajas ", "repository":{ "type": "git", - "url": "git://github.com/wuchengwei/node-wget.git" + "url": "git://github.com/bearjaws/node-wget.git" }, - "main": "./index.js", + "contributors": [ + { + "name": "Michael Barajas" + } + ], + "homepage": "https://github.com/bearjaws/wget-improved", "bin": { "nwget": "./bin/nwget.js" }, From 554548bbb26a822876bd38370bd2bf79dcf9823a Mon Sep 17 00:00:00 2001 From: Michael Barajas Date: Sat, 20 Dec 2014 14:59:29 -0500 Subject: [PATCH 02/15] Updated package.json --- package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/package.json b/package.json index e8e5087..48d6975 100644 --- a/package.json +++ b/package.json @@ -14,9 +14,6 @@ } ], "homepage": "https://github.com/bearjaws/wget-improved", - "bin": { - "nwget": "./bin/nwget.js" - }, "dependencies": { "tunnel": "0.0.2" }, From f6f5d6eb6eb5cf6008571cef7d963e37782995fd Mon Sep 17 00:00:00 2001 From: Michael Barajas Date: Sat, 20 Dec 2014 15:06:40 -0500 Subject: [PATCH 03/15] fixing readme.md and package.json --- README.md | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a293a33..22d274e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ wget-improved simplifies retrieving files from any URL Improvements over old wget module - - Handles 302 redirects (including infinite redirects) + - Handles 302 redirects (including infinite redirect loops) - Passes URL parameters - Better error reporting @@ -13,7 +13,7 @@ Improvements over old wget module npm install wget-improved --save ``` ```js -var wget = require('wget'); +var wget = require('wget-improved'); var src = 'http://nodejs.org/images/logo.svg'; var output = '/tmp/logo.svg'; var options = { @@ -73,6 +73,6 @@ options = {} options.proxy.protocol = 'http'; options.proxy.host = 'someproxy.org'; options.proxy.port = 1337; - options.proxy.proxyAuth = '{basic auth}; + options.proxy.proxyAuth = '{basic auth}'; options.proxy.headers = {'User-Agent': 'Node'}; ``` diff --git a/package.json b/package.json index 48d6975..72a757e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wget-improved", - "version": "0.9.0", + "version": "0.9.2", "description": "wget in nodejs, forked from wget to add improvements and help maintain the project", "keywords": ["download", "http", "https", "ftp", "proxy", "wget"], "author": "Michael Barajas ", From af4f6d7a40011d5f23cc9164c9070406362fe264 Mon Sep 17 00:00:00 2001 From: Michael Barajas Date: Sat, 20 Dec 2014 15:12:55 -0500 Subject: [PATCH 04/15] Improve redirect code --- lib/wget.js | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/wget.js b/lib/wget.js index 94890bb..8f55062 100644 --- a/lib/wget.js +++ b/lib/wget.js @@ -9,6 +9,9 @@ var fs = require('fs'); var EventEmitter = require('events').EventEmitter; function download(src, output, options, redirects) { + if(typeof redirects === "undefined") { + redirects = 0; + } var downloader = new EventEmitter(), srcUrl, tunnelAgent, diff --git a/package.json b/package.json index 72a757e..b8d17c7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wget-improved", - "version": "0.9.2", + "version": "0.9.3", "description": "wget in nodejs, forked from wget to add improvements and help maintain the project", "keywords": ["download", "http", "https", "ftp", "proxy", "wget"], "author": "Michael Barajas ", From 4ecef7f79e2bb001e9f5cc226c6afa34c87c38ca Mon Sep 17 00:00:00 2001 From: Michael Barajas Date: Sun, 21 Dec 2014 14:10:29 -0500 Subject: [PATCH 05/15] Added support for gzip, updated version, fixed recursion not closing req properly --- README.md | 8 ++++++++ lib/wget.js | 48 ++++++++++++++++++++++++++++++++++++++---------- package.json | 2 +- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 22d274e..be201f7 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Improvements over old wget module - Handles 302 redirects (including infinite redirect loops) - Passes URL parameters - Better error reporting + - Does not write using append (now uses w+ identical to wget) + - Handles gzip compression, allow you to automatically gunzip the stream # Installing @@ -69,6 +71,8 @@ req.on('error', function(err) { ```js options = {} + // Set to true to have any gzip stream automatically decompressed before saving + options.gunzip = false; options.proxy = {}; options.proxy.protocol = 'http'; options.proxy.host = 'someproxy.org'; @@ -76,3 +80,7 @@ options = {} options.proxy.proxyAuth = '{basic auth}'; options.proxy.headers = {'User-Agent': 'Node'}; ``` + + +#Todo +Enable gzip when using request method \ No newline at end of file diff --git a/lib/wget.js b/lib/wget.js index 8f55062..7f4f174 100644 --- a/lib/wget.js +++ b/lib/wget.js @@ -4,15 +4,24 @@ var http = require('http'); var https = require('https'); var tunnel = require('tunnel'); var url = require('url'); -var util = require('util'); +var zlib = require('zlib'); var fs = require('fs'); var EventEmitter = require('events').EventEmitter; -function download(src, output, options, redirects) { +/** + * Downloads a file using http get and request + * @param {string} src - The http URL to download from + * @param {string} output - The filepath to save to + * @param {object} options - Options object + * @param {object} _parentEvent - Used for when their is a 302 redirect and need to maintain state to a new request + * @param {number} redirects - The number of redirects, used to prevent infinite loops + * @returns {*|EventEmitter} + */ +function download(src, output, options, _parentEvent, redirects) { if(typeof redirects === "undefined") { redirects = 0; } - var downloader = new EventEmitter(), + var downloader = _parentEvent || new EventEmitter(), srcUrl, tunnelAgent, req; @@ -32,20 +41,22 @@ function download(src, output, options, redirects) { method: 'GET' }, function(res) { var fileSize, writeStream, downloadedSize; + var gunzip = zlib.createGunzip(); + // Handle 302 redirects if(res.statusCode === 302) { redirects++; - if(redirects >= 6) { - return downloader.emit('error', 'Infinite redirect loop detected'); + if(redirects >= 10) { + downloader.emit('error', 'Infinite redirect loop detected'); } - return download(res.headers.location, output, options, redirects); + download(res.headers.location, output, options, downloader, redirects); } if (res.statusCode === 200) { downloadedSize = 0; fileSize = res.headers['content-length']; writeStream = fs.createWriteStream(output, { - flags: 'a', + flags: 'w+', encoding: 'binary' }); @@ -53,21 +64,38 @@ function download(src, output, options, redirects) { writeStream.end(); downloader.emit('error', err); }); + + var encoding = ""; + if(typeof res.headers['content-encoding'] === "string") { + encoding = res.headers['content-encoding']; + } + + // If the user has specified to unzip, and the file is gzip encoded, pipe to gunzip + if(options.gunzip === true && encoding === "gzip") { + res.pipe(gunzip); + } else { + res.pipe(writeStream); + } + // Data handlers res.on('data', function(chunk) { downloadedSize += chunk.length; downloader.emit('progress', downloadedSize/fileSize); + }); + gunzip.on('data', function(chunk) { writeStream.write(chunk); }); - res.on('end', function() { + + writeStream.on('finish', function() { writeStream.end(); - downloader.emit('end', output); + downloader.emit('end', "Finished writing to disk"); + req.end('finished'); }); } else { downloader.emit('error', 'Server respond ' + res.statusCode); } }); - req.end(); + req.end('done'); req.on('error', function(err) { downloader.emit('error', err); }); diff --git a/package.json b/package.json index b8d17c7..e749ba8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wget-improved", - "version": "0.9.3", + "version": "1.0.0", "description": "wget in nodejs, forked from wget to add improvements and help maintain the project", "keywords": ["download", "http", "https", "ftp", "proxy", "wget"], "author": "Michael Barajas ", From 637c218ce3c07f6df11c292a8c510b6327cae6ca Mon Sep 17 00:00:00 2001 From: Michael Barajas Date: Sun, 21 Dec 2014 15:03:22 -0500 Subject: [PATCH 06/15] Fixed emitting error when redirecting, fixed options being undefined in some cases --- README.md | 2 +- lib/wget.js | 10 ++++++++-- package.json | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index be201f7..dad19df 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Improvements over old wget module - Handles 302 redirects (including infinite redirect loops) - Passes URL parameters - Better error reporting - - Does not write using append (now uses w+ identical to wget) + - Does not write using append (uses w+ identical to wget) - Handles gzip compression, allow you to automatically gunzip the stream diff --git a/lib/wget.js b/lib/wget.js index 7f4f174..396061c 100644 --- a/lib/wget.js +++ b/lib/wget.js @@ -28,6 +28,10 @@ function download(src, output, options, _parentEvent, redirects) { if (options) { options = parseOptions('download', options); + } else { + options = { + gunzip: false + }; } srcUrl = url.parse(src); srcUrl.protocol = cleanProtocol(srcUrl.protocol); @@ -90,8 +94,8 @@ function download(src, output, options, _parentEvent, redirects) { downloader.emit('end', "Finished writing to disk"); req.end('finished'); }); - } else { - downloader.emit('error', 'Server respond ' + res.statusCode); + } else if(res.statusCode !== 200 && res.statusCode !== 302) { + downloader.emit('error', 'Server responded with unhandled status: ' + res.statusCode); } }); @@ -186,6 +190,8 @@ function parseOptions(type, options) { options.proxy.headers = {'User-Agent': 'Node'}; } } + + options.gunzip = options.gunzip || false; return options; } } diff --git a/package.json b/package.json index e749ba8..8495660 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wget-improved", - "version": "1.0.0", + "version": "1.0.1", "description": "wget in nodejs, forked from wget to add improvements and help maintain the project", "keywords": ["download", "http", "https", "ftp", "proxy", "wget"], "author": "Michael Barajas ", From accfd0f6082e9842213a98f38937351404b8116c Mon Sep 17 00:00:00 2001 From: Michael Barajas Date: Tue, 28 Jul 2015 10:47:14 -0400 Subject: [PATCH 07/15] Added the request object to wget object, fixed not handling 301 redirects DOH --- lib/wget.js | 7 +++++-- package.json | 3 ++- test/test.js | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/wget.js b/lib/wget.js index 396061c..cf366e3 100644 --- a/lib/wget.js +++ b/lib/wget.js @@ -48,7 +48,7 @@ function download(src, output, options, _parentEvent, redirects) { var gunzip = zlib.createGunzip(); // Handle 302 redirects - if(res.statusCode === 302) { + if(res.statusCode === 301 || res.statusCode === 302) { redirects++; if(redirects >= 10) { downloader.emit('error', 'Infinite redirect loop detected'); @@ -94,7 +94,7 @@ function download(src, output, options, _parentEvent, redirects) { downloader.emit('end', "Finished writing to disk"); req.end('finished'); }); - } else if(res.statusCode !== 200 && res.statusCode !== 302) { + } else if(res.statusCode !== 200 && res.statusCode !== 301 && res.statusCode !== 302) { downloader.emit('error', 'Server responded with unhandled status: ' + res.statusCode); } }); @@ -103,6 +103,9 @@ function download(src, output, options, _parentEvent, redirects) { req.on('error', function(err) { downloader.emit('error', err); }); + // Attach request to our EventEmitter for backwards compatibility, enables actions such as + // req.abort(); + downloader.req = req; return downloader; } diff --git a/package.json b/package.json index 8495660..4f413f0 100644 --- a/package.json +++ b/package.json @@ -17,5 +17,6 @@ "dependencies": { "tunnel": "0.0.2" }, - "engines": { "node": ">= 0.6.18" } + "engines": { "node": ">= 0.6.18" }, + "readme": "# wget-improved\nwget-improved simplifies retrieving files from any URL\nImprovements over old wget module\n - Handles 302 redirects (including infinite redirect loops)\n - Passes URL parameters\n - Better error reporting\n - Does not write using append (uses w+ identical to wget)\n - Handles gzip compression, allow you to automatically gunzip the stream\n# Installing\n```\nnpm install wget-improved --save\n```\n```js\nvar wget = require('wget-improved');\nvar src = 'http://nodejs.org/images/logo.svg';\nvar output = '/tmp/logo.svg';\nvar options = {\n//see options below\n};\nvar download = wget.download(src, output, options);\ndownload.on('error', function(err) {\n console.log(err);\n});\ndownload.on('end', function(output) {\n console.log(output);\n});\ndownload.on('progress', function(progress) {\n // code to show progress bar\n});\n```\n\n## request(options, callback)\n```js\nvar wget = require('wget');\nvar options = {\n protocol: 'https',\n host: 'raw.github.com',\n path: '/Fyrd/caniuse/master/data.json',\n proxy: 'http://host:port',\n method: 'GET'\n};\nvar req = wget.request(options, function(res) {\n var content = '';\n if (res.statusCode === 200) {\n res.on('error', function(err) {\n console.log(err);\n });\n res.on('data', function(chunk) {\n content += chunk;\n });\n res.on('end', function() {\n console.log(content);\n });\n } else {\n console.log('Server respond ' + res.statusCode);\n }\n});\nreq.end();\nreq.on('error', function(err) {\n console.log(err);\n});\n```\n#download and request method options\n```js\noptions = {}\n // Set to true to have any gzip stream automatically decompressed before saving\n options.gunzip = false;\n options.proxy = {};\n options.proxy.protocol = 'http';\n options.proxy.host = 'someproxy.org';\n options.proxy.port = 1337;\n options.proxy.proxyAuth = '{basic auth}';\n options.proxy.headers = {'User-Agent': 'Node'};\n```\n#Todo\nEnable gzip when using request method" } diff --git a/test/test.js b/test/test.js index 9e5dbae..590487b 100644 --- a/test/test.js +++ b/test/test.js @@ -1,6 +1,6 @@ var wget = require('../lib/wget'); -var download = wget.download('https://raw.github.com/Fyrd/caniuse/master/data.json', '/tmp/README.md'); +var download = wget.download('https://www.npmjs.com/static/images/npm-logo.svg', '/tmp/README.md'); // with a proxy: // var download = wget.download('https://raw.github.com/Fyrd/caniuse/master/data.json', '/tmp/README.md', {proxy: 'http://proxyhost:port'}); download.on('error', function(err) { @@ -8,6 +8,7 @@ download.on('error', function(err) { }); download.on('end', function(output) { console.log(output); + process.exit(); }); download.on('progress', function(progress) { console.log(progress); From fddd59b863f95f6afdb6574fe7db247284e8be6c Mon Sep 17 00:00:00 2001 From: Michael Barajas Date: Tue, 28 Jul 2015 10:47:45 -0400 Subject: [PATCH 08/15] increased package version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4f413f0..801f499 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wget-improved", - "version": "1.0.1", + "version": "1.1.1", "description": "wget in nodejs, forked from wget to add improvements and help maintain the project", "keywords": ["download", "http", "https", "ftp", "proxy", "wget"], "author": "Michael Barajas ", From aac9bab96b97aed0aa1e63c985d40809aa2e3ac5 Mon Sep 17 00:00:00 2001 From: Greg Linklater Date: Thu, 3 Dec 2015 16:31:23 +0200 Subject: [PATCH 09/15] feat(wget): handle temporary redirects --- lib/wget.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wget.js b/lib/wget.js index cf366e3..c99a0f3 100644 --- a/lib/wget.js +++ b/lib/wget.js @@ -48,7 +48,7 @@ function download(src, output, options, _parentEvent, redirects) { var gunzip = zlib.createGunzip(); // Handle 302 redirects - if(res.statusCode === 301 || res.statusCode === 302) { + if(res.statusCode === 301 || res.statusCode === 302 || res.statusCode === 307) { redirects++; if(redirects >= 10) { downloader.emit('error', 'Infinite redirect loop detected'); @@ -204,4 +204,4 @@ function cleanProtocol(str) { } exports.download = download; -exports.request = request; \ No newline at end of file +exports.request = request; From 0597ab13f0b0d4976b0a1b88567452e23c246e31 Mon Sep 17 00:00:00 2001 From: Michael Barajas Date: Thu, 3 Dec 2015 12:50:44 -0500 Subject: [PATCH 10/15] Update package.json Updated version to enable bug fix from https://github.com/bearjaws/node-wget/pull/1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 801f499..8a7346a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wget-improved", - "version": "1.1.1", + "version": "1.2.1", "description": "wget in nodejs, forked from wget to add improvements and help maintain the project", "keywords": ["download", "http", "https", "ftp", "proxy", "wget"], "author": "Michael Barajas ", From 7a97689253080a7968a8b9ca80b80cf184e39df0 Mon Sep 17 00:00:00 2001 From: DudeNamedBen Date: Thu, 10 Dec 2015 08:33:55 +0100 Subject: [PATCH 11/15] emit start event when starting the download so the user knows the file-size he's going to download --- lib/wget.js | 4 ++++ test/test.js | 3 +++ 2 files changed, 7 insertions(+) diff --git a/lib/wget.js b/lib/wget.js index c99a0f3..0934644 100644 --- a/lib/wget.js +++ b/lib/wget.js @@ -80,6 +80,10 @@ function download(src, output, options, _parentEvent, redirects) { } else { res.pipe(writeStream); } + + //emit a start event so the user knows the file-size he's gonna receive + downloader.emit('start', fileSize); + // Data handlers res.on('data', function(chunk) { downloadedSize += chunk.length; diff --git a/test/test.js b/test/test.js index 590487b..409d80d 100644 --- a/test/test.js +++ b/test/test.js @@ -6,6 +6,9 @@ var download = wget.download('https://www.npmjs.com/static/images/npm-logo.svg', download.on('error', function(err) { console.log(err); }); +download.on('start', function(fileSize) { + console.log(fileSize); +}); download.on('end', function(output) { console.log(output); process.exit(); From 51037f5ff55158b265fa0dbcf05b91a31dbbc295 Mon Sep 17 00:00:00 2001 From: DudeNamedBen Date: Thu, 10 Dec 2015 08:38:07 +0100 Subject: [PATCH 12/15] emit start event when starting the download so the user knows the file-size he's going to download --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index dad19df..ea8b9a8 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ var download = wget.download(src, output, options); download.on('error', function(err) { console.log(err); }); +download.on('start', function(fileSize) { + console.log(fileSize); +}); download.on('end', function(output) { console.log(output); }); From 890a6ff724be1a74934dca23d2833e8ec17bbdab Mon Sep 17 00:00:00 2001 From: Michael Barajas Date: Wed, 20 Jan 2016 09:10:19 -0500 Subject: [PATCH 13/15] Updated readme, published new NPM version --- README.md | 2 ++ package.json | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ea8b9a8..7fe592a 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,9 @@ download.on('progress', function(progress) { }); ``` + + ## request(options, callback) ```js diff --git a/package.json b/package.json index 8a7346a..fc89cb5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wget-improved", - "version": "1.2.1", + "version": "1.3.0", "description": "wget in nodejs, forked from wget to add improvements and help maintain the project", "keywords": ["download", "http", "https", "ftp", "proxy", "wget"], "author": "Michael Barajas ", @@ -17,6 +17,5 @@ "dependencies": { "tunnel": "0.0.2" }, - "engines": { "node": ">= 0.6.18" }, - "readme": "# wget-improved\nwget-improved simplifies retrieving files from any URL\nImprovements over old wget module\n - Handles 302 redirects (including infinite redirect loops)\n - Passes URL parameters\n - Better error reporting\n - Does not write using append (uses w+ identical to wget)\n - Handles gzip compression, allow you to automatically gunzip the stream\n# Installing\n```\nnpm install wget-improved --save\n```\n```js\nvar wget = require('wget-improved');\nvar src = 'http://nodejs.org/images/logo.svg';\nvar output = '/tmp/logo.svg';\nvar options = {\n//see options below\n};\nvar download = wget.download(src, output, options);\ndownload.on('error', function(err) {\n console.log(err);\n});\ndownload.on('end', function(output) {\n console.log(output);\n});\ndownload.on('progress', function(progress) {\n // code to show progress bar\n});\n```\n\n## request(options, callback)\n```js\nvar wget = require('wget');\nvar options = {\n protocol: 'https',\n host: 'raw.github.com',\n path: '/Fyrd/caniuse/master/data.json',\n proxy: 'http://host:port',\n method: 'GET'\n};\nvar req = wget.request(options, function(res) {\n var content = '';\n if (res.statusCode === 200) {\n res.on('error', function(err) {\n console.log(err);\n });\n res.on('data', function(chunk) {\n content += chunk;\n });\n res.on('end', function() {\n console.log(content);\n });\n } else {\n console.log('Server respond ' + res.statusCode);\n }\n});\nreq.end();\nreq.on('error', function(err) {\n console.log(err);\n});\n```\n#download and request method options\n```js\noptions = {}\n // Set to true to have any gzip stream automatically decompressed before saving\n options.gunzip = false;\n options.proxy = {};\n options.proxy.protocol = 'http';\n options.proxy.host = 'someproxy.org';\n options.proxy.port = 1337;\n options.proxy.proxyAuth = '{basic auth}';\n options.proxy.headers = {'User-Agent': 'Node'};\n```\n#Todo\nEnable gzip when using request method" + "engines": { "node": ">= 0.6.18" } } From 40081894b733659c31cc1660829d8eef5bcbfc87 Mon Sep 17 00:00:00 2001 From: Grant Nestor Date: Mon, 10 Oct 2016 08:55:52 -0700 Subject: [PATCH 14/15] Add CLI --- README.md | 40 ++++++++++++++++++++++++++-------------- bin/nwget | 42 ++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 69 insertions(+), 14 deletions(-) create mode 100755 bin/nwget diff --git a/README.md b/README.md index 7fe592a..2d9f731 100644 --- a/README.md +++ b/README.md @@ -2,24 +2,27 @@ wget-improved simplifies retrieving files from any URL -Improvements over old wget module - - Handles 302 redirects (including infinite redirect loops) - - Passes URL parameters - - Better error reporting - - Does not write using append (uses w+ identical to wget) - - Handles gzip compression, allow you to automatically gunzip the stream +Improvements over [wuchengwei/node-wget](https://github.com/wuchengwei/node-wget) +- Handles 302 redirects (including infinite redirect loops) +- Passes URL parameters +- Better error reporting +- Does not write using append (uses w+ identical to wget) +- Handles gzip compression, allow you to automatically gunzip the stream +## Install -# Installing ``` npm install wget-improved --save ``` + +## download(src, output, options) + ```js var wget = require('wget-improved'); var src = 'http://nodejs.org/images/logo.svg'; var output = '/tmp/logo.svg'; var options = { -//see options below + // see options below }; var download = wget.download(src, output, options); download.on('error', function(err) { @@ -36,9 +39,6 @@ download.on('progress', function(progress) { }); ``` - - - ## request(options, callback) ```js @@ -72,7 +72,9 @@ req.on('error', function(err) { console.log(err); }); ``` -#download and request method options + +## options + ```js options = {} @@ -86,6 +88,16 @@ options = {} options.proxy.headers = {'User-Agent': 'Node'}; ``` +## CLI + +```bash +# If installed globally +nwget https://raw.github.com/Fyrd/caniuse/master/data.json -O /tmp/data.json + +# If not installed globally +./node_modules/.bin/nwget https://raw.github.com/Fyrd/caniuse/master/data.json -O /tmp/data.json +``` + +## Todo -#Todo -Enable gzip when using request method \ No newline at end of file +- Enable gzip when using request method diff --git a/bin/nwget b/bin/nwget new file mode 100755 index 0000000..8881d62 --- /dev/null +++ b/bin/nwget @@ -0,0 +1,42 @@ +#!/usr/bin/env node + +/** + +Usage: +nwget [URL] [OPTIONS] + +Example: +nwget https://raw.github.com/Fyrd/caniuse/master/data.json -O /tmp/data.json + +**/ + +var wget = require('../lib/wget'); +var path = require('path'); + +var args = require('minimist')(process.argv); +var url = args._[2] +var output = args.O + +if (args.help) return console.log(`Usage: wget [URL] [OPTIONS] + +Download: + -O, --output-document=FILE write documents to FILE +`) + +if (!url) return console.error('The first argument must be a URL to a downloadable resource (e.g. nwget https://raw.github.com/Fyrd/caniuse/master/data.json)') + +if (!output) return console.error('The second argument must be a file path for the downloaded resource (e.g. nwget https://raw.github.com/Fyrd/caniuse/master/data.json -O /tmp/data.json)') + +// console.log(url, output); + +var download = wget.download(url, path.resolve(output)); + +download.on('error', function(err) { + console.error(err); +}); +download.on('end', function(output) { + console.log(output); +}); +download.on('progress', function(progress) { + console.log(progress); +}); diff --git a/package.json b/package.json index fc89cb5..0870dac 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ ], "homepage": "https://github.com/bearjaws/wget-improved", "dependencies": { + "minimist": "^1.2.0", "tunnel": "0.0.2" }, "engines": { "node": ">= 0.6.18" } From d6e50f2d5ea0195ea6fb3311ec9d4682b39f8008 Mon Sep 17 00:00:00 2001 From: Grant Nestor Date: Mon, 10 Oct 2016 09:18:01 -0700 Subject: [PATCH 15/15] Add bin to package.json --- package.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 0870dac..61754cc 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "wget-improved", "version": "1.3.0", - "description": "wget in nodejs, forked from wget to add improvements and help maintain the project", + "description": "wget in nodejs, forked from wuchengwei/node-wget to add improvements and help maintain the project", "keywords": ["download", "http", "https", "ftp", "proxy", "wget"], "author": "Michael Barajas ", "repository":{ @@ -13,7 +13,10 @@ "name": "Michael Barajas" } ], - "homepage": "https://github.com/bearjaws/wget-improved", + "homepage": "https://github.com/bearjaws/node-wget", + "bin": { + "nwget": "./bin/nwget" + }, "dependencies": { "minimist": "^1.2.0", "tunnel": "0.0.2"