updating to import module

Author: Chris Wilson

index.js

@@ -1,13 +1,14 @@
-const fs = require('graceful-fs');
-const path = require('path');
-const querystring = require("querystring");
-const fetch = require('node-fetch');
-const request = require('postman-request');
-const urlparse = require('url');
-const mkdirp = require('mkdirp');
-const log = require('npmlog');
-
-const RateLimiter = require('limiter').RateLimiter;
+import fs from 'graceful-fs';
+import path from 'path';
+import querystring from "querystring";
+import fetch from 'node-fetch';
+import request from 'postman-request';
+import urlparse from 'url';
+import mkdirp from 'mkdirp';
+import log from 'npmlog';
+
+// import rl from 'limiter';
+// const RateLimiter = rl.RateLimiter;
 
 /* OPTIONS */
 /*
@@ -26,7 +27,7 @@ post: POST request
 let limiter, opts, queueDownload, download;
 
 // create a filepath out of url, same as wget
-const url_to_path = module.exports.url_to_path = function(url, opts) {
+const url_to_path = function(url, opts) {
 	let parsedUrl = urlparse.parse(url);
 	let p = path.join(parsedUrl.hostname, parsedUrl.path);
 	// exorcise any trailing "/"
@@ -51,7 +52,7 @@ const url_to_path = module.exports.url_to_path = function(url, opts) {
 // downcache("http://example.com", function(err, resp, body) {} )
 
 // we want to be flexible with the order of arguments since there are only three feasible types
-module.exports = function() {
+const Downcache = function() {
 	// this doesn't currently fire.
 	let callback = function(err, resp, body) {
 		log.info("This is the default downcache callback since you didn't provide one.");
@@ -100,7 +101,7 @@ module.exports = function() {
 
 	log.level = opts.log;
 
-	limiter = new RateLimiter(1, opts.limit);
+	// limiter = new RateLimiter(1, opts.limit);
 
 	if (!opts.useRequest) {
 		download = downloadWithFetch;
@@ -109,13 +110,14 @@ module.exports = function() {
 	}
 
 	queueDownload = function(opts, callback) {
-		limiter.removeTokens(1, function(err, remainingRequests) {
-			if (err) {
-				log.info("rate limited " + opts.url);
-				return callback("rate limited");
-			}
-			download(opts, callback);
-		});
+		// limiter.removeTokens(1, function(err, remainingRequests) {
+		// 	if (err) {
+		// 		log.info("rate limited " + opts.url);
+		// 		return callback("rate limited");
+		// 	}
+		// 	download(opts, callback);
+		// });
+		download(opts, callback);
 	}
 
 	// copy `url` and `encoding` to the headers object
@@ -162,7 +164,7 @@ let fireCallback = function(opts, err, resp, body, callback) {
 }
 
 // check if the file is in cache
-let retrieve = module.exports.retrieve = function(opts, callback) {
+let retrieve = function(opts, callback) {
 	if (opts.force) {
 		queueDownload(opts, callback);
 		return;
@@ -301,13 +303,15 @@ function savePage(opts, resp, body) {
 } // savePage
 
 // update the global settings that get used in absense of a specification in the individual call
-module.exports.set = function(property, value) {
+function SetProperty(property, value) {
 	if (typeof property == "string" && typeof value == "string") {
 		opts[property] = value;
 	} else if (typeof property == "object") {
 		opts = Object.assign(property, opts);
 	}
 	if (property == "limit" || property.limit) {
-		limiter = new RateLimiter(1, opts.limit);
+		// limiter = new RateLimiter(1, opts.limit);
 	}
 }
+
+export { Downcache, SetProperty };

package.json

@@ -4,6 +4,7 @@
   "author": "Chris Wilson <christopher.e.wilson@gmail.com>",
   "description": "Download and cache webpages and write them to disk for fast future retrieval",
   "main": "index.js",
+  "type": "module",
   "repository": {
     "type": "git",
     "url": "https://github.com/wilson428/downcache"
@@ -15,13 +16,13 @@
     "test": "test/test.js"
   },
   "dependencies": {
-    "fetch-cookie": "^0.11.0",
-    "graceful-fs": "^4.2.4",
-    "limiter": "^1.1.5",
+    "fetch-cookie": "^1.0.0",
+    "graceful-fs": "^4.2.8",
+    "limiter": "^2.1.0",
     "mkdirp": "^1.0.4",
-    "node-fetch": "^2.6.1",
-    "npmlog": "^4.1.2",
-    "postman-request": "^2.88.1-postman.29",
+    "node-fetch": "^3.1.0",
+    "npmlog": "^6.0.0",
+    "postman-request": "^2.88.1-postman.30",
     "request": "^2.88.2"
   },
   "devDependencies": {

test/test.js

@@ -1,39 +1,39 @@
 #!/usr/bin/env node
 
-var downcache = require("../index"),
-	assert = require("assert"),
-	rimraf = require("rimraf");
+import { Downcache, SetProperty } from "../index.js";
+import { strict as assert } from 'assert';
+import rimraf from "rimraf";
 
-downcache.set({
+SetProperty({
 	log: "verbose",
 	limit: 2000
 });
 
-assert.equal(downcache.url_to_path("http://example.com", {}), "example.com/index.html");
+//assert.equal(Downcache.url_to_path("http://example.com", {}), "example.com/index.html");
 
 // remove old cache from previous tests
 rimraf("./cache", function(err) {
 	// see what happens when we get a status code other than 200
-	downcache("http://api.meetup.com/2/members?group_id=741891&key=dsafsadfsadfasf", function(err, resp, html) {
+	Downcache("http://api.meetup.com/2/members?group_id=741891&key=dsafsadfsadfasf", function(err, resp, html) {
 		console.log(err);
 	});
 
-	downcache("http://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=Jimmy%20Rollins&rvprop=content&format=json", function(err, resp, html) {
+	Downcache("http://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=Jimmy%20Rollins&rvprop=content&format=json", function(err, resp, html) {
 		//console.log(resp);
 		//console.log(Object.keys(resp.response));
 		console.log(resp.type, resp.sub_type);
 	});
 
-	downcache("http://www.imdb.com", function(err, resp, html) {
-		downcache("http://www.imdb.com/title/tt0068646/", function(err, resp, html) {
+	Downcache("http://www.imdb.com", function(err, resp, html) {
+		Downcache("http://www.imdb.com/title/tt0068646/", function(err, resp, html) {
 			console.log("Downloaded", html.length, "characters from", resp.url);
-			downcache("http://www.imdb.com/title/tt0068646/fullcredits", function(err, resp, html) {
+			Downcache("http://www.imdb.com/title/tt0068646/fullcredits", function(err, resp, html) {
 				console.log("Downloaded", html.length, "characters from", resp.url);
 			});
 		});
 	});
 
-	downcache("http://time.com/67280/mean-girls-mythology/", {
+	Downcache("http://time.com/67280/mean-girls-mythology/", {
 		path: "great-articles/meangirls.html",
 		log: "verbose" // will override just for this call
 	}, function(err, resp, html) {
@@ -42,7 +42,7 @@ rimraf("./cache", function(err) {
 		}
 
 		// try again, see it load from cache
-		downcache("http://time.com/67280/mean-girls-mythology/", {
+		Downcache("http://time.com/67280/mean-girls-mythology/", {
 			path: "great-articles/meangirls.html",
 			log: "verbose"
 		}, function(err, resp, html) {
@@ -51,7 +51,7 @@ rimraf("./cache", function(err) {
 	});
 
 	// testing POST 
-	downcache("https://statements.mevaker.gov.il/Handler/GuarantyDonationPublisherHandler.ashx", {
+	Downcache("https://statements.mevaker.gov.il/Handler/GuarantyDonationPublisherHandler.ashx", {
 		post: {
 			action: "gcbp",
 			p: 3
@@ -61,7 +61,7 @@ rimraf("./cache", function(err) {
 	});
 	/*
 	// still working on this
-	downcache("http://mlb.mlb.com/images/players/mugshot/ph_424324.jpg", function(err, resp, body) {
+	Downcache("http://mlb.mlb.com/images/players/mugshot/ph_424324.jpg", function(err, resp, body) {
 		console.log(resp.type, resp.sub_type);
 	});
 	*/

test/test2.js

@@ -1,6 +1,6 @@
-const downcache = require('../index');
+import { Downcache } from '../index.js';
 
-downcache("https://electproject.github.io/Early-Vote-2020G/index.html", {
+Downcache("https://electproject.github.io/Early-Vote-2020G/index.html", {
 	dir: "my_cache",
 	log: "verbose"
 }, function(err, resp, body) {
@@ -9,7 +9,7 @@ downcache("https://electproject.github.io/Early-Vote-2020G/index.html", {
 });
 
 
-downcache("http://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=Jimmy%20Rollins&rvprop=content&format=json", { dir: "json", json: true }, function(err, resp, json) {
+Downcache("http://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=Jimmy%20Rollins&rvprop=content&format=json", { dir: "json", json: true }, function(err, resp, json) {
 	console.log(json);
 	console.log(typeof json);
 });