From 60852bc770ff6d6e1a2ca0b029ba805f9e5ffcd3 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 21 Oct 2012 23:53:01 +0200 Subject: [PATCH 01/42] renamed test.js to benchmark.js (which is more correct) --- README.markdown | 2 +- test.js => benchmark.js | 0 package.json | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename test.js => benchmark.js (100%) diff --git a/README.markdown b/README.markdown index 474ef46..18396d4 100644 --- a/README.markdown +++ b/README.markdown @@ -20,7 +20,7 @@ Benchmarks on my 2011 Macbook Air whilst running lots of software. The test too ## How to use 1. git clone https://github.com/benlowry/node-geoip-native 2. cd node-geoip-native -3. node test.js +3. node benchmark.js or just ```npm install geoip-native``` diff --git a/test.js b/benchmark.js similarity index 100% rename from test.js rename to benchmark.js diff --git a/package.json b/package.json index d7a2fb4..6c30e8a 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "url": "http://opensource.org/licenses/mit-license.php" } ], - "readme": "# Node GeoIP Native\n\nThis package is a lightning-fast, native JavaScript geoip lookup built on [MaxMind](http://www.maxmind.com/)'s free country database.\n\nIt is non-blocking and operates without any IO after initially loading the data into memory.\n\nResults are 4 - 5 times faster than [geoip-lite](https://github.com/bluesmoon/node-geoip) with the caveat that it takes 2 or 3 times longer to initialize and uses 60 or 70 megabytes memory.\n\nThis is used in production at [Playtomic](https://playtomic.com/) in a [high volume API](https://success.heroku.com/playtomic) where performance matters.\n\nBenchmarks on my 2011 Macbook Air whilst running lots of software. The test took the middle 10 results from 20 iterations and averaged them. The APIs are interchangeable so tests were identical.\n\n\tgeoip-native:\taverage: 1540.3ms / million lookups\n\tgeoip-lite: \taverage: 8375.3ms / million lookups\n\n## Requires\n\n1. Comes with the [standard CSV database by MaxMind](http://www.maxmind.com/app/geolite) which may require updating.\n\n## How to use\n1. git clone https://github.com/benlowry/node-geoip-native\n2. cd node-geoip-native\n3. node test.js\n\nor just ```npm install geoip-native```\n\n## Methods\n\nNode GeoIP Native provides methods for:\n\n1. ```lookup``` performs the lookup, takes the ip address as a parameter\n\n## Examples\n\n\tvar geoip = require(\"geoip-native\");\n\tvar ip = \"123.123.123.123\";\n\tgeoip.lookup(ip);\n\tconsole.log(\"country: \" + ip.name + \" / \" + ip.code);\n\n\t// in practice you'd want:\n\t// ip = request.headers[\"x-forwarded-for\"] || request.connection.remoteAddress,\n\n### What's missing\nBe neat to expand this to include cities.\n\n### License\nCopyright [Playtomic Inc](https://playtomic.com), 2012. Licensed under the MIT license. Certain portions may come from 3rd parties and carry their own licensing terms and are referenced where applicable.\n\nThis product includes GeoLite data created by MaxMind, available from http://www.maxmind.com\n", + "readme": "# Node GeoIP Native\n\nThis package is a lightning-fast, native JavaScript geoip lookup built on [MaxMind](http://www.maxmind.com/)'s free country database.\n\nIt is non-blocking and operates without any IO after initially loading the data into memory.\n\nResults are 4 - 5 times faster than [geoip-lite](https://github.com/bluesmoon/node-geoip) with the caveat that it takes 2 or 3 times longer to initialize and uses 60 or 70 megabytes memory.\n\nThis is used in production at [Playtomic](https://playtomic.com/) in a [high volume API](https://success.heroku.com/playtomic) where performance matters.\n\nBenchmarks on my 2011 Macbook Air whilst running lots of software. The test took the middle 10 results from 20 iterations and averaged them. The APIs are interchangeable so tests were identical.\n\n\tgeoip-native:\taverage: 1540.3ms / million lookups\n\tgeoip-lite: \taverage: 8375.3ms / million lookups\n\n## Requires\n\n1. Comes with the [standard CSV database by MaxMind](http://www.maxmind.com/app/geolite) which may require updating.\n\n## How to use\n1. git clone https://github.com/benlowry/node-geoip-native\n2. cd node-geoip-native\n3. node benchmark.js\n\nor just ```npm install geoip-native```\n\n## Methods\n\nNode GeoIP Native provides methods for:\n\n1. ```lookup``` performs the lookup, takes the ip address as a parameter\n\n## Examples\n\n\tvar geoip = require(\"geoip-native\");\n\tvar ip = \"123.123.123.123\";\n\tgeoip.lookup(ip);\n\tconsole.log(\"country: \" + ip.name + \" / \" + ip.code);\n\n\t// in practice you'd want:\n\t// ip = request.headers[\"x-forwarded-for\"] || request.connection.remoteAddress,\n\n### What's missing\nBe neat to expand this to include cities.\n\n### License\nCopyright [Playtomic Inc](https://playtomic.com), 2012. Licensed under the MIT license. Certain portions may come from 3rd parties and carry their own licensing terms and are referenced where applicable.\n\nThis product includes GeoLite data created by MaxMind, available from http://www.maxmind.com\n", "_id": "geoip-native@0.0.2", "_from": "geoip-native" } From b79555d6db2f52b25bcb63a2680c57c8cba14343 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Mon, 22 Oct 2012 00:45:48 +0200 Subject: [PATCH 02/42] added a NodeUnit test for proving correctness --- test/unit/lookup.js | 67 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 test/unit/lookup.js diff --git a/test/unit/lookup.js b/test/unit/lookup.js new file mode 100644 index 0000000..9d2a7ae --- /dev/null +++ b/test/unit/lookup.js @@ -0,0 +1,67 @@ +// use Node-Unit (https://github.com/caolan/nodeunit) to run this test + +module.exports = { + setUp:function (callback) { + this.geoip = require("../../geoip.js"); + // wait until ready + callback(); + }, + + tearDown:function (callback) { + // clean up + callback(); + }, + + test_do_warm_up:function (test) { + function waitalittle() { + test.done(); + } + + setTimeout(waitalittle, 3000); + }, + + test_check_if_warmed_up:function (test) { + test.notEqual(this.geoip.lookup("127.0.0.1").code, "N/A", "No yet warmed up, please use more time for warmup"); + test.done(); + }, + + test_ip_in_the_lower_range:function (test) { + var ip; + var actual; + // "1.0.64.0","1.0.127.255","16793600","16809983","JP","Japan" + ip = "1.0.64.0"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Japan"); + test.equals(actual.code, "JP"); + ip = "1.0.90.90"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Japan"); + test.equals(actual.code, "JP"); + ip = "1.0.127.255"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Japan"); + test.equals(actual.code, "JP"); + + test.done(); + }, + + test_ip_in_upper_range:function (test) { + var ip; + var actual; + // "223.223.168.0","223.223.175.255","3755976704","3755978751","KH","Cambodia" + ip = "223.223.168.0"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Cambodia"); + test.equals(actual.code, "KH"); + ip = "223.223.170.170"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Cambodia"); + test.equals(actual.code, "KH"); + ip = "223.223.175.255"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Cambodia"); + test.equals(actual.code, "KH"); + + test.done(); + } +}; \ No newline at end of file From 1d5086ed004443cf88849b37a2dfe23bab87ffbc Mon Sep 17 00:00:00 2001 From: nitram509 Date: Wed, 31 Oct 2012 21:49:17 +0100 Subject: [PATCH 03/42] renamed lookup.js to lookupTest.js --- test/unit/{lookup.js => lookupTest.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/unit/{lookup.js => lookupTest.js} (100%) diff --git a/test/unit/lookup.js b/test/unit/lookupTest.js similarity index 100% rename from test/unit/lookup.js rename to test/unit/lookupTest.js From 0eaa06632b45482a2ebf412197b0a00f6e44927c Mon Sep 17 00:00:00 2001 From: nitram509 Date: Wed, 31 Oct 2012 21:54:37 +0100 Subject: [PATCH 04/42] added some hints about nodeunit --- README.markdown | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.markdown b/README.markdown index 18396d4..a45e319 100644 --- a/README.markdown +++ b/README.markdown @@ -43,6 +43,14 @@ Node GeoIP Native provides methods for: ### What's missing Be neat to expand this to include cities. +### Testing +First, you have to install nodeunit (https://github.com/caolan/nodeunit) + $> npm install nodeunit + +Second, run the unit test: + $> ./node_modules/nodeunit/bin/nodeunit test/unit/lookupTest.js + + ### License Copyright [Playtomic Inc](https://playtomic.com), 2012. Licensed under the MIT license. Certain portions may come from 3rd parties and carry their own licensing terms and are referenced where applicable. From 1188781a2239f1639e199c3352bab20bdcf142f9 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Wed, 31 Oct 2012 21:55:25 +0100 Subject: [PATCH 05/42] cosmetics --- README.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.markdown b/README.markdown index a45e319..83eb627 100644 --- a/README.markdown +++ b/README.markdown @@ -45,9 +45,11 @@ Be neat to expand this to include cities. ### Testing First, you have to install nodeunit (https://github.com/caolan/nodeunit) + $> npm install nodeunit Second, run the unit test: + $> ./node_modules/nodeunit/bin/nodeunit test/unit/lookupTest.js From 51d7aecf5192af1fef3e2d44a1bd87e320104451 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Wed, 31 Oct 2012 21:56:20 +0100 Subject: [PATCH 06/42] cosmetics --- README.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index 83eb627..435082b 100644 --- a/README.markdown +++ b/README.markdown @@ -46,11 +46,11 @@ Be neat to expand this to include cities. ### Testing First, you have to install nodeunit (https://github.com/caolan/nodeunit) - $> npm install nodeunit + $> npm install nodeunit Second, run the unit test: - $> ./node_modules/nodeunit/bin/nodeunit test/unit/lookupTest.js + $> ./node_modules/nodeunit/bin/nodeunit test/unit/lookupTest.js ### License From 101f943845eaf1810dff358699c8fdb0073968c9 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Wed, 31 Oct 2012 21:57:24 +0100 Subject: [PATCH 07/42] reordered paragraphs --- README.markdown | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/README.markdown b/README.markdown index 435082b..c67ca8a 100644 --- a/README.markdown +++ b/README.markdown @@ -30,6 +30,15 @@ Node GeoIP Native provides methods for: 1. ```lookup``` performs the lookup, takes the ip address as a parameter +## Testing +First, you have to install nodeunit (https://github.com/caolan/nodeunit) + + $> npm install nodeunit + +Second, run the unit test: + + $> ./node_modules/nodeunit/bin/nodeunit test/unit/lookupTest.js + ## Examples var geoip = require("geoip-native"); @@ -43,16 +52,6 @@ Node GeoIP Native provides methods for: ### What's missing Be neat to expand this to include cities. -### Testing -First, you have to install nodeunit (https://github.com/caolan/nodeunit) - - $> npm install nodeunit - -Second, run the unit test: - - $> ./node_modules/nodeunit/bin/nodeunit test/unit/lookupTest.js - - ### License Copyright [Playtomic Inc](https://playtomic.com), 2012. Licensed under the MIT license. Certain portions may come from 3rd parties and carry their own licensing terms and are referenced where applicable. From f1e93b7f55a78e0c98d3cce5848e4c5b339b9f12 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Wed, 31 Oct 2012 21:59:51 +0100 Subject: [PATCH 08/42] added IntelliJIdea working files to git ignore list --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index a98ecab..448e893 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ mongodb node_modules .DS_Store +.idea +*.iml From cb89a8f2b94ab963107e815088cc2a200535911a Mon Sep 17 00:00:00 2001 From: nitram509 Date: Thu, 1 Nov 2012 00:01:52 +0100 Subject: [PATCH 09/42] made separate test cases for each corner case FIX: country name contains a '\r' on windows --- geoip.js | 4 +- test/unit/lookupTest.js | 156 ++++++++++++++++++++++++---------------- 2 files changed, 97 insertions(+), 63 deletions(-) diff --git a/geoip.js b/geoip.js index 140e6c2..8a9122e 100644 --- a/geoip.js +++ b/geoip.js @@ -102,7 +102,9 @@ function find(ip) { for(var i=0; i 5) { + countries.push({ipstart: parseInt(entry[2]), code: entry[4], name: entry[5].trim()}); + } } countries.sort(function(a, b) { diff --git a/test/unit/lookupTest.js b/test/unit/lookupTest.js index 9d2a7ae..3a38f3e 100644 --- a/test/unit/lookupTest.js +++ b/test/unit/lookupTest.js @@ -1,67 +1,99 @@ // use Node-Unit (https://github.com/caolan/nodeunit) to run this test module.exports = { - setUp:function (callback) { - this.geoip = require("../../geoip.js"); - // wait until ready - callback(); - }, - - tearDown:function (callback) { - // clean up - callback(); - }, - - test_do_warm_up:function (test) { - function waitalittle() { - test.done(); - } - - setTimeout(waitalittle, 3000); - }, - - test_check_if_warmed_up:function (test) { - test.notEqual(this.geoip.lookup("127.0.0.1").code, "N/A", "No yet warmed up, please use more time for warmup"); - test.done(); - }, - - test_ip_in_the_lower_range:function (test) { - var ip; - var actual; - // "1.0.64.0","1.0.127.255","16793600","16809983","JP","Japan" - ip = "1.0.64.0"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Japan"); - test.equals(actual.code, "JP"); - ip = "1.0.90.90"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Japan"); - test.equals(actual.code, "JP"); - ip = "1.0.127.255"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Japan"); - test.equals(actual.code, "JP"); - - test.done(); - }, - - test_ip_in_upper_range:function (test) { - var ip; - var actual; - // "223.223.168.0","223.223.175.255","3755976704","3755978751","KH","Cambodia" - ip = "223.223.168.0"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Cambodia"); - test.equals(actual.code, "KH"); - ip = "223.223.170.170"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Cambodia"); - test.equals(actual.code, "KH"); - ip = "223.223.175.255"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Cambodia"); - test.equals(actual.code, "KH"); - - test.done(); + setUp:function (callback) { + this.geoip = require("../../geoip.js"); + // wait until ready + callback(); + }, + + tearDown:function (callback) { + // clean up + callback(); + }, + + test_do_warm_up:function (test) { + function waitalittle() { + test.done(); } + + setTimeout(waitalittle, 3000); + }, + + test_check_if_warmed_up:function (test) { + test.notEqual(this.geoip.lookup("127.0.0.1").code, "N/A", "No yet warmed up, please use more time for warmup"); + test.done(); + }, + + test_ip_in_the_lower_range_and_leftmost_border:function (test) { + var ip; + var actual; + // "1.0.64.0","1.0.127.255","16793600","16809983","JP","Japan" + ip = "1.0.64.0"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Japan"); + test.equals(actual.code, "JP"); + + test.done(); + }, + + test_ip_in_the_lower_range_and_middle:function (test) { + var ip; + var actual; + // "1.0.64.0","1.0.127.255","16793600","16809983","JP","Japan" + ip = "1.0.90.90"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Japan"); + test.equals(actual.code, "JP"); + + test.done(); + }, + + test_ip_in_the_lower_range_and_rightmost_border:function (test) { + var ip; + var actual; + // "1.0.64.0","1.0.127.255","16793600","16809983","JP","Japan" + ip = "1.0.64.0"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Japan"); + test.equals(actual.code, "JP"); + + test.done(); + }, + + test_ip_in_upper_range_and_leftmost_border:function (test) { + var ip; + var actual; + // "223.223.168.0","223.223.175.255","3755976704","3755978751","KH","Cambodia" + ip = "223.223.168.0"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Cambodia"); + test.equals(actual.code, "KH"); + + test.done(); + }, + + test_ip_in_upper_range_and_middle:function (test) { + var ip; + var actual; + // "223.223.168.0","223.223.175.255","3755976704","3755978751","KH","Cambodia" + ip = "223.223.170.170"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Cambodia"); + test.equals(actual.code, "KH"); + + test.done(); + }, + + test_ip_in_upper_range_and_rightmost_border:function (test) { + var ip; + var actual; + // "223.223.168.0","223.223.175.255","3755976704","3755978751","KH","Cambodia" + ip = "223.223.175.255"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "Cambodia"); + test.equals(actual.code, "KH"); + + test.done(); + } }; \ No newline at end of file From 115770bd21de1e8824d792363c1998125cb5a4ee Mon Sep 17 00:00:00 2001 From: nitram509 Date: Thu, 1 Nov 2012 00:05:28 +0100 Subject: [PATCH 10/42] FIX: wrong calculation of midpoints --- geoip.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geoip.js b/geoip.js index 8a9122e..2e21ae5 100644 --- a/geoip.js +++ b/geoip.js @@ -111,9 +111,9 @@ function find(ip) { return a.ipstart - b.ipstart; }); - var n = Math.floor(countries.length / 2); + var n = countries.length; while(n >= 1) { - n = Math.floor(n / 2); + n = n >> 1; midpoints.push(n); } From 3f5eaf34f9915d796c64edc79be9f361f9bf8938 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Thu, 1 Nov 2012 00:13:18 +0100 Subject: [PATCH 11/42] only reformatting --- geoip.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/geoip.js b/geoip.js index 2e21ae5..05b478c 100644 --- a/geoip.js +++ b/geoip.js @@ -61,7 +61,7 @@ function find(ip) { // we're either current, next or previous depending on which is closest to ipl var cd = Math.abs(ipl - current.ipstart); - var nd = next && next.ipstart< ipl ? ipl - next.ipstart : 1000000000; + var nd = next && next.ipstart < ipl ? ipl - next.ipstart : 1000000000; var pd = prev && prev.ipstart < ipl ? ipl - prev.ipstart : 1000000000; // current wins @@ -72,7 +72,6 @@ function find(ip) { // next wins if(nd < cd && nd < pd) { return next; - } // prev wins @@ -111,14 +110,14 @@ function find(ip) { return a.ipstart - b.ipstart; }); - var n = countries.length; + numcountries = countries.length; + var n = numcountries; while(n >= 1) { n = n >> 1; midpoints.push(n); } - numcountries = countries.length; - ready = true; + ready = true; }); }()); \ No newline at end of file From c3dac81a2f37f260d63c3cdc71b6d3b8f2a434f3 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Thu, 1 Nov 2012 00:33:27 +0100 Subject: [PATCH 12/42] only reformatting and renaming variables for better readability --- geoip.js | 102 +++++++++++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/geoip.js b/geoip.js index 05b478c..9330ff3 100644 --- a/geoip.js +++ b/geoip.js @@ -25,57 +25,57 @@ function find(ip) { var n = midpoints[0]; var step; var parts = ip.split("."); - var ipl = parseInt(parts[3], 10) + - (parseInt(parts[2], 10) * 256) + - (parseInt(parts[1], 10) * 65536) + - (parseInt(parts[0], 10) * 16777216); - - var current; - var next; - var prev; - var nn; - var pn; - - while(true) { - - mpi++; - step = midpoints[mpi]; - current = countries[n]; - nn = n + 1; - pn = n - 1; - - next = nn < numcountries ? countries[nn] : null; - prev = pn > -1 ? countries[pn] : null; - - // take another step? - if(step > 0) { - - if(!next || next.ipstart < ipl) { - n += step; - } else { - n -= step; - } - - continue; - } - - // we're either current, next or previous depending on which is closest to ipl - var cd = Math.abs(ipl - current.ipstart); - var nd = next && next.ipstart < ipl ? ipl - next.ipstart : 1000000000; - var pd = prev && prev.ipstart < ipl ? ipl - prev.ipstart : 1000000000; - - // current wins - if(cd < nd && cd < pd) { - return current; - } - - // next wins - if(nd < cd && nd < pd) { - return next; - } - - // prev wins - return prev; + var target_ip = parseInt(parts[3], 10) + + (parseInt(parts[2], 10) * 256) + + (parseInt(parts[1], 10) * 65536) + + (parseInt(parts[0], 10) * 16777216); + + var current; + var next; + var prev; + var nn; + var pn; + + while(true) { + + mpi++; + step = midpoints[mpi]; + current = countries[n]; + nn = n + 1; + pn = n - 1; + + next = nn < numcountries ? countries[nn] : null; + prev = pn > -1 ? countries[pn] : null; + + // take another step? + if(step > 0) { + + if(!next || next.ipstart < target_ip) { + n += step; + } else { + n -= step; + } + + continue; + } + + // we're either current, next or previous depending on which is closest to target_ip + var curr_ip_diff = Math.abs(target_ip - current.ipstart); + var next_ip_diff = next && next.ipstart < target_ip ? target_ip - next.ipstart : 1000000000; + var prev_ip_diff = prev && prev.ipstart < target_ip ? target_ip - prev.ipstart : 1000000000; + + // current wins + if(curr_ip_diff < next_ip_diff && curr_ip_diff < prev_ip_diff) { + return current; + } + + // next wins + if(next_ip_diff < curr_ip_diff && next_ip_diff < prev_ip_diff) { + return next; + } + + // prev wins + return prev; } } From 32ad6cd8c12d61ea1bdd06d8583f1a24e2846d46 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Thu, 1 Nov 2012 00:34:36 +0100 Subject: [PATCH 13/42] FIX: lower IP ranges, leftmost and rightmost borders are correctly detected, as unit tests are green now --- geoip.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geoip.js b/geoip.js index 9330ff3..09cfd74 100644 --- a/geoip.js +++ b/geoip.js @@ -62,7 +62,7 @@ function find(ip) { // we're either current, next or previous depending on which is closest to target_ip var curr_ip_diff = Math.abs(target_ip - current.ipstart); var next_ip_diff = next && next.ipstart < target_ip ? target_ip - next.ipstart : 1000000000; - var prev_ip_diff = prev && prev.ipstart < target_ip ? target_ip - prev.ipstart : 1000000000; + var prev_ip_diff = prev && prev.ipstart <= target_ip ? target_ip - prev.ipstart : 1000000000; // current wins if(curr_ip_diff < next_ip_diff && curr_ip_diff < prev_ip_diff) { From 24a1d33a80d1f6f0252c79413a550f9ed9c52de7 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 4 Nov 2012 19:17:06 +0100 Subject: [PATCH 14/42] reformatted code, only --- geoip.js | 142 +++++++++++++++++++++++++++---------------------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/geoip.js b/geoip.js index 09cfd74..d9ff313 100644 --- a/geoip.js +++ b/geoip.js @@ -1,16 +1,16 @@ -var countries = [], - midpoints = [], - numcountries = 0, - ready = false; +var countries = []; +var midpoints = []; +var numcountries = 0; +var ready = false; module.exports = geoip = { - lookup: function(ip) { - if(!ready) { - console.log("geoip warming up"); - return {code: "N/A", name: "UNKNOWN"}; - } - - return find(ip); + lookup:function (ip) { + if (!ready) { + console.log("geoip warming up"); + return {code:"N/A", name:"UNKNOWN"}; + } + + return find(ip); } }; @@ -26,17 +26,17 @@ function find(ip) { var step; var parts = ip.split("."); var target_ip = parseInt(parts[3], 10) + - (parseInt(parts[2], 10) * 256) + - (parseInt(parts[1], 10) * 65536) + - (parseInt(parts[0], 10) * 16777216); + (parseInt(parts[2], 10) * 256) + + (parseInt(parts[1], 10) * 65536) + + (parseInt(parts[0], 10) * 16777216); - var current; - var next; - var prev; - var nn; - var pn; + var current; + var next; + var prev; + var nn; + var pn; - while(true) { + while (true) { mpi++; step = midpoints[mpi]; @@ -48,15 +48,15 @@ function find(ip) { prev = pn > -1 ? countries[pn] : null; // take another step? - if(step > 0) { + if (step > 0) { - if(!next || next.ipstart < target_ip) { - n += step; - } else { - n -= step; - } + if (!next || next.ipstart < target_ip) { + n += step; + } else { + n -= step; + } - continue; + continue; } // we're either current, next or previous depending on which is closest to target_ip @@ -65,59 +65,59 @@ function find(ip) { var prev_ip_diff = prev && prev.ipstart <= target_ip ? target_ip - prev.ipstart : 1000000000; // current wins - if(curr_ip_diff < next_ip_diff && curr_ip_diff < prev_ip_diff) { - return current; + if (curr_ip_diff < next_ip_diff && curr_ip_diff < prev_ip_diff) { + return current; } // next wins - if(next_ip_diff < curr_ip_diff && next_ip_diff < prev_ip_diff) { - return next; + if (next_ip_diff < curr_ip_diff && next_ip_diff < prev_ip_diff) { + return next; } // prev wins return prev; - } + } } /** -* Prepare the data. This uses the standard free GeoIP CSV database -* from MaxMind, you should be able to update it at any time by just -* overwriting GeoIPCountryWhois.csv with a new version. -*/ -(function() { - - var fs = require("fs"); - var sys = require("sys"); - var stream = fs.createReadStream(__dirname + "/GeoIPCountryWhois.csv"); - var buffer = ""; - - stream.addListener("data", function(data) { - buffer += data.toString().replace(/"/g, ""); - }); - - stream.addListener("end", function() { - - var entries = buffer.split("\n"); - - for(var i=0; i 5) { - countries.push({ipstart: parseInt(entry[2]), code: entry[4], name: entry[5].trim()}); - } - } - - countries.sort(function(a, b) { - return a.ipstart - b.ipstart; - }); - - numcountries = countries.length; - var n = numcountries; - while(n >= 1) { - n = n >> 1; - midpoints.push(n); - } - - ready = true; - }); + * Prepare the data. This uses the standard free GeoIP CSV database + * from MaxMind, you should be able to update it at any time by just + * overwriting GeoIPCountryWhois.csv with a new version. + */ +(function () { + + var fs = require("fs"); + var sys = require("sys"); + var stream = fs.createReadStream(__dirname + "/GeoIPCountryWhois.csv"); + var buffer = ""; + + stream.addListener("data", function (data) { + buffer += data.toString().replace(/"/g, ""); + }); + + stream.addListener("end", function () { + + var entries = buffer.split("\n"); + + for (var i = 0; i < entries.length; i++) { + var entry = entries[i].split(","); + if (entry.length > 5) { + countries.push({ipstart:parseInt(entry[2]), code:entry[4], name:entry[5].trim()}); + } + } + + countries.sort(function (a, b) { + return a.ipstart - b.ipstart; + }); + + numcountries = countries.length; + var n = numcountries; + while (n >= 1) { + n = n >> 1; + midpoints.push(n); + } + + ready = true; + }); }()); \ No newline at end of file From a843b2c73e35d1f3bb3349309be138b73df4e41e Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 4 Nov 2012 19:21:52 +0100 Subject: [PATCH 15/42] renamed variable --- geoip.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/geoip.js b/geoip.js index d9ff313..8bdeeda 100644 --- a/geoip.js +++ b/geoip.js @@ -1,6 +1,6 @@ var countries = []; var midpoints = []; -var numcountries = 0; +var countriesLength = 0; var ready = false; module.exports = geoip = { @@ -44,7 +44,7 @@ function find(ip) { nn = n + 1; pn = n - 1; - next = nn < numcountries ? countries[nn] : null; + next = nn < countriesLength ? countries[nn] : null; prev = pn > -1 ? countries[pn] : null; // take another step? @@ -110,8 +110,8 @@ function find(ip) { return a.ipstart - b.ipstart; }); - numcountries = countries.length; - var n = numcountries; + countriesLength = countries.length; + var n = countriesLength; while (n >= 1) { n = n >> 1; midpoints.push(n); From ea898f524fc2d2d1775afb391a338abe924eba30 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 4 Nov 2012 20:52:34 +0100 Subject: [PATCH 16/42] generating country maps and lists and saving them to .JS source files using country-names list with index, which results in half the memory usage --- .gitignore | 1 + generate_sources.js | 85 +++++++++++++++++++++++++++++++++++++++++++++ geoip.js | 67 ++++++++--------------------------- 3 files changed, 101 insertions(+), 52 deletions(-) create mode 100644 generate_sources.js diff --git a/.gitignore b/.gitignore index 448e893..2f70496 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ node_modules .DS_Store .idea *.iml +generated-*.js diff --git a/generate_sources.js b/generate_sources.js new file mode 100644 index 0000000..3e605cd --- /dev/null +++ b/generate_sources.js @@ -0,0 +1,85 @@ +var countries = []; +var countryNamesAndCodes = []; + +function read_csv_file_and_prepare_data() { + + var fs = require("fs"); + var data = fs.readFileSync(__dirname + "/GeoIPCountryWhois.csv") + var buffer = ""; + buffer += data.toString().replace(/"/g, ""); + + var entries = buffer.split("\n"); + var offsetCounter = 0; + var countrySet = {}; + + for (var i = 0; i < entries.length; i++) { + var entry = entries[i].split(","); + if (entry.length > 5) { + var countryName = entry[5].trim(); + var countryCode = entry[4]; + var countryIndex = 0; + if (!countrySet[countryName]) { + countryIndex = offsetCounter++; + countrySet[countryName] = {index:countryIndex}; + countryNamesAndCodes.push(countryName); + countryNamesAndCodes.push(countryCode); + } else { + countryIndex = countrySet[countryName].index; + } + countries.push({ipstart:parseInt(entry[2]), code:countryCode, name:countryName, index:(countryIndex * 2)}); + } + } + + countries.sort(function (a, b) { + return a.ipstart - b.ipstart; + }); + +} + +function write_sourceFile_countryNamesAndCodes() { + var data = ""; + data += "// AUTOGENERATED CODE - DO NOT MODIFY! " + new Date() + " \n"; + data += "var countryNamesAndCodes = [];\n"; + data += "module.exports = {\n"; + data += " countryNamesAndCodes:countryNamesAndCodes\n"; + data += "};\n"; + data += "(function () {\n"; + for (var i = 0, len = countryNamesAndCodes.length; i < len; i++) { + var item = countryNamesAndCodes[i]; + item = item.replace("'", "\\'"); + data += "countryNamesAndCodes.push('" + item + "');\n"; + } + data += "})();\n"; + + var fs = require("fs"); + fs.writeFileSync('generated-namesandcodes.js', data, 'utf8'); +} + +function write_sourceFile_countries() { + var data = ""; + data += "// AUTOGENERATED CODE - DO NOT MODIFY! " + new Date() + " \n"; + data += "var countries = [];\n"; + data += "module.exports = {\n"; + data += " countries:countries\n"; + data += "};\n"; + data += "(function () {\n"; + for (var i = 0, len = countries.length; i < len; i++) { + var country = countries[i]; + var item = "{ipstart:" + country.ipstart + ",idx:" + country.index + "}"; + data += "countries.push(" + item + ");//" + countryNamesAndCodes[country.index + 1] + "\n"; + } + data += "})();\n"; + + var fs = require("fs"); + fs.writeFileSync('generated-countries.js', data, 'utf8'); +} + +/** + * Prepare the data. This uses the standard free GeoIP CSV database + * from MaxMind, you should be able to update it at any time by just + * overwriting GeoIPCountryWhois.csv with a new version. + */ + +read_csv_file_and_prepare_data() +write_sourceFile_countryNamesAndCodes(); +write_sourceFile_countries(); diff --git a/geoip.js b/geoip.js index 8bdeeda..60dc190 100644 --- a/geoip.js +++ b/geoip.js @@ -1,21 +1,16 @@ -var countries = []; var midpoints = []; -var countriesLength = 0; -var ready = false; +var countryNamesAndCodes = require('./generated-namesandcodes.js').countryNamesAndCodes; +var countries = require('./generated-countries.js').countries; +var countriesLength = countries.length; module.exports = geoip = { lookup:function (ip) { - if (!ready) { - console.log("geoip warming up"); - return {code:"N/A", name:"UNKNOWN"}; - } - return find(ip); } }; /** - * A qcuick little binary search + * A quick little binary search * @param ip the ip we're looking for * @return {*} */ @@ -66,58 +61,26 @@ function find(ip) { // current wins if (curr_ip_diff < next_ip_diff && curr_ip_diff < prev_ip_diff) { - return current; + return {ipstart:current.ipstart, name:countryNamesAndCodes[current.idx], code:countryNamesAndCodes[current.idx + 1]}; } // next wins if (next_ip_diff < curr_ip_diff && next_ip_diff < prev_ip_diff) { - return next; + return {ipstart:next.ipstart, name:countryNamesAndCodes[next.idx], code:countryNamesAndCodes[next.idx + 1]}; } // prev wins - return prev; + return {ipstart:prev.ipstart, name:countryNamesAndCodes[prev.idx], code:countryNamesAndCodes[prev.idx + 1]}; } } -/** - * Prepare the data. This uses the standard free GeoIP CSV database - * from MaxMind, you should be able to update it at any time by just - * overwriting GeoIPCountryWhois.csv with a new version. +/* + prepare midpoints .... */ (function () { - - var fs = require("fs"); - var sys = require("sys"); - var stream = fs.createReadStream(__dirname + "/GeoIPCountryWhois.csv"); - var buffer = ""; - - stream.addListener("data", function (data) { - buffer += data.toString().replace(/"/g, ""); - }); - - stream.addListener("end", function () { - - var entries = buffer.split("\n"); - - for (var i = 0; i < entries.length; i++) { - var entry = entries[i].split(","); - if (entry.length > 5) { - countries.push({ipstart:parseInt(entry[2]), code:entry[4], name:entry[5].trim()}); - } - } - - countries.sort(function (a, b) { - return a.ipstart - b.ipstart; - }); - - countriesLength = countries.length; - var n = countriesLength; - while (n >= 1) { - n = n >> 1; - midpoints.push(n); - } - - ready = true; - }); - -}()); \ No newline at end of file + var n = countriesLength; + while (n >= 1) { + n = n >> 1; + midpoints.push(n); + } +})(); \ No newline at end of file From e918eb97e27b94cafc4d05cb95df270f963fb383 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 4 Nov 2012 21:06:05 +0100 Subject: [PATCH 17/42] reworked benchmark.js, so it's no more recursive and easier to understand --- benchmark.js | 89 ++++++++++++++++++++++++++-------------------------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/benchmark.js b/benchmark.js index 53887e2..5e70159 100644 --- a/benchmark.js +++ b/benchmark.js @@ -1,53 +1,54 @@ -var geoip = require("./geoip.js"); -//var geoip = require("geoip-lite"); +var geoip = require("./geoip.js"); // needed because of async data loading -var test1 = true; - -function test() { - - var total = 0; - var numtests = 20; - var numiterations = 1000000; - - console.log("starting test: " + (test1 ? "geoip-native" : "geoip-lite")); - - for(var t=0; t 4 && t < 15) { - total += (finish - start); - console.log("time " + (finish - start)); - } - } - - console.log("average: " + (total / 10)); +/* + @param geoipLibrary: the library from which to use the lookup method + */ +function benchmark_IP_lookup(geoipLibrary) { + + var total = 0; + var numtests = 20; + var numiterations = 1000000; + console.log("----------------------------"); + for (var t = 0; t < numtests; t++) { + + var start = new Date().getTime(); + + for (var i = 0; i < numiterations; i++) { + + var o1 = 1 + Math.round(Math.random() * 254); + var o2 = 1 + Math.round(Math.random() * 254); + var o3 = 1 + Math.round(Math.random() * 254); + var o4 = 1 + Math.round(Math.random() * 254); + var ip = o1 + "." + o2 + "." + o3 + "." + o4; + geoipLibrary.lookup(ip); + } + + var finish = new Date().getTime(); + + if (t > 4 && t < 15) { + total += (finish - start); + console.log("time " + (finish - start)); + } + } + console.log("average: " + (total / 10)); + console.log("----------------------------"); +} - if(!test1) { - return; - } +function run_all_benchmarks() { + console.log("starting test: geoip-native"); + geoip = require("./geoip.js"); + benchmark_IP_lookup(geoip); - geoip = require("geoip-lite"); - test1 = false; - test(); + console.log("starting test: geoip-lite"); + geoip = require("geoip-lite"); + benchmark_IP_lookup(geoip); } -setTimeout(test, 3000); +// waiting 3 seconds to finish async loading of data +setTimeout(run_all_benchmarks, 3000); /* -benchmark results: + benchmark results: geoip-native time 1500 @@ -73,4 +74,4 @@ benchmark results: time 8303 time 8416 time 8261 - average: 8375.3*/ \ No newline at end of file + average: 8375.3*/ From 6085abd6e65399aa8c975bfecb53ea2dabb983e8 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 4 Nov 2012 21:24:48 +0100 Subject: [PATCH 18/42] reworked benchmark.js, by using generated sources there's no need to wait/warmup --- benchmark.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/benchmark.js b/benchmark.js index 5e70159..d73adef 100644 --- a/benchmark.js +++ b/benchmark.js @@ -1,5 +1,3 @@ -var geoip = require("./geoip.js"); // needed because of async data loading - /* @param geoipLibrary: the library from which to use the lookup method */ @@ -35,17 +33,24 @@ function benchmark_IP_lookup(geoipLibrary) { } function run_all_benchmarks() { + console.log("starting test: geoip-native"); - geoip = require("./geoip.js"); + var start = new Date().getTime(); + var geoip = require("./geoip.js"); + var finish = new Date().getTime(); + console.log("loading geoip-native took " + (finish - start) + "ms"); benchmark_IP_lookup(geoip); + console.log("starting test: geoip-lite"); + start = new Date().getTime(); geoip = require("geoip-lite"); + finish = new Date().getTime(); + console.log("loading geoip-lite took " + (finish - start) + "ms"); benchmark_IP_lookup(geoip); } -// waiting 3 seconds to finish async loading of data -setTimeout(run_all_benchmarks, 3000); +run_all_benchmarks(); /* benchmark results: From 3a58a3b0b60eead3818c607539cfc01900d890c3 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 4 Nov 2012 21:25:19 +0100 Subject: [PATCH 19/42] direct array generation speeds up loading of the source file --- generate_sources.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/generate_sources.js b/generate_sources.js index 3e605cd..82c0786 100644 --- a/generate_sources.js +++ b/generate_sources.js @@ -58,17 +58,15 @@ function write_sourceFile_countryNamesAndCodes() { function write_sourceFile_countries() { var data = ""; data += "// AUTOGENERATED CODE - DO NOT MODIFY! " + new Date() + " \n"; - data += "var countries = [];\n"; - data += "module.exports = {\n"; - data += " countries:countries\n"; - data += "};\n"; - data += "(function () {\n"; + data += "var countries = [\n"; for (var i = 0, len = countries.length; i < len; i++) { var country = countries[i]; - var item = "{ipstart:" + country.ipstart + ",idx:" + country.index + "}"; - data += "countries.push(" + item + ");//" + countryNamesAndCodes[country.index + 1] + "\n"; + data += "{ipstart:" + country.ipstart + ",idx:" + country.index + "},//" + countryNamesAndCodes[country.index + 1] + "\n"; } - data += "})();\n"; + data += "];\n"; + data += "module.exports = {\n"; + data += " countries:countries\n"; + data += "};\n"; var fs = require("fs"); fs.writeFileSync('generated-countries.js', data, 'utf8'); From 9a4384d6390838c5f8fb774626adb6556fb93d05 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 4 Nov 2012 21:35:41 +0100 Subject: [PATCH 20/42] direct array generation speeds up loading of the source file --- generate_sources.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/generate_sources.js b/generate_sources.js index 82c0786..3ee1b50 100644 --- a/generate_sources.js +++ b/generate_sources.js @@ -39,17 +39,17 @@ function read_csv_file_and_prepare_data() { function write_sourceFile_countryNamesAndCodes() { var data = ""; data += "// AUTOGENERATED CODE - DO NOT MODIFY! " + new Date() + " \n"; - data += "var countryNamesAndCodes = [];\n"; + data += "var countryNamesAndCodes = [\n"; + for (var i = 0, len = countryNamesAndCodes.length >> 1; i < len; i++) { + var countryName = countryNamesAndCodes[i<<1].replace("'", "\\'"); + var countryCode = countryNamesAndCodes[(i<<1)+1].replace("'", "\\'"); + data += "'" + countryName + "','"+countryCode+"',\n"; + data += "'" + countryName + "','"+countryCode+"',\n"; + } + data += "];\n"; data += "module.exports = {\n"; data += " countryNamesAndCodes:countryNamesAndCodes\n"; data += "};\n"; - data += "(function () {\n"; - for (var i = 0, len = countryNamesAndCodes.length; i < len; i++) { - var item = countryNamesAndCodes[i]; - item = item.replace("'", "\\'"); - data += "countryNamesAndCodes.push('" + item + "');\n"; - } - data += "})();\n"; var fs = require("fs"); fs.writeFileSync('generated-namesandcodes.js', data, 'utf8'); From 40099f508cd7e98cc60585f9028b3973d69e41c5 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 4 Nov 2012 21:54:58 +0100 Subject: [PATCH 21/42] added generate_sources.js as script to build phase within package.json --- package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package.json b/package.json index 6c30e8a..5edd5db 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,9 @@ "node": ">=0.6.0" }, "main" : "./index.js", + "scripts" : { + "build" : "./generate_sources.js" + }, "licenses": [ { "type": "MIT", From ee1918b12fc5f0a1ac1dee7d750427224f98484c Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 4 Nov 2012 21:59:37 +0100 Subject: [PATCH 22/42] removed warm up test, not needed anymore --- test/unit/lookupTest.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/unit/lookupTest.js b/test/unit/lookupTest.js index 3a38f3e..50eabe2 100644 --- a/test/unit/lookupTest.js +++ b/test/unit/lookupTest.js @@ -12,14 +12,6 @@ module.exports = { callback(); }, - test_do_warm_up:function (test) { - function waitalittle() { - test.done(); - } - - setTimeout(waitalittle, 3000); - }, - test_check_if_warmed_up:function (test) { test.notEqual(this.geoip.lookup("127.0.0.1").code, "N/A", "No yet warmed up, please use more time for warmup"); test.done(); From 4ddf3929aef8e3f9e755a5f88c79aba653cab804 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 4 Nov 2012 22:00:11 +0100 Subject: [PATCH 23/42] fix: don't write the line twice --- generate_sources.js | 1 - 1 file changed, 1 deletion(-) diff --git a/generate_sources.js b/generate_sources.js index 3ee1b50..1defcc8 100644 --- a/generate_sources.js +++ b/generate_sources.js @@ -44,7 +44,6 @@ function write_sourceFile_countryNamesAndCodes() { var countryName = countryNamesAndCodes[i<<1].replace("'", "\\'"); var countryCode = countryNamesAndCodes[(i<<1)+1].replace("'", "\\'"); data += "'" + countryName + "','"+countryCode+"',\n"; - data += "'" + countryName + "','"+countryCode+"',\n"; } data += "];\n"; data += "module.exports = {\n"; From 68c9e1f16c49b9bffccad08ea491761fa3f02da4 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Mon, 5 Nov 2012 00:25:27 +0100 Subject: [PATCH 24/42] removed warm up test, not needed anymore --- test/unit/lookupTest.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/unit/lookupTest.js b/test/unit/lookupTest.js index 50eabe2..07a4df1 100644 --- a/test/unit/lookupTest.js +++ b/test/unit/lookupTest.js @@ -12,11 +12,6 @@ module.exports = { callback(); }, - test_check_if_warmed_up:function (test) { - test.notEqual(this.geoip.lookup("127.0.0.1").code, "N/A", "No yet warmed up, please use more time for warmup"); - test.done(); - }, - test_ip_in_the_lower_range_and_leftmost_border:function (test) { var ip; var actual; From bd057314431fdc83f3225f6ea0612a6eb46ae28d Mon Sep 17 00:00:00 2001 From: nitram509 Date: Mon, 5 Nov 2012 00:34:08 +0100 Subject: [PATCH 25/42] reworked binary search (see wikipedia) and FIXed last 'red' test --- geoip.js | 77 ++++++++++++++++++-------------------------------------- 1 file changed, 24 insertions(+), 53 deletions(-) diff --git a/geoip.js b/geoip.js index 09cfd74..ed9ebb4 100644 --- a/geoip.js +++ b/geoip.js @@ -21,62 +21,33 @@ module.exports = geoip = { */ function find(ip) { - var mpi = 0; - var n = midpoints[0]; - var step; var parts = ip.split("."); var target_ip = parseInt(parts[3], 10) + - (parseInt(parts[2], 10) * 256) + - (parseInt(parts[1], 10) * 65536) + - (parseInt(parts[0], 10) * 16777216); - - var current; - var next; - var prev; - var nn; - var pn; - - while(true) { - - mpi++; - step = midpoints[mpi]; - current = countries[n]; - nn = n + 1; - pn = n - 1; - - next = nn < numcountries ? countries[nn] : null; - prev = pn > -1 ? countries[pn] : null; - - // take another step? - if(step > 0) { - - if(!next || next.ipstart < target_ip) { - n += step; - } else { - n -= step; - } - - continue; + (parseInt(parts[2], 10) * 256) + + (parseInt(parts[1], 10) * 65536) + + (parseInt(parts[0], 10) * 16777216); + + var idxMin = 0; + var idxMiddle = 0; + var idxMax = numcountries - 1; + var pickedCountry = undefined; + while (idxMin < idxMax) { + idxMiddle = (idxMax + idxMin) >> 1; + pickedCountry = countries[idxMiddle]; + // determine which subarray to search + if (pickedCountry.ipstart < target_ip) { + // change min index to search upper subarray + idxMin = idxMiddle + 1; + } else if (pickedCountry.ipstart > target_ip) { + // change max index to search lower subarray + idxMax = idxMiddle - 1; + } else { + // key found at index imid + return pickedCountry; } - - // we're either current, next or previous depending on which is closest to target_ip - var curr_ip_diff = Math.abs(target_ip - current.ipstart); - var next_ip_diff = next && next.ipstart < target_ip ? target_ip - next.ipstart : 1000000000; - var prev_ip_diff = prev && prev.ipstart <= target_ip ? target_ip - prev.ipstart : 1000000000; - - // current wins - if(curr_ip_diff < next_ip_diff && curr_ip_diff < prev_ip_diff) { - return current; - } - - // next wins - if(next_ip_diff < curr_ip_diff && next_ip_diff < prev_ip_diff) { - return next; - } - - // prev wins - return prev; - } + } + // return previous found country. + return pickedCountry; } /** From a356fa8d17823a054070a1b6d27e592a917efaa5 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Mon, 5 Nov 2012 00:40:05 +0100 Subject: [PATCH 26/42] simplyfied return statement --- geoip.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geoip.js b/geoip.js index b9a646a..e07811f 100644 --- a/geoip.js +++ b/geoip.js @@ -37,7 +37,7 @@ function find(ip) { idxMax = idxMiddle - 1; } else { // key found at index imid - return {ipstart:pickedCountry.ipstart, name:countryNamesAndCodes[pickedCountry.idx], code:countryNamesAndCodes[pickedCountry.idx + 1]}; + break; } } // return previous found country. From 1e7969f6b71beb1ee1e3b0ffedf5dbabe66a76e6 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 24 Mar 2013 21:10:45 +0100 Subject: [PATCH 27/42] updated package.json - fixed preinstall hook --- package.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 5edd5db..42fb422 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,12 @@ "author": { "name": "Playtomic Inc" }, + "contributors": [ + { + "name" : "Martin W. Kirst", + "url": "https://github.com/nitram509" + } + ], "repository": { "type": "git", "url": "http://github.com/benlowry/node-geoip-native.git" @@ -20,7 +26,7 @@ }, "main" : "./index.js", "scripts" : { - "build" : "./generate_sources.js" + "preinstall" : "node generate_sources.js" }, "licenses": [ { From 21fee47b51b561c70ae6dcc69cfe7135769dc094 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 24 Mar 2013 21:20:01 +0100 Subject: [PATCH 28/42] shortened the generated code --- generate_sources.js | 2 +- geoip.js | 55 ++++++++++++++++++++++++--------------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/generate_sources.js b/generate_sources.js index 1defcc8..6045471 100644 --- a/generate_sources.js +++ b/generate_sources.js @@ -60,7 +60,7 @@ function write_sourceFile_countries() { data += "var countries = [\n"; for (var i = 0, len = countries.length; i < len; i++) { var country = countries[i]; - data += "{ipstart:" + country.ipstart + ",idx:" + country.index + "},//" + countryNamesAndCodes[country.index + 1] + "\n"; + data += "{ip:" + country.ipstart + ",idx:" + country.index + "},//" + countryNamesAndCodes[country.index + 1] + "\n"; } data += "];\n"; data += "module.exports = {\n"; diff --git a/geoip.js b/geoip.js index e07811f..c639473 100644 --- a/geoip.js +++ b/geoip.js @@ -3,9 +3,9 @@ var countries = require('./generated-countries.js').countries; var countriesLength = countries.length; module.exports = geoip = { - lookup:function (ip) { - return find(ip); - } + lookup: function (ip) { + return find(ip); + } }; /** @@ -15,31 +15,34 @@ module.exports = geoip = { */ function find(ip) { - var parts = ip.split("."); - var target_ip = parseInt(parts[3], 10) + + var parts = ip.split("."); + var target_ip = parseInt(parts[3], 10) + (parseInt(parts[2], 10) * 256) + (parseInt(parts[1], 10) * 65536) + (parseInt(parts[0], 10) * 16777216); - var idxMin = 0; - var idxMiddle = 0; - var idxMax = countriesLength - 1; - var pickedCountry = undefined; - while (idxMin < idxMax) { - idxMiddle = (idxMax + idxMin) >> 1; - pickedCountry = countries[idxMiddle]; - // determine which subarray to search - if (pickedCountry.ipstart < target_ip) { - // change min index to search upper subarray - idxMin = idxMiddle + 1; - } else if (pickedCountry.ipstart > target_ip) { - // change max index to search lower subarray - idxMax = idxMiddle - 1; - } else { - // key found at index imid - break; - } - } - // return previous found country. - return {ipstart:pickedCountry.ipstart, name:countryNamesAndCodes[pickedCountry.idx], code:countryNamesAndCodes[pickedCountry.idx + 1]}; + var idxMin = 0; + var idxMiddle = 0; + var idxMax = countriesLength - 1; + var pickedCountry = undefined; + while (idxMin < idxMax) { + idxMiddle = (idxMax + idxMin) >> 1; + pickedCountry = countries[idxMiddle]; + // determine which subarray to search + if (pickedCountry.ip < target_ip) { + // change min index to search upper subarray + idxMin = idxMiddle + 1; + } else if (pickedCountry.ip > target_ip) { + // change max index to search lower subarray + idxMax = idxMiddle - 1; + } else { + // key found at index imid + break; + } + } + // return previous found country. + return { + ipstart: pickedCountry.ip, + name: countryNamesAndCodes[pickedCountry.idx], + code: countryNamesAndCodes[pickedCountry.idx + 1]}; } From 90212261c73fb3b04cfb4e21462ff90678e12f8b Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 24 Mar 2013 21:28:17 +0100 Subject: [PATCH 29/42] fixed example in readme --- README.markdown | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.markdown b/README.markdown index c67ca8a..fa30b3f 100644 --- a/README.markdown +++ b/README.markdown @@ -20,7 +20,8 @@ Benchmarks on my 2011 Macbook Air whilst running lots of software. The test too ## How to use 1. git clone https://github.com/benlowry/node-geoip-native 2. cd node-geoip-native -3. node benchmark.js +3. npm install +4. node benchmark.js or just ```npm install geoip-native``` @@ -41,10 +42,10 @@ Second, run the unit test: ## Examples - var geoip = require("geoip-native"); - var ip = "123.123.123.123"; - geoip.lookup(ip); - console.log("country: " + ip.name + " / " + ip.code); + var geoip = require("./geoip.js"); + var ip = geoip.lookup("134.12.12.123"); + console.log("numeric ip value: " + ip.ipstart); + console.log("country: " + ip.name + " / " + ip.code); // in practice you'd want: // ip = request.headers["x-forwarded-for"] || request.connection.remoteAddress, From d5fc3dfd6fd49ace30836d3248b80dfccf290cbe Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 24 Mar 2013 21:29:11 +0100 Subject: [PATCH 30/42] fixed example in readme --- README.markdown | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/README.markdown b/README.markdown index fa30b3f..22471a7 100644 --- a/README.markdown +++ b/README.markdown @@ -42,13 +42,10 @@ Second, run the unit test: ## Examples - var geoip = require("./geoip.js"); - var ip = geoip.lookup("134.12.12.123"); - console.log("numeric ip value: " + ip.ipstart); - console.log("country: " + ip.name + " / " + ip.code); - - // in practice you'd want: - // ip = request.headers["x-forwarded-for"] || request.connection.remoteAddress, + var geoip = require("./geoip.js"); + var ip = geoip.lookup("134.12.12.123"); + console.log("numeric ip value: " + ip.ipstart); + console.log("country: " + ip.name + " / " + ip.code); ### What's missing Be neat to expand this to include cities. From dbedc8f3e26cdd031a49e617ce5e7743a22ad54d Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 24 Mar 2013 21:59:57 +0100 Subject: [PATCH 31/42] cleanups: tools folder for benchmark.js, consistence naming --- README.markdown | 13 +++++++++---- geoip.js => geoip-native.js | 7 ++++--- index.js | 1 - package.json | 2 +- benchmark.js => tools/benchmark.js | 0 5 files changed, 14 insertions(+), 9 deletions(-) rename geoip.js => geoip-native.js (92%) delete mode 100644 index.js rename benchmark.js => tools/benchmark.js (100%) diff --git a/README.markdown b/README.markdown index 22471a7..35c12a9 100644 --- a/README.markdown +++ b/README.markdown @@ -17,14 +17,19 @@ Benchmarks on my 2011 Macbook Air whilst running lots of software. The test too 1. Comes with the [standard CSV database by MaxMind](http://www.maxmind.com/app/geolite) which may require updating. -## How to use + +## Install + +Simple run + + npm install geoip-native + +## Install from source 1. git clone https://github.com/benlowry/node-geoip-native 2. cd node-geoip-native 3. npm install 4. node benchmark.js -or just ```npm install geoip-native``` - ## Methods Node GeoIP Native provides methods for: @@ -42,7 +47,7 @@ Second, run the unit test: ## Examples - var geoip = require("./geoip.js"); + var geoip = require("geoip-native"); var ip = geoip.lookup("134.12.12.123"); console.log("numeric ip value: " + ip.ipstart); console.log("country: " + ip.name + " / " + ip.code); diff --git a/geoip.js b/geoip-native.js similarity index 92% rename from geoip.js rename to geoip-native.js index c639473..3d47f6b 100644 --- a/geoip.js +++ b/geoip-native.js @@ -4,7 +4,7 @@ var countriesLength = countries.length; module.exports = geoip = { lookup: function (ip) { - return find(ip); + return _lookup(ip); } }; @@ -13,7 +13,7 @@ module.exports = geoip = { * @return {*} * @see http://en.wikipedia.org/wiki/Binary_search_algorithm (Iterative approach) */ -function find(ip) { +function _lookup(ip) { var parts = ip.split("."); var target_ip = parseInt(parts[3], 10) + @@ -44,5 +44,6 @@ function find(ip) { return { ipstart: pickedCountry.ip, name: countryNamesAndCodes[pickedCountry.idx], - code: countryNamesAndCodes[pickedCountry.idx + 1]}; + code: countryNamesAndCodes[pickedCountry.idx + 1] + }; } diff --git a/index.js b/index.js deleted file mode 100644 index 8fba889..0000000 --- a/index.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./geoip.js"); \ No newline at end of file diff --git a/package.json b/package.json index 42fb422..a62c2a3 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "engines": { "node": ">=0.6.0" }, - "main" : "./index.js", + "main" : "./geoip-native.js", "scripts" : { "preinstall" : "node generate_sources.js" }, diff --git a/benchmark.js b/tools/benchmark.js similarity index 100% rename from benchmark.js rename to tools/benchmark.js From 0ef86039c76811a999907638d70f0596449d1f1b Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sun, 24 Mar 2013 22:02:30 +0100 Subject: [PATCH 32/42] fixed require statement in benchmark.js --- tools/benchmark.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/benchmark.js b/tools/benchmark.js index d73adef..6649ace 100644 --- a/tools/benchmark.js +++ b/tools/benchmark.js @@ -36,7 +36,7 @@ function run_all_benchmarks() { console.log("starting test: geoip-native"); var start = new Date().getTime(); - var geoip = require("./geoip.js"); + var geoip = require("../geoip-native.js"); var finish = new Date().getTime(); console.log("loading geoip-native took " + (finish - start) + "ms"); benchmark_IP_lookup(geoip); From 7a4f992ed89db6418bcf458b362929b26b8e5824 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Thu, 18 Apr 2013 22:16:18 +0200 Subject: [PATCH 33/42] fixed geoip require statement --- test/unit/lookupTest.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/lookupTest.js b/test/unit/lookupTest.js index 07a4df1..e267e0b 100644 --- a/test/unit/lookupTest.js +++ b/test/unit/lookupTest.js @@ -2,7 +2,7 @@ module.exports = { setUp:function (callback) { - this.geoip = require("../../geoip.js"); + this.geoip = require("../../geoip-native.js"); // wait until ready callback(); }, From 4ee1fe8f48b6d8c39f4d9fe125523e382173e890 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Thu, 18 Apr 2013 23:32:15 +0200 Subject: [PATCH 34/42] fixed binary search, by using Deferred detection of equality approach and combining this with more range/equation checks --- geoip-native.js | 61 +++++++++++++++++++++++++++++------------ test/unit/lookupTest.js | 11 ++++++++ 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/geoip-native.js b/geoip-native.js index 3d47f6b..e93e392 100644 --- a/geoip-native.js +++ b/geoip-native.js @@ -11,39 +11,64 @@ module.exports = geoip = { /** * @param ip the ip we're looking for * @return {*} - * @see http://en.wikipedia.org/wiki/Binary_search_algorithm (Iterative approach) + * @see http://en.wikipedia.org/wiki/Binary_search_algorithm (Deferred detection of equality approach) */ function _lookup(ip) { var parts = ip.split("."); var target_ip = parseInt(parts[3], 10) + - (parseInt(parts[2], 10) * 256) + - (parseInt(parts[1], 10) * 65536) + - (parseInt(parts[0], 10) * 16777216); + (parseInt(parts[2], 10) * 256) + + (parseInt(parts[1], 10) * 65536) + + (parseInt(parts[0], 10) * 16777216); var idxMin = 0; var idxMiddle = 0; var idxMax = countriesLength - 1; - var pickedCountry = undefined; + while (idxMin < idxMax) { idxMiddle = (idxMax + idxMin) >> 1; - pickedCountry = countries[idxMiddle]; - // determine which subarray to search - if (pickedCountry.ip < target_ip) { - // change min index to search upper subarray + if (!(idxMiddle < idxMax)) { + throw "assertion error: idxMiddle is not lower then idxMax" + } + if (countries[idxMiddle].ip < target_ip) { idxMin = idxMiddle + 1; - } else if (pickedCountry.ip > target_ip) { - // change max index to search lower subarray - idxMax = idxMiddle - 1; } else { - // key found at index imid - break; + idxMax = idxMiddle; } } - // return previous found country. + + var pickedCountry = countries[idxMin]; + if ((idxMax == idxMin) && (pickedCountry.ip == target_ip)) { + pickedCountry = countries[idxMin]; + return { + ipstart: pickedCountry.ip, + name: countryNamesAndCodes[pickedCountry.idx], + code: countryNamesAndCodes[pickedCountry.idx + 1] + }; + } + + if ((idxMiddle > 0) && (countries[idxMiddle - 1].ip < target_ip) && (target_ip < countries[idxMiddle].ip)) { + pickedCountry = countries[idxMiddle - 1] + return { + ipstart: pickedCountry.ip, + name: countryNamesAndCodes[pickedCountry.idx], + code: countryNamesAndCodes[pickedCountry.idx + 1] + }; + } + + if ((idxMiddle < countriesLength - 2) && (countries[idxMiddle].ip < target_ip) && (target_ip < countries[idxMiddle + 1].ip)) { + pickedCountry = countries[idxMiddle] + return { + ipstart: pickedCountry.ip, + name: countryNamesAndCodes[pickedCountry.idx], + code: countryNamesAndCodes[pickedCountry.idx + 1] + }; + } + return { - ipstart: pickedCountry.ip, - name: countryNamesAndCodes[pickedCountry.idx], - code: countryNamesAndCodes[pickedCountry.idx + 1] + ipstart: -1, + name: "UNKNOWN", + code: "N/A" }; + } diff --git a/test/unit/lookupTest.js b/test/unit/lookupTest.js index e267e0b..5d03d58 100644 --- a/test/unit/lookupTest.js +++ b/test/unit/lookupTest.js @@ -81,6 +81,17 @@ module.exports = { test.equals(actual.name, "Cambodia"); test.equals(actual.code, "KH"); + test.done(); + }, + + test_in_the_middle_of_two_ips:function (test) { + var ip; + var actual; + ip = "63.155.159.123"; + actual = this.geoip.lookup(ip); + test.equals(actual.name, "United States"); + test.equals(actual.code, "US"); + test.done(); } }; \ No newline at end of file From 592baed31193804f0cdd8cd891003b0511744c78 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sat, 27 Apr 2013 16:16:33 +0200 Subject: [PATCH 35/42] fixed range check --- geoip-native.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geoip-native.js b/geoip-native.js index e93e392..c233246 100644 --- a/geoip-native.js +++ b/geoip-native.js @@ -56,7 +56,7 @@ function _lookup(ip) { }; } - if ((idxMiddle < countriesLength - 2) && (countries[idxMiddle].ip < target_ip) && (target_ip < countries[idxMiddle + 1].ip)) { + if ((idxMiddle < idxMax) && (countries[idxMiddle].ip < target_ip) && (target_ip < countries[idxMiddle + 1].ip)) { pickedCountry = countries[idxMiddle] return { ipstart: pickedCountry.ip, From c72fe1c1481295c81d3b9764a5fbfd2d8bc26ade Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sat, 27 Apr 2013 19:08:22 +0200 Subject: [PATCH 36/42] re-ident to two spaces --- generate_sources.js | 106 ++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/generate_sources.js b/generate_sources.js index 6045471..bc17867 100644 --- a/generate_sources.js +++ b/generate_sources.js @@ -3,72 +3,72 @@ var countryNamesAndCodes = []; function read_csv_file_and_prepare_data() { - var fs = require("fs"); - var data = fs.readFileSync(__dirname + "/GeoIPCountryWhois.csv") - var buffer = ""; - buffer += data.toString().replace(/"/g, ""); + var fs = require("fs"); + var data = fs.readFileSync(__dirname + "/GeoIPCountryWhois.csv") + var buffer = ""; + buffer += data.toString().replace(/"/g, ""); - var entries = buffer.split("\n"); - var offsetCounter = 0; - var countrySet = {}; + var entries = buffer.split("\n"); + var offsetCounter = 0; + var countrySet = {}; - for (var i = 0; i < entries.length; i++) { - var entry = entries[i].split(","); - if (entry.length > 5) { - var countryName = entry[5].trim(); - var countryCode = entry[4]; - var countryIndex = 0; - if (!countrySet[countryName]) { - countryIndex = offsetCounter++; - countrySet[countryName] = {index:countryIndex}; - countryNamesAndCodes.push(countryName); - countryNamesAndCodes.push(countryCode); - } else { - countryIndex = countrySet[countryName].index; - } - countries.push({ipstart:parseInt(entry[2]), code:countryCode, name:countryName, index:(countryIndex * 2)}); + for (var i = 0; i < entries.length; i++) { + var entry = entries[i].split(","); + if (entry.length > 5) { + var countryName = entry[5].trim(); + var countryCode = entry[4]; + var countryIndex = 0; + if (!countrySet[countryName]) { + countryIndex = offsetCounter++; + countrySet[countryName] = {index: countryIndex}; + countryNamesAndCodes.push(countryName); + countryNamesAndCodes.push(countryCode); + } else { + countryIndex = countrySet[countryName].index; } - } + countries.push({ipstart: parseInt(entry[2]), code: countryCode, name: countryName, index: (countryIndex * 2)}); + } + } - countries.sort(function (a, b) { - return a.ipstart - b.ipstart; - }); + countries.sort(function (a, b) { + return a.ipstart - b.ipstart; + }); } function write_sourceFile_countryNamesAndCodes() { - var data = ""; - data += "// AUTOGENERATED CODE - DO NOT MODIFY! " + new Date() + " \n"; - data += "var countryNamesAndCodes = [\n"; - for (var i = 0, len = countryNamesAndCodes.length >> 1; i < len; i++) { - var countryName = countryNamesAndCodes[i<<1].replace("'", "\\'"); - var countryCode = countryNamesAndCodes[(i<<1)+1].replace("'", "\\'"); - data += "'" + countryName + "','"+countryCode+"',\n"; - } - data += "];\n"; - data += "module.exports = {\n"; - data += " countryNamesAndCodes:countryNamesAndCodes\n"; - data += "};\n"; + var data = ""; + data += "// AUTOGENERATED CODE - DO NOT MODIFY! " + new Date() + " \n"; + data += "var countryNamesAndCodes = [\n"; + for (var i = 0, len = countryNamesAndCodes.length >> 1; i < len; i++) { + var countryName = countryNamesAndCodes[i << 1].replace("'", "\\'"); + var countryCode = countryNamesAndCodes[(i << 1) + 1].replace("'", "\\'"); + data += "'" + countryName + "','" + countryCode + "',\n"; + } + data += "];\n"; + data += "module.exports = {\n"; + data += " countryNamesAndCodes:countryNamesAndCodes\n"; + data += "};\n"; - var fs = require("fs"); - fs.writeFileSync('generated-namesandcodes.js', data, 'utf8'); + var fs = require("fs"); + fs.writeFileSync('generated-namesandcodes.js', data, 'utf8'); } function write_sourceFile_countries() { - var data = ""; - data += "// AUTOGENERATED CODE - DO NOT MODIFY! " + new Date() + " \n"; - data += "var countries = [\n"; - for (var i = 0, len = countries.length; i < len; i++) { - var country = countries[i]; - data += "{ip:" + country.ipstart + ",idx:" + country.index + "},//" + countryNamesAndCodes[country.index + 1] + "\n"; - } - data += "];\n"; - data += "module.exports = {\n"; - data += " countries:countries\n"; - data += "};\n"; + var data = ""; + data += "// AUTOGENERATED CODE - DO NOT MODIFY! " + new Date() + " \n"; + data += "var countries = [\n"; + for (var i = 0, len = countries.length; i < len; i++) { + var country = countries[i]; + data += "{ip:" + country.ipstart + ",idx:" + country.index + "},//" + countryNamesAndCodes[country.index + 1] + "\n"; + } + data += "];\n"; + data += "module.exports = {\n"; + data += " countries:countries\n"; + data += "};\n"; - var fs = require("fs"); - fs.writeFileSync('generated-countries.js', data, 'utf8'); + var fs = require("fs"); + fs.writeFileSync('generated-countries.js', data, 'utf8'); } /** From d8634caf2fb8c5d4da629bdd78ca38a5eb11ac4a Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sat, 27 Apr 2013 19:18:57 +0200 Subject: [PATCH 37/42] refactored: extracted methods for better readability --- generate_sources.js | 15 ++++++++++----- geoip-native.js | 28 +++++++++------------------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/generate_sources.js b/generate_sources.js index bc17867..2e6cf09 100644 --- a/generate_sources.js +++ b/generate_sources.js @@ -1,14 +1,19 @@ +"use strict"; + var countries = []; var countryNamesAndCodes = []; function read_csv_file_and_prepare_data() { - var fs = require("fs"); - var data = fs.readFileSync(__dirname + "/GeoIPCountryWhois.csv") - var buffer = ""; - buffer += data.toString().replace(/"/g, ""); + function load_CSV_file() { + var fs = require("fs"); + var data = fs.readFileSync(__dirname + "/GeoIPCountryWhois.csv") + var buffer = ""; + buffer += data.toString().replace(/"/g, ""); + return buffer; + } - var entries = buffer.split("\n"); + var entries = load_CSV_file().split("\n"); var offsetCounter = 0; var countrySet = {}; diff --git a/geoip-native.js b/geoip-native.js index c233246..bd1b014 100644 --- a/geoip-native.js +++ b/geoip-native.js @@ -40,35 +40,25 @@ function _lookup(ip) { var pickedCountry = countries[idxMin]; if ((idxMax == idxMin) && (pickedCountry.ip == target_ip)) { pickedCountry = countries[idxMin]; - return { - ipstart: pickedCountry.ip, - name: countryNamesAndCodes[pickedCountry.idx], - code: countryNamesAndCodes[pickedCountry.idx + 1] - }; + return createCountry(pickedCountry.ip, countryNamesAndCodes[pickedCountry.idx], countryNamesAndCodes[pickedCountry.idx + 1]); } if ((idxMiddle > 0) && (countries[idxMiddle - 1].ip < target_ip) && (target_ip < countries[idxMiddle].ip)) { pickedCountry = countries[idxMiddle - 1] - return { - ipstart: pickedCountry.ip, - name: countryNamesAndCodes[pickedCountry.idx], - code: countryNamesAndCodes[pickedCountry.idx + 1] - }; + return createCountry(pickedCountry.ip, countryNamesAndCodes[pickedCountry.idx], countryNamesAndCodes[pickedCountry.idx + 1]); } if ((idxMiddle < idxMax) && (countries[idxMiddle].ip < target_ip) && (target_ip < countries[idxMiddle + 1].ip)) { pickedCountry = countries[idxMiddle] - return { - ipstart: pickedCountry.ip, - name: countryNamesAndCodes[pickedCountry.idx], - code: countryNamesAndCodes[pickedCountry.idx + 1] - }; + return createCountry(pickedCountry.ip, countryNamesAndCodes[pickedCountry.idx], countryNamesAndCodes[pickedCountry.idx + 1]); } + return createCountry(target_ip, "UNKNOWN", "N/A"); +} +function createCountry(ipstart, countryName, countryCode) { return { - ipstart: -1, - name: "UNKNOWN", - code: "N/A" + ipstart: ipstart, + name: countryName, + code: countryCode }; - } From 2751af7f9f91f8f29ad0c4957f9a68f618e7c50f Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sat, 27 Apr 2013 20:15:48 +0200 Subject: [PATCH 38/42] improved lookupTest, and uncovered a bug: last country in the list will not be catched --- test/unit/lookupTest.js | 174 +++++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 81 deletions(-) diff --git a/test/unit/lookupTest.js b/test/unit/lookupTest.js index 5d03d58..a9b32aa 100644 --- a/test/unit/lookupTest.js +++ b/test/unit/lookupTest.js @@ -1,97 +1,109 @@ // use Node-Unit (https://github.com/caolan/nodeunit) to run this test -module.exports = { - setUp:function (callback) { - this.geoip = require("../../geoip-native.js"); - // wait until ready - callback(); - }, - - tearDown:function (callback) { - // clean up - callback(); - }, - - test_ip_in_the_lower_range_and_leftmost_border:function (test) { - var ip; - var actual; - // "1.0.64.0","1.0.127.255","16793600","16809983","JP","Japan" - ip = "1.0.64.0"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Japan"); - test.equals(actual.code, "JP"); +"use strict"; - test.done(); - }, +module.exports.setUp = function (callback) { + this.geoip = require("../../geoip-native.js"); - test_ip_in_the_lower_range_and_middle:function (test) { - var ip; - var actual; - // "1.0.64.0","1.0.127.255","16793600","16809983","JP","Japan" - ip = "1.0.90.90"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Japan"); - test.equals(actual.code, "JP"); + callback(); +}; - test.done(); - }, +module.exports.tearDown = function (callback) { + // clean up + callback(); +}; - test_ip_in_the_lower_range_and_rightmost_border:function (test) { - var ip; - var actual; - // "1.0.64.0","1.0.127.255","16793600","16809983","JP","Japan" - ip = "1.0.64.0"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Japan"); - test.equals(actual.code, "JP"); +function createTestFunction(ipFrom, ipTo, int32From, int32To, countryName, countryCode) { + return function (test) { + var actual = this.geoip.lookup(ipFrom); + test.equals(actual.name, countryName); + test.equals(actual.code, countryCode); + test.equals(actual.ipstart, int32From); + + var actual = this.geoip.lookup(ipTo); + test.equals(actual.name, countryName); + test.equals(actual.code, countryCode); + test.equals(actual.ipstart, int32From); test.done(); - }, + } +} + +prepare_Tests_from_data_provider(); +function prepare_Tests_from_data_provider() { + + var records = load_CSV_file().split("\n"); + + var testRanges = [ + [0, 99], + [records.length / 2 - 50, records.length / 2 + 50], + [records.length - 100, records.length - 1], + ]; + + for (var i = 0; i < records.length; i++) { + var shouldContinue = false; + for (var z = 0; z < testRanges.length; z++) { + var testRange = testRanges[z]; + shouldContinue |= (testRange[0] <= i && i <= testRange[1]); + } + if (!shouldContinue) continue; + + var record = records[i].toString().trim(); + if (record.length < 1) { + continue; + } + var dataParts = record.split(/,/); + + var ipFrom = dataParts[0].toString().replace(/"/g, ""); + var ipTo = dataParts[1].toString().replace(/"/g, ""); + var int32From = parseInt(dataParts[2].toString().replace(/"/g, ""), 10); + var int32To = parseInt(dataParts[3].toString().replace(/"/g, ""), 10); + var countryCode = dataParts[4].toString().replace(/"/g, ""); + var countryName = dataParts[5].toString().replace(/"/g, ""); + + var testMethodName = ("test_record_index_" + i + "_ipFrom_" + ipFrom + "_name_" + countryName); + module.exports[testMethodName] = createTestFunction(ipFrom, ipTo, int32From, int32To, countryName, countryCode); + } +} - test_ip_in_upper_range_and_leftmost_border:function (test) { - var ip; - var actual; - // "223.223.168.0","223.223.175.255","3755976704","3755978751","KH","Cambodia" - ip = "223.223.168.0"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Cambodia"); - test.equals(actual.code, "KH"); +module.exports.test_unknown_low_ip_should_give_an_UNKNONW_country = function (test) { - test.done(); - }, + var ip = "0.1.2.3"; - test_ip_in_upper_range_and_middle:function (test) { - var ip; - var actual; - // "223.223.168.0","223.223.175.255","3755976704","3755978751","KH","Cambodia" - ip = "223.223.170.170"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Cambodia"); - test.equals(actual.code, "KH"); + var actual = this.geoip.lookup(ip); - test.done(); - }, + test.equals(actual.name, "UNKNOWN"); + test.equals(actual.code, "N/A"); + test.equals(actual.ipstart, ip2int(ip)); - test_ip_in_upper_range_and_rightmost_border:function (test) { - var ip; - var actual; - // "223.223.168.0","223.223.175.255","3755976704","3755978751","KH","Cambodia" - ip = "223.223.175.255"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "Cambodia"); - test.equals(actual.code, "KH"); + test.done(); +} - test.done(); - }, +module.exports.test_unknown_high_ip_should_give_an_UNKNONW_country = function (test) { - test_in_the_middle_of_two_ips:function (test) { - var ip; - var actual; - ip = "63.155.159.123"; - actual = this.geoip.lookup(ip); - test.equals(actual.name, "United States"); - test.equals(actual.code, "US"); + var ip = "254.255.254.255"; - test.done(); - } -}; \ No newline at end of file + var actual = this.geoip.lookup(ip); + + test.equals(actual.name, "UNKNOWN"); + test.equals(actual.code, "N/A"); + test.equals(actual.ipstart, ip2int(ip)); + + test.done(); +} + +function ip2int(ipAsString) { + var parts = ipAsString.split(/[.]/); + return parseInt(parts[0], 10) * (1 << 24) + + parseInt(parts[1], 10) * (1 << 16) + + parseInt(parts[2], 10) * (1 << 8) + + parseInt(parts[3], 10); +} + +function load_CSV_file() { + var fs = require("fs"); + var data = fs.readFileSync(__dirname + "/../../GeoIPCountryWhois.csv") + var buffer = ""; + buffer += data.toString().replace(/"/g, ""); + return buffer; +} \ No newline at end of file From c74d4df4fdfbcfa622af2cd99e1c7ffc1c3080b2 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sat, 27 Apr 2013 20:45:45 +0200 Subject: [PATCH 39/42] improved lookupTest, and uncovered one more bug: "Virgin Islands" and "Virgin Islands, British" are treated the same --- test/unit/lookupTest.js | 43 +++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/test/unit/lookupTest.js b/test/unit/lookupTest.js index a9b32aa..4fdbcfa 100644 --- a/test/unit/lookupTest.js +++ b/test/unit/lookupTest.js @@ -2,14 +2,15 @@ "use strict"; +prepare_Tests_from_data_provider(); + module.exports.setUp = function (callback) { this.geoip = require("../../geoip-native.js"); - callback(); }; module.exports.tearDown = function (callback) { - // clean up + this.geoip = null; callback(); }; @@ -29,31 +30,35 @@ function createTestFunction(ipFrom, ipTo, int32From, int32To, countryName, count } } -prepare_Tests_from_data_provider(); +function inMinimizedTestRange(maxLength, index) { + var testRanges_fromIndex_toIndex = [ + [0, 99], + [maxLength / 2 - 50, maxLength / 2 + 50], + [maxLength - 100, maxLength - 1], + ]; + var inRange = false; + for (var z = 0; z < testRanges_fromIndex_toIndex.length; z++) { + var testRange = testRanges_fromIndex_toIndex[z]; + inRange |= (testRange[0] <= index && index <= testRange[1]); + } + return inRange; +} + function prepare_Tests_from_data_provider() { var records = load_CSV_file().split("\n"); - var testRanges = [ - [0, 99], - [records.length / 2 - 50, records.length / 2 + 50], - [records.length - 100, records.length - 1], - ]; for (var i = 0; i < records.length; i++) { - var shouldContinue = false; - for (var z = 0; z < testRanges.length; z++) { - var testRange = testRanges[z]; - shouldContinue |= (testRange[0] <= i && i <= testRange[1]); - } - if (!shouldContinue) continue; - var record = records[i].toString().trim(); - if (record.length < 1) { - continue; - } var dataParts = record.split(/,/); + // warning: testing the complete Range will take more than one minute ! + var shouldContinue = inMinimizedTestRange(records.length, i); + var recordContainsComma = dataParts.length > 6; + if (!shouldContinue && !recordContainsComma) continue; + if (record.length < 1) continue; + var ipFrom = dataParts[0].toString().replace(/"/g, ""); var ipTo = dataParts[1].toString().replace(/"/g, ""); var int32From = parseInt(dataParts[2].toString().replace(/"/g, ""), 10); @@ -61,7 +66,7 @@ function prepare_Tests_from_data_provider() { var countryCode = dataParts[4].toString().replace(/"/g, ""); var countryName = dataParts[5].toString().replace(/"/g, ""); - var testMethodName = ("test_record_index_" + i + "_ipFrom_" + ipFrom + "_name_" + countryName); + var testMethodName = ("test: record index=" + i + " ipFrom=" + ipFrom + " name=" + countryName); module.exports[testMethodName] = createTestFunction(ipFrom, ipTo, int32From, int32To, countryName, countryCode); } } From bfa84f428a16d0d6078536af599db264dc2ddff2 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sat, 27 Apr 2013 21:24:20 +0200 Subject: [PATCH 40/42] Fixed problem of commas in country names (example: "Virgin Island" vs. "Virgin Island, British") --- generate_sources.js | 24 ++++++++++++++++++------ test/unit/lookupTest.js | 18 +++++++++++++++--- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/generate_sources.js b/generate_sources.js index 2e6cf09..10c9018 100644 --- a/generate_sources.js +++ b/generate_sources.js @@ -9,7 +9,7 @@ function read_csv_file_and_prepare_data() { var fs = require("fs"); var data = fs.readFileSync(__dirname + "/GeoIPCountryWhois.csv") var buffer = ""; - buffer += data.toString().replace(/"/g, ""); + buffer += data.toString(); return buffer; } @@ -17,11 +17,23 @@ function read_csv_file_and_prepare_data() { var offsetCounter = 0; var countrySet = {}; + function extractPartsFromCsv(line) { + var matches = line.match(/("(?:[^"]|"")*"|[^,]*)/g); + var result = []; + for (var i = 0; i < matches.length; i++) { + var part = matches[i].replace(/"/g, "").trim(); + if (part.length > 0) { + result.push(part); + } + } + return result; + } + for (var i = 0; i < entries.length; i++) { - var entry = entries[i].split(","); - if (entry.length > 5) { - var countryName = entry[5].trim(); - var countryCode = entry[4]; + var parts = extractPartsFromCsv(entries[i]); + if (parts.length > 5) { + var countryName = parts[5].trim(); + var countryCode = parts[4]; var countryIndex = 0; if (!countrySet[countryName]) { countryIndex = offsetCounter++; @@ -31,7 +43,7 @@ function read_csv_file_and_prepare_data() { } else { countryIndex = countrySet[countryName].index; } - countries.push({ipstart: parseInt(entry[2]), code: countryCode, name: countryName, index: (countryIndex * 2)}); + countries.push({ipstart: parseInt(parts[2]), code: countryCode, name: countryName, index: (countryIndex * 2)}); } } diff --git a/test/unit/lookupTest.js b/test/unit/lookupTest.js index 4fdbcfa..202e596 100644 --- a/test/unit/lookupTest.js +++ b/test/unit/lookupTest.js @@ -34,7 +34,7 @@ function inMinimizedTestRange(maxLength, index) { var testRanges_fromIndex_toIndex = [ [0, 99], [maxLength / 2 - 50, maxLength / 2 + 50], - [maxLength - 100, maxLength - 1], + [maxLength - 100, maxLength - 1] ]; var inRange = false; for (var z = 0; z < testRanges_fromIndex_toIndex.length; z++) { @@ -51,7 +51,7 @@ function prepare_Tests_from_data_provider() { for (var i = 0; i < records.length; i++) { var record = records[i].toString().trim(); - var dataParts = record.split(/,/); + var dataParts = extractPartsFromCsv(record); // warning: testing the complete Range will take more than one minute ! var shouldContinue = inMinimizedTestRange(records.length, i); @@ -109,6 +109,18 @@ function load_CSV_file() { var fs = require("fs"); var data = fs.readFileSync(__dirname + "/../../GeoIPCountryWhois.csv") var buffer = ""; - buffer += data.toString().replace(/"/g, ""); + buffer += data.toString(); return buffer; +} + +function extractPartsFromCsv(line) { + var matches = line.match(/("(?:[^"]|"")*"|[^,]*)/g); + var result = []; + for (var i = 0; i < matches.length; i++) { + var part = matches[i].replace(/"/g, "").trim(); + if (part.length > 0) { + result.push(part); + } + } + return result; } \ No newline at end of file From 3a21f8c146559e3fd2966ee7d3652a61dfbc8c21 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sat, 27 Apr 2013 21:49:27 +0200 Subject: [PATCH 41/42] Fixed problem that last records in database weren't resolved correctly --- generate_sources.js | 39 ++++++++++++++++++++++++++++++++------- geoip-native.js | 19 ++++++++++--------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/generate_sources.js b/generate_sources.js index 10c9018..ff5ab0b 100644 --- a/generate_sources.js +++ b/generate_sources.js @@ -13,10 +13,6 @@ function read_csv_file_and_prepare_data() { return buffer; } - var entries = load_CSV_file().split("\n"); - var offsetCounter = 0; - var countrySet = {}; - function extractPartsFromCsv(line) { var matches = line.match(/("(?:[^"]|"")*"|[^,]*)/g); var result = []; @@ -29,12 +25,17 @@ function read_csv_file_and_prepare_data() { return result; } + var entries = load_CSV_file().split("\n"); + var offsetCounter = 0; + var countryIndex = 0; + var countrySet = {}; + var lastIpRangeEnd = 0; + for (var i = 0; i < entries.length; i++) { var parts = extractPartsFromCsv(entries[i]); if (parts.length > 5) { var countryName = parts[5].trim(); var countryCode = parts[4]; - var countryIndex = 0; if (!countrySet[countryName]) { countryIndex = offsetCounter++; countrySet[countryName] = {index: countryIndex}; @@ -43,16 +44,37 @@ function read_csv_file_and_prepare_data() { } else { countryIndex = countrySet[countryName].index; } - countries.push({ipstart: parseInt(parts[2]), code: countryCode, name: countryName, index: (countryIndex * 2)}); + countries.push(createCountryInformation(parseInt(parts[2]), parseInt(parts[3]), countryCode, countryName, (countryIndex * 2))); + lastIpRangeEnd = parseInt(parts[3]); } } + // add a special country, which indicates the END + countryIndex = offsetCounter; + countrySet["UNKNOWN"] = {index: countryIndex}; + countryNamesAndCodes.push("UNKNOWN"); + countryNamesAndCodes.push("N/A"); + countries.push( + createCountryInformation(lastIpRangeEnd + 1, lastIpRangeEnd + 1, "N/A", "UNKNOWN", countryIndex * 2) + ); + countries.sort(function (a, b) { return a.ipstart - b.ipstart; }); } +function createCountryInformation(ipStart, ipEnd, countryCode, countryName, index) { + return { + ipstart: ipStart, + ipend: ipEnd, + code: countryCode, + name: countryName, + index: index + }; +} + + function write_sourceFile_countryNamesAndCodes() { var data = ""; data += "// AUTOGENERATED CODE - DO NOT MODIFY! " + new Date() + " \n"; @@ -77,7 +99,10 @@ function write_sourceFile_countries() { data += "var countries = [\n"; for (var i = 0, len = countries.length; i < len; i++) { var country = countries[i]; - data += "{ip:" + country.ipstart + ",idx:" + country.index + "},//" + countryNamesAndCodes[country.index + 1] + "\n"; + data += "{s:" + country.ipstart + ","; + data += "e:" + country.ipend + ","; + data += "i:" + country.index + "},"; + data += "//" + countryNamesAndCodes[country.index + 1] + "\n"; } data += "];\n"; data += "module.exports = {\n"; diff --git a/geoip-native.js b/geoip-native.js index bd1b014..839e44e 100644 --- a/geoip-native.js +++ b/geoip-native.js @@ -30,7 +30,7 @@ function _lookup(ip) { if (!(idxMiddle < idxMax)) { throw "assertion error: idxMiddle is not lower then idxMax" } - if (countries[idxMiddle].ip < target_ip) { + if (countries[idxMiddle].s < target_ip) { idxMin = idxMiddle + 1; } else { idxMax = idxMiddle; @@ -38,26 +38,27 @@ function _lookup(ip) { } var pickedCountry = countries[idxMin]; - if ((idxMax == idxMin) && (pickedCountry.ip == target_ip)) { + if ((idxMax == idxMin) && (pickedCountry.s == target_ip)) { pickedCountry = countries[idxMin]; - return createCountry(pickedCountry.ip, countryNamesAndCodes[pickedCountry.idx], countryNamesAndCodes[pickedCountry.idx + 1]); + return createCountry(pickedCountry.s, pickedCountry.e, countryNamesAndCodes[pickedCountry.i], countryNamesAndCodes[pickedCountry.i + 1]); } - if ((idxMiddle > 0) && (countries[idxMiddle - 1].ip < target_ip) && (target_ip < countries[idxMiddle].ip)) { + if ((idxMiddle > 0) && (countries[idxMiddle - 1].s < target_ip) && (target_ip < countries[idxMiddle].s)) { pickedCountry = countries[idxMiddle - 1] - return createCountry(pickedCountry.ip, countryNamesAndCodes[pickedCountry.idx], countryNamesAndCodes[pickedCountry.idx + 1]); + return createCountry(pickedCountry.s, pickedCountry.e, countryNamesAndCodes[pickedCountry.i], countryNamesAndCodes[pickedCountry.i + 1]); } - if ((idxMiddle < idxMax) && (countries[idxMiddle].ip < target_ip) && (target_ip < countries[idxMiddle + 1].ip)) { + if ((idxMiddle < idxMax) && (countries[idxMiddle].s < target_ip) && (target_ip < countries[idxMiddle + 1].s)) { pickedCountry = countries[idxMiddle] - return createCountry(pickedCountry.ip, countryNamesAndCodes[pickedCountry.idx], countryNamesAndCodes[pickedCountry.idx + 1]); + return createCountry(pickedCountry.s, pickedCountry.e, countryNamesAndCodes[pickedCountry.i], countryNamesAndCodes[pickedCountry.i + 1]); } - return createCountry(target_ip, "UNKNOWN", "N/A"); + return createCountry(target_ip, target_ip, "UNKNOWN", "N/A"); } -function createCountry(ipstart, countryName, countryCode) { +function createCountry(ipstart, ipend, countryName, countryCode) { return { ipstart: ipstart, + ipend: ipend, name: countryName, code: countryCode }; From 73262d97821a11226fd5abd87ef47115609ade91 Mon Sep 17 00:00:00 2001 From: nitram509 Date: Sat, 27 Apr 2013 21:53:21 +0200 Subject: [PATCH 42/42] added one more assertion for ending range --- test/unit/lookupTest.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/lookupTest.js b/test/unit/lookupTest.js index 202e596..2f2e5d4 100644 --- a/test/unit/lookupTest.js +++ b/test/unit/lookupTest.js @@ -20,11 +20,13 @@ function createTestFunction(ipFrom, ipTo, int32From, int32To, countryName, count test.equals(actual.name, countryName); test.equals(actual.code, countryCode); test.equals(actual.ipstart, int32From); + test.equals(actual.ipend, int32To); var actual = this.geoip.lookup(ipTo); test.equals(actual.name, countryName); test.equals(actual.code, countryCode); test.equals(actual.ipstart, int32From); + test.equals(actual.ipend, int32To); test.done(); }