From cb91319a8116f1583451099ede7dabf32e932acb Mon Sep 17 00:00:00 2001 From: Daniel Kraft Date: Wed, 10 Dec 2014 20:32:11 +0100 Subject: [PATCH 1/7] Implement REST API as backend for NMControl. --- lib/backendDataRest.py | 48 ++++++++++++++++++++++++++++++++++++++++++ plugin/pluginData.py | 3 ++- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 lib/backendDataRest.py diff --git a/lib/backendDataRest.py b/lib/backendDataRest.py new file mode 100644 index 0000000..ab20485 --- /dev/null +++ b/lib/backendDataRest.py @@ -0,0 +1,48 @@ +from common import * + +import httplib +import json +import urllib +import urlparse + +class backendData(): + validURL = False + + def __init__(self, conf): + url = urlparse.urlparse(conf) + if url.scheme == 'http': + self.validURL = True + self.host = url.hostname + self.port = url.port + elif app['debug']: + print "Unsupported scheme for REST URL:", url.scheme + + def getAllNames(self): + # The REST API doesn't support enumerating the names. + return False + + def getName(self, name): + if not self.validURL: + return "invalid REST URL", None + + encoded = urllib.quote_plus(name) + data = self._queryHttpGet("/rest/name/" + encoded + ".json") + + if data is None: + return "query failed", None + return None, json.loads(data) + + def _queryHttpGet(self, path): + assert self.validURL + conn = httplib.HTTPConnection(self.host, self.port) + conn.request('GET', path) + + res = conn.getresponse() + if res.status != 200: + if app['debug']: + print "REST returned error code:", res.status + print res.read() + return None + + return res.read() + diff --git a/plugin/pluginData.py b/plugin/pluginData.py index 6b7d752..4a5635b 100644 --- a/plugin/pluginData.py +++ b/plugin/pluginData.py @@ -17,10 +17,11 @@ class pluginData(plugin.PluginThread): {'import.namecoin': ['Path of namecoin.conf', platformDep.getNamecoinDir() + os.sep + 'namecoin.conf']}, {'update.mode': ['Update mode', 'ondemand', '']}, - {'update.from': ['Update data from', 'namecoin', '']}, + {'update.from': ['Update data from', 'namecoin', '']}, {'update.freq': ['Update data if older than', '30m', '[h|m|s]']}, {'update.file': ['Update data from file ', 'data' + os.sep + 'namecoin.dat']}, {'update.namecoin': ['Path of namecoin.conf', platformDep.getNamecoinDir() + os.sep + 'namecoin.conf']}, + {'update.rest': ['REST API to query', 'http://localhost:8336']}, {'export.mode': ['Export mode', 'none', '']}, {'export.to': ['Export data to', 'file']}, From 276c26c59500687a0535588b145f5912c72b818f Mon Sep 17 00:00:00 2001 From: Daniel Kraft Date: Thu, 11 Dec 2014 21:58:35 +0100 Subject: [PATCH 2/7] Support REST over TLS. --- lib/backendDataRest.py | 92 +++++++++++++++++++++++++++++++++++++++++- lib/utils.py | 13 ++++++ plugin/pluginData.py | 2 +- plugin/pluginDns.py | 15 ++----- 4 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 lib/utils.py diff --git a/lib/backendDataRest.py b/lib/backendDataRest.py index ab20485..baaa68c 100644 --- a/lib/backendDataRest.py +++ b/lib/backendDataRest.py @@ -1,19 +1,84 @@ from common import * +from utils import * +import hashlib import httplib import json +import socket +import ssl import urllib import urlparse +class MyHTTPSConnection(httplib.HTTPConnection): + """ + Make a HTTPS connection, but support checking the + server's certificate fingerprint. + """ + + def __init__(self, host, port): + httplib.HTTPConnection.__init__(self, host, port) + + def connect(self): + sock = socket.create_connection((self.host, self.port)) + self.sock = ssl.wrap_socket(sock) + + def verifyCert(self, fprs): + """ + Check whether the server's certificate fingerprint + matches the given one. 'fprs' should be a dict with + keys corresponding to digest methods and values being + the digest value. + """ + + hasher = None + fpr = None + + if 'sha256' in fprs: + hasher = hashlib.sha256() + fpr = fprs['sha256'] + elif 'sha1' in fprs: + hasher = hashlib.sha1() + fpr = fprs['sha1'] + + # Be strict here. If this routine is called, it means that at least + # some parameters were given in the REST URI. If we can't verify the + # fingerprint because the parameters were invalid, fail to make sure + # that the user does not expect security but doesn't get it due + # to an operational mistake. + if hasher is None: + if app['debug']: + print "no recognised fingerprint given, failing check" + return False + assert fpr is not None + + cert = self.sock.getpeercert(True) + hasher.update(cert) + digest = hasher.hexdigest() + + if sanitiseFingerprint(digest) != sanitiseFingerprint(fpr): + if app['debug']: + print "Fingerprint mismatch:" + print " expected:", fpr + print " got:", digest + return False + + return True + class backendData(): validURL = False def __init__(self, conf): url = urlparse.urlparse(conf) - if url.scheme == 'http': + if url.scheme == 'http' or url.scheme == 'https': self.validURL = True + self.tls = (url.scheme == 'https') self.host = url.hostname self.port = url.port + + if url.params == '': + self.fprs = None + else: + self.fprs = self._parseFprOptions(url.params) elif app['debug']: print "Unsupported scheme for REST URL:", url.scheme @@ -34,9 +99,16 @@ def getName(self, name): def _queryHttpGet(self, path): assert self.validURL - conn = httplib.HTTPConnection(self.host, self.port) + if self.tls: + conn = MyHTTPSConnection(self.host, self.port) + else: + conn = httplib.HTTPConnection(self.host, self.port) conn.request('GET', path) + if self.tls and (self.fprs is not None): + if not conn.verifyCert(self.fprs): + return "TLS fingerprint wrong", None + res = conn.getresponse() if res.status != 200: if app['debug']: @@ -46,3 +118,19 @@ def _queryHttpGet(self, path): return res.read() + def _parseFprOptions(self, s): + """ + Parse the REST URI params string that includes (optionally) + the TLS certificate fingerprints. + """ + + pieces = s.split(',') + + res = {} + for p in pieces: + parts = p.split('=', 1) + assert len (parts) <= 2 + if len (parts) == 2: + res[parts[0]] = parts[1] + + return res diff --git a/lib/utils.py b/lib/utils.py new file mode 100644 index 0000000..ee8f6fe --- /dev/null +++ b/lib/utils.py @@ -0,0 +1,13 @@ +def sanitiseFingerprint(fpr): + """ + Sanitise a fingerprint (of a TLS certificate, for instance) for + comparison. This removes colons, spaces and makes the string + upper case. + """ + + #fpr = fpr.translate (None, ': ') + fpr = fpr.replace (":", "") + fpr = fpr.replace (" ", "") + fpr = fpr.upper () + + return fpr diff --git a/plugin/pluginData.py b/plugin/pluginData.py index 4a5635b..89f5bbd 100644 --- a/plugin/pluginData.py +++ b/plugin/pluginData.py @@ -21,7 +21,7 @@ class pluginData(plugin.PluginThread): {'update.freq': ['Update data if older than', '30m', '[h|m|s]']}, {'update.file': ['Update data from file ', 'data' + os.sep + 'namecoin.dat']}, {'update.namecoin': ['Path of namecoin.conf', platformDep.getNamecoinDir() + os.sep + 'namecoin.conf']}, - {'update.rest': ['REST API to query', 'http://localhost:8336']}, + {'update.rest': ['REST API to query', 'http://localhost:8336/']}, {'export.mode': ['Export mode', 'none', '']}, {'export.to': ['Export data to', 'file']}, diff --git a/plugin/pluginDns.py b/plugin/pluginDns.py index 814a5cf..b687379 100644 --- a/plugin/pluginDns.py +++ b/plugin/pluginDns.py @@ -1,4 +1,5 @@ from common import * +from utils import * import plugin #import DNS #import json, base64, types, random, traceback @@ -184,9 +185,9 @@ def verifyFingerprint (self, domain, fpr): "is not a list" return False - fpr = self._sanitiseFingerprint (fpr) + fpr = sanitiseFingerprint (fpr) for a in allowable: - if self._sanitiseFingerprint (a) == fpr: + if sanitiseFingerprint (a) == fpr: return True if app['debug']: @@ -289,13 +290,3 @@ def _getSubDomainTlsFingerprint(self,domain,protocol,port): return tls except: continue - - # Sanitise a fingerprint for comparison. This makes it - # all upper-case and removes colons and spaces. - def _sanitiseFingerprint (self, fpr): - #fpr = fpr.translate (None, ': ') - fpr = fpr.replace (":", "") - fpr = fpr.replace (" ", "") - fpr = fpr.upper () - - return fpr From 040b612b6ccfc3ef161cbcd51b05ae4b14c8bb26 Mon Sep 17 00:00:00 2001 From: JeremyRand Date: Tue, 24 Feb 2015 02:43:56 -0600 Subject: [PATCH 3/7] REST code with python-requests. --- lib/backendDataRest.py | 165 +++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 90 deletions(-) diff --git a/lib/backendDataRest.py b/lib/backendDataRest.py index baaa68c..490e8bc 100644 --- a/lib/backendDataRest.py +++ b/lib/backendDataRest.py @@ -1,123 +1,108 @@ from common import * from utils import * -import hashlib -import httplib -import json -import socket -import ssl +import requests import urllib import urlparse -class MyHTTPSConnection(httplib.HTTPConnection): - """ - Make a HTTPS connection, but support checking the - server's certificate fingerprint. - """ +# This backend requires non-default modules loaded. +# If not using TLS, you can do this on Fedora with: +# yum install python-requests +# If using TLS (experimental), you can do this on Fedora with: +# yum install python-requests pyOpenSSL python-ndg_httpsclient python-pyasn1 - def __init__(self, host, port): - httplib.HTTPConnection.__init__(self, host, port) +# TODO: Finish testing the TLS code. Note that Bitcoin Core is probably removing TLS support soon, so TLS support is solely for things like Nginx proxies. - def connect(self): - sock = socket.create_connection((self.host, self.port)) - self.sock = ssl.wrap_socket(sock) +fp_sha256 = "" - def verifyCert(self, fprs): - """ - Check whether the server's certificate fingerprint - matches the given one. 'fprs' should be a dict with - keys corresponding to digest methods and values being - the digest value. - """ - - hasher = None - fpr = None - - if 'sha256' in fprs: - hasher = hashlib.sha256() - fpr = fprs['sha256'] - elif 'sha1' in fprs: - hasher = hashlib.sha1() - fpr = fprs['sha1'] - - # Be strict here. If this routine is called, it means that at least - # some parameters were given in the REST URI. If we can't verify the - # fingerprint because the parameters were invalid, fail to make sure - # that the user does not expect security but doesn't get it due - # to an operational mistake. - if hasher is None: - if app['debug']: - print "no recognised fingerprint given, failing check" - return False - assert fpr is not None - - cert = self.sock.getpeercert(True) - hasher.update(cert) - digest = hasher.hexdigest() - - if sanitiseFingerprint(digest) != sanitiseFingerprint(fpr): - if app['debug']: - print "Fingerprint mismatch:" - print " expected:", fpr - print " got:", digest - return False - - return True +def assert_fingerprint(connection, x509, errnum, errdepth, ok): + # Accept a cert if verification is forced off, or if it's a non-primary CA cert (the main cert will still be verified), or if the SHA256 matches + return fp_sha256.lower() == "none" or errdepth > 0 or sanitiseFingerprint(x509.digest("sha256")) == sanitiseFingerprint(fp_sha256) class backendData(): validURL = False def __init__(self, conf): + + global fp_sha256 + url = urlparse.urlparse(conf) if url.scheme == 'http' or url.scheme == 'https': self.validURL = True + self.scheme = url.scheme self.tls = (url.scheme == 'https') self.host = url.hostname self.port = url.port + + # Sessions let us reuse TCP connections, while keeping unique identities on different TCP connections + self.sessions = {} + + if self.tls: + try: + # Set ciphers and enable fingerprint verification via PyOpenSSL + requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST = "EDH+aRSA+AES256:EECDH+aRSA+AES256:!SSLv3" + requests.packages.urllib3.contrib.pyopenssl._verify_callback = assert_fingerprint + requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() + except: + if app['debug']: + print "ERROR: Failed to load PyOpenSSL; make sure you have the right packages installed." + print "On Fedora, run:" + print "sudo yum install pyOpenSSL python-ndg_httpsclient python-pyasn1" + print "Other distros/OS's may be similar" + + if app['debug']: + print "WARNING: You are using the experimental REST over TLS feature. This is probably broken and should not be used in production." if url.params == '': - self.fprs = None + self.fprs = {} else: self.fprs = self._parseFprOptions(url.params) + + if "sha256" in self.fprs: + fp_sha256 = self.fprs["sha256"] + + if self.tls and fp_sha256 == "": + if app['debug']: + print "ERROR: REST SHA256 fingerprint missing in plugin-data.conf; REST lookups will fail." + + if "testTlsConfig" in self.fprs: + testResults = self._queryHttpGet("https://www.ssllabs.com/ssltest/viewMyClient.html", "").text + print "TLS test result:" + print testResults + import os + os._exit(0) elif app['debug']: - print "Unsupported scheme for REST URL:", url.scheme + print "ERROR: Unsupported scheme for REST URL:", url.scheme def getAllNames(self): # The REST API doesn't support enumerating the names. - return False - - def getName(self, name): - if not self.validURL: - return "invalid REST URL", None + if app['debug']: + print 'ERROR: REST data backend does not support name enumeration; set import.mode=none or switch to a different import.from backend.' + return (True, None) # TODO: Should this be True rather than False? See the data plugin for usage. + def getName(self, name, sessionId = ""): + encoded = urllib.quote_plus(name) - data = self._queryHttpGet("/rest/name/" + encoded + ".json") - - if data is None: - return "query failed", None - return None, json.loads(data) - - def _queryHttpGet(self, path): - assert self.validURL - if self.tls: - conn = MyHTTPSConnection(self.host, self.port) - else: - conn = httplib.HTTPConnection(self.host, self.port) - conn.request('GET', path) - - if self.tls and (self.fprs is not None): - if not conn.verifyCert(self.fprs): - return "TLS fingerprint wrong", None - - res = conn.getresponse() - if res.status != 200: + + result = self._queryHttpGet(self.scheme + "://" + self.host + ":" + str(self.port) + "/rest/name/" + encoded + ".json", sessionId) + + try: + resultJson = result.json() + except ValueError: + raise Exception("Error parsing REST response. Make sure that Namecoin Core is running with -rest option.") + + return (None, resultJson) + + def _queryHttpGet(self, url, sessionId): + + # set up a session if we haven't yet for this identity (Tor users will use multiple identities) + if sessionId not in self.sessions: if app['debug']: - print "REST returned error code:", res.status - print res.read() - return None - - return res.read() - + print 'Creating new REST identity = "' + sessionId + '"' + self.sessions[sessionId] = requests.Session() + + return self.sessions[sessionId].get(url) + def _parseFprOptions(self, s): """ Parse the REST URI params string that includes (optionally) @@ -133,4 +118,4 @@ def _parseFprOptions(self, s): if len (parts) == 2: res[parts[0]] = parts[1] - return res + return res \ No newline at end of file From 8fb890dbb2de46c49dcdca35719c570c48a712a5 Mon Sep 17 00:00:00 2001 From: JeremyRand Date: Tue, 24 Feb 2015 04:15:56 -0600 Subject: [PATCH 4/7] Added newline at end of REST backend file. --- lib/backendDataRest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/backendDataRest.py b/lib/backendDataRest.py index 490e8bc..9f99dfc 100644 --- a/lib/backendDataRest.py +++ b/lib/backendDataRest.py @@ -118,4 +118,4 @@ def _parseFprOptions(self, s): if len (parts) == 2: res[parts[0]] = parts[1] - return res \ No newline at end of file + return res From 55136b26152fe71f5c57efea8a79037642908ff2 Mon Sep 17 00:00:00 2001 From: JeremyRand Date: Sun, 8 Mar 2015 20:14:59 -0500 Subject: [PATCH 5/7] Fix REST data backend for systems running old versions of python-requests, e.g. Debian Wheezy. --- lib/backendDataRest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/backendDataRest.py b/lib/backendDataRest.py index 9f99dfc..40ab52c 100644 --- a/lib/backendDataRest.py +++ b/lib/backendDataRest.py @@ -4,6 +4,7 @@ import requests import urllib import urlparse +import json # Hack for Debian Wheezy compatibility; remove this import when Wheezy is phased out # This backend requires non-default modules loaded. # If not using TLS, you can do this on Fedora with: @@ -87,7 +88,8 @@ def getName(self, name, sessionId = ""): result = self._queryHttpGet(self.scheme + "://" + self.host + ":" + str(self.port) + "/rest/name/" + encoded + ".json", sessionId) try: - resultJson = result.json() + resultJson = json.loads(result.text) # Hack for Debian Wheezy compatibility; use the following line instead when Wheezy is phased out + # resultJson = result.json() except ValueError: raise Exception("Error parsing REST response. Make sure that Namecoin Core is running with -rest option.") From 6a3245a02b34e677dd09cb0095102486db41d122 Mon Sep 17 00:00:00 2001 From: JeremyRand Date: Wed, 11 Mar 2015 21:41:23 -0500 Subject: [PATCH 6/7] Split off TLS code from REST data backend to its own module. --- lib/backendDataRest.py | 36 ++++++++++++++---------------------- lib/requests_fp.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 22 deletions(-) create mode 100644 lib/requests_fp.py diff --git a/lib/backendDataRest.py b/lib/backendDataRest.py index 40ab52c..35f1162 100644 --- a/lib/backendDataRest.py +++ b/lib/backendDataRest.py @@ -1,7 +1,7 @@ from common import * from utils import * -import requests +import requests_fp import urllib import urlparse import json # Hack for Debian Wheezy compatibility; remove this import when Wheezy is phased out @@ -14,19 +14,11 @@ # TODO: Finish testing the TLS code. Note that Bitcoin Core is probably removing TLS support soon, so TLS support is solely for things like Nginx proxies. -fp_sha256 = "" - -def assert_fingerprint(connection, x509, errnum, errdepth, ok): - # Accept a cert if verification is forced off, or if it's a non-primary CA cert (the main cert will still be verified), or if the SHA256 matches - return fp_sha256.lower() == "none" or errdepth > 0 or sanitiseFingerprint(x509.digest("sha256")) == sanitiseFingerprint(fp_sha256) - class backendData(): validURL = False def __init__(self, conf): - global fp_sha256 - url = urlparse.urlparse(conf) if url.scheme == 'http' or url.scheme == 'https': self.validURL = True @@ -39,17 +31,17 @@ def __init__(self, conf): self.sessions = {} if self.tls: + # Init the TLS security settings try: - # Set ciphers and enable fingerprint verification via PyOpenSSL - requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST = "EDH+aRSA+AES256:EECDH+aRSA+AES256:!SSLv3" - requests.packages.urllib3.contrib.pyopenssl._verify_callback = assert_fingerprint - requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() + requests_fp.init() except: - if app['debug']: - print "ERROR: Failed to load PyOpenSSL; make sure you have the right packages installed." - print "On Fedora, run:" - print "sudo yum install pyOpenSSL python-ndg_httpsclient python-pyasn1" - print "Other distros/OS's may be similar" + print "ERROR: Failed to load PyOpenSSL." + print "Make sure you have the right packages installed." + print "On Fedora, run:" + print "sudo yum install pyOpenSSL python-ndg_httpsclient python-pyasn1" + print "Other distros/OS's may be similar" + import os + os._exit(-1) if app['debug']: print "WARNING: You are using the experimental REST over TLS feature. This is probably broken and should not be used in production." @@ -60,14 +52,14 @@ def __init__(self, conf): self.fprs = self._parseFprOptions(url.params) if "sha256" in self.fprs: - fp_sha256 = self.fprs["sha256"] + requests_fp.add_fingerprint(self.host, self.fprs["sha256"]) - if self.tls and fp_sha256 == "": + if self.tls and "sha256" not in self.fprs: if app['debug']: print "ERROR: REST SHA256 fingerprint missing in plugin-data.conf; REST lookups will fail." if "testTlsConfig" in self.fprs: - testResults = self._queryHttpGet("https://www.ssllabs.com/ssltest/viewMyClient.html", "").text + testResults = requests_fp.test_tls_config() print "TLS test result:" print testResults import os @@ -101,7 +93,7 @@ def _queryHttpGet(self, url, sessionId): if sessionId not in self.sessions: if app['debug']: print 'Creating new REST identity = "' + sessionId + '"' - self.sessions[sessionId] = requests.Session() + self.sessions[sessionId] = requests_fp.Session() return self.sessions[sessionId].get(url) diff --git a/lib/requests_fp.py b/lib/requests_fp.py new file mode 100644 index 0000000..0eece98 --- /dev/null +++ b/lib/requests_fp.py @@ -0,0 +1,36 @@ +from common import * +from utils import * + +from requests import * + +# Stores desired fingerprints +fp_sha256 = {} + +# PyOpenSSL callback +def verify_fingerprint(connection, x509, errnum, errdepth, ok): + host = connection.get_servername() + seen_fp = sanitiseFingerprint(x509.digest("sha256")) + + if app['debug']: + print "Checking TLS cert", seen_fp, "for", host + + # Accept a cert if verification is forced off, or if it's a non-primary CA cert (the main cert will still be verified), or if the SHA256 matches + return (host in fp_sha256 and sanitiseFingerprint("NONE") in fp_sha256[host]) or errdepth > 0 or (host in fp_sha256 and seen_fp in fp_sha256[host]) + +# Add a fingerprint to the whitelist +def add_fingerprint(host, fp): + if host not in fp_sha256: + fp_sha256[host] = [] + + fp_sha256[host].append(sanitiseFingerprint(fp)) + +# Returns HTML analysis from SSLLabs. Output this to a file and view with Javascript disabled. +def test_tls_config(): + return get("https://www.ssllabs.com/ssltest/viewMyClient.html").text + +def init(): + # Set ciphers and enable fingerprint verification via PyOpenSSL + packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST = "EDH+aRSA+AES256:EECDH+aRSA+AES256:!SSLv3" + packages.urllib3.contrib.pyopenssl._verify_callback = verify_fingerprint + packages.urllib3.contrib.pyopenssl.inject_into_urllib3() + From d77c60a7303ba9259115298ef82f71d7ce6726f5 Mon Sep 17 00:00:00 2001 From: JeremyRand Date: Tue, 12 May 2015 15:40:20 -0500 Subject: [PATCH 7/7] REST TLS backend now shows useful error message when using broken PyOpenSSL v0.14. --- lib/requests_fp.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/requests_fp.py b/lib/requests_fp.py index 0eece98..735d6b3 100644 --- a/lib/requests_fp.py +++ b/lib/requests_fp.py @@ -8,7 +8,12 @@ # PyOpenSSL callback def verify_fingerprint(connection, x509, errnum, errdepth, ok): - host = connection.get_servername() + + try: + host = connection.get_servername() + except AttributeError, e: + raise Exception("ERROR: You appear to be on a broken PyOpenSSL version such as 0.14. Please upgrade PyOpenSSL if you wish to use TLS validation. " + str(e)) + seen_fp = sanitiseFingerprint(x509.digest("sha256")) if app['debug']: