From 45f501a6c3fde5a1be6aba2108e24a45280f7dcd Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 3 Jul 2017 16:50:36 +0100 Subject: [PATCH 01/11] Initial txaws prototype --- setup.py | 3 + src/txacme/challenges/_route53.py | 181 ++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 src/txacme/challenges/_route53.py diff --git a/setup.py b/setup.py index ea196f4..d9e4a28 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,9 @@ def read(*parts): 'libcloud': [ 'apache-libcloud', ], + 'txaws': [ + 'txaws', + ], 'test': [ 'fixtures>=1.4.0', 'hypothesis>=3.1.0,<4.0.0', diff --git a/src/txacme/challenges/_route53.py b/src/txacme/challenges/_route53.py new file mode 100644 index 0000000..359bc0a --- /dev/null +++ b/src/txacme/challenges/_route53.py @@ -0,0 +1,181 @@ +import hashlib + +import attr + +from acme import jose +from txaws import AWSServiceRegion +from txaws.route53.model import ( + RRSetKey, RRSet, Name, TXT, create_rrset, upsert_rrset, delete_rrset +) +from zope.interface import implementer + +from txacme.errors import ZoneNotFound +from txacme.interfaces import IResponder + + +def _validation(response): + """ + Get the validation value for a challenge response. + """ + # TODO: This is just duplicated directly from _libcloud.py. Should we hoist + # this out to a common utility module? + h = hashlib.sha256(response.key_authorization.encode("utf-8")) + return jose.b64encode(h.digest()).decode() + + +def _add_txt_record(args, full_name, validation, client): + """ + Adds a TXT record for full_name to the appropriate resource record set in + Route 53, with the value set to validation. + + This is implemented by doing an UPSERT of the RR set if it already exists: + otherwise, we'll need to create a new one. + """ + zone_id, rr_sets = args + + # Right off the bat we can create the record we're going to insert. A + # quirk of Route53 is that these need to be surrounded by dquotes. + resource_record = TXT(texts=(u'"%s"' % validation,)) + + # We're interested only in the TXT RR Set. If it exists, we're going to + # update ('upsert') it. If it does not exist, we're going to create it. + key = RRSetKey(label=Name(full_name), type=u'TXT') + + try: + rr_set = rr_sets[key] + except KeyError: + rr_set = RRSet( + name=full_name, type=u'TXT', ttl=300, records=set(resource_record)) + + rr_set.records.add(resource_record) + rr_set_update = upsert_rrset(rr_set) + + return client.change_resource_record_sets(zone_id, rr_set_update) + + +def _delete_txt_record(args, full_name, validation, client): + """ + Deletes a TXT record for full_name from the appropriate resource record set + in Route 53, with the value set to validation. + + This is implemented by doing an UPSERT of any RR set with multiple values, + or by deleting a complete RR set if there are no other values. + """ + zone_id, rr_sets = args + + # Right off the bat we can create the record we're going to remove. A + # quirk of Route53 is that these need to be surrounded by dquotes. + resource_record = TXT(texts=(u'"%s"' % validation,)) + + # We're interested only in the TXT RR Set. We expect this to exist: if it + # doesn't, we'll just quietly exit as we have no work to do. + key = RRSetKey(label=Name(full_name), type=u'TXT') + + try: + rr_set = rr_sets[key] + except KeyError: + return + + # Now we want to check that the record is in the RR set. If it isn't, we + # again quietly exit. + if resource_record not in rr_set: + return + + if len(rr_set) == 1: + rr_set_update = delete_rrset(rr_set) + else: + rr_set.records.remove(resource_record) + rr_set_update = upsert_rrset(rr_set) + + return client.change_resource_record_sets(zone_id, rr_set_update) + + +def _get_rr_sets_for_zone(zone_id, client): + """ + Given a single zone ID, returns a tuple of that zone ID and the RRSets for + that zone. + """ + d = client.list_resource_record_sets(zone_id) + d.addCallback(lambda rr_sets: (zone_id, rr_sets)) + return d + + +def _get_zone_id(zones, server_name): + """ + Given the collection of zones in Route53, returns the zone ID of the one + that is appropriate for use with this challenge. If no zones are, then this + raises a ZoneNotFound error. + """ + for zone in zones: + if server_name.endswith(zone.name): + return zone.identifier + + raise ZoneNotFound(u"Unable to find zone for %s" % server_name) + + +@attr.s(hash=False) +@implementer(IResponder) +class Route53DNSResponder(object): + """ + A ``dns-01`` challenge responder using txaws and Route53. + """ + challenge_type = u'dns-01' + + _client = attr.ib() + access_key = attr.ib() + secret_key = attr.ib() + settle_delay = attr.ib() + + @classmethod + def create(cls, access_key, secret_key, settle_delay=60.0): + """ + Create a responder. + + :param str access_key: The AWS IAM access key to use. + :param str secret_key: The AWS IAM secret key to use. + :param float settle_delay: The time, in seconds, to allow for the DNS + provider to propagate record changes. + """ + region = AWSServiceRegion(access_key=access_key, secret_key=secret_key) + return cls( + client=region.get_route53_client(), + settle_delay=settle_delay) + + def start_responding(self, server_name, challenge, response): + """ + Install a TXT challenge response record. + """ + validation = _validation(response) + full_name = challenge.validation_domain_name(server_name) + + d = self._client.list_hosted_zones() + d.addCallback(_get_zone_id, server_name=full_name) + d.addCallback(_get_rr_sets_for_zone, client=self._client) + d.addCallback( + _add_txt_record, + full_name=full_name, + validation=validation, + client=self._client + ) + + return d + + def stop_responding(self, server_name, challenge, response): + """ + Remove a TXT challenge response record. + """ + validation = _validation(response) + full_name = challenge.validation_domain_name(server_name) + + d = self._client.list_hosted_zones() + d.addCallback(_get_zone_id, server_name=full_name) + d.addCallback(_get_rr_sets_for_zone, client=self._client) + d.addCallback( + _delete_txt_record, + full_name=full_name, + validation=validation, + client=self._client + ) + + +__all__ = ['Route53DNSResponder'] From 3f7770b540b675a24b19f69bcfcf18ee47ef1cc1 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 3 Jul 2017 16:58:44 +0100 Subject: [PATCH 02/11] Add a settle delay --- src/txacme/challenges/_route53.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/txacme/challenges/_route53.py b/src/txacme/challenges/_route53.py index 359bc0a..94445c7 100644 --- a/src/txacme/challenges/_route53.py +++ b/src/txacme/challenges/_route53.py @@ -3,6 +3,7 @@ import attr from acme import jose +from twisted.internet.defer import Deferred from txaws import AWSServiceRegion from txaws.route53.model import ( RRSetKey, RRSet, Name, TXT, create_rrset, upsert_rrset, delete_rrset @@ -13,6 +14,18 @@ from txacme.interfaces import IResponder +def _sleep(rval, reactor, delay): + """ + Returns a Deferred that does not fire until delay seconds have passed. This + is used simply to encourage a Twisted application to wait for the DNS + propagation to settle. This is expected to be used from + Deferred.addCallback, so it propagates the value it was called with to the + rest of the defer chain. + """ + d = Deferred() + reactor.callLater(delay, d.callback, rval) + return d + def _validation(response): """ Get the validation value for a challenge response. @@ -121,16 +134,16 @@ class Route53DNSResponder(object): """ challenge_type = u'dns-01' + _reactor = attr.ib() _client = attr.ib() - access_key = attr.ib() - secret_key = attr.ib() settle_delay = attr.ib() @classmethod - def create(cls, access_key, secret_key, settle_delay=60.0): + def create(cls, reactor, access_key, secret_key, settle_delay=60.0): """ Create a responder. + :param reactor: The Twisted reactor to use for delay support. :param str access_key: The AWS IAM access key to use. :param str secret_key: The AWS IAM secret key to use. :param float settle_delay: The time, in seconds, to allow for the DNS @@ -138,6 +151,7 @@ def create(cls, access_key, secret_key, settle_delay=60.0): """ region = AWSServiceRegion(access_key=access_key, secret_key=secret_key) return cls( + reactor=reactor, client=region.get_route53_client(), settle_delay=settle_delay) @@ -157,6 +171,7 @@ def start_responding(self, server_name, challenge, response): validation=validation, client=self._client ) + d.addCallback(_sleep, reactor=self._reactor, delay=self.settle_delay) return d From d27af487bd3aa1fd8cc394aad8d6f27ef7e52d79 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 3 Jul 2017 17:04:43 +0100 Subject: [PATCH 03/11] Export Route53DNSResponder --- src/txacme/challenges/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/txacme/challenges/__init__.py b/src/txacme/challenges/__init__.py index 48920da..8ff8b71 100644 --- a/src/txacme/challenges/__init__.py +++ b/src/txacme/challenges/__init__.py @@ -9,4 +9,12 @@ pass -__all__ = ['HTTP01Responder', 'LibcloudDNSResponder', 'TLSSNI01Responder'] +try: + from ._route53 import Route53DNSResponder +except ImportError: + # txaws may not be installed + pass + + +__all__ = ['HTTP01Responder', 'LibcloudDNSResponder', 'TLSSNI01Responder', + 'Route53DNSResponder'] From b7ed669e75370a259d9519e71b40fba29c1d111a Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 3 Jul 2017 17:57:35 +0100 Subject: [PATCH 04/11] Miscellaneous cleanups --- src/txacme/challenges/_route53.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/txacme/challenges/_route53.py b/src/txacme/challenges/_route53.py index 94445c7..ca94f17 100644 --- a/src/txacme/challenges/_route53.py +++ b/src/txacme/challenges/_route53.py @@ -4,7 +4,7 @@ from acme import jose from twisted.internet.defer import Deferred -from txaws import AWSServiceRegion +from txaws.service import AWSServiceRegion from txaws.route53.model import ( RRSetKey, RRSet, Name, TXT, create_rrset, upsert_rrset, delete_rrset ) @@ -48,7 +48,7 @@ def _add_txt_record(args, full_name, validation, client): # Right off the bat we can create the record we're going to insert. A # quirk of Route53 is that these need to be surrounded by dquotes. - resource_record = TXT(texts=(u'"%s"' % validation,)) + resource_record = TXT(texts=(validation,)) # We're interested only in the TXT RR Set. If it exists, we're going to # update ('upsert') it. If it does not exist, we're going to create it. @@ -58,12 +58,16 @@ def _add_txt_record(args, full_name, validation, client): rr_set = rr_sets[key] except KeyError: rr_set = RRSet( - name=full_name, type=u'TXT', ttl=300, records=set(resource_record)) + label=Name(full_name), + type=u'TXT', + ttl=300, + records=set([resource_record]) + ) rr_set.records.add(resource_record) rr_set_update = upsert_rrset(rr_set) - return client.change_resource_record_sets(zone_id, rr_set_update) + return client.change_resource_record_sets(zone_id, [rr_set_update]) def _delete_txt_record(args, full_name, validation, client): @@ -78,7 +82,7 @@ def _delete_txt_record(args, full_name, validation, client): # Right off the bat we can create the record we're going to remove. A # quirk of Route53 is that these need to be surrounded by dquotes. - resource_record = TXT(texts=(u'"%s"' % validation,)) + resource_record = TXT(texts=(validation,)) # We're interested only in the TXT RR Set. We expect this to exist: if it # doesn't, we'll just quietly exit as we have no work to do. @@ -91,16 +95,16 @@ def _delete_txt_record(args, full_name, validation, client): # Now we want to check that the record is in the RR set. If it isn't, we # again quietly exit. - if resource_record not in rr_set: + if resource_record not in rr_set.records: return - if len(rr_set) == 1: + if len(rr_set.records) == 1: rr_set_update = delete_rrset(rr_set) else: rr_set.records.remove(resource_record) rr_set_update = upsert_rrset(rr_set) - return client.change_resource_record_sets(zone_id, rr_set_update) + return client.change_resource_record_sets(zone_id, [rr_set_update]) def _get_rr_sets_for_zone(zone_id, client): @@ -160,7 +164,7 @@ def start_responding(self, server_name, challenge, response): Install a TXT challenge response record. """ validation = _validation(response) - full_name = challenge.validation_domain_name(server_name) + full_name = challenge.validation_domain_name(server_name) + u'.' d = self._client.list_hosted_zones() d.addCallback(_get_zone_id, server_name=full_name) @@ -180,7 +184,7 @@ def stop_responding(self, server_name, challenge, response): Remove a TXT challenge response record. """ validation = _validation(response) - full_name = challenge.validation_domain_name(server_name) + full_name = challenge.validation_domain_name(server_name) + u'.' d = self._client.list_hosted_zones() d.addCallback(_get_zone_id, server_name=full_name) From 3a8671f01ec7ac59c0264a5de6e04e350d6503f7 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 3 Jul 2017 17:57:55 +0100 Subject: [PATCH 05/11] This comment is lies --- src/txacme/challenges/_route53.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/txacme/challenges/_route53.py b/src/txacme/challenges/_route53.py index ca94f17..40f17aa 100644 --- a/src/txacme/challenges/_route53.py +++ b/src/txacme/challenges/_route53.py @@ -46,8 +46,7 @@ def _add_txt_record(args, full_name, validation, client): """ zone_id, rr_sets = args - # Right off the bat we can create the record we're going to insert. A - # quirk of Route53 is that these need to be surrounded by dquotes. + # Right off the bat we can create the record we're going to insert. resource_record = TXT(texts=(validation,)) # We're interested only in the TXT RR Set. If it exists, we're going to @@ -80,8 +79,7 @@ def _delete_txt_record(args, full_name, validation, client): """ zone_id, rr_sets = args - # Right off the bat we can create the record we're going to remove. A - # quirk of Route53 is that these need to be surrounded by dquotes. + # Right off the bat we can create the record we're going to remove. resource_record = TXT(texts=(validation,)) # We're interested only in the TXT RR Set. We expect this to exist: if it From c1c1dd9ca5fb899569b68c5f668465ce1f7c66d9 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 4 Jul 2017 18:41:04 +0100 Subject: [PATCH 06/11] A better _sleep --- src/txacme/challenges/_route53.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/txacme/challenges/_route53.py b/src/txacme/challenges/_route53.py index 40f17aa..75d599f 100644 --- a/src/txacme/challenges/_route53.py +++ b/src/txacme/challenges/_route53.py @@ -4,6 +4,7 @@ from acme import jose from twisted.internet.defer import Deferred +from twisted.internet.task import deferLater from txaws.service import AWSServiceRegion from txaws.route53.model import ( RRSetKey, RRSet, Name, TXT, create_rrset, upsert_rrset, delete_rrset @@ -22,9 +23,7 @@ def _sleep(rval, reactor, delay): Deferred.addCallback, so it propagates the value it was called with to the rest of the defer chain. """ - d = Deferred() - reactor.callLater(delay, d.callback, rval) - return d + return deferLater(reactor, delay, lambda: rval) def _validation(response): """ From 759b00b6d1165410b66c500346c8c4fe397fa109 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 4 Jul 2017 18:43:24 +0100 Subject: [PATCH 07/11] Factor out common DNS utility code --- src/txacme/challenges/_dnsutil.py | 13 +++++++++++++ src/txacme/challenges/_libcloud.py | 11 +---------- src/txacme/challenges/_route53.py | 13 +------------ 3 files changed, 15 insertions(+), 22 deletions(-) create mode 100644 src/txacme/challenges/_dnsutil.py diff --git a/src/txacme/challenges/_dnsutil.py b/src/txacme/challenges/_dnsutil.py new file mode 100644 index 0000000..639cb45 --- /dev/null +++ b/src/txacme/challenges/_dnsutil.py @@ -0,0 +1,13 @@ +import hashlib + +from acme import jose + + +def _validation(response): + """ + Get the validation value for a challenge response. + """ + # TODO: This is just duplicated directly from _libcloud.py. Should we hoist + # this out to a common utility module? + h = hashlib.sha256(response.key_authorization.encode("utf-8")) + return jose.b64encode(h.digest()).decode() diff --git a/src/txacme/challenges/_libcloud.py b/src/txacme/challenges/_libcloud.py index 4960b8a..abe7372 100644 --- a/src/txacme/challenges/_libcloud.py +++ b/src/txacme/challenges/_libcloud.py @@ -1,15 +1,14 @@ -import hashlib import time from threading import Thread import attr -from acme import jose from libcloud.dns.providers import get_driver from twisted._threads import pool from twisted.internet.defer import Deferred from twisted.python.failure import Failure from zope.interface import implementer +from txacme.challenges._dnsutil import _validation from txacme.errors import NotInZone, ZoneNotFound from txacme.interfaces import IResponder from txacme.util import const @@ -90,14 +89,6 @@ def _get_existing(driver, zone_name, server_name, validation): return zone, existing, subdomain -def _validation(response): - """ - Get the validation value for a challenge response. - """ - h = hashlib.sha256(response.key_authorization.encode("utf-8")) - return jose.b64encode(h.digest()).decode() - - @attr.s(hash=False) @implementer(IResponder) class LibcloudDNSResponder(object): diff --git a/src/txacme/challenges/_route53.py b/src/txacme/challenges/_route53.py index 75d599f..475bb3f 100644 --- a/src/txacme/challenges/_route53.py +++ b/src/txacme/challenges/_route53.py @@ -1,8 +1,5 @@ -import hashlib - import attr -from acme import jose from twisted.internet.defer import Deferred from twisted.internet.task import deferLater from txaws.service import AWSServiceRegion @@ -11,6 +8,7 @@ ) from zope.interface import implementer +from txacme.challenges._dnsutil import _validation from txacme.errors import ZoneNotFound from txacme.interfaces import IResponder @@ -25,15 +23,6 @@ def _sleep(rval, reactor, delay): """ return deferLater(reactor, delay, lambda: rval) -def _validation(response): - """ - Get the validation value for a challenge response. - """ - # TODO: This is just duplicated directly from _libcloud.py. Should we hoist - # this out to a common utility module? - h = hashlib.sha256(response.key_authorization.encode("utf-8")) - return jose.b64encode(h.digest()).decode() - def _add_txt_record(args, full_name, validation, client): """ From 1aa2d33083d6021adefaa9bf08c76c15e9c166e7 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 4 Jul 2017 18:46:14 +0100 Subject: [PATCH 08/11] Make settle_delay private --- src/txacme/challenges/_route53.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/txacme/challenges/_route53.py b/src/txacme/challenges/_route53.py index 475bb3f..fb8dcee 100644 --- a/src/txacme/challenges/_route53.py +++ b/src/txacme/challenges/_route53.py @@ -1,6 +1,6 @@ import attr -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, succeed from twisted.internet.task import deferLater from txaws.service import AWSServiceRegion from txaws.route53.model import ( @@ -77,12 +77,12 @@ def _delete_txt_record(args, full_name, validation, client): try: rr_set = rr_sets[key] except KeyError: - return + return succeed(None) # Now we want to check that the record is in the RR set. If it isn't, we # again quietly exit. if resource_record not in rr_set.records: - return + return succeed(None) if len(rr_set.records) == 1: rr_set_update = delete_rrset(rr_set) @@ -126,10 +126,10 @@ class Route53DNSResponder(object): _reactor = attr.ib() _client = attr.ib() - settle_delay = attr.ib() + _settle_delay = attr.ib() @classmethod - def create(cls, reactor, access_key, secret_key, settle_delay=60.0): + def create(cls, reactor, access_key, secret_key): """ Create a responder. @@ -139,6 +139,11 @@ def create(cls, reactor, access_key, secret_key, settle_delay=60.0): :param float settle_delay: The time, in seconds, to allow for the DNS provider to propagate record changes. """ + # This isn't publicly exposed because we want to wait for txaws to + # support the DNS change status API: + # http://docs.aws.amazon.com/Route53/latest/APIReference/API_GetChange.html + # Until that time, we hard-code a settle delay. + settle_delay = 60.0 region = AWSServiceRegion(access_key=access_key, secret_key=secret_key) return cls( reactor=reactor, @@ -161,7 +166,7 @@ def start_responding(self, server_name, challenge, response): validation=validation, client=self._client ) - d.addCallback(_sleep, reactor=self._reactor, delay=self.settle_delay) + d.addCallback(_sleep, reactor=self._reactor, delay=self._settle_delay) return d From 0e9ff801bf17250029b29446fef1e1b0f038b682 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 4 Jul 2017 18:48:22 +0100 Subject: [PATCH 09/11] Remove redundant docstring --- src/txacme/challenges/_route53.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/txacme/challenges/_route53.py b/src/txacme/challenges/_route53.py index fb8dcee..13e1cd4 100644 --- a/src/txacme/challenges/_route53.py +++ b/src/txacme/challenges/_route53.py @@ -136,8 +136,6 @@ def create(cls, reactor, access_key, secret_key): :param reactor: The Twisted reactor to use for delay support. :param str access_key: The AWS IAM access key to use. :param str secret_key: The AWS IAM secret key to use. - :param float settle_delay: The time, in seconds, to allow for the DNS - provider to propagate record changes. """ # This isn't publicly exposed because we want to wait for txaws to # support the DNS change status API: From b3e774ba5f7380113e4d403fcf1ef90ad7239009 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 6 Jul 2017 09:55:42 +0100 Subject: [PATCH 10/11] Remove unused imports --- src/txacme/challenges/_route53.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/txacme/challenges/_route53.py b/src/txacme/challenges/_route53.py index 13e1cd4..41ad153 100644 --- a/src/txacme/challenges/_route53.py +++ b/src/txacme/challenges/_route53.py @@ -1,10 +1,10 @@ import attr -from twisted.internet.defer import Deferred, succeed +from twisted.internet.defer import succeed from twisted.internet.task import deferLater from txaws.service import AWSServiceRegion from txaws.route53.model import ( - RRSetKey, RRSet, Name, TXT, create_rrset, upsert_rrset, delete_rrset + RRSetKey, RRSet, Name, TXT, upsert_rrset, delete_rrset ) from zope.interface import implementer From d5aaeed44f48b149d3d463ba5f8998b641d80de8 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 6 Jul 2017 10:06:07 +0100 Subject: [PATCH 11/11] Also txaws for documentation --- requirements-doc.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-doc.txt b/requirements-doc.txt index 9b03fc8..ce08fbc 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,4 +1,4 @@ -.[test,libcloud] +.[test,libcloud,txaws] sphinx sphinx_rtd_theme repoze.sphinx.autointerface>=0.8