Skip to content

Bug fixes and enhancements #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
syskill wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Bug fixes and enhancements #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
958032b
Fix use of global variables as function default argument values
syskill Jan 21, 2022
27230d0
Remove extraneous settings from config file
syskill Jan 21, 2022
3074872
Close HTTP response and error objects after reading
syskill Jan 21, 2022
d7ba17a
Enable IMDSv2
syskill Jan 21, 2022
c417342
Option to get credentials from ECS metadata service
syskill Jan 21, 2022
5509aa1
Default plugin configuration should be empty
syskill Jan 26, 2022
35b8aab
Add command-line option for relative path in container metadata service
syskill Jan 26, 2022
22218a2
Expect that option parsing might be unavailable
syskill Feb 10, 2022
7957148
Append newline to URL during request canonicaliztion
syskill Jun 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 5 additions & 14 deletions cob.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,11 @@
; limitations under the License.
;
[main]
cachedir=/var/cache/yum/$basearch/$releasever
keepcache=1
debuglevel=4
logfile=/var/log/yum.log
exactarch=1
obsoletes=0
gpgcheck=0
plugins=1
distroverpkg=centos-release
enabled=1

[aws]
# access_key =
# secret_key =
timeout = 60
retries = 5
metadata_server = http://169.254.169.254
# metadata_server = http://192.0.2.169 ; alternate URL for metadata server
# timeout = 60
# retries = 5
# access_key = ; AWS credentials may be configured here, rather than
# secret_key = ; retrieved from metadata server
126 changes: 94 additions & 32 deletions cob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,16 @@

__all__ = ['requires_api_version',
'plugin_type',
'init_hook']
'init_hook',
'prereposetup_hook']

requires_api_version = '2.5'
plugin_type = yum.plugins.TYPE_CORE

timeout = 60
retries = 5
metadata_server = "http://169.254.169.254"
imds_token = None

EMPTY_SHA256_HASH = (
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
Expand Down Expand Up @@ -160,7 +162,7 @@ def signed_headers(self, headers_to_sign):
def canonical_request(self, request):
cr = [request.method.upper()]
path = self._normalize_url_path(urlsplit(request.url).path)
cr.append(path)
cr.append(path + '\n')
headers_to_sign = self.headers_to_sign(request)
cr.append(self.canonical_headers(headers_to_sign) + '\n')
cr.append(self.signed_headers(headers_to_sign))
Expand Down Expand Up @@ -274,7 +276,7 @@ def get_region_from_s3url(url):
return "us-east-1"


def retry_url(url, retry_on_404=False, num_retries=retries, timeout=timeout):
def retry_url(url, retry_on_404=False, method=None, add_headers=[]):
"""
Retry a url. This is specifically used for accessing the metadata
service on an instance. Since this address should never be proxied
Expand All @@ -285,48 +287,73 @@ def retry_url(url, retry_on_404=False, num_retries=retries, timeout=timeout):
original = socket.getdefaulttimeout()
socket.setdefaulttimeout(timeout)

for i in range(0, num_retries):
add_headers = list(add_headers)
if imds_token:
add_headers.append(('X-aws-ec2-metadata-token', imds_token))

for i in range(0, retries):
try:
proxy_handler = urllib2.ProxyHandler({})
opener = urllib2.build_opener(proxy_handler)
if add_headers:
opener.addheaders = add_headers
req = urllib2.Request(url)
if method:
req.get_method = lambda: method
r = opener.open(req)
result = r.read()
r.close()
return result
except urllib2.HTTPError as e:
# in 2.6 you use getcode(), in 2.5 and earlier you use code
if hasattr(e, 'getcode'):
code = e.getcode()
else:
code = e.code
e.close()
if code == 404 and not retry_on_404:
return None
except Exception as e:
pass
print '[ERROR] Caught exception reading instance data'
# If not on the last iteration of the loop then sleep.
if i + 1 != num_retries:
if i + 1 != retries:
time.sleep(2 ** i)
print '[ERROR] Unable to read instance data, giving up'
return None


def get_region(url=metadata_server, version="latest",
def get_imds_token(version="latest",
params="api/token",
ttl=21600):
"""
Get an IMDSv2 token.
"""
url = urlparse.urljoin(metadata_server, "/".join([version, params]))
result = retry_url(url, method="PUT", add_headers=[('X-aws-ec2-metadata-token-ttl-seconds', str(ttl))])
if result is None:
#print "Could not get IMDSv2 token; is IMDSv2 enabled?"
return None
else:
return result


def get_region(version="latest",
params="meta-data/placement/availability-zone/"):
"""
Fetch the region from AWS metadata store.
"""
url = urlparse.urljoin(url, "/".join([version, params]))
url = urlparse.urljoin(metadata_server, "/".join([version, params]))
result = retry_url(url)
return result[:-1].strip()


def get_iam_role(url=metadata_server, version="latest",
def get_iam_role(version="latest",
params="meta-data/iam/security-credentials/"):
"""
Read IAM role from AWS metadata store.
"""
url = urlparse.urljoin(url, "/".join([version, params]))
url = urlparse.urljoin(metadata_server, "/".join([version, params]))
result = retry_url(url)
if result is None:
# print "No IAM role found in the machine"
Expand All @@ -335,17 +362,14 @@ def get_iam_role(url=metadata_server, version="latest",
return result


def get_credentials_from_iam_role(url=metadata_server,
version="latest",
params="meta-data/iam/security-credentials",
iam_role=None):
def get_credentials_from_path(path):
"""
Read IAM credentials from AWS metadata store.
Read IAM credentials from a given path in the AWS metadata store.
"""
url = urlparse.urljoin(url, "/".join([version, params, iam_role]))
url = urlparse.urljoin(metadata_server, path)
result = retry_url(url)
if result is None:
# print "No IAM credentials found in the machine"
# print "No credentials found at URL", repr(url)
return None
try:
data = json.loads(result)
Expand All @@ -365,7 +389,26 @@ def get_credentials_from_iam_role(url=metadata_server,
token.encode("utf-8"))


def get_credentials_for_iam_role(iam_role,
version="latest",
params="meta-data/iam/security-credentials"):
"""
Read IAM role credentials from AWS metadata store.
"""
return get_credentials_from_path("/".join([version, params, iam_role]))


def init_hook(conduit):
"""
Add argument for relative path in container credentials metadata service
"""
parser = conduit.getOptParser()
if parser:
parser.add_option("--aws-container-credentials-relative-uri",
dest='aws_container_credentials_relative_uri')


def prereposetup_hook(conduit):
"""
Setup the S3 repositories
"""
Expand Down Expand Up @@ -459,7 +502,7 @@ def _getFile(self, url=None, relative=None, local=None,
def set_region(self):

# Fetch params from local config file
global timeout, retries, metadata_server
global timeout, retries, metadata_server, imds_token
timeout = self.conduit.confInt('aws', 'timeout', default=timeout)
retries = self.conduit.confInt('aws', 'retries', default=retries)
metadata_server = self.conduit.confString('aws',
Expand All @@ -474,6 +517,9 @@ def set_region(self):
if self.region:
return True

# Try to get IMDSv2 token
imds_token = imds_token or get_imds_token()

# Fetch region from meta data
region = get_region()
if region is None:
Expand All @@ -487,7 +533,7 @@ def set_region(self):
def set_credentials(self):

# Fetch params from local config file
global timeout, retries, metadata_server
global timeout, retries, metadata_server, imds_token
timeout = self.conduit.confInt('aws', 'timeout', default=timeout)
retries = self.conduit.confInt('aws', 'retries', default=retries)
metadata_server = self.conduit.confString('aws',
Expand All @@ -505,27 +551,43 @@ def set_credentials(self):
if self.access_key and self.secret_key:
return True

# Fetch credentials from iam role meta data
iam_role = get_iam_role()
if iam_role is None:
self.conduit.info(3, "[ERROR] No credentials in the plugin conf "
"for the repo '%s'" % self.repoid)
raise IncorrectCredentialsError

credentials = get_credentials_from_iam_role(iam_role=iam_role)
if credentials is None:
self.conduit.info(3, "[ERROR] Fail to get IAM credentials"
"for the repo '%s'" % self.repoid)
raise IncorrectCredentialsError
opts, cmd = self.conduit.getCmdLine()
if opts and opts.aws_container_credentials_relative_uri:
# Reload metadata server address, default to ECS metadata service
metadata_server = self.conduit.confString('aws',
'metadata_server',
default="http://169.254.170.2")

# Fetch credentials from given path
credentials = get_credentials_from_path(opts.aws_container_credentials_relative_uri)
if credentials is None:
self.conduit.info(3, "[ERROR] Fail to get container credentials"
"for the repo '%s'" % self.repoid)
raise IncorrectCredentialsError
else:
# Try to get IMDSv2 token
imds_token = imds_token or get_imds_token()

# Fetch credentials from iam role meta data
iam_role = get_iam_role()
if iam_role is None:
self.conduit.info(3, "[ERROR] No credentials in the plugin conf "
"for the repo '%s'" % self.repoid)
raise IncorrectCredentialsError

credentials = get_credentials_for_iam_role(iam_role)
if credentials is None:
self.conduit.info(3, "[ERROR] Fail to get IAM credentials"
"for the repo '%s'" % self.repoid)
raise IncorrectCredentialsError

self.access_key, self.secret_key, self.token = credentials
return True

def fetch_headers(self, url, path):
headers = {}

# "\n" in the url, required by AWS S3 Auth v4
url = urlparse.urljoin(url, urllib2.quote(path)) + "\n"
url = urlparse.urljoin(url, urllib2.quote(path))
credentials = Credentials(self.access_key, self.secret_key, self.token)
request = HTTPRequest("GET", url)
signer = S3SigV4Auth(credentials, "s3", self.region, self.conduit)
Expand Down