diff --git a/.gitignore b/.gitignore index c4ec99a..f9eb7ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ *~ *.cache .#* -.DS_Store \ No newline at end of file +.DS_Store +*.pyc +build/output/* diff --git a/README.md b/README.md index 68bf121..83e7566 100644 --- a/README.md +++ b/README.md @@ -17,3 +17,14 @@ CForm is now available via the Sublime Text package control. See https://packag 6. Save it with the extension `cform`, `template` or `cloudformation` and run it through the CloudFormation console 7. Profit +# Updating the snippets + +## Requirements + +1. You will need ```Python 2.7.10+``` +2. Run ```pip install -r build/requirements.txt``` + +## Updating the docs from the latest + +1. Go inside the build dir from the comand line +2. Run ```rm -rf *.pyc && python build-snippets.py``` diff --git a/build/build-snippets.py b/build/build-snippets.py new file mode 100644 index 0000000..c13f568 --- /dev/null +++ b/build/build-snippets.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +# vim: ai ts=4 sts=4 et sw=4 +"""Build snippets from documentation from AWS.""" +from __future__ import unicode_literals + +import requests +import os +import sys +from collections import OrderedDict +from templating import build_with_template +from lxml import html +from colorama import init +from colorama import Fore +from fabulous.widget import ProgressBar + +BASE_HREF = 'http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/' + + +def print_header(snippets_found, headers): + """Header print function.""" + print('\033c' + Fore.GREEN + + """ + _ _ _ _ __ __ + | | | | (_) | / / / _| + | |__ ___ __ _| | ___ __ _| |_ / /__| |_ ___ _ __ _ __ ___ + | '_ \ / _ \/ _` | |/ / '_ \| | __| / / __| _/ _ \| '__| '_ ` _ ` + | |_) | __/ (_| | <| | | | | |_ / / (__| || (_) | | | | | | | | + |_.__/ \___|\__,_|_|\_\_| |_|_|\__/_/ \___|_| \___/|_| |_| |_| |_|""") + + print(' Updated: ' + Fore.GREEN + '{}'.format(headers['last-modified'])) + print(' Snippets founds: ' + Fore.GREEN + '{}\n'.format(snippets_found)) + + +def build_index(toc_uri, elem_expr): + """Build the main index with key and url for the services.""" + page = requests.get(toc_uri) + doc = html.fromstring(page.text.decode('utf-8')) + tree = doc.xpath(elem_expr) + index = OrderedDict() + + print_header(len(tree), page.headers) + + for e in tree: + arn = e.text_content() + title = e.text_content().replace('::', '-') \ + .replace('AWS-', "") \ + .replace('Fn-', "") \ + .lower() + href = e.get('href') + full_href = BASE_HREF + href + index[arn] = (arn, title, href, full_href) + + return index + + +def generate(index): + """Will traverse index and generate all the snippets.""" + total = len(index) + i = 0 + percent = 0 + progress = ProgressBar() + progress.update(i, 'Generating snippets') + for k, v in index.iteritems(): + (arn, title, href, full_href) = v + snippet = createSnippet(arn, title, href, full_href) + i += 1 + percent = i * 100 / total + progress.update(percent, 'Creating ' + Fore.GREEN + arn) + writeToOutput(title, snippet) + progress.update(percent, 'Snippets completed') + + +def createSnippet(arn, title, href, full_href): + """Create a snippet with the args.""" + elem_expr = '//*[@id="main-col-body"]/div/div/pre/code' + + # uses an old documentation format + if (arn == 'AWS::SDB::Domain'): + elem_expr = '//*[@id="main-col-body"]/div/pre' + + # oddly addition + if (arn == 'AWS::CertificateManager::Certificate'): + elem_expr = '//*[@id="JSON"]/pre/code' + + page = requests.get(full_href) + doc = html.fromstring(page.text) + tree = doc.xpath(elem_expr) + # print 'creating {} => {}'.format(arn, title) + return build_with_template(arn, title, tree, full_href) + + +def safedebug(index, k): + """Debug helper for scrapping problems.""" + (arn, title, href, full_href) = index[k] + snippet = createSnippet(arn, title, href, full_href) + print snippet + + +def writeToOutput(title, snippet): + """Write the snippet to the output folder.""" + default_folder = "./output/" + default_suffix = ".sublime-snippet" + + if (not os.path.exists(default_folder)): + os.makedirs(default_folder) + + filename = default_folder + title + default_suffix + out = file(filename, 'w') + out.write(snippet.encode('utf8', 'replace')) + out.close() + + +def generate_functions(): + """Function are a special cases, they're scaterred all over the docs.""" + toc_functions = [ + ('Fn::Select', 'fn-select', '{ "Fn::Select" : [ index, listOfObjects ] }', 'intrinsic-function-reference-select.html', BASE_HREF + 'intrinsic-function-reference-select.html'), + ('Ref', 'ref', '"Ref" : "logicalName"', 'intrinsic-function-reference-ref.html', BASE_HREF + 'intrinsic-function-reference-ref.html'), + ('Fn::Join', 'fn-join', '"Fn::Join" : [ "delimiter", [ comma-delimited list of values ] ]', 'intrinsic-function-reference-join.html', BASE_HREF + 'intrinsic-function-reference-join.html'), + ('Fn::GetAZs', 'fn-getAZs', '"Fn::GetAZs" : "region"', 'intrinsic-function-reference-getavailabilityzones.html', BASE_HREF + 'intrinsic-function-reference-getavailabilityzones.html'), + ('Fn::GetAtt', 'fn-getatt', '"Fn::GetAtt" : [ "logicalNameOfResource", "attributeName" ]', 'intrinsic-function-reference-getatt.html', BASE_HREF + 'intrinsic-function-reference-getatt.html'), + ('Fn::FindInMap', 'fn-findInMap', '"Fn::FindInMap" : [ "MapName", "TopLevelKey", "SecondLevelKey"]', 'intrinsic-function-reference-findinmap.html', BASE_HREF + 'intrinsic-function-reference-findinmap.html') + ] + i = 0 + percent = 0 + total = len(toc_functions) + print(' Functions found: ' + Fore.GREEN + str(total)) + + progress = ProgressBar() + progress.update(i, 'Generating functions') + + for v in toc_functions: + (arn, title, body, href, full_href) = v + i += 1 + percent = i * 100 / total + progress.update(percent, 'Creating ' + Fore.GREEN + arn) + writeToOutput(title, build_with_template(arn, title, body, full_href)) + + progress.update(percent, 'Functions generated.') + + +def main(): + """Main doc generation.""" + init(autoreset=True) # colorama + + # build topics + toc_uri = BASE_HREF + 'aws-template-resource-type-ref.html' + elem_expr = '//*[@id="main-col-body"]/div[2]/div/ul/li/a' + index = build_index(toc_uri, elem_expr) + # safedebug(index, 'AWS::AutoScaling::ScheduledAction') + # sys.exit(0) + generate(index) + + # build functions + generate_functions() + print('\n Snippets generated, have fun.') + print(' Got questions or want something on this? ' + + ' https://github.com/beaknit/cform') + + +if __name__ == "__main__": + main() diff --git a/build/requirements.txt b/build/requirements.txt new file mode 100644 index 0000000..eef2bec --- /dev/null +++ b/build/requirements.txt @@ -0,0 +1,4 @@ +requests +lxml +colorama +fabulous diff --git a/build/templating.py b/build/templating.py new file mode 100644 index 0000000..f61af5a --- /dev/null +++ b/build/templating.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# vim: ai ts=4 sts=4 et sw=4 +"""Templating operations for the snippets created by the build script.""" + +from __future__ import unicode_literals + + +def build_with_template(trigger, title, body, full_href): + """ + Template generation. + + With all args give, transform body with param list and return + a complete snippet. + """ + # print trigger + # print body + if (isinstance(body, list)): + processed_body = """"${{1:-}}" : {}""".format((body[0]).text_content()) + else: + processed_body = """"${{1:-}}" : {}""".format((body)) + + template = """ + + + + + {} + + source.cloudformation + + """.format(full_href, processed_body, title) + + transformed_template = transform(template) + + return transformed_template + + +def transform(template): + """Transform the template to adapt the parameters of the template.""" + return template \ + .replace('JSON object', '[]') \ + .replace(': String', '""') \ + .replace(': Integer', '') \ + .replace(': Boolean', '${true | false}')