From 7ee85e9097819cea6e64373b2582c85685df101b Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Mon, 1 Aug 2016 22:44:56 -0300 Subject: [PATCH 01/14] initial commit of the scrapper for beaknit/cform#34 --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c4ec99a..9cc8f07 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ *~ *.cache .#* -.DS_Store \ No newline at end of file +.DS_Store From d36dcb0db32a292beb60363e50b4526db5735a31 Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Tue, 2 Aug 2016 11:57:35 -0300 Subject: [PATCH 02/14] Initial commit of the build script --- build/build-snippets.py | 46 +++++++++++++++++++++++++++++++++++++++++ build/requirements.txt | 2 ++ build/templating.py | 17 +++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 build/build-snippets.py create mode 100644 build/requirements.txt create mode 100644 build/templating.py diff --git a/build/build-snippets.py b/build/build-snippets.py new file mode 100644 index 0000000..ec65569 --- /dev/null +++ b/build/build-snippets.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# vim: ai ts=4 sts=4 et sw=4 +"""Build snippets from documentation from AWS.""" +import requests +from templating import build_with_template +from lxml import html + + +BASE_HREF = 'http://docs.aws.amazon.com/pt_br/AWSCloudFormation/latest/UserGuide/' + + +def build_index(): + """Build the main index with key and url for the services.""" + toc_uri = BASE_HREF + 'aws-template-resource-type-ref.html' + elem_expr = '//*[@id="main-col-body"]/div[2]/div[2]/ul/li/a' + page = requests.get(toc_uri) + doc = html.fromstring(page.content) + tree = doc.xpath(elem_expr) + index = {} + + for e in tree: + arn = e.text_content() + title = e.text_content().replace('::', '-').replace('AWS-', "").lower() + href = e.get('href') + full_href = BASE_HREF + href + index[arn] = (arn, title, href, full_href) + return index + +def generate(index): + """Will traverse index and generate all the snippets.""" + default_suffix = '.sublime-snippet' + for k, v in index.iteritems(): + (arn, title, href, full_href) = v + snippet = createSnippet(arn, title, href, full_href) + +def createSnippet(arn, title, href, full_href): + """Create a snippet with the args.""" + print build_with_template(arn, title, '') + +def main(): + """Main doc generation.""" + index = build_index() + generate(index) + +if __name__ == "__main__": + main() diff --git a/build/requirements.txt b/build/requirements.txt new file mode 100644 index 0000000..828889b --- /dev/null +++ b/build/requirements.txt @@ -0,0 +1,2 @@ +requests +lxml diff --git a/build/templating.py b/build/templating.py new file mode 100644 index 0000000..3c70d44 --- /dev/null +++ b/build/templating.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# vim: ai ts=4 sts=4 et sw=4 +"""Templating operations for the snippets created by the build script.""" + + +def build_with_template(trigger, title, body): + template = """ + + + + {} + + source.cloudformation + + """.format(body, title) + + return template From 2d3c6e35c848d41123aea8040fdce037dfec07fe Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Tue, 2 Aug 2016 17:14:58 -0300 Subject: [PATCH 03/14] Snippet script almost rewriten, still buggy cause of unicode chars contained in the body of the syntax. beaknit/cform#34 --- .gitignore | 1 + build/build-snippets.py | 19 +++++++++++++++---- build/templating.py | 9 ++++++++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 9cc8f07..165aa57 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.cache .#* .DS_Store +*.pyc diff --git a/build/build-snippets.py b/build/build-snippets.py index ec65569..9582f10 100644 --- a/build/build-snippets.py +++ b/build/build-snippets.py @@ -2,6 +2,8 @@ # vim: ai ts=4 sts=4 et sw=4 """Build snippets from documentation from AWS.""" import requests +import sys +import collections from templating import build_with_template from lxml import html @@ -12,11 +14,11 @@ def build_index(): """Build the main index with key and url for the services.""" toc_uri = BASE_HREF + 'aws-template-resource-type-ref.html' - elem_expr = '//*[@id="main-col-body"]/div[2]/div[2]/ul/li/a' + elem_expr = '//*[@id="main-col-body"]/div[2]/div/ul/li/a' page = requests.get(toc_uri) - doc = html.fromstring(page.content) + doc = html.fromstring(page.text.decode('utf-8')) tree = doc.xpath(elem_expr) - index = {} + index = collections.OrderedDict() for e in tree: arn = e.text_content() @@ -24,8 +26,10 @@ def build_index(): href = e.get('href') full_href = BASE_HREF + href index[arn] = (arn, title, href, full_href) + return index + def generate(index): """Will traverse index and generate all the snippets.""" default_suffix = '.sublime-snippet' @@ -33,9 +37,16 @@ def generate(index): (arn, title, href, full_href) = v snippet = createSnippet(arn, title, href, full_href) + def createSnippet(arn, title, href, full_href): """Create a snippet with the args.""" - print build_with_template(arn, title, '') + elem_expr = '//*[@id="main-col-body"]/div/div/pre/code' + page = requests.get(full_href) + doc = doc = html.fromstring(page.text.decode('utf-8')) + tree = doc.xpath(elem_expr) + print 'creating {} {} {}'.format(arn, title, tree) + return build_with_template(arn, title, tree) + def main(): """Main doc generation.""" diff --git a/build/templating.py b/build/templating.py index 3c70d44..758dfc7 100644 --- a/build/templating.py +++ b/build/templating.py @@ -3,7 +3,14 @@ """Templating operations for the snippets created by the build script.""" +import sys + + def build_with_template(trigger, title, body): + """With all args give, transform body with param list and return a complete snippet.""" + + processed_body = """"${{1:-}}" : {}""".format((body[0]).text_content()) + template = """ @@ -12,6 +19,6 @@ def build_with_template(trigger, title, body): source.cloudformation - """.format(body, title) + """.format(processed_body, title) return template From dd2f935024fe72cce7a0b1d68b89581f489a3747 Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Wed, 3 Aug 2016 10:51:47 -0300 Subject: [PATCH 04/14] Almost done, need just add the parameter matching and writing the snippet to the output. beaknit/cform#34 --- build/build-snippets.py | 35 +++++++++++++++++++++++++++-------- build/templating.py | 15 ++++++++++----- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/build/build-snippets.py b/build/build-snippets.py index 9582f10..74d4f21 100644 --- a/build/build-snippets.py +++ b/build/build-snippets.py @@ -2,13 +2,12 @@ # vim: ai ts=4 sts=4 et sw=4 """Build snippets from documentation from AWS.""" import requests -import sys import collections from templating import build_with_template from lxml import html -BASE_HREF = 'http://docs.aws.amazon.com/pt_br/AWSCloudFormation/latest/UserGuide/' +BASE_HREF = 'http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/' def build_index(): @@ -22,7 +21,9 @@ def build_index(): for e in tree: arn = e.text_content() - title = e.text_content().replace('::', '-').replace('AWS-', "").lower() + title = e.text_content().replace('::', '-') \ + .replace('AWS-', "") \ + .lower() href = e.get('href') full_href = BASE_HREF + href index[arn] = (arn, title, href, full_href) @@ -32,26 +33,44 @@ def build_index(): def generate(index): """Will traverse index and generate all the snippets.""" - default_suffix = '.sublime-snippet' for k, v in index.iteritems(): (arn, title, href, full_href) = v snippet = createSnippet(arn, title, href, full_href) + writeToOutput(arn, snippet) def createSnippet(arn, title, href, full_href): """Create a snippet with the args.""" elem_expr = '//*[@id="main-col-body"]/div/div/pre/code' + + # uses an old documentation format + if (arn == 'AWS::SDB::Domain'): + elem_expr = '//*[@id="main-col-body"]/div/pre' + page = requests.get(full_href) - doc = doc = html.fromstring(page.text.decode('utf-8')) + doc = html.fromstring(page.text) tree = doc.xpath(elem_expr) - print 'creating {} {} {}'.format(arn, title, tree) - return build_with_template(arn, title, tree) + print 'creating {} => {}'.format(arn, title) + return build_with_template(arn, title, tree, full_href) + + +def safedebug(index, k): + """Debug helper for scrapping problems.""" + (arn, title, href, full_href) = index[k] + snippet = createSnippet(arn, title, href, full_href) + print snippet + + +def writeToOutput(arn, snippet): + """Write the snippet to the output folder.""" + default_suffix = ".sublime-snippet" def main(): """Main doc generation.""" index = build_index() - generate(index) + # generate(index) + safedebug(index, 'AWS::SSM::Document') if __name__ == "__main__": main() diff --git a/build/templating.py b/build/templating.py index 758dfc7..ca9e76b 100644 --- a/build/templating.py +++ b/build/templating.py @@ -2,16 +2,16 @@ # vim: ai ts=4 sts=4 et sw=4 """Templating operations for the snippets created by the build script.""" - +from __future__ import unicode_literals import sys - -def build_with_template(trigger, title, body): +def build_with_template(trigger, title, body, full_href): """With all args give, transform body with param list and return a complete snippet.""" processed_body = """"${{1:-}}" : {}""".format((body[0]).text_content()) template = """ + @@ -19,6 +19,11 @@ def build_with_template(trigger, title, body): source.cloudformation - """.format(processed_body, title) + """.format(full_href, processed_body, title) + + transformed_template = transform(template) + + return transformed_template - return template +def transform(template): + return template.replace('JSON object', '[]') From 4bf686814c21669372927e19009c289ff970c405 Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Wed, 3 Aug 2016 10:53:19 -0300 Subject: [PATCH 05/14] import cleaning beaknit/cform#34 --- build/build-snippets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/build-snippets.py b/build/build-snippets.py index 74d4f21..6f7cbae 100644 --- a/build/build-snippets.py +++ b/build/build-snippets.py @@ -2,7 +2,7 @@ # vim: ai ts=4 sts=4 et sw=4 """Build snippets from documentation from AWS.""" import requests -import collections +from collections import OrderedDict from templating import build_with_template from lxml import html @@ -17,7 +17,7 @@ def build_index(): page = requests.get(toc_uri) doc = html.fromstring(page.text.decode('utf-8')) tree = doc.xpath(elem_expr) - index = collections.OrderedDict() + index = OrderedDict() for e in tree: arn = e.text_content() From 527210e556600a4be13c2bb039dee840d50352ee Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Thu, 4 Aug 2016 11:28:28 -0300 Subject: [PATCH 06/14] generation is complete, lacks parameter interpolation beankit/cform#34 --- .gitignore | 1 + build/build-snippets.py | 16 ++++++++++++---- build/templating.py | 14 ++++++++++---- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 165aa57..f9eb7ac 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .#* .DS_Store *.pyc +build/output/* diff --git a/build/build-snippets.py b/build/build-snippets.py index 6f7cbae..9854202 100644 --- a/build/build-snippets.py +++ b/build/build-snippets.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- # vim: ai ts=4 sts=4 et sw=4 """Build snippets from documentation from AWS.""" +from __future__ import unicode_literals import requests +# import sys from collections import OrderedDict from templating import build_with_template from lxml import html @@ -36,7 +38,8 @@ def generate(index): for k, v in index.iteritems(): (arn, title, href, full_href) = v snippet = createSnippet(arn, title, href, full_href) - writeToOutput(arn, snippet) + writeToOutput(title, snippet) + # sys.exit(0) def createSnippet(arn, title, href, full_href): @@ -61,16 +64,21 @@ def safedebug(index, k): print snippet -def writeToOutput(arn, snippet): +def writeToOutput(title, snippet): """Write the snippet to the output folder.""" + default_folder = "./output/" default_suffix = ".sublime-snippet" + filename = default_folder + title + default_suffix + out = file(filename, 'w') + out.write(snippet.encode('utf8', 'replace')) + out.close() def main(): """Main doc generation.""" index = build_index() - # generate(index) - safedebug(index, 'AWS::SSM::Document') + generate(index) + # safedebug(index, 'AWS::SSM::Document') if __name__ == "__main__": main() diff --git a/build/templating.py b/build/templating.py index ca9e76b..4514ffb 100644 --- a/build/templating.py +++ b/build/templating.py @@ -3,20 +3,24 @@ """Templating operations for the snippets created by the build script.""" from __future__ import unicode_literals -import sys + def build_with_template(trigger, title, body, full_href): - """With all args give, transform body with param list and return a complete snippet.""" + """ + Template generation. + With all args give, transform body with param list and return + a complete snippet. + """ processed_body = """"${{1:-}}" : {}""".format((body[0]).text_content()) template = """ - + {} - + source.cloudformation """.format(full_href, processed_body, title) @@ -25,5 +29,7 @@ def build_with_template(trigger, title, body, full_href): return transformed_template + def transform(template): + """Transform the template to adapt the parameters of the template.""" return template.replace('JSON object', '[]') From b88cc48a44377b6b56519a4666f198ceb23d45f6 Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Thu, 4 Aug 2016 11:32:03 -0300 Subject: [PATCH 07/14] Readme updated. beankit/cform#34 --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 68bf121..8554312 100644 --- a/README.md +++ b/README.md @@ -17,3 +17,14 @@ CForm is now available via the Sublime Text package control. See https://packag 6. Save it with the extension `cform`, `template` or `cloudformation` and run it through the CloudFormation console 7. Profit +# Update the snippets + +## Requirements + +1. You will need ```Python 2.7.10+``` +2. Run ```pip install -r build/requirements.txt``` + +## Updating the docs from the latest + +1. Go inside the build dir from the comand line +2. Run ```rm -rf *.pyc && python build-snippets.py``` From 0fd0436a7fe03038ad7320e4245c418f02d52440 Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Thu, 4 Aug 2016 11:33:24 -0300 Subject: [PATCH 08/14] Readme updated. beankit/cform#34 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8554312..83e7566 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ CForm is now available via the Sublime Text package control. See https://packag 6. Save it with the extension `cform`, `template` or `cloudformation` and run it through the CloudFormation console 7. Profit -# Update the snippets +# Updating the snippets ## Requirements From f1669daf05cbc86338b74da713b82e1e29a8d5cc Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Mon, 8 Aug 2016 15:35:44 -0300 Subject: [PATCH 09/14] Generation fixed in case of non-existent outputdir. Fancy counter added. beankit/cform#34 --- build/build-snippets.py | 44 +++++++++++++++++++++++++++++++++++++---- build/requirements.txt | 2 ++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/build/build-snippets.py b/build/build-snippets.py index 9854202..7a80e7d 100644 --- a/build/build-snippets.py +++ b/build/build-snippets.py @@ -2,16 +2,34 @@ # vim: ai ts=4 sts=4 et sw=4 """Build snippets from documentation from AWS.""" from __future__ import unicode_literals + import requests -# import sys +import os from collections import OrderedDict from templating import build_with_template from lxml import html - +from colorama import init +from colorama import Fore +from fabulous.widget import ProgressBar BASE_HREF = 'http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/' +def print_header(snippets_found, headers): + """Header print function.""" + print('\033c' + Fore.GREEN + + """ + _ _ _ _ __ __ + | | | | (_) | / / / _| + | |__ ___ __ _| | ___ __ _| |_ / /__| |_ ___ _ __ _ __ ___ + | '_ \ / _ \/ _` | |/ / '_ \| | __| / / __| _/ _ \| '__| '_ ` _ ` + | |_) | __/ (_| | <| | | | | |_ / / (__| || (_) | | | | | | | | + |_.__/ \___|\__,_|_|\_\_| |_|_|\__/_/ \___|_| \___/|_| |_| |_| |_|""") + + print(' Updated: ' + Fore.GREEN + '{}'.format(headers['last-modified'])) + print(' Snippets founds: ' + Fore.GREEN + '{}\n'.format(snippets_found)) + + def build_index(): """Build the main index with key and url for the services.""" toc_uri = BASE_HREF + 'aws-template-resource-type-ref.html' @@ -21,6 +39,8 @@ def build_index(): tree = doc.xpath(elem_expr) index = OrderedDict() + print_header(len(tree), page.headers) + for e in tree: arn = e.text_content() title = e.text_content().replace('::', '-') \ @@ -35,11 +55,22 @@ def build_index(): def generate(index): """Will traverse index and generate all the snippets.""" + total = len(index) + i = 0 + percent = 0 + progress = ProgressBar() + progress.update(i, 'Generating snippets') for k, v in index.iteritems(): (arn, title, href, full_href) = v snippet = createSnippet(arn, title, href, full_href) + i += 1 + percent = i * 100 / total + progress.update(percent, 'Creating ' + Fore.GREEN + arn) writeToOutput(title, snippet) - # sys.exit(0) + progress.update(percent, 'Generation completed') + print(' Snippets generated, have fun.') + print(' Got questions or want something on this? ' + + ' https://github.com/beaknit/cform') def createSnippet(arn, title, href, full_href): @@ -53,7 +84,7 @@ def createSnippet(arn, title, href, full_href): page = requests.get(full_href) doc = html.fromstring(page.text) tree = doc.xpath(elem_expr) - print 'creating {} => {}'.format(arn, title) + # print 'creating {} => {}'.format(arn, title) return build_with_template(arn, title, tree, full_href) @@ -68,6 +99,10 @@ def writeToOutput(title, snippet): """Write the snippet to the output folder.""" default_folder = "./output/" default_suffix = ".sublime-snippet" + + if (not os.path.exists(default_folder)): + os.makedirs(default_folder) + filename = default_folder + title + default_suffix out = file(filename, 'w') out.write(snippet.encode('utf8', 'replace')) @@ -76,6 +111,7 @@ def writeToOutput(title, snippet): def main(): """Main doc generation.""" + init(autoreset=True) # colorama index = build_index() generate(index) # safedebug(index, 'AWS::SSM::Document') diff --git a/build/requirements.txt b/build/requirements.txt index 828889b..eef2bec 100644 --- a/build/requirements.txt +++ b/build/requirements.txt @@ -1,2 +1,4 @@ requests lxml +colorama +fabulous From fe05632ea111a01ba9c35db984ec27b0f212404f Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Mon, 8 Aug 2016 15:42:21 -0300 Subject: [PATCH 10/14] Depdencies updated beankit/cform#34 --- build/templating.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/templating.py b/build/templating.py index 4514ffb..e56cd35 100644 --- a/build/templating.py +++ b/build/templating.py @@ -32,4 +32,8 @@ def build_with_template(trigger, title, body, full_href): def transform(template): """Transform the template to adapt the parameters of the template.""" - return template.replace('JSON object', '[]') + return template \ + .replace('JSON object', '[]') \ + .replace(': String', '""') \ + .replace(': Integer', '') + .replace(': Boolean', '${true | false}') From 84b604fa9f77deb60798d1e3ce93e53ea0967b97 Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Mon, 8 Aug 2016 16:32:22 -0300 Subject: [PATCH 11/14] Function generation included beaknit/cform#34 --- build/build-snippets.py | 50 ++++++++++++++++++++++++++++++++++------- build/templating.py | 7 ++++-- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/build/build-snippets.py b/build/build-snippets.py index 7a80e7d..1017d55 100644 --- a/build/build-snippets.py +++ b/build/build-snippets.py @@ -30,10 +30,8 @@ def print_header(snippets_found, headers): print(' Snippets founds: ' + Fore.GREEN + '{}\n'.format(snippets_found)) -def build_index(): +def build_index(toc_uri, elem_expr): """Build the main index with key and url for the services.""" - toc_uri = BASE_HREF + 'aws-template-resource-type-ref.html' - elem_expr = '//*[@id="main-col-body"]/div[2]/div/ul/li/a' page = requests.get(toc_uri) doc = html.fromstring(page.text.decode('utf-8')) tree = doc.xpath(elem_expr) @@ -45,6 +43,7 @@ def build_index(): arn = e.text_content() title = e.text_content().replace('::', '-') \ .replace('AWS-', "") \ + .replace('Fn-', "") \ .lower() href = e.get('href') full_href = BASE_HREF + href @@ -68,9 +67,6 @@ def generate(index): progress.update(percent, 'Creating ' + Fore.GREEN + arn) writeToOutput(title, snippet) progress.update(percent, 'Generation completed') - print(' Snippets generated, have fun.') - print(' Got questions or want something on this? ' - + ' https://github.com/beaknit/cform') def createSnippet(arn, title, href, full_href): @@ -109,12 +105,50 @@ def writeToOutput(title, snippet): out.close() +def generate_functions(): + """Function are a special cases, they're scaterred all over the docs.""" + toc_functions = [ + ('Fn::Select', 'fn-select', '{ "Fn::Select" : [ index, listOfObjects ] }', 'intrinsic-function-reference-select.html', BASE_HREF + 'intrinsic-function-reference-select.html'), + ('Ref', 'ref', '"Ref" : "logicalName"', 'intrinsic-function-reference-ref.html', BASE_HREF + 'intrinsic-function-reference-ref.html'), + ('Fn::Join', 'fn-join', '"Fn::Join" : [ "delimiter", [ comma-delimited list of values ] ]', 'intrinsic-function-reference-join.html', BASE_HREF + 'intrinsic-function-reference-join.html'), + ('Fn::GetAZs', 'fn-getAZs', '"Fn::GetAZs" : "region"', 'intrinsic-function-reference-getavailabilityzones.html', BASE_HREF + 'intrinsic-function-reference-getavailabilityzones.html'), + ('Fn::GetAtt', 'fn-getatt', '"Fn::GetAtt" : [ "logicalNameOfResource", "attributeName" ]', 'intrinsic-function-reference-getatt.html', BASE_HREF + 'intrinsic-function-reference-getatt.html'), + ('Fn::FindInMap', 'fn-findInMap', '"Fn::FindInMap" : [ "MapName", "TopLevelKey", "SecondLevelKey"]', 'intrinsic-function-reference-findinmap.html', BASE_HREF + 'intrinsic-function-reference-findinmap.html') + ] + i = 0 + percent = 0 + progress = ProgressBar() + progress.update(i, 'Generating functions') + total = len(toc_functions) + + print(' Functions found: ' + Fore.GREEN + str(total)) + + for v in toc_functions: + (arn, title, body, href, full_href) = v + writeToOutput(title, build_with_template(arn, title, body, full_href)) + i += 1 + percent = i * 100 / total + progress.update(percent, 'Creating ' + Fore.GREEN + arn) + + progress.update(percent, 'Generation completed') + + def main(): """Main doc generation.""" init(autoreset=True) # colorama - index = build_index() + + # build topics + toc_uri = BASE_HREF + 'aws-template-resource-type-ref.html' + elem_expr = '//*[@id="main-col-body"]/div[2]/div/ul/li/a' + index = build_index(toc_uri, elem_expr) generate(index) - # safedebug(index, 'AWS::SSM::Document') + + # build functions + generate_functions() + print('\n Snippets generated, have fun.') + print(' Got questions or want something on this? ' + + ' https://github.com/beaknit/cform') + if __name__ == "__main__": main() diff --git a/build/templating.py b/build/templating.py index e56cd35..7275e19 100644 --- a/build/templating.py +++ b/build/templating.py @@ -12,7 +12,10 @@ def build_with_template(trigger, title, body, full_href): With all args give, transform body with param list and return a complete snippet. """ - processed_body = """"${{1:-}}" : {}""".format((body[0]).text_content()) + if (isinstance(body, list)): + processed_body = """"${{1:-}}" : {}""".format((body[0]).text_content()) + else: + processed_body = """"${{1:-}}" : {}""".format((body)) template = """ @@ -35,5 +38,5 @@ def transform(template): return template \ .replace('JSON object', '[]') \ .replace(': String', '""') \ - .replace(': Integer', '') + .replace(': Integer', '') \ .replace(': Boolean', '${true | false}') From b9ad038a64fdabc1301c26af8e403dc9aa4e587c Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Mon, 8 Aug 2016 16:44:48 -0300 Subject: [PATCH 12/14] Output polishing beaknit/cform#34 --- build/build-snippets.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build/build-snippets.py b/build/build-snippets.py index 1017d55..c266b2d 100644 --- a/build/build-snippets.py +++ b/build/build-snippets.py @@ -61,12 +61,12 @@ def generate(index): progress.update(i, 'Generating snippets') for k, v in index.iteritems(): (arn, title, href, full_href) = v - snippet = createSnippet(arn, title, href, full_href) + # snippet = createSnippet(arn, title, href, full_href) i += 1 percent = i * 100 / total progress.update(percent, 'Creating ' + Fore.GREEN + arn) - writeToOutput(title, snippet) - progress.update(percent, 'Generation completed') + # writeToOutput(title, snippet) + progress.update(percent, 'Snippets completed') def createSnippet(arn, title, href, full_href): @@ -117,12 +117,12 @@ def generate_functions(): ] i = 0 percent = 0 - progress = ProgressBar() - progress.update(i, 'Generating functions') total = len(toc_functions) - print(' Functions found: ' + Fore.GREEN + str(total)) + progress = ProgressBar() + progress.update(i, 'Generating functions') + for v in toc_functions: (arn, title, body, href, full_href) = v writeToOutput(title, build_with_template(arn, title, body, full_href)) @@ -130,7 +130,7 @@ def generate_functions(): percent = i * 100 / total progress.update(percent, 'Creating ' + Fore.GREEN + arn) - progress.update(percent, 'Generation completed') + progress.update(percent, 'Functions generated.') def main(): From 95aa3e495b054cc785ce50332252a0410fe090f6 Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Tue, 9 Aug 2016 09:49:39 -0300 Subject: [PATCH 13/14] Oopsie, left things uncommited beaknit/cform#34 --- build/build-snippets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/build-snippets.py b/build/build-snippets.py index c266b2d..09c6bb1 100644 --- a/build/build-snippets.py +++ b/build/build-snippets.py @@ -61,11 +61,11 @@ def generate(index): progress.update(i, 'Generating snippets') for k, v in index.iteritems(): (arn, title, href, full_href) = v - # snippet = createSnippet(arn, title, href, full_href) + snippet = createSnippet(arn, title, href, full_href) i += 1 percent = i * 100 / total progress.update(percent, 'Creating ' + Fore.GREEN + arn) - # writeToOutput(title, snippet) + writeToOutput(title, snippet) progress.update(percent, 'Snippets completed') From c6e9db773926e683e8d309888348d1caea24e4a7 Mon Sep 17 00:00:00 2001 From: dgomesbr Date: Tue, 9 Aug 2016 16:25:28 -0300 Subject: [PATCH 14/14] So AWS::CertificateManager::Certificate isn't using default template as well. Damn beaknit/cform#34 --- build/build-snippets.py | 9 ++++++++- build/templating.py | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/build/build-snippets.py b/build/build-snippets.py index 09c6bb1..c13f568 100644 --- a/build/build-snippets.py +++ b/build/build-snippets.py @@ -5,6 +5,7 @@ import requests import os +import sys from collections import OrderedDict from templating import build_with_template from lxml import html @@ -77,6 +78,10 @@ def createSnippet(arn, title, href, full_href): if (arn == 'AWS::SDB::Domain'): elem_expr = '//*[@id="main-col-body"]/div/pre' + # oddly addition + if (arn == 'AWS::CertificateManager::Certificate'): + elem_expr = '//*[@id="JSON"]/pre/code' + page = requests.get(full_href) doc = html.fromstring(page.text) tree = doc.xpath(elem_expr) @@ -125,10 +130,10 @@ def generate_functions(): for v in toc_functions: (arn, title, body, href, full_href) = v - writeToOutput(title, build_with_template(arn, title, body, full_href)) i += 1 percent = i * 100 / total progress.update(percent, 'Creating ' + Fore.GREEN + arn) + writeToOutput(title, build_with_template(arn, title, body, full_href)) progress.update(percent, 'Functions generated.') @@ -141,6 +146,8 @@ def main(): toc_uri = BASE_HREF + 'aws-template-resource-type-ref.html' elem_expr = '//*[@id="main-col-body"]/div[2]/div/ul/li/a' index = build_index(toc_uri, elem_expr) + # safedebug(index, 'AWS::AutoScaling::ScheduledAction') + # sys.exit(0) generate(index) # build functions diff --git a/build/templating.py b/build/templating.py index 7275e19..f61af5a 100644 --- a/build/templating.py +++ b/build/templating.py @@ -12,6 +12,8 @@ def build_with_template(trigger, title, body, full_href): With all args give, transform body with param list and return a complete snippet. """ + # print trigger + # print body if (isinstance(body, list)): processed_body = """"${{1:-}}" : {}""".format((body[0]).text_content()) else: