diff --git a/runtime/scripts/wskgen_v1 b/runtime/scripts/wskgen_v1 new file mode 100644 index 0000000..bde61aa --- /dev/null +++ b/runtime/scripts/wskgen_v1 @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 + +import requests +import argparse +import json +import subprocess +import random +import os +import yaml + +from glob import glob + +dir_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = os.path.join(dir_path, '../lib') + +# Function for extracting the meta info +def metaline(line, filetype): + if filetype == '.py': + if line.startswith('#@ '): + return True, line[3:] + elif filetype == '.yaml': + return True, line + return False, None + + +def getMetaFromCode(meta, line): + def findMeta(startpattern, endpattern, line): + t=[] + s = line.find(startpattern) + if s != -1: + e = line.find(endpattern, s + len(startpattern)) + t = line[s + len(startpattern):e].replace("'", "").replace('"', "").split(", ") + return t + + #Get Memory Element + t = findMeta("action.get_transport(",")", line) + if len(t)>1: + meta["corunning"][t[0]] = {"trans":t[0], "type":t[1] } + + #Get Parent Element + t = findMeta("params[", "]", line) + if len(t) > 0: + meta["parents"].append(t[-1]) + + +def AutoGenrateFilemeta(f): + metaphase = True + name = os.path.basename(f).split('.')[0] + filetype = os.path.splitext(f)[1] + metadictionary = {} + metalist = [] + sourcelist = [] + metadictionary["type"] = "memory" + if filetype == '.py': + metadictionary["type"] = "compute" + metadictionary["corunning"] = {} + + metadictionary["name"] = name + metadictionary["filetype"] = filetype + metadictionary["dependents"] =[] + metadictionary["parents"] = [] + print("processing object file", f, "type", filetype) + with open(f) as source: + for line in source.readlines(): + is_meta, meta = metaline(line, filetype) + if is_meta: + metalist.append(meta) + else: + sourcelist.append(line) + getMetaFromCode(metadictionary,line) + + metaFromComment = yaml.load(''.join(metalist), Loader=yaml.Loader) + if metaFromComment: + for key in metaFromComment: + if key not in metadictionary: + metadictionary[key] = metaFromComment[key] + + return metaphase, metadictionary, ''.join(sourcelist) + + + +def gen_obj(meta, source): + # process memory objects + memory_source = "def main(_, action):\n t = action.get_transport('memory', 'rdma_server')\n t.serve()" + + kind = 'python:3' + annotations = {} + annotations['exec'] = 'python' + if meta['type'] == 'memory': + source = memory_source + + name = meta.get('name') + + # process limits + limits = { + "concurrency": 1, + "logs": 10, + "resources": { + "cpu": meta.get('limits', {}).get('cpu', 1.0), + "mem": meta.get('limits', {}).get('mem', '128 MB'), + "storage": meta.get('limits', {}).get('storage', '512 MB') + }, + "timeout": 60000 + } + + # change corunning dict to list + corunning = [k for k in meta.get('corunning', {})] + + # import + for filename in meta.get('import', []): + fullname = os.path.join(dir_path, "../lib/", filename) + print('import', fullname) + with open(fullname) as f: + source = f.read() + '\n' + source + + obj = { + 'annotations': [{'key': k, 'value': v} for k, v in annotations.items()], + 'exec': { + 'kind': kind, + 'code': source, + 'binary': True, + 'main': 'main' + }, + 'limits': limits, + 'name': name, + 'parameters': [], + 'publish': False, + 'porusParams': { + 'relationships': { + 'corunning': corunning, + 'dependents': meta.get('dependents', []), + 'parents': meta.get('parents', []) + }, + 'runtimeType': meta['type'], + 'withMerged': meta.get('withMerged', []), + }, + 'version': "0.0.1" + } + + if 'parallelism' in meta: + obj['porusParams']['parallelism'] = meta['parallelism'] + + return obj + +# l : meta, l +def gen_json(funcname, l): + func = { + 'name': funcname, + 'objects': [], + "publish": True, + } + dependent = {} + for _, meta, _ in l: + parents = meta['parents'] + for p in parents: + if p not in dependent: + dependent[p] = set() + dependent[p].add(meta["name"]) + + for _, meta, source in l: + meta["dependents"] = list(dependent[meta["name"]]) if meta["name"] in dependent else [] + func['objects'].append(gen_obj(meta, source)) + return {'functions': [func], "publish": True} + + +def generate(path, targetFile): + if path.endswith('/'): + path = path[:-1] + dirname = os.path.basename(path) + objects = glob(os.path.join(path, "*.o.*")) + config = gen_json(dirname, [AutoGenrateFilemeta(f) for f in objects]) + + print('generating config file:', targetFile, 'from folder', dirname) + + with open(targetFile, 'w') as outfile: + json.dump(config, outfile, indent = 2) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description='Generate disaggregated json from folder') + parser.add_argument('-o', '--output', type=str, default='action.json', + help='Json file with disaggregated openwhisk type') + parser.add_argument('path', type=str, + help='path of the object files') + args = parser.parse_args() + + generate(args.path, args.output)