From 016ba6ec812870b46118ebf1676894e353612142 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Sun, 12 Apr 2015 16:19:24 +0200 Subject: [PATCH 01/16] Moved plugins-config loader to plugin section --- .gitignore | 1 + config.go => plugin/config.go | 0 config_unix.go => plugin/config_unix.go | 0 config_windows.go => plugin/config_windows.go | 0 4 files changed, 1 insertion(+) rename config.go => plugin/config.go (100%) rename config_unix.go => plugin/config_unix.go (100%) rename config_windows.go => plugin/config_windows.go (100%) diff --git a/.gitignore b/.gitignore index 23d04d4..fa1a54d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ src pkg bin docs/api +public/*.js diff --git a/config.go b/plugin/config.go similarity index 100% rename from config.go rename to plugin/config.go diff --git a/config_unix.go b/plugin/config_unix.go similarity index 100% rename from config_unix.go rename to plugin/config_unix.go diff --git a/config_windows.go b/plugin/config_windows.go similarity index 100% rename from config_windows.go rename to plugin/config_windows.go From 49c16ed07c76926361d2cb38265970be19842737 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Wed, 15 Apr 2015 21:43:23 +0200 Subject: [PATCH 02/16] Added workflow model, routers, controllers, templates; refactoring --- api.go | 6 +- plugin/{ => config}/config.go | 2 +- plugin/{ => config}/config_unix.go | 2 +- plugin/{ => config}/config_windows.go | 2 +- resource/entry.go | 1 - resource/feed.go | 38 ++--- resource/plugin.go | 1 + resource/resource.go | 24 ++- resource/workflow.go | 108 ++++++++++++++ service/store/v1/controller/workflow.go | 137 ++++++++++++++++++ service/store/v1/router/default.go | 1 + service/store/v1/router/workflow.go | 11 ++ service/store/v1/template/workflow/request.go | 59 ++++++++ .../store/v1/template/workflow/response.go | 128 ++++++++++++++++ workflow/manager.go | 7 +- workflow/workflow.go | 25 ++-- 16 files changed, 506 insertions(+), 46 deletions(-) rename plugin/{ => config}/config.go (99%) rename plugin/{ => config}/config_unix.go (98%) rename plugin/{ => config}/config_windows.go (98%) create mode 100644 resource/plugin.go create mode 100644 resource/workflow.go create mode 100644 service/store/v1/controller/workflow.go create mode 100644 service/store/v1/router/workflow.go create mode 100644 service/store/v1/template/workflow/request.go create mode 100644 service/store/v1/template/workflow/response.go diff --git a/api.go b/api.go index afed378..b515977 100644 --- a/api.go +++ b/api.go @@ -9,6 +9,10 @@ import ( "github.com/feedlabs/elasticfeed/elasticfeed" ) +var ( + ServerEngine *elasticfeed.Elasticfeed +) + func main() { rm := resource.NewResourceManager() em := event.NewEventManager() @@ -16,6 +20,6 @@ func main() { wm := workflow.NewWorkflowManager(nil, pm, em) sm := service.NewServiceManager() - ServerEngine := elasticfeed.NewElasticfeed(rm, em, sm, pm, wm) + ServerEngine = elasticfeed.NewElasticfeed(rm, em, sm, pm, wm) ServerEngine.Run() } diff --git a/plugin/config.go b/plugin/config/config.go similarity index 99% rename from plugin/config.go rename to plugin/config/config.go index 76f0ae8..4e50078 100644 --- a/plugin/config.go +++ b/plugin/config/config.go @@ -1,4 +1,4 @@ -package main +package plugin import ( "encoding/json" diff --git a/plugin/config_unix.go b/plugin/config/config_unix.go similarity index 98% rename from plugin/config_unix.go rename to plugin/config/config_unix.go index 2c8a7a3..7460985 100644 --- a/plugin/config_unix.go +++ b/plugin/config/config_unix.go @@ -1,6 +1,6 @@ // +build darwin freebsd linux netbsd openbsd -package main +package plugin import ( "bytes" diff --git a/plugin/config_windows.go b/plugin/config/config_windows.go similarity index 98% rename from plugin/config_windows.go rename to plugin/config/config_windows.go index fa3ab94..80cc4be 100644 --- a/plugin/config_windows.go +++ b/plugin/config/config_windows.go @@ -1,6 +1,6 @@ // +build windows -package main +package plugin import ( "path/filepath" diff --git a/resource/entry.go b/resource/entry.go index fdc2f48..df6fbee 100644 --- a/resource/entry.go +++ b/resource/entry.go @@ -6,7 +6,6 @@ import ( "encoding/json" "github.com/feedlabs/feedify/graph" - "github.com/feedlabs/elasticfeed/service/stream/controller/room" ) diff --git a/resource/feed.go b/resource/feed.go index bad14cd..46f5d00 100644 --- a/resource/feed.go +++ b/resource/feed.go @@ -5,33 +5,33 @@ import ( "strconv" "github.com/feedlabs/feedify/graph" - "github.com/feedlabs/elasticfeed/service/stream/controller/room" - "github.com/feedlabs/elasticfeed/workflow" ) func (this *Feed) AddEntry(entry Entry) (EntryId string, err error) { return AddEntry(entry, this.Id, this.Application.Id, this.Application.Org.Id) } +func (this *Feed) AddWorkflow(workflow Workflow) (WorkflowId string, err error) { + return AddWorkflow(workflow, this.Id, this.Application.Id, this.Application.Org.Id) +} + func (this *Feed) GetEntryList() (entries []*Entry, err error) { return GetEntryList(this.Id, this.Application.Id, this.Application.Org.Id) } -func (this *Feed) SetWorkflowfile(data map[string]interface{}) { - this.Workflowfile = data +func (this *Feed) GetWorkflowList() (entries []*Workflow, err error) { + return GetWorkflowList(this.Id, this.Application.Id, this.Application.Org.Id) } -func (this *Feed) GetWorkflowfile() map[string]interface{} { - return this.Workflowfile -} +func (this *Feed) GetWorkflow() *Workflow { + w, err := GetWorkflowList(this.Id, this.Application.Id, this.Application.Org.Id) -func (this *Feed) InitWorkflow(wm *workflow.WorkflowManager) { - this.Workflow = wm.CreateFeedWorkflow(this, this.GetWorkflowfile()) -} + if err == nil { + return w[0] + } -func (this *Feed) GetWorkflow() *workflow.Workflow { - return this.Workflow + return nil } func GetFeedList(ApplicationId string, OrgId string) (feedList []*Feed, err error) { @@ -48,9 +48,10 @@ func GetFeedList(ApplicationId string, OrgId string) (feedList []*Feed, err erro for _, rel := range _rels { data := rel.EndNode.Data["data"].(string) id := strconv.Itoa(rel.EndNode.Id) - rels, _ := storage.RelationshipsNode(rel.EndNode.Id, "entry") + entry_rels, _ := storage.RelationshipsNode(rel.EndNode.Id, "entry") + workflow_rels, _ := storage.RelationshipsNode(rel.EndNode.Id, "workflow") - feed := NewFeed(id , app, data, len(rels)) + feed := NewFeed(id, app, data, len(entry_rels), len(workflow_rels)) feeds = append(feeds, feed) } @@ -76,8 +77,9 @@ func GetFeed(id string, applicationId string, orgId string) (feed *Feed, err err if node != nil && Contains(node.Labels, RESOURCE_FEED_LABEL) && app.Id == node.Data["applicationId"].(string) { data := node.Data["data"].(string) - rels, _ := storage.RelationshipsNode(node.Id, "entry") - return NewFeed(strconv.Itoa(node.Id), app, data, len(rels)), nil + entry_rels, _ := storage.RelationshipsNode(node.Id, "entry") + workflow_rels, _ := storage.RelationshipsNode(node.Id, "workflow") + return NewFeed(strconv.Itoa(node.Id), app, data, len(entry_rels), len(workflow_rels)), nil } return nil, errors.New("FeedId not exist for ApplicationId `"+applicationId+"`") @@ -129,6 +131,6 @@ func ActionEmptyFeed(id string) { room.Publish <- room.NewFeedEvent(room.FEED_EMPTY, id, "empty") } -func NewFeed(id string, app *Application, data string, entries int) *Feed { - return &Feed{id, app, data, entries, nil, nil} +func NewFeed(id string, app *Application, data string, entries int, workflows int) *Feed { + return &Feed{id, app, data, entries, workflows} } diff --git a/resource/plugin.go b/resource/plugin.go new file mode 100644 index 0000000..958e354 --- /dev/null +++ b/resource/plugin.go @@ -0,0 +1 @@ +package resource diff --git a/resource/resource.go b/resource/resource.go index f075123..f10d585 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -13,8 +13,6 @@ import ( "github.com/feedlabs/elasticfeed/service/stream/controller/room" "github.com/feedlabs/elasticfeed/service/stream/model" "github.com/feedlabs/elasticfeed/plugin/pipeline" - - "github.com/feedlabs/elasticfeed/workflow" ) const ( @@ -24,8 +22,10 @@ const ( RESOURCE_APPLICATION_LABEL = "application" RESOURCE_FEED_LABEL = "feed" RESOURCE_ENTRY_LABEL = "entry" - RESOURCE_METRIC_LABEL = "metric" - RESOURCE_VIEWER_LABEL = "viewer" + RESOURCE_METRIC_LABEL = "metric" + RESOURCE_VIEWER_LABEL = "viewer" + RESOURCE_WORKFLOW_LABEL = "workflow" + RESOURCE_PLUGIN_LABEL = "plugin" ) var ( @@ -37,6 +37,8 @@ var ( Entries map[string]*Entry Metrics map[string]*Metric Viewers map[string]*Viewer + Workflows map[string]*Workflow + Plugins map[string]*Plugin message *stream.StreamMessage storage *graph.GraphStorage @@ -82,10 +84,8 @@ type Feed struct { Application *Application Data string - Entries int - - Workflow *workflow.Workflow - Workflowfile map[string]interface{} + Entries int + Workflows int } type Entry struct { @@ -98,6 +98,14 @@ type Viewer struct {} type Metric struct {} +type Workflow struct { + Id string + Feed *Feed + Data string +} + +type Plugin struct {} + func ResourceStreamManager() { for { select { diff --git a/resource/workflow.go b/resource/workflow.go new file mode 100644 index 0000000..67a2077 --- /dev/null +++ b/resource/workflow.go @@ -0,0 +1,108 @@ +package resource + +import ( + "strconv" + "errors" + + "github.com/feedlabs/feedify/graph" +) + +func (this *Workflow) GetRawData() map[string]interface{} { + return make(map[string]interface{}) +} + +func (this *Workflow) GetProfilerRawData() map[string]string { + return this.GetRawData()["profiler"].(map[string]string) +} + +func GetWorkflowList(FeedId string, ApplicationId string, OrgId string) (feedWorkflows []*Workflow, err error) { + feed, err := GetFeed(FeedId, ApplicationId, OrgId) + if err != nil { + return nil, err + } + + _id, _ := strconv.Atoi(feed.Id) + _rels, _ := storage.RelationshipsNode(_id, "workflow") + + var workflows []*Workflow + + for _, rel := range _rels { + data := rel.EndNode.Data["data"].(string) + workflow := NewWorkflow(strconv.Itoa(rel.EndNode.Id), feed, data) + if workflow != nil && Contains(rel.EndNode.Labels, RESOURCE_WORKFLOW_LABEL) && feed.Id == rel.EndNode.Data["feedId"].(string) { + workflows = append(workflows, workflow) + } + } + + if workflows == nil { + workflows = make([]*Workflow, 0) + } + + return workflows, nil +} + +func GetWorkflow(id string, FeedId string, ApplicationId string, OrgId string) (feedWorkflow *Workflow, err error) { + feed, err := GetFeed(FeedId, ApplicationId, OrgId) + if err != nil { + return nil, err + } + + _id, err := strconv.Atoi(id) + workflow, err := storage.Node(_id) + + if err != nil { + return nil, err + } + + if workflow != nil && Contains(workflow.Labels, RESOURCE_WORKFLOW_LABEL) && feed.Id == workflow.Data["feedId"].(string) { + data := workflow.Data["data"].(string) + return NewWorkflow(strconv.Itoa(workflow.Id), feed, data), nil + } + + return nil, errors.New("WorkflowId `"+id+"` not exist") +} + +func AddWorkflow(feedWorkflow Workflow, FeedId string, ApplicationId string, OrgId string) (WorkflowId string, err error) { + + // get feed + feed, err := GetFeed(FeedId, ApplicationId, OrgId) + if err != nil { + return "0", err + } + + // add feed-workflow + properties := graph.Props{ + "feedId": feed.Id, + "data": feedWorkflow.Data, + } + + workflow, err := storage.NewNode(properties, RESOURCE_WORKFLOW_LABEL) + + if err != nil { + return "0", err + } + + // create relation + _feedId, _ := strconv.Atoi(feed.Id) + rel, err := storage.RelateNodes(_feedId, workflow.Id, "workflow", nil) + + if err != nil || rel.Type == "" { + return "0", err + } + + feedWorkflow.Id = strconv.Itoa(workflow.Id) + + return feedWorkflow.Id, nil +} + +func UpdateWorkflow(id string, FeedId string, ApplicationId string, OrgId string, data string) (err error) { + return nil +} + +func DeleteWorkflow(id string, FeedId string, ApplicationId string, OrgId string) (error) { + return nil +} + +func NewWorkflow(id string, feed *Feed, data string) *Workflow { + return &Workflow{id, feed, data} +} diff --git a/service/store/v1/controller/workflow.go b/service/store/v1/controller/workflow.go new file mode 100644 index 0000000..e16d252 --- /dev/null +++ b/service/store/v1/controller/workflow.go @@ -0,0 +1,137 @@ +package controller + +import ( + "encoding/json" + + "github.com/feedlabs/elasticfeed/resource" + "github.com/feedlabs/elasticfeed/service/store/v1/template/workflow" +) + +type WorkflowController struct { + DefaultController +} + +/** + * @api {get} application/:applicationId/feed/:feedId/workflow Get List (Feed) + * @apiVersion 1.0.0 + * @apiName GetWorkflowListFeed + * @apiGroup Workflow + * @apiDescription This will return a list of all entries per feed you have created. + * + * @apiUse WorkflowGetListRequest + * @apiUse WorkflowGetListResponse + */ +func (this *WorkflowController) GetList() { + workflow.RequestGetList(this.GetInput()) + + appId := this.Ctx.Input.Params[":applicationId"] + feedId := this.Ctx.Input.Params[":feedId"] + feed, err := resource.GetFeed(feedId, appId, this.GetAdminOrgId()) + obs, err := feed.GetWorkflowList() + + if err != nil { + this.Data["json"] = map[string]string{"result": err.Error(), "status": "error"} + } else { + this.Data["json"] = obs + } + + workflow.ResponseGetList() + this.Controller.ServeJson() +} + +/** + * + * @api {post} application/:applicationId/feed/:feedId/workflow/ Add (Feed) + * @apiVersion 1.0.0 + * @apiName PostWorkflowFeedAdd + * @apiGroup Workflow + * @apiDescription Add a workflow to the feed which is already store in the system. + * + * @apiUse WorkflowAddToFeedRequest + * @apiUse WorkflowAddToFeedResponse + */ +func (this *WorkflowController) Post() { + workflow.RequestPost(this.GetInput()) + + appId := this.Ctx.Input.Params[":applicationId"] + feedId := this.Ctx.Input.Params[":feedId"] + + var ob resource.Workflow + data := this.Ctx.Input.CopyBody() + json.Unmarshal(data, &ob) + + app, err := resource.GetApplication(appId, this.GetAdminOrgId()) + feed, err := app.GetFeed(feedId) + workflowId, err := feed.AddWorkflow(ob) + + if err != nil { + this.Data["json"] = map[string]string{"result": err.Error(), "status": "error"} + } else { + this.Data["json"] = map[string]string{"id": workflowId} + } + + workflow.ResponsePost() + this.Controller.ServeJson() +} + +/** + * @api {put} application/:applicationId/workflow/:workflowId Update (Global) + * @apiVersion 1.0.0 + * @apiName PutWorkflow + * @apiGroup Workflow + * @apiDescription Update a specific workflow. + * + * @apiUse WorkflowPutRequest + * @apiUse WorkflowPutResponse + */ +func (this *WorkflowController) Put() { + workflow.RequestPut(this.GetInput()) + + appId := this.Ctx.Input.Params[":applicationId"] + feedId := this.Ctx.Input.Params[":feedId"] + feedWorkflowId := this.Ctx.Input.Params[":feedWorkflowId"] + + var ob resource.Workflow + + data := this.Ctx.Input.CopyBody() + json.Unmarshal(data, &ob) + + err := resource.UpdateWorkflow(feedWorkflowId, feedId, appId, this.GetAdminOrgId(), ob.Data) + if err != nil { + this.Data["json"] = map[string]string{"result": err.Error(), "status": "error"} + } else { + this.Data["json"] = map[string]string{"result": "update success", "status": "ok"} + } + + workflow.ResponsePut() + this.Controller.ServeJson() +} + +/** + * @api {delete} application/:applicationId/workflow/:workflowId Delete (Global) + * @apiVersion 1.0.0 + * @apiName DeleteWorkflow + * @apiGroup Workflow + * @apiDescription Delete a specific workflow. (will also remove the workflow from all feeds) + * + * @apiUse WorkflowDeleteRequest + * @apiUse WorkflowDeleteResponse + */ +func (this *WorkflowController) Delete() { + workflow.RequestDelete(this.GetInput()) + + appId := this.Ctx.Input.Params[":applicationId"] + feedId := this.Ctx.Input.Params[":feedId"] + feedWorkflowId := this.Ctx.Input.Params[":feedWorkflowId"] + + err := resource.DeleteWorkflow(feedWorkflowId, feedId, appId, this.GetAdminOrgId()) + + if err != nil { + this.Data["json"] = map[string]string{"result": err.Error(), "status": "error"} + } else { + this.Data["json"] = map[string]string{"result": "delete success", "status": "ok"} + } + + workflow.ResponseDelete() + this.Controller.ServeJson() +} diff --git a/service/store/v1/router/default.go b/service/store/v1/router/default.go index eb0b3ca..dc1b5de 100644 --- a/service/store/v1/router/default.go +++ b/service/store/v1/router/default.go @@ -15,6 +15,7 @@ func InitRouters() { InitAdminRouters() InitApplicationRouters() InitEntryRouters() + InitEntryWorkflows() InitFeedRouters() InitOrgRouters() InitTokenRouters() diff --git a/service/store/v1/router/workflow.go b/service/store/v1/router/workflow.go new file mode 100644 index 0000000..b69746a --- /dev/null +++ b/service/store/v1/router/workflow.go @@ -0,0 +1,11 @@ +package router + +import ( + "github.com/feedlabs/feedify" + "github.com/feedlabs/elasticfeed/service/store/v1/controller" +) + +func InitEntryWorkflows() { + feedify.Router("/v1/application/:applicationId:string/feed/:feedId:int/workflow", &controller.WorkflowController{}, "get:GetList;post:Post") + feedify.Router("/v1/application/:applicationId:string/feed/:feedId:int/workflow/:feedWorkflowId:int", &controller.WorkflowController{}, "get:Get;delete:Delete;put:Put") +} diff --git a/service/store/v1/template/workflow/request.go b/service/store/v1/template/workflow/request.go new file mode 100644 index 0000000..1c7a6d7 --- /dev/null +++ b/service/store/v1/template/workflow/request.go @@ -0,0 +1,59 @@ +package workflow + +import ( + "github.com/feedlabs/feedify/context" +) + + +/** + * @apiDefine WorkflowGetListByFeedRequest + * + * @apiParam {String} applicationId The application id + * @apiParam {String} feedId The application id + */ +func RequestGetList(input *context.Input) { + +} + +/** + * @apiDefine WorkflowGetRequest + * + * @apiParam {String} applicationId The application id + * @apiParam {String} workflowId The workflow id + */ +func RequestGet(input *context.Input) { + +} + +/** + * @apiDefine WorkflowPostRequest + * + * @apiParam {String} applicationId The application id + * @apiParam {String} data The data of the workflow + * @apiParam {String[]} [tagList] Tags of the workflow + */ +func RequestPost(input *context.Input) { + +} + +/** + * @apiDefine WorkflowPutRequest + * + * @apiParam {String} applicationId The application id + * @apiParam {String} workflowId The workflow id + * @apiParam {String} data The data of the workflow + * @apiParam {String[]} [tagList] Tags of the workflow + */ +func RequestPut(input *context.Input) { + +} + +/** + * @apiDefine WorkflowDeleteRequest + * + * @apiParam {String} applicationId The application id + * @apiParam {String} workflowId The workflow id + */ +func RequestDelete(input *context.Input) { + +} diff --git a/service/store/v1/template/workflow/response.go b/service/store/v1/template/workflow/response.go new file mode 100644 index 0000000..782c7ce --- /dev/null +++ b/service/store/v1/template/workflow/response.go @@ -0,0 +1,128 @@ +package workflow + + +/** + * @apiDefine WorkflowGetListByFeedResponse + * + * @apiSuccess {Object} feed + * @apiSuccess {String} feed.id The application Id + * @apiSuccess {String} feed.name The name of the application + * @apiSuccess {String} feed.channelId The channel Id of the feed + * @apiSuccess {String} feed.applicationId The id of the application the feed belongs to + * @apiSuccess {Int} feed.createStamp Unix time stamp of create time + * @apiSuccess {Object[]} workflowList Array of all workflows + * @apiSuccess {String} workflowList.id The workflow Id + * @apiSuccess {Int} workflowList.createStamp Unix time stamp of create time + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "feed": { + * "id": "KAJFDA7GFTRE87FDS78F7", + * "name": "Son Goku", + * "channelId": "ASJDH86ASD678ASDASD768", + * "applicationId": "KAJFDA786FDS87FDS78F6", + * "createStamp": "1415637736", + * } + * "workflowList": [ + * { + * "id": "KAJFDA7GFTRE87FDS78F7", + * "createStamp": "1415637736", + * }, + * ... + * ] + * } + */ +func ResponseGetList() { + +} + +/** + * @apiDefine WorkflowGetResponse + * + * @apiSuccess {String} id The feed Id + * @apiSuccess {String} applicationId The application id + * @apiSuccess {String} data The data of the workflow + * @apiSuccess {String[]} tagList List of set tags + * @apiSuccess {Int} createStamp Unix time stamp of create time + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "id": "KAJFDA7GFTRE87FDS78F7", + * "applicationId": "KAJDFE7GFTRE87FDS78F7", + * "data": "Hello, I'm Son Gocu and this is my first post.", + * "tagList": [ + * "First", + * "Awesome" + * ], + * "createStamp": "1415637736", + * } + */ +func ResponseGet() { + +} + +/** + * @apiDefine WorkflowPostResponse + * + * @apiSuccess {String} id The feed Id + * @apiSuccess {String} applicationId The application id + * @apiSuccess {String} data The data of the workflow + * @apiSuccess {String[]} tagList List of set tags + * @apiSuccess {Int} createStamp Unix time stamp of create time + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "id": "KAJFDA7GFTRE87FDS78F7", + * "applicationId": "KAJDFE7GFTRE87FDS78F7", + * "data": "Hello, I'm Son Gocu and this is my first post.", + * "tagList": [ + * "First", + * "Awesome" + * ], + * "createStamp": "1415637736", + * } + */ +func ResponsePost() { + +} + +/** + * @apiDefine WorkflowPutResponse + * + * @apiSuccess {String} id The feed Id + * @apiSuccess {String} applicationId The application id + * @apiSuccess {String} [feedId] The feed id + * @apiSuccess {String} data The data of the workflow + * @apiSuccess {String[]} tagList List of set tags + * @apiSuccess {Int} createStamp Unix time stamp of create time + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "id": "KAJFDA7GFTRE87FDS78F7", + * "applicationId": "KAJDFE7GFTRE87FDS78F7", + * "feedId": "KAJDFE7GFTRE87FDS78F7", + * "data": "Hello, I'm Son Gocu and this is my first post.", + * "tagList": [ + * "First", + * "Awesome" + * ], + * "createStamp": "1415637736", + * } + */ +func ResponsePut() { + +} + +/** + * @apiDefine WorkflowDeleteResponse + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + */ +func ResponseDelete() { + +} diff --git a/workflow/manager.go b/workflow/manager.go index 066d46d..cf3cdd1 100644 --- a/workflow/manager.go +++ b/workflow/manager.go @@ -3,13 +3,14 @@ package workflow import ( "github.com/feedlabs/elasticfeed/plugin" "github.com/feedlabs/elasticfeed/event" + "github.com/feedlabs/elasticfeed/resource" ) type WorkflowManager struct { pManager *plugin.PluginManager eManager *event.EventManager - workflows []*Workflow + workflows []*WorkflowController template interface{} } @@ -19,8 +20,8 @@ func (this *WorkflowManager) InitTemplate(t interface{}) { this.template = t } -func (this *WorkflowManager) CreateFeedWorkflow(f interface {}, data map[string]interface {}) *Workflow { - w := NewWorkflow(data, f, this) +func (this *WorkflowManager) CreateFeedWorkflow(feed *resource.Feed) *WorkflowController { + w := NewWorkflowController(feed, this) w.Init() this.workflows = append(this.workflows, w) return w diff --git a/workflow/workflow.go b/workflow/workflow.go index 727c95c..9cf7608 100644 --- a/workflow/workflow.go +++ b/workflow/workflow.go @@ -2,46 +2,47 @@ package workflow import ( "github.com/feedlabs/elasticfeed/plugin" + "github.com/feedlabs/elasticfeed/resource" ) -type Workflow struct { - feed interface{} +type WorkflowController struct { + feed *resource.Feed manager *WorkflowManager profiler *plugin.Profiler - data map[string]interface{} } -func (this *Workflow) GetManager() *WorkflowManager { +func (this *WorkflowController) GetManager() *WorkflowManager { return nil } -func (this *Workflow) GetFeed() *interface{} { +func (this *WorkflowController) GetFeed() *interface{} { return nil } -func (this *Workflow) GetProfiler() *plugin.Profiler { +func (this *WorkflowController) GetProfiler() *plugin.Profiler { return this.profiler } -func (this *Workflow) Init() { +func (this *WorkflowController) Init() { // verify Feed.Workflowfile stricture; does match WorkflowManager Templating // verify plugins availability: this.manager.findPlugin() // run Plugins if require specific Profiler // bind Feed to system Events: this.manager.BindToSystemEvents() } -func (this *Workflow) DispatchIndexerHook(data interface{}) interface{} { +func (this *WorkflowController) DispatchIndexerHook(data interface{}) interface{} { return data } -func (this *Workflow) DispatchPipelineHook(data interface{}) interface{} { +func (this *WorkflowController) DispatchPipelineHook(data interface{}) interface{} { return data } -func NewWorkflow(data map[string]interface{}, f interface{}, wm *WorkflowManager) *Workflow { - p := plugin.NewProfiler(data["profiler"].(map[string]string)) - w := &Workflow{f, wm, p, data} +func NewWorkflowController(feed *resource.Feed, wm *WorkflowManager) *WorkflowController { + data := feed.GetWorkflow().GetProfilerRawData() + p := plugin.NewProfiler(data) + w := &WorkflowController{feed, wm, p} w.Init() From ef806e3dab15fc1dffcdcb9c33f6f064206249a0 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Fri, 17 Apr 2015 13:22:47 +0200 Subject: [PATCH 03/16] Fixed entry routers and controllers --- service/store/v1/controller/entry.go | 60 ++-------------------------- service/store/v1/router/entry.go | 7 +--- 2 files changed, 6 insertions(+), 61 deletions(-) diff --git a/service/store/v1/controller/entry.go b/service/store/v1/controller/entry.go index 7bb3772..b5763d0 100644 --- a/service/store/v1/controller/entry.go +++ b/service/store/v1/controller/entry.go @@ -21,7 +21,7 @@ type EntryController struct { * @apiUse EntryGetListByFeedRequest * @apiUse EntryGetListByFeedResponse */ -func (this *EntryController) GetListByFeed() { +func (this *EntryController) GetList() { entry.RequestGetListByFeed(this.GetInput()) appId := this.Ctx.Input.Params[":applicationId"] @@ -40,7 +40,7 @@ func (this *EntryController) GetListByFeed() { } /** - * @api {get} application/:applicationId/entry/:entryId Get (Global) + * @api {get} application/:applicationId/feed/:feedId/entry/:entryId Get (Global) * @apiVersion 1.0.0 * @apiName GetEntry * @apiGroup Entry @@ -49,16 +49,6 @@ func (this *EntryController) GetListByFeed() { * @apiUse EntryGetRequest * @apiUse EntryGetResponse */ -/** - * @api {get} application/:applicationId/feed/:feedId/entry/:entryId Get (Feed) - * @apiVersion 1.0.0 - * @apiName GetEntryFeed - * @apiGroup Entry - * @apiDescription This will return a specific entry. - * - * @apiUse EntryGetByFeedRequest - * @apiUse EntryGetByFeedResponse - */ func (this *EntryController) Get() { // two different usages both // 1: get entry from specific feed (includes feedId) @@ -81,33 +71,6 @@ func (this *EntryController) Get() { this.Controller.ServeJson() } -/** - * @api {post} application/:applicationId/entry Create (Global) - * @apiVersion 1.0.0 - * @apiName PostEntry - * @apiGroup Entry - * @apiDescription Create a entry on the global feed. This could be used to store a element in the cloud system and re-use it later. - * - * @apiUse EntryPostRequest - * @apiUse EntryPostResponse - */ -func (this *EntryController) Post() { - // global entry; should be added to APP no to the FEED - entry.RequestPost(this.GetInput()) - entry.ResponsePost() - this.Controller.ServeJson() -} - -/** - * @api {post} application/:applicationId/feed/:feedId/entry Create (Feed) - * @apiVersion 1.0.0 - * @apiName PostEntryFeed - * @apiGroup Entry - * @apiDescription Create a entry in the global feed and link it automatically to a feed. - * - * @apiUse EntryPostToFeedRequest - * @apiUse EntryPostToFeedResponse - */ /** * * @api {post} application/:applicationId/feed/:feedId/entry/ Add (Feed) @@ -119,7 +82,7 @@ func (this *EntryController) Post() { * @apiUse EntryAddToFeedRequest * @apiUse EntryAddToFeedResponse */ -func (this *EntryController) PostToFeed() { +func (this *EntryController) Post() { // two different usages // 1: post new data and create entry directly to feed // 2: post just a entryId which will be added to feed @@ -181,7 +144,7 @@ func (this *EntryController) Put() { } /** - * @api {delete} application/:applicationId/entry/:entryId Delete (Global) + * @api {delete} application/:applicationId/feed/:feedId/entry/:entryId Delete (Global) * @apiVersion 1.0.0 * @apiName DeleteEntry * @apiGroup Entry @@ -209,18 +172,3 @@ func (this *EntryController) Delete() { this.Controller.ServeJson() } -/** - * @api {delete} application/:applicationId/feed/:feedId/entry/:entryId Remove (Feed) - * @apiVersion 1.0.0 - * @apiName RemoveEntry - * @apiGroup Entry - * @apiDescription Removes a specific entry from a feed. - * - * @apiUse EntryRemoveRequest - * @apiUse EntryRemoveResponse - */ -func (this *EntryController) Remove() { - entry.RequestRemove(this.GetInput()) - entry.ResponseRemove() - this.Controller.ServeJson() -} diff --git a/service/store/v1/router/entry.go b/service/store/v1/router/entry.go index 44d1398..042f755 100644 --- a/service/store/v1/router/entry.go +++ b/service/store/v1/router/entry.go @@ -6,9 +6,6 @@ import ( ) func InitEntryRouters() { - feedify.Router("/v1/application/:applicationId:string/entry", &controller.EntryController{}, "post:Post") - feedify.Router("/v1/application/:applicationId:string/entry/:feedEntryId:int", &controller.EntryController{}, "get:Get;delete:Delete;put:Put") - - feedify.Router("/v1/application/:applicationId:string/feed/:feedId:int/entry", &controller.EntryController{}, "get:GetListByFeed;post:PostToFeed") - feedify.Router("/v1/application/:applicationId:string/feed/:feedId:int/entry/:feedEntryId:int", &controller.EntryController{}, "get:Get;delete:Remove;put:Put") + feedify.Router("/v1/application/:applicationId:string/feed/:feedId:int/entry", &controller.EntryController{}, "get:GetList;post:Post") + feedify.Router("/v1/application/:applicationId:string/feed/:feedId:int/entry/:feedEntryId:int", &controller.EntryController{}, "get:Get;delete:Delete;put:Put") } From ee08f69d10fce8759eff45fb60c155abab897ea2 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Fri, 17 Apr 2015 22:32:02 +0200 Subject: [PATCH 04/16] Added workflow update --- resource/workflow.go | 12 +++++++++++- service/stream/model/event.go | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/resource/workflow.go b/resource/workflow.go index 67a2077..e54f902 100644 --- a/resource/workflow.go +++ b/resource/workflow.go @@ -96,7 +96,17 @@ func AddWorkflow(feedWorkflow Workflow, FeedId string, ApplicationId string, Org } func UpdateWorkflow(id string, FeedId string, ApplicationId string, OrgId string, data string) (err error) { - return nil + workflow, err := GetWorkflow(id, FeedId, ApplicationId, OrgId) + + if err != nil { + return err + } + + // update workflow + workflow.Data = data + + _id, _ := strconv.Atoi(workflow.Id) + return storage.SetPropertyNode(_id, "data", data) } func DeleteWorkflow(id string, FeedId string, ApplicationId string, OrgId string) (error) { diff --git a/service/stream/model/event.go b/service/stream/model/event.go index 40a287c..d3312c9 100644 --- a/service/stream/model/event.go +++ b/service/stream/model/event.go @@ -24,7 +24,7 @@ type SocketEvent struct { OrgId string } -const archiveSize = 1 +const archiveSize = 10 var Archive = list.New() From 2d3dcd782f1da001f5391a715d22b962e542200f Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Mon, 20 Apr 2015 17:43:17 +0200 Subject: [PATCH 05/16] Implemented workflow remove --- resource/workflow.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/resource/workflow.go b/resource/workflow.go index e54f902..43b41a5 100644 --- a/resource/workflow.go +++ b/resource/workflow.go @@ -110,7 +110,20 @@ func UpdateWorkflow(id string, FeedId string, ApplicationId string, OrgId string } func DeleteWorkflow(id string, FeedId string, ApplicationId string, OrgId string) (error) { - return nil + workflow, err := GetWorkflow(id, FeedId, ApplicationId, OrgId) + + if err != nil { + return err + } + + _id, _ := strconv.Atoi(workflow.Id) + _rels, _ := storage.RelationshipsNode(_id, "workflow") + + for _, rel := range _rels { + storage.DeleteRelation(rel.Id) + } + + return storage.DeleteNode(_id) } func NewWorkflow(id string, feed *Feed, data string) *Workflow { From d68f7ecf2282da289a221b342a62d3b8e944c18c Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Mon, 20 Apr 2015 20:16:16 +0200 Subject: [PATCH 06/16] Added plugin skeleteon --- resource/org.go | 4 - resource/plugin.go | 89 +++++++++++++ resource/resource.go | 8 +- service/system/controller/default.go | 24 ++++ service/system/controller/plugin.go | 146 +++++++++++++++++++++ service/system/router/plugin.go | 11 ++ service/system/template/const.go | 12 ++ service/system/template/default.go | 29 ++++ service/system/template/plugin/request.go | 72 ++++++++++ service/system/template/plugin/response.go | 107 +++++++++++++++ service/system/template/request.go | 9 ++ service/system/template/response.go | 56 ++++++++ 12 files changed, 562 insertions(+), 5 deletions(-) create mode 100644 service/system/controller/default.go create mode 100644 service/system/controller/plugin.go create mode 100644 service/system/router/plugin.go create mode 100644 service/system/template/const.go create mode 100644 service/system/template/default.go create mode 100644 service/system/template/plugin/request.go create mode 100644 service/system/template/plugin/response.go create mode 100644 service/system/template/request.go create mode 100644 service/system/template/response.go diff --git a/resource/org.go b/resource/org.go index 685bb47..77d206c 100644 --- a/resource/org.go +++ b/resource/org.go @@ -103,10 +103,6 @@ func DeleteOrg(id string) (error) { return storage.DeleteNode(_id) } -func init() { - Orgs = make(map[string]*Org) -} - // TOKEN PART func GetOrgTokenList(orgId string) (orgList []*Token, err error) { diff --git a/resource/plugin.go b/resource/plugin.go index 958e354..5ad551a 100644 --- a/resource/plugin.go +++ b/resource/plugin.go @@ -1 +1,90 @@ package resource + +import ( + "errors" + "strconv" + + "github.com/feedlabs/feedify/graph" +) + +func GetPluginList() (pluginList []*Plugin, err error) { + nodes, err := storage.FindNodesByLabel(RESOURCE_ORG_LABEL) + if err != nil { + nodes = nil + } + + var plugins []*Plugin + + for _, node := range nodes { + id := strconv.Itoa(node.Id) + + if node.Data["name"] == nil { + node.Data["name"] = "" + } + + plugin := NewPlugin(id , node.Data["name"].(string)) + plugins = append(plugins, plugin) + } + + if plugins == nil { + plugins = make([]*Plugin, 0) + } + + return plugins, nil +} + +func GetPlugin(id string) (plugin *Plugin, err error) { + _id, err := strconv.Atoi(id) + node, err := storage.Node(_id) + + if err != nil { + return nil, err + } + + if node != nil && Contains(node.Labels, RESOURCE_ORG_LABEL) { + + if node.Data["name"] == nil { + node.Data["name"] = "" + } + + return NewPlugin(id , node.Data["name"].(string)), nil + } + + return nil, errors.New("PluginId `"+id+"` not exist") +} + +func AddPlugin(plugin *Plugin) (err error) { + properties := graph.Props { + "name": plugin.Name, + "version": plugin.Version, + "path": plugin.Path, + } + + _plugin, err := storage.NewNode(properties, RESOURCE_ORG_LABEL) + + if err != nil { + return err + } + + plugin.Id = strconv.Itoa(_plugin.Id) + + return nil +} + +func UpdatePlugin(plugin *Plugin) (err error) { + _id, _ := strconv.Atoi(plugin.Id) + return storage.SetPropertyNode(_id, "name", plugin.Name) +} + +func DeletePlugin(id string) (error) { + _id, _ := strconv.Atoi(id) + return storage.DeleteNode(_id) +} + +func NewPlugin(id string, name string) *Plugin { + _type := "sensor" + _version := "1" + _path := "/tmp" + + return &Plugin{id, name, _type, _version, _path} +} diff --git a/resource/resource.go b/resource/resource.go index f10d585..f33e6d3 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -104,7 +104,13 @@ type Workflow struct { Data string } -type Plugin struct {} +type Plugin struct { + Id string + Name string + Type string + Version string + Path string +} func ResourceStreamManager() { for { diff --git a/service/system/controller/default.go b/service/system/controller/default.go new file mode 100644 index 0000000..fe583aa --- /dev/null +++ b/service/system/controller/default.go @@ -0,0 +1,24 @@ +package controller + +import ( + "github.com/feedlabs/feedify" +) + +type DefaultController struct { + feedify.Controller +} + +func (this *DefaultController) Get() { + this.Data["json"] = map[string]string{"succes": "ok"} + this.Controller.ServeJson() +} + +func (this *DefaultController) ServeJson(data interface{}, status int) { + this.Data["json"] = data + this.SetResponseStatusCode(status) + this.Controller.ServeJson() +} + +func (this *DefaultController) SetResponseStatusCode(code int) { + this.Controller.Ctx.Output.SetStatus(code) +} diff --git a/service/system/controller/plugin.go b/service/system/controller/plugin.go new file mode 100644 index 0000000..1d0029e --- /dev/null +++ b/service/system/controller/plugin.go @@ -0,0 +1,146 @@ +package controller + +import ( + "encoding/json" + + "github.com/feedlabs/elasticfeed/resource" + template "github.com/feedlabs/elasticfeed/service/system/template/plugin" +) + +type PluginController struct { + DefaultController +} + +/** + * @api {get} plugin Get List + * @apiVersion 1.0.0 + * @apiName GetPluginList + * @apiGroup Plugin + * @apiDescription This will return a list of all plugins. + * + * @apiUse PluginGetListRequest + * @apiUse PluginGetListResponse + */ +func (this *PluginController) GetList() { + formatter, err := template.RequestGetList(this.GetInput()) + if err != nil { + this.ServeJson(template.GetError(err)) + return + } + + obs, err := resource.GetPluginList() + if err != nil { + this.ServeJson(template.GetError(err)) + } else { + this.ServeJson(template.ResponseGetList(obs, formatter)) + } +} + +/** + * @api {get} plugin/:pluginId Get + * @apiVersion 1.0.0 + * @apiName GetPlugin + * @apiGroup Plugin + * @apiDescription This will return a specific plugin. + * + * @apiUse PluginGetRequest + * @apiUse PluginGetResponse + */ +func (this *PluginController) Get() { + formatter, err := template.RequestGet(this.GetInput()) + if err != nil { + this.ServeJson(template.GetError(err)) + return + } + + pluginId := this.Ctx.Input.Params[":pluginId"] + + ob, err := resource.GetPlugin(pluginId) + if err != nil { + this.ServeJson(template.GetError(err)) + } else { + this.ServeJson(template.ResponseGet(ob, formatter)) + } +} + +/** + * @api {post} plugin Create + * @apiVersion 1.0.0 + * @apiName PostPlugin + * @apiGroup Plugin + * @apiDescription Create a plugin. + * + * @apiUse PluginPostRequest + * @apiUse PluginPostResponse + */ +func (this *PluginController) Post() { + formatter, err := template.RequestPost(this.GetInput()) + if err != nil { + this.ServeJson(template.GetError(err)) + return + } + + var plugin resource.Plugin + + data := this.Ctx.Input.CopyBody() + json.Unmarshal(data, &plugin) + + err = resource.AddPlugin(&plugin) + if err != nil { + this.ServeJson(template.GetError(err)) + } else { + this.ServeJson(template.ResponsePost(&plugin, formatter)) + } +} + +/** + * @api {put} plugin/:pluginId Update + * @apiVersion 1.0.0 + * @apiName PutPlugin + * @apiGroup Plugin + * @apiDescription Update a specific plugin. + * + * @apiUse PluginPutRequest + * @apiUse PluginPutResponse + */ + +func (this *PluginController) Put() { + formatter, err := template.RequestPut(this.GetInput()) + if err != nil { + this.ServeJson(template.GetError(err)) + return + } + + var plugin resource.Plugin + plugin.Id = this.Ctx.Input.Params[":pluginId"] + + data := this.Ctx.Input.CopyBody() + json.Unmarshal(data, &plugin) + + err = resource.UpdatePlugin(&plugin) + if err != nil { + this.ServeJson(template.GetError(err)) + } else { + this.ServeJson(template.ResponsePut(&plugin, formatter)) + } +} + +/** + * @api {delete} plugin/:pluginId Delete + * @apiVersion 1.0.0 + * @apiName DeletePlugin + * @apiGroup Plugin + * @apiDescription Delete a specific plugin. + + * @apiUse PluginDeleteRequest + * @apiUse PluginDeleteResponse + */ +func (this *PluginController) Delete() { + formatter, err := template.RequestDelete(this.GetInput()) + if err != nil { + this.ServeJson(template.GetError(err)) + return + } else { + this.ServeJson(template.ResponseDelete("Plugin has been deleted", formatter)) + } +} diff --git a/service/system/router/plugin.go b/service/system/router/plugin.go new file mode 100644 index 0000000..aadb847 --- /dev/null +++ b/service/system/router/plugin.go @@ -0,0 +1,11 @@ +package router + +import ( + "github.com/feedlabs/feedify" + "github.com/feedlabs/elasticfeed/service/system/controller" +) + +func InitOrgRouters() { + feedify.Router("/v1/plugin", &controller.PluginController{}, "get:GetList;post:Post") + feedify.Router("/v1/plugin/:pluginId:string", &controller.PluginController{}, "get:Get;delete:Delete;put:Put") +} diff --git a/service/system/template/const.go b/service/system/template/const.go new file mode 100644 index 0000000..8e811ae --- /dev/null +++ b/service/system/template/const.go @@ -0,0 +1,12 @@ +package template + +const HTTP_CODE_VALID_REQUEST = 200 +const HTTP_CODE_ENTITY_CREATED = 201 +const HTTP_CODE_ENTITY_NOEXIST = 404 +const HTTP_CODE_ENTITY_CONFLICT = 409 +const HTTP_CODE_INVALID_REQUEST = 400 +const HTTP_CODE_ACCESS_UNAUTHORIZED = 401 +const HTTP_CODE_ACCESS_FORBIDDEN = 403 +const HTTP_CODE_NOALLOWED_REQUEST = 405 +const HTTP_CODE_TOOMANY_REQUEST = 429 +const HTTP_CODE_SERVER_ERROR = 500 diff --git a/service/system/template/default.go b/service/system/template/default.go new file mode 100644 index 0000000..eeac146 --- /dev/null +++ b/service/system/template/default.go @@ -0,0 +1,29 @@ +package template + +func Error(err error) (entry map[string]interface{}, code int) { + entry = make(map[string]interface{}) + entry["result"] = err.Error() + entry["status"] = "error" + + return entry, HTTP_CODE_ENTITY_NOEXIST +} + +func Success(msg string) (entry map[string]string, code int) { + return map[string]string{"result": msg, "status": "ok"}, HTTP_CODE_VALID_REQUEST +} + +func GetOK() int { + return HTTP_CODE_VALID_REQUEST +} + +func PostOK() int { + return HTTP_CODE_ENTITY_CREATED +} + +func PutOK() int { + return HTTP_CODE_ENTITY_CREATED +} + +func DeleteOK() int { + return HTTP_CODE_VALID_REQUEST +} diff --git a/service/system/template/plugin/request.go b/service/system/template/plugin/request.go new file mode 100644 index 0000000..21f84ba --- /dev/null +++ b/service/system/template/plugin/request.go @@ -0,0 +1,72 @@ +package plugin + +import ( + "errors" + "github.com/feedlabs/feedify/context" + "github.com/feedlabs/elasticfeed/service/system/template" +) + +func CheckRequiredParams() { + // pluginId +} + +func GetResponseDefinition(input *context.Input) (*template.ResponseDefinition) { + return template.NewResponseDefinition(input) +} + +/** + * @apiDefine OrgGetListRequest + * + */ +func RequestGetList(input *context.Input) (formatter *template.ResponseDefinition, err error) { + if template.QueryParamsCount(input.Request.URL) > 4 { + return nil, errors.New("Too many params in URI query") + } + return GetResponseDefinition(input), nil +} + +/** + * @apiDefine OrgGetRequest + * + * @apiParam {String} pluginId The plugin id + */ +func RequestGet(input *context.Input) (formatter *template.ResponseDefinition, err error) { + if template.QueryParamsCount(input.Request.URL) != 1 { + return nil, errors.New("Too many params in URI query") + } + return GetResponseDefinition(input), nil +} + +/** + * @apiDefine OrgPostRequest + */ +func RequestPost(input *context.Input) (formatter *template.ResponseDefinition, err error) { + if template.QueryParamsCount(input.Request.URL) != 0 { + return nil, errors.New("Too many params in URI query") + } + return GetResponseDefinition(input), nil +} + +/** + * @apiDefine OrgPutRequest + * + * @apiParam {String} pluginId The plugin id + */ +func RequestPut(input *context.Input) (formatter *template.ResponseDefinition, err error) { + if template.QueryParamsCount(input.Request.URL) != 1 { + return nil, errors.New("Too many params in URI query") + } + return GetResponseDefinition(input), nil +} + +/** + * @apiDefine OrgDeleteRequest + * + * @apiParam {String} pluginId The plugin id + */ +func RequestDelete(input *context.Input) (formatter *template.ResponseDefinition, err error) { + if template.QueryParamsCount(input.Request.URL) != 1 { + return nil, errors.New("Too many params in URI query") + } + return GetResponseDefinition(input), nil +} diff --git a/service/system/template/plugin/response.go b/service/system/template/plugin/response.go new file mode 100644 index 0000000..87e0002 --- /dev/null +++ b/service/system/template/plugin/response.go @@ -0,0 +1,107 @@ +package plugin + +import ( + "github.com/feedlabs/elasticfeed/resource" + "github.com/feedlabs/elasticfeed/service/system/template" +) + +func GetEntry(plugin *resource.Plugin) (entry map[string]interface{}) { + entry = make(map[string]interface{}) + + return entry +} + +func GetError(err error) (entry map[string]interface{}, code int) { + return template.Error(err) +} + +func GetSuccess(msg string) (entry map[string]string, code int) { + return template.Success(msg) +} + +/** + * @apiDefine PluginGetListResponse + * + * @apiSuccess {Object[]} pluginList Array of all pluginanisations + * @apiSuccess {String} pluginList.id The plugin id + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * "pluginList": [ + * { + * "id": "KAJFDA7GFTRE87FDS78F7", + * "createStamp": "1415637736", + * }, + * ... + * ] + * } + */ +func ResponseGetList(pluginList []*resource.Plugin, formatter *template.ResponseDefinition) (entryList []map[string]interface{}, code int) { + var output []map[string]interface{} + + output = make([]map[string]interface{}, 0) + + return output, template.GetOK() +} + +/** + * @apiDefine PluginGetResponse + * + * @apiSuccess {String} id The id + * @apiSuccess {Int} createStamp Unix time stamp of create time + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "id": "KAJFDA7GFTRE87FDS78F7", + * ... + * "createStamp": "1415637736", + * } + */ +func ResponseGet(plugin *resource.Plugin, formatter *template.ResponseDefinition) (entry map[string]interface{}, code int) { + return GetEntry(plugin), template.GetOK() +} + +/** + * @apiDefine PluginPostResponse + * + * @apiSuccess {String} id The id + * @apiSuccess {Int} createStamp Unix time stamp of create time + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "id": "KAJFDA7GFTRE87FDS78F7", + * "createStamp": "1415637736", + * } + */ +func ResponsePost(plugin *resource.Plugin, formatter *template.ResponseDefinition) (entry map[string]interface{}, code int) { + return GetEntry(plugin), template.PostOK() +} + +/** + * @apiDefine PluginPutResponse + * + * @apiSuccess {String} id The id + * @apiSuccess {Int} createStamp Unix time stamp of create time + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "id": "KAJFDA7GFTRE87FDS78F7", + * "createStamp": "1415637736", + * } + */ +func ResponsePut(plugin *resource.Plugin, formatter *template.ResponseDefinition) (entry map[string]interface{}, code int) { + return GetEntry(plugin), template.PutOK() +} + +/** + * @apiDefine PluginDeleteResponse + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + */ +func ResponseDelete(msg string, formatter *template.ResponseDefinition) (entry map[string]string, code int) { + return GetSuccess(msg) +} diff --git a/service/system/template/request.go b/service/system/template/request.go new file mode 100644 index 0000000..52a8849 --- /dev/null +++ b/service/system/template/request.go @@ -0,0 +1,9 @@ +package template + +import ( + net "net/url" +) + +func QueryParamsCount(url *net.URL) int { + return len(url.Query()) +} diff --git a/service/system/template/response.go b/service/system/template/response.go new file mode 100644 index 0000000..4e3fe81 --- /dev/null +++ b/service/system/template/response.go @@ -0,0 +1,56 @@ +package template + +import ( + "strconv" + "github.com/feedlabs/feedify/context" +) + +type ResponseDefinition struct { + orderby string + orderdir string + page int + limit int +} + +func (this *ResponseDefinition) GetOrderBy() string { + return this.orderby +} + +func (this *ResponseDefinition) GetOrderDir() string { + return this.orderdir +} + +func (this *ResponseDefinition) GetPage() int { + return this.page +} + +func (this *ResponseDefinition) GetLimit() int { + return this.limit +} + +func NewResponseDefinition(input *context.Input) *ResponseDefinition { + orderby := input.Request.URL.Query().Get("orderby") + if orderby == "" { + orderby = "id" + } + + orderdir := input.Request.URL.Query().Get("orderdir") + if orderdir == "" { + orderdir = "asc" + } + + page := input.Request.URL.Query().Get("page") + if page == "" { + page = "0" + } + + limit := input.Request.URL.Query().Get("limit") + if limit == "" { + limit = "100" + } + + pageInt, _ := strconv.Atoi(page) + limitInt, _ := strconv.Atoi(limit) + + return &ResponseDefinition{orderby, orderdir, pageInt, limitInt} +} From c9ad6eb4975949f88b0280ee137fc2d9eef29810 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Tue, 21 Apr 2015 14:27:11 +0200 Subject: [PATCH 07/16] Added plugins logic; Refactored system service with versioning --- resource/plugin.go | 6 +++--- service/store/{api.go => store.go} | 0 service/system/router/plugin.go | 11 ----------- service/system/router/status.go | 10 ---------- service/system/system.go | 3 ++- service/system/{ => v1}/controller/default.go | 0 service/system/{ => v1}/controller/plugin.go | 2 +- service/system/{ => v1}/controller/status.go | 0 service/system/v1/router/plugin.go | 11 +++++++++++ service/system/v1/router/status.go | 10 ++++++++++ service/system/{ => v1}/template/const.go | 0 service/system/{ => v1}/template/default.go | 0 service/system/{ => v1}/template/plugin/request.go | 2 +- service/system/{ => v1}/template/plugin/response.go | 2 +- service/system/{ => v1}/template/request.go | 0 service/system/{ => v1}/template/response.go | 0 16 files changed, 29 insertions(+), 28 deletions(-) rename service/store/{api.go => store.go} (100%) delete mode 100644 service/system/router/plugin.go delete mode 100644 service/system/router/status.go rename service/system/{ => v1}/controller/default.go (100%) rename service/system/{ => v1}/controller/plugin.go (97%) rename service/system/{ => v1}/controller/status.go (100%) create mode 100644 service/system/v1/router/plugin.go create mode 100644 service/system/v1/router/status.go rename service/system/{ => v1}/template/const.go (100%) rename service/system/{ => v1}/template/default.go (100%) rename service/system/{ => v1}/template/plugin/request.go (96%) rename service/system/{ => v1}/template/plugin/response.go (97%) rename service/system/{ => v1}/template/request.go (100%) rename service/system/{ => v1}/template/response.go (100%) diff --git a/resource/plugin.go b/resource/plugin.go index 5ad551a..6b46c96 100644 --- a/resource/plugin.go +++ b/resource/plugin.go @@ -8,7 +8,7 @@ import ( ) func GetPluginList() (pluginList []*Plugin, err error) { - nodes, err := storage.FindNodesByLabel(RESOURCE_ORG_LABEL) + nodes, err := storage.FindNodesByLabel(RESOURCE_PLUGIN_LABEL) if err != nil { nodes = nil } @@ -41,7 +41,7 @@ func GetPlugin(id string) (plugin *Plugin, err error) { return nil, err } - if node != nil && Contains(node.Labels, RESOURCE_ORG_LABEL) { + if node != nil && Contains(node.Labels, RESOURCE_PLUGIN_LABEL) { if node.Data["name"] == nil { node.Data["name"] = "" @@ -60,7 +60,7 @@ func AddPlugin(plugin *Plugin) (err error) { "path": plugin.Path, } - _plugin, err := storage.NewNode(properties, RESOURCE_ORG_LABEL) + _plugin, err := storage.NewNode(properties, RESOURCE_PLUGIN_LABEL) if err != nil { return err diff --git a/service/store/api.go b/service/store/store.go similarity index 100% rename from service/store/api.go rename to service/store/store.go diff --git a/service/system/router/plugin.go b/service/system/router/plugin.go deleted file mode 100644 index aadb847..0000000 --- a/service/system/router/plugin.go +++ /dev/null @@ -1,11 +0,0 @@ -package router - -import ( - "github.com/feedlabs/feedify" - "github.com/feedlabs/elasticfeed/service/system/controller" -) - -func InitOrgRouters() { - feedify.Router("/v1/plugin", &controller.PluginController{}, "get:GetList;post:Post") - feedify.Router("/v1/plugin/:pluginId:string", &controller.PluginController{}, "get:Get;delete:Delete;put:Put") -} diff --git a/service/system/router/status.go b/service/system/router/status.go deleted file mode 100644 index c6ba3cf..0000000 --- a/service/system/router/status.go +++ /dev/null @@ -1,10 +0,0 @@ -package router - -import ( - "github.com/feedlabs/feedify" - "github.com/feedlabs/elasticfeed/service/system/controller" -) - -func InitRouters() { - feedify.Router("/system/status", &controller.StatusController{}, "get:Get") -} diff --git a/service/system/system.go b/service/system/system.go index bb8dca2..1087cb6 100644 --- a/service/system/system.go +++ b/service/system/system.go @@ -1,13 +1,14 @@ package system import ( - "github.com/feedlabs/elasticfeed/service/system/router" + "github.com/feedlabs/elasticfeed/service/system/v1/router" ) type SystemService struct {} func (this *SystemService) Init() { router.InitRouters() + router.InitPluginRouters() } func NewMetricService() *SystemService { diff --git a/service/system/controller/default.go b/service/system/v1/controller/default.go similarity index 100% rename from service/system/controller/default.go rename to service/system/v1/controller/default.go diff --git a/service/system/controller/plugin.go b/service/system/v1/controller/plugin.go similarity index 97% rename from service/system/controller/plugin.go rename to service/system/v1/controller/plugin.go index 1d0029e..8603027 100644 --- a/service/system/controller/plugin.go +++ b/service/system/v1/controller/plugin.go @@ -4,7 +4,7 @@ import ( "encoding/json" "github.com/feedlabs/elasticfeed/resource" - template "github.com/feedlabs/elasticfeed/service/system/template/plugin" + template "github.com/feedlabs/elasticfeed/service/system/v1/template/plugin" ) type PluginController struct { diff --git a/service/system/controller/status.go b/service/system/v1/controller/status.go similarity index 100% rename from service/system/controller/status.go rename to service/system/v1/controller/status.go diff --git a/service/system/v1/router/plugin.go b/service/system/v1/router/plugin.go new file mode 100644 index 0000000..95f37d7 --- /dev/null +++ b/service/system/v1/router/plugin.go @@ -0,0 +1,11 @@ +package router + +import ( + "github.com/feedlabs/feedify" + "github.com/feedlabs/elasticfeed/service/system/v1/controller" +) + +func InitPluginRouters() { + feedify.Router("/v1/system/plugin", &controller.PluginController{}, "get:GetList;post:Post") + feedify.Router("/v1/system/plugin/:pluginId:string", &controller.PluginController{}, "get:Get;delete:Delete;put:Put") +} diff --git a/service/system/v1/router/status.go b/service/system/v1/router/status.go new file mode 100644 index 0000000..dbb68c2 --- /dev/null +++ b/service/system/v1/router/status.go @@ -0,0 +1,10 @@ +package router + +import ( + "github.com/feedlabs/feedify" + "github.com/feedlabs/elasticfeed/service/system/v1/controller" +) + +func InitRouters() { + feedify.Router("/v1/system/status", &controller.StatusController{}, "get:Get") +} diff --git a/service/system/template/const.go b/service/system/v1/template/const.go similarity index 100% rename from service/system/template/const.go rename to service/system/v1/template/const.go diff --git a/service/system/template/default.go b/service/system/v1/template/default.go similarity index 100% rename from service/system/template/default.go rename to service/system/v1/template/default.go diff --git a/service/system/template/plugin/request.go b/service/system/v1/template/plugin/request.go similarity index 96% rename from service/system/template/plugin/request.go rename to service/system/v1/template/plugin/request.go index 21f84ba..74bd17e 100644 --- a/service/system/template/plugin/request.go +++ b/service/system/v1/template/plugin/request.go @@ -3,7 +3,7 @@ package plugin import ( "errors" "github.com/feedlabs/feedify/context" - "github.com/feedlabs/elasticfeed/service/system/template" + "github.com/feedlabs/elasticfeed/service/system/v1/template" ) func CheckRequiredParams() { diff --git a/service/system/template/plugin/response.go b/service/system/v1/template/plugin/response.go similarity index 97% rename from service/system/template/plugin/response.go rename to service/system/v1/template/plugin/response.go index 87e0002..ca828a1 100644 --- a/service/system/template/plugin/response.go +++ b/service/system/v1/template/plugin/response.go @@ -2,7 +2,7 @@ package plugin import ( "github.com/feedlabs/elasticfeed/resource" - "github.com/feedlabs/elasticfeed/service/system/template" + "github.com/feedlabs/elasticfeed/service/system/v1/template" ) func GetEntry(plugin *resource.Plugin) (entry map[string]interface{}) { diff --git a/service/system/template/request.go b/service/system/v1/template/request.go similarity index 100% rename from service/system/template/request.go rename to service/system/v1/template/request.go diff --git a/service/system/template/response.go b/service/system/v1/template/response.go similarity index 100% rename from service/system/template/response.go rename to service/system/v1/template/response.go From 6bf243ce48c8ff45ea3e4316a609631043b33d77 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Tue, 21 Apr 2015 17:44:38 +0200 Subject: [PATCH 08/16] Implemented more plugin logic --- resource/plugin.go | 37 +++++++++++++++---- resource/resource.go | 3 +- service/system/v1/controller/plugin.go | 5 +++ service/system/v1/template/plugin/response.go | 11 +++++- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/resource/plugin.go b/resource/plugin.go index 6b46c96..6ca40be 100644 --- a/resource/plugin.go +++ b/resource/plugin.go @@ -22,7 +22,19 @@ func GetPluginList() (pluginList []*Plugin, err error) { node.Data["name"] = "" } - plugin := NewPlugin(id , node.Data["name"].(string)) + if node.Data["group"] == nil { + node.Data["group"] = "" + } + + if node.Data["version"] == nil { + node.Data["version"] = "" + } + + if node.Data["path"] == nil { + node.Data["path"] = "" + } + + plugin := NewPlugin(id , node.Data["name"].(string), node.Data["group"].(string), node.Data["version"].(string), node.Data["path"].(string)) plugins = append(plugins, plugin) } @@ -47,7 +59,19 @@ func GetPlugin(id string) (plugin *Plugin, err error) { node.Data["name"] = "" } - return NewPlugin(id , node.Data["name"].(string)), nil + if node.Data["group"] == nil { + node.Data["group"] = "" + } + + if node.Data["version"] == nil { + node.Data["version"] = "" + } + + if node.Data["path"] == nil { + node.Data["path"] = "" + } + + return NewPlugin(id , node.Data["name"].(string), node.Data["group"].(string), node.Data["version"].(string), node.Data["path"].(string)), nil } return nil, errors.New("PluginId `"+id+"` not exist") @@ -56,6 +80,7 @@ func GetPlugin(id string) (plugin *Plugin, err error) { func AddPlugin(plugin *Plugin) (err error) { properties := graph.Props { "name": plugin.Name, + "group": plugin.Group, "version": plugin.Version, "path": plugin.Path, } @@ -81,10 +106,6 @@ func DeletePlugin(id string) (error) { return storage.DeleteNode(_id) } -func NewPlugin(id string, name string) *Plugin { - _type := "sensor" - _version := "1" - _path := "/tmp" - - return &Plugin{id, name, _type, _version, _path} +func NewPlugin(id string, name string, group string, version string, path string) *Plugin { + return &Plugin{id, name, group, version, path} } diff --git a/resource/resource.go b/resource/resource.go index f33e6d3..1cab690 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -107,7 +107,7 @@ type Workflow struct { type Plugin struct { Id string Name string - Type string + Group string Version string Path string } @@ -231,6 +231,7 @@ func InitResources() { Tokens = make(map[string]*Token) Metrics = make(map[string]*Metric) Viewers = make(map[string]*Viewer) + Plugins = make(map[string]*Plugin) } func InitStorage() { diff --git a/service/system/v1/controller/plugin.go b/service/system/v1/controller/plugin.go index 8603027..acee3b8 100644 --- a/service/system/v1/controller/plugin.go +++ b/service/system/v1/controller/plugin.go @@ -137,6 +137,11 @@ func (this *PluginController) Put() { */ func (this *PluginController) Delete() { formatter, err := template.RequestDelete(this.GetInput()) + + pluginId := this.Ctx.Input.Params[":pluginId"] + + err = resource.DeletePlugin(pluginId) + if err != nil { this.ServeJson(template.GetError(err)) return diff --git a/service/system/v1/template/plugin/response.go b/service/system/v1/template/plugin/response.go index ca828a1..a44e3da 100644 --- a/service/system/v1/template/plugin/response.go +++ b/service/system/v1/template/plugin/response.go @@ -8,6 +8,13 @@ import ( func GetEntry(plugin *resource.Plugin) (entry map[string]interface{}) { entry = make(map[string]interface{}) + entry["id"] = plugin.Id + entry["name"] = plugin.Name + entry["group"] = plugin.Group + entry["version"] = plugin.Version + entry["status"] = "running" + entry["errors"] = "no errors" + return entry } @@ -39,7 +46,9 @@ func GetSuccess(msg string) (entry map[string]string, code int) { func ResponseGetList(pluginList []*resource.Plugin, formatter *template.ResponseDefinition) (entryList []map[string]interface{}, code int) { var output []map[string]interface{} - output = make([]map[string]interface{}, 0) + for _, plugin := range pluginList { + output = append(output, GetEntry(plugin)) + } return output, template.GetOK() } From ede524dcd06b0980280e65c2d7d323ec135c2af6 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Tue, 21 Apr 2015 19:18:53 +0200 Subject: [PATCH 09/16] Extended workflow resource, controller --- resource/resource.go | 17 ++-- resource/workflow.go | 20 ++++- service/store/v1/controller/workflow.go | 87 ++++++++++++------- service/store/v1/template/workflow/request.go | 63 ++++++++------ .../store/v1/template/workflow/response.go | 50 +++++++++-- service/system/v1/template/plugin/request.go | 10 +-- 6 files changed, 167 insertions(+), 80 deletions(-) diff --git a/resource/resource.go b/resource/resource.go index 1cab690..eb6c8c3 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -99,17 +99,18 @@ type Viewer struct {} type Metric struct {} type Workflow struct { - Id string - Feed *Feed - Data string + Id string + Feed *Feed + Default bool + Data string } type Plugin struct { - Id string - Name string - Group string - Version string - Path string + Id string + Name string + Group string + Version string + Path string } func ResourceStreamManager() { diff --git a/resource/workflow.go b/resource/workflow.go index 43b41a5..9f24e88 100644 --- a/resource/workflow.go +++ b/resource/workflow.go @@ -28,7 +28,13 @@ func GetWorkflowList(FeedId string, ApplicationId string, OrgId string) (feedWor for _, rel := range _rels { data := rel.EndNode.Data["data"].(string) - workflow := NewWorkflow(strconv.Itoa(rel.EndNode.Id), feed, data) + def := rel.EndNode.Data["default"] + + if def == nil { + def = false + } + + workflow := NewWorkflow(strconv.Itoa(rel.EndNode.Id), feed, def.(bool), data) if workflow != nil && Contains(rel.EndNode.Labels, RESOURCE_WORKFLOW_LABEL) && feed.Id == rel.EndNode.Data["feedId"].(string) { workflows = append(workflows, workflow) } @@ -56,7 +62,13 @@ func GetWorkflow(id string, FeedId string, ApplicationId string, OrgId string) ( if workflow != nil && Contains(workflow.Labels, RESOURCE_WORKFLOW_LABEL) && feed.Id == workflow.Data["feedId"].(string) { data := workflow.Data["data"].(string) - return NewWorkflow(strconv.Itoa(workflow.Id), feed, data), nil + def := workflow.Data["default"] + + if def == nil { + def = false + } + + return NewWorkflow(strconv.Itoa(workflow.Id), feed, def.(bool), data), nil } return nil, errors.New("WorkflowId `"+id+"` not exist") @@ -126,6 +138,6 @@ func DeleteWorkflow(id string, FeedId string, ApplicationId string, OrgId string return storage.DeleteNode(_id) } -func NewWorkflow(id string, feed *Feed, data string) *Workflow { - return &Workflow{id, feed, data} +func NewWorkflow(id string, feed *Feed, def bool, data string) *Workflow { + return &Workflow{id, feed, def, data} } diff --git a/service/store/v1/controller/workflow.go b/service/store/v1/controller/workflow.go index e16d252..6f3f958 100644 --- a/service/store/v1/controller/workflow.go +++ b/service/store/v1/controller/workflow.go @@ -4,7 +4,7 @@ import ( "encoding/json" "github.com/feedlabs/elasticfeed/resource" - "github.com/feedlabs/elasticfeed/service/store/v1/template/workflow" + template "github.com/feedlabs/elasticfeed/service/store/v1/template/workflow" ) type WorkflowController struct { @@ -22,21 +22,47 @@ type WorkflowController struct { * @apiUse WorkflowGetListResponse */ func (this *WorkflowController) GetList() { - workflow.RequestGetList(this.GetInput()) + formatter, err := template.RequestGetList(this.GetInput()) + if err != nil { + this.ServeJson(template.GetError(err)) + return + } appId := this.Ctx.Input.Params[":applicationId"] feedId := this.Ctx.Input.Params[":feedId"] feed, err := resource.GetFeed(feedId, appId, this.GetAdminOrgId()) - obs, err := feed.GetWorkflowList() + obs, err := feed.GetWorkflowList() if err != nil { - this.Data["json"] = map[string]string{"result": err.Error(), "status": "error"} + this.ServeJson(template.GetError(err)) } else { - this.Data["json"] = obs + this.ServeJson(template.ResponseGetList(obs, formatter)) } +} + +/** + * @api {get} application/:applicationId/feed/:feedId/workflow/:workflowId Get + * @apiVersion 1.0.0 + * @apiName GetWorkflow + * @apiGroup Workflow + * @apiDescription This will return a specific workflow. + * + * @apiUse WorkflowGetRequest + * @apiUse WorkflowGetResponse + */ +func (this *WorkflowController) Get() { + formatter, err := template.RequestGet(this.GetInput()) + + appId := this.Ctx.Input.Params[":applicationId"] + feedId := this.Ctx.Input.Params[":feedId"] + feedWorkflowId := this.Ctx.Input.Params[":feedWorkflowId"] - workflow.ResponseGetList() - this.Controller.ServeJson() + ob, err := resource.GetWorkflow(feedWorkflowId, feedId, appId, this.GetAdminOrgId()) + if err != nil { + this.ServeJson(template.GetError(err)) + } else { + this.ServeJson(template.ResponseGet(ob, formatter)) + } } /** @@ -51,7 +77,11 @@ func (this *WorkflowController) GetList() { * @apiUse WorkflowAddToFeedResponse */ func (this *WorkflowController) Post() { - workflow.RequestPost(this.GetInput()) + formatter, err := template.RequestPost(this.GetInput()) + if err != nil { + this.ServeJson(template.GetError(err)) + return + } appId := this.Ctx.Input.Params[":applicationId"] feedId := this.Ctx.Input.Params[":feedId"] @@ -62,16 +92,13 @@ func (this *WorkflowController) Post() { app, err := resource.GetApplication(appId, this.GetAdminOrgId()) feed, err := app.GetFeed(feedId) - workflowId, err := feed.AddWorkflow(ob) + _, err = feed.AddWorkflow(ob) if err != nil { - this.Data["json"] = map[string]string{"result": err.Error(), "status": "error"} + this.ServeJson(template.GetError(err)) } else { - this.Data["json"] = map[string]string{"id": workflowId} + this.ServeJson(template.ResponsePost(&ob, formatter)) } - - workflow.ResponsePost() - this.Controller.ServeJson() } /** @@ -85,7 +112,11 @@ func (this *WorkflowController) Post() { * @apiUse WorkflowPutResponse */ func (this *WorkflowController) Put() { - workflow.RequestPut(this.GetInput()) + formatter, err := template.RequestPut(this.GetInput()) + if err != nil { + this.ServeJson(template.GetError(err)) + return + } appId := this.Ctx.Input.Params[":applicationId"] feedId := this.Ctx.Input.Params[":feedId"] @@ -96,15 +127,12 @@ func (this *WorkflowController) Put() { data := this.Ctx.Input.CopyBody() json.Unmarshal(data, &ob) - err := resource.UpdateWorkflow(feedWorkflowId, feedId, appId, this.GetAdminOrgId(), ob.Data) + err = resource.UpdateWorkflow(feedWorkflowId, feedId, appId, this.GetAdminOrgId(), ob.Data) if err != nil { - this.Data["json"] = map[string]string{"result": err.Error(), "status": "error"} + this.ServeJson(template.GetError(err)) } else { - this.Data["json"] = map[string]string{"result": "update success", "status": "ok"} + this.ServeJson(template.ResponsePut(&ob, formatter)) } - - workflow.ResponsePut() - this.Controller.ServeJson() } /** @@ -118,20 +146,21 @@ func (this *WorkflowController) Put() { * @apiUse WorkflowDeleteResponse */ func (this *WorkflowController) Delete() { - workflow.RequestDelete(this.GetInput()) + formatter, err := template.RequestDelete(this.GetInput()) + if err != nil { + this.ServeJson(template.GetError(err)) + return + } appId := this.Ctx.Input.Params[":applicationId"] feedId := this.Ctx.Input.Params[":feedId"] feedWorkflowId := this.Ctx.Input.Params[":feedWorkflowId"] - err := resource.DeleteWorkflow(feedWorkflowId, feedId, appId, this.GetAdminOrgId()) - + err = resource.DeleteWorkflow(feedWorkflowId, feedId, appId, this.GetAdminOrgId()) if err != nil { - this.Data["json"] = map[string]string{"result": err.Error(), "status": "error"} + this.ServeJson(template.GetError(err)) + return } else { - this.Data["json"] = map[string]string{"result": "delete success", "status": "ok"} + this.ServeJson(template.ResponseDelete("Org has been deleted", formatter)) } - - workflow.ResponseDelete() - this.Controller.ServeJson() } diff --git a/service/store/v1/template/workflow/request.go b/service/store/v1/template/workflow/request.go index 1c7a6d7..ec4b22a 100644 --- a/service/store/v1/template/workflow/request.go +++ b/service/store/v1/template/workflow/request.go @@ -1,59 +1,72 @@ package workflow import ( + "errors" "github.com/feedlabs/feedify/context" + "github.com/feedlabs/elasticfeed/service/store/v1/template" ) +func CheckRequiredParams() { + // workflowId +} + +func GetResponseDefinition(input *context.Input) (*template.ResponseDefinition) { + return template.NewResponseDefinition(input) +} /** - * @apiDefine WorkflowGetListByFeedRequest + * @apiDefine WorkflowGetListRequest * - * @apiParam {String} applicationId The application id - * @apiParam {String} feedId The application id */ -func RequestGetList(input *context.Input) { - +func RequestGetList(input *context.Input) (formatter *template.ResponseDefinition, err error) { + if template.QueryParamsCount(input.Request.URL) > 4 { + return nil, errors.New("Too many params in URI query") + } + return GetResponseDefinition(input), nil } /** * @apiDefine WorkflowGetRequest * - * @apiParam {String} applicationId The application id - * @apiParam {String} workflowId The workflow id + * @apiParam {String} pluginId The plugin id */ -func RequestGet(input *context.Input) { - +func RequestGet(input *context.Input) (formatter *template.ResponseDefinition, err error) { + if template.QueryParamsCount(input.Request.URL) != 1 { + return nil, errors.New("Too many params in URI query") + } + return GetResponseDefinition(input), nil } /** * @apiDefine WorkflowPostRequest - * - * @apiParam {String} applicationId The application id - * @apiParam {String} data The data of the workflow - * @apiParam {String[]} [tagList] Tags of the workflow */ -func RequestPost(input *context.Input) { - +func RequestPost(input *context.Input) (formatter *template.ResponseDefinition, err error) { + if template.QueryParamsCount(input.Request.URL) != 0 { + return nil, errors.New("Too many params in URI query") + } + return GetResponseDefinition(input), nil } /** * @apiDefine WorkflowPutRequest * - * @apiParam {String} applicationId The application id - * @apiParam {String} workflowId The workflow id - * @apiParam {String} data The data of the workflow - * @apiParam {String[]} [tagList] Tags of the workflow + * @apiParam {String} pluginId The plugin id */ -func RequestPut(input *context.Input) { - +func RequestPut(input *context.Input) (formatter *template.ResponseDefinition, err error) { + if template.QueryParamsCount(input.Request.URL) != 1 { + return nil, errors.New("Too many params in URI query") + } + return GetResponseDefinition(input), nil } /** * @apiDefine WorkflowDeleteRequest * - * @apiParam {String} applicationId The application id - * @apiParam {String} workflowId The workflow id + * @apiParam {String} pluginId The plugin id */ -func RequestDelete(input *context.Input) { - +func RequestDelete(input *context.Input) (formatter *template.ResponseDefinition, err error) { + if template.QueryParamsCount(input.Request.URL) != 1 { + return nil, errors.New("Too many params in URI query") + } + return GetResponseDefinition(input), nil } diff --git a/service/store/v1/template/workflow/response.go b/service/store/v1/template/workflow/response.go index 782c7ce..f083e44 100644 --- a/service/store/v1/template/workflow/response.go +++ b/service/store/v1/template/workflow/response.go @@ -1,6 +1,31 @@ package workflow +import ( + "github.com/feedlabs/elasticfeed/resource" + "github.com/feedlabs/elasticfeed/service/store/v1/template" +) +func GetEntry(workflow *resource.Workflow) (entry map[string]interface{}) { + entry = make(map[string]interface{}) + + entry["id"] = workflow.Id + entry["applicationId"] = workflow.Feed.Application.Id + entry["feedId"] = workflow.Feed.Id + entry["default"] = workflow.Default + entry["data"] = workflow.Data + entry["status"] = "running" + entry["errors"] = "no errors" + + return entry +} + +func GetError(err error) (entry map[string]interface{}, code int) { + return template.Error(err) +} + +func GetSuccess(msg string) (entry map[string]string, code int) { + return template.Success(msg) +} /** * @apiDefine WorkflowGetListByFeedResponse * @@ -33,8 +58,14 @@ package workflow * ] * } */ -func ResponseGetList() { +func ResponseGetList(workflowList []*resource.Workflow, formatter *template.ResponseDefinition) (entryList []map[string]interface{}, code int) { + var output []map[string]interface{} + for _, plugin := range workflowList { + output = append(output, GetEntry(plugin)) + } + + return output, template.GetOK() } /** @@ -59,8 +90,8 @@ func ResponseGetList() { * "createStamp": "1415637736", * } */ -func ResponseGet() { - +func ResponseGet(workflow *resource.Workflow, formatter *template.ResponseDefinition) (entry map[string]interface{}, code int) { + return GetEntry(workflow), template.GetOK() } /** @@ -85,8 +116,8 @@ func ResponseGet() { * "createStamp": "1415637736", * } */ -func ResponsePost() { - +func ResponsePost(workflow *resource.Workflow, formatter *template.ResponseDefinition) (entry map[string]interface{}, code int) { + return GetEntry(workflow), template.PostOK() } /** @@ -113,8 +144,8 @@ func ResponsePost() { * "createStamp": "1415637736", * } */ -func ResponsePut() { - +func ResponsePut(workflow *resource.Workflow, formatter *template.ResponseDefinition) (entry map[string]interface{}, code int) { + return GetEntry(workflow), template.PutOK() } /** @@ -123,6 +154,7 @@ func ResponsePut() { * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK */ -func ResponseDelete() { - +func ResponseDelete(msg string, formatter *template.ResponseDefinition) (entry map[string]string, code int) { + return GetSuccess(msg) } + diff --git a/service/system/v1/template/plugin/request.go b/service/system/v1/template/plugin/request.go index 74bd17e..67b9162 100644 --- a/service/system/v1/template/plugin/request.go +++ b/service/system/v1/template/plugin/request.go @@ -15,7 +15,7 @@ func GetResponseDefinition(input *context.Input) (*template.ResponseDefinition) } /** - * @apiDefine OrgGetListRequest + * @apiDefine PluginGetListRequest * */ func RequestGetList(input *context.Input) (formatter *template.ResponseDefinition, err error) { @@ -26,7 +26,7 @@ func RequestGetList(input *context.Input) (formatter *template.ResponseDefinitio } /** - * @apiDefine OrgGetRequest + * @apiDefine PluginGetRequest * * @apiParam {String} pluginId The plugin id */ @@ -38,7 +38,7 @@ func RequestGet(input *context.Input) (formatter *template.ResponseDefinition, e } /** - * @apiDefine OrgPostRequest + * @apiDefine PluginPostRequest */ func RequestPost(input *context.Input) (formatter *template.ResponseDefinition, err error) { if template.QueryParamsCount(input.Request.URL) != 0 { @@ -48,7 +48,7 @@ func RequestPost(input *context.Input) (formatter *template.ResponseDefinition, } /** - * @apiDefine OrgPutRequest + * @apiDefine PluginPutRequest * * @apiParam {String} pluginId The plugin id */ @@ -60,7 +60,7 @@ func RequestPut(input *context.Input) (formatter *template.ResponseDefinition, e } /** - * @apiDefine OrgDeleteRequest + * @apiDefine PluginDeleteRequest * * @apiParam {String} pluginId The plugin id */ From 4163489c73723500f74ff7740c96ead65bace86d Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Tue, 21 Apr 2015 19:57:03 +0200 Subject: [PATCH 10/16] Return instance of workflow after POST --- resource/workflow.go | 15 +++++++++++---- service/store/v1/controller/workflow.go | 6 +++--- service/store/v1/template/workflow/request.go | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/resource/workflow.go b/resource/workflow.go index 9f24e88..2c33f7e 100644 --- a/resource/workflow.go +++ b/resource/workflow.go @@ -107,18 +107,25 @@ func AddWorkflow(feedWorkflow Workflow, FeedId string, ApplicationId string, Org return feedWorkflow.Id, nil } -func UpdateWorkflow(id string, FeedId string, ApplicationId string, OrgId string, data string) (err error) { - workflow, err := GetWorkflow(id, FeedId, ApplicationId, OrgId) +func UpdateWorkflow(id string, FeedId string, ApplicationId string, OrgId string, data string) (workflow *Workflow, err error) { + workflow, err = GetWorkflow(id, FeedId, ApplicationId, OrgId) if err != nil { - return err + return nil, err } // update workflow workflow.Data = data _id, _ := strconv.Atoi(workflow.Id) - return storage.SetPropertyNode(_id, "data", data) + + err = storage.SetPropertyNode(_id, "data", data) + + if err != nil { + return nil, err + } + + return workflow, nil } func DeleteWorkflow(id string, FeedId string, ApplicationId string, OrgId string) (error) { diff --git a/service/store/v1/controller/workflow.go b/service/store/v1/controller/workflow.go index 6f3f958..211093d 100644 --- a/service/store/v1/controller/workflow.go +++ b/service/store/v1/controller/workflow.go @@ -122,16 +122,16 @@ func (this *WorkflowController) Put() { feedId := this.Ctx.Input.Params[":feedId"] feedWorkflowId := this.Ctx.Input.Params[":feedWorkflowId"] - var ob resource.Workflow + var ob *resource.Workflow data := this.Ctx.Input.CopyBody() json.Unmarshal(data, &ob) - err = resource.UpdateWorkflow(feedWorkflowId, feedId, appId, this.GetAdminOrgId(), ob.Data) + _ob, err := resource.UpdateWorkflow(feedWorkflowId, feedId, appId, this.GetAdminOrgId(), ob.Data) if err != nil { this.ServeJson(template.GetError(err)) } else { - this.ServeJson(template.ResponsePut(&ob, formatter)) + this.ServeJson(template.ResponsePut(_ob, formatter)) } } diff --git a/service/store/v1/template/workflow/request.go b/service/store/v1/template/workflow/request.go index ec4b22a..29f28c6 100644 --- a/service/store/v1/template/workflow/request.go +++ b/service/store/v1/template/workflow/request.go @@ -53,7 +53,7 @@ func RequestPost(input *context.Input) (formatter *template.ResponseDefinition, * @apiParam {String} pluginId The plugin id */ func RequestPut(input *context.Input) (formatter *template.ResponseDefinition, err error) { - if template.QueryParamsCount(input.Request.URL) != 1 { + if template.QueryParamsCount(input.Request.URL) > 4 { return nil, errors.New("Too many params in URI query") } return GetResponseDefinition(input), nil From df63acb238b8294e21da87af5603aae5d0aecc64 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Tue, 21 Apr 2015 23:44:25 +0200 Subject: [PATCH 11/16] Added plugin binary upload --- resource/plugin.go | 26 +++++++++--- resource/resource.go | 11 ++--- service/system/v1/controller/plugin.go | 41 +++++++++++++++++++ service/system/v1/router/plugin.go | 1 + service/system/v1/template/plugin/response.go | 2 + 5 files changed, 71 insertions(+), 10 deletions(-) diff --git a/resource/plugin.go b/resource/plugin.go index 6ca40be..f52351e 100644 --- a/resource/plugin.go +++ b/resource/plugin.go @@ -34,7 +34,11 @@ func GetPluginList() (pluginList []*Plugin, err error) { node.Data["path"] = "" } - plugin := NewPlugin(id , node.Data["name"].(string), node.Data["group"].(string), node.Data["version"].(string), node.Data["path"].(string)) + if node.Data["license"] == nil { + node.Data["license"] = "" + } + + plugin := NewPlugin(id , node.Data["name"].(string), node.Data["group"].(string), node.Data["version"].(string), node.Data["path"].(string), node.Data["license"].(string)) plugins = append(plugins, plugin) } @@ -71,7 +75,11 @@ func GetPlugin(id string) (plugin *Plugin, err error) { node.Data["path"] = "" } - return NewPlugin(id , node.Data["name"].(string), node.Data["group"].(string), node.Data["version"].(string), node.Data["path"].(string)), nil + if node.Data["license"] == nil { + node.Data["license"] = "" + } + + return NewPlugin(id , node.Data["name"].(string), node.Data["group"].(string), node.Data["version"].(string), node.Data["path"].(string), node.Data["license"].(string)), nil } return nil, errors.New("PluginId `"+id+"` not exist") @@ -83,6 +91,7 @@ func AddPlugin(plugin *Plugin) (err error) { "group": plugin.Group, "version": plugin.Version, "path": plugin.Path, + "license": plugin.License, } _plugin, err := storage.NewNode(properties, RESOURCE_PLUGIN_LABEL) @@ -98,7 +107,14 @@ func AddPlugin(plugin *Plugin) (err error) { func UpdatePlugin(plugin *Plugin) (err error) { _id, _ := strconv.Atoi(plugin.Id) - return storage.SetPropertyNode(_id, "name", plugin.Name) + + err = storage.SetPropertyNode(_id, "name", plugin.Name) + err = storage.SetPropertyNode(_id, "group", plugin.Group) + err = storage.SetPropertyNode(_id, "version", plugin.Version) + err = storage.SetPropertyNode(_id, "path", plugin.Path) + err = storage.SetPropertyNode(_id, "license", plugin.License) + + return err } func DeletePlugin(id string) (error) { @@ -106,6 +122,6 @@ func DeletePlugin(id string) (error) { return storage.DeleteNode(_id) } -func NewPlugin(id string, name string, group string, version string, path string) *Plugin { - return &Plugin{id, name, group, version, path} +func NewPlugin(id string, name string, group string, version string, path string, license string) *Plugin { + return &Plugin{id, name, group, version, path, license} } diff --git a/resource/resource.go b/resource/resource.go index eb6c8c3..bab2c6e 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -106,11 +106,12 @@ type Workflow struct { } type Plugin struct { - Id string - Name string - Group string - Version string - Path string + Id string + Name string + Group string + Version string + Path string + License string } func ResourceStreamManager() { diff --git a/service/system/v1/controller/plugin.go b/service/system/v1/controller/plugin.go index acee3b8..d4b6ec7 100644 --- a/service/system/v1/controller/plugin.go +++ b/service/system/v1/controller/plugin.go @@ -2,6 +2,7 @@ package controller import ( "encoding/json" + "io/ioutil" "github.com/feedlabs/elasticfeed/resource" template "github.com/feedlabs/elasticfeed/service/system/v1/template/plugin" @@ -125,6 +126,46 @@ func (this *PluginController) Put() { } } +/** + * @api {put} plugin/:pluginId/upload Upload + * @apiVersion 1.0.0 + * @apiName PutPluginFile + * @apiGroup Plugin + * @apiDescription Update a specific plugin. + * + * @apiUse PluginPutFileRequest + * @apiUse PluginPutFileResponse + */ +func (this *PluginController) PutFile() { + + formatter, err := template.RequestPut(this.GetInput()) + if err != nil { + this.ServeJson(template.GetError(err)) + return + } + + pluginId := this.Ctx.Input.Params[":pluginId"] + + data := this.Ctx.Input.CopyBody() + + // write whole the body + path := "/tmp/output-" + pluginId + err = ioutil.WriteFile(path, data, 0644) + if err != nil { + panic(err) + } + + plugin, err := resource.GetPlugin(pluginId) + plugin.Path = path + + err = resource.UpdatePlugin(plugin) + if err != nil { + this.ServeJson(template.GetError(err)) + } else { + this.ServeJson(template.ResponsePut(plugin, formatter)) + } +} + /** * @api {delete} plugin/:pluginId Delete * @apiVersion 1.0.0 diff --git a/service/system/v1/router/plugin.go b/service/system/v1/router/plugin.go index 95f37d7..4fdab9f 100644 --- a/service/system/v1/router/plugin.go +++ b/service/system/v1/router/plugin.go @@ -8,4 +8,5 @@ import ( func InitPluginRouters() { feedify.Router("/v1/system/plugin", &controller.PluginController{}, "get:GetList;post:Post") feedify.Router("/v1/system/plugin/:pluginId:string", &controller.PluginController{}, "get:Get;delete:Delete;put:Put") + feedify.Router("/v1/system/plugin/:pluginId:string/upload", &controller.PluginController{}, "put:PutFile") } diff --git a/service/system/v1/template/plugin/response.go b/service/system/v1/template/plugin/response.go index a44e3da..9a47faa 100644 --- a/service/system/v1/template/plugin/response.go +++ b/service/system/v1/template/plugin/response.go @@ -12,6 +12,8 @@ func GetEntry(plugin *resource.Plugin) (entry map[string]interface{}) { entry["name"] = plugin.Name entry["group"] = plugin.Group entry["version"] = plugin.Version + entry["license"] = plugin.License + entry["path"] = plugin.Path entry["status"] = "running" entry["errors"] = "no errors" From 97487354111335a00a72763349b0ca4168aa6f28 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Tue, 21 Apr 2015 23:57:59 +0200 Subject: [PATCH 12/16] Updated plugin response template --- service/system/v1/template/plugin/response.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/service/system/v1/template/plugin/response.go b/service/system/v1/template/plugin/response.go index 9a47faa..a140b81 100644 --- a/service/system/v1/template/plugin/response.go +++ b/service/system/v1/template/plugin/response.go @@ -13,9 +13,14 @@ func GetEntry(plugin *resource.Plugin) (entry map[string]interface{}) { entry["group"] = plugin.Group entry["version"] = plugin.Version entry["license"] = plugin.License - entry["path"] = plugin.Path - entry["status"] = "running" - entry["errors"] = "no errors" + + if plugin.Path == "" { + entry["status"] = "error" + entry["errors"] = "File path is missing" + } else { + entry["status"] = "running" + entry["errors"] = "no errors" + } return entry } From 19bc288c881f49ea59dc4e9bb741bf4af2c00544 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Wed, 22 Apr 2015 13:44:21 +0200 Subject: [PATCH 13/16] Create config paths on app starts --- common/config/general.go | 32 ++++++++++++++++++++++++++ conf/app.conf | 3 +++ service/system/v1/controller/plugin.go | 5 ++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/common/config/general.go b/common/config/general.go index b1cc1dd..2c9bf1a 100644 --- a/common/config/general.go +++ b/common/config/general.go @@ -1,6 +1,8 @@ package config import ( + "fmt" + "os" "github.com/feedlabs/feedify" ) @@ -23,3 +25,33 @@ func GetAuthType() string { func GetAuthRealm() string { return feedify.GetConfigKey("auth::realm") } + +func GetPluginStoragePath() string { + return GetHomeAbsolutePath() + "/" + feedify.GetConfigKey("plugin-manager::storage") +} + +func GetHomeAbsolutePath() string { + pwd, _ := os.Getwd() + return pwd +} + +func init() { + + _, err := os.Getwd() + if err != nil { + fmt.Println("Cannot read working directory path!") + os.Exit(1) + } + + path := GetHomeAbsolutePath() + "/" + feedify.GetConfigKey("plugin-manager::storage") + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(path, 0777) + if err != nil { + fmt.Println("Cannot create plugins storage directory!") + os.Exit(1) + } + } + } + +} diff --git a/conf/app.conf b/conf/app.conf index 636f8ba..6117fe7 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -61,3 +61,6 @@ storage_adapter=neo4j_cypher host=localhost port=1111 topic=feed + +[plugin-manager] +storage=public/userfiles/plugin/imports diff --git a/service/system/v1/controller/plugin.go b/service/system/v1/controller/plugin.go index d4b6ec7..ff8731f 100644 --- a/service/system/v1/controller/plugin.go +++ b/service/system/v1/controller/plugin.go @@ -4,6 +4,8 @@ import ( "encoding/json" "io/ioutil" + "github.com/feedlabs/elasticfeed/common/config" + "github.com/feedlabs/elasticfeed/common/uuid" "github.com/feedlabs/elasticfeed/resource" template "github.com/feedlabs/elasticfeed/service/system/v1/template/plugin" ) @@ -148,8 +150,7 @@ func (this *PluginController) PutFile() { data := this.Ctx.Input.CopyBody() - // write whole the body - path := "/tmp/output-" + pluginId + path := config.GetPluginStoragePath() + "/" + uuid.TimeOrderedUUID() + "-" + pluginId err = ioutil.WriteFile(path, data, 0644) if err != nil { panic(err) From 7ad3058055a38779e1fc01a9f3da4ff583f87c6a Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Wed, 22 Apr 2015 21:13:38 +0200 Subject: [PATCH 14/16] Migrated Packer RPC communication into Elasticfeed Plugin Manager --- common/config/config.go | 14 + {plugin => common}/config/config_unix.go | 2 +- {plugin => common}/config/config_windows.go | 2 +- common/config/general.go | 2 +- plugin/client.go | 350 ++++++++++++++++++ plugin/cmd_indexer.go | 46 +++ plugin/config/config.go | 224 ----------- plugin/indexer.go | 5 - plugin/manager.go | 221 ++++++++++- plugin/model/artifact.go | 15 + plugin/model/cache.go | 120 ++++++ plugin/{ => model}/crawler.go | 2 +- plugin/model/helper.go | 5 + plugin/model/hook.go | 5 + plugin/model/indexer.go | 9 + plugin/{ => model}/pipeline.go | 2 +- plugin/{ => model}/profiler.go | 2 +- plugin/{ => model}/resource_api.go | 2 +- plugin/{ => model}/scenario.go | 2 +- plugin/{ => model}/sensor.go | 2 +- plugin/plugin.go | 29 +- plugin/{ => plugins}/compiler/js.go | 0 .../pipeline/ai_neural_network.go | 0 plugin/{ => plugins}/pipeline/pipeline.go | 0 .../{ => plugins}/pipeline/random_animator.go | 0 .../scenario/ai_neural_network_trainer.go | 0 .../{math => plugins/scenario}/rpc_client.go | 0 .../{provider => plugins/sensor}/sensors.go | 2 +- plugin/rpc_artifact.go | 88 +++++ plugin/rpc_cache.go | 78 ++++ plugin/rpc_client.go | 88 +++++ plugin/rpc_dial.go | 33 ++ plugin/rpc_error.go | 21 ++ plugin/rpc_indexer.go | 112 ++++++ plugin/rpc_init.go | 10 + plugin/rpc_mux_broker.go | 191 ++++++++++ plugin/rpc_server.go | 109 ++++++ plugin/server.go | 139 +++++++ resource/plugin.go | 9 + resource/resource.go | 2 +- service/system/v1/controller/plugin.go | 11 +- service/system/v1/template/plugin/response.go | 3 +- {docs => website/docs}/PLUGIN-NOTES.md | 0 {docs => website/docs}/WORKFLOW-NOTES.md | 0 workflow/workflow.go | 8 +- 45 files changed, 1695 insertions(+), 270 deletions(-) create mode 100644 common/config/config.go rename {plugin => common}/config/config_unix.go (98%) rename {plugin => common}/config/config_windows.go (98%) create mode 100644 plugin/client.go create mode 100644 plugin/cmd_indexer.go delete mode 100644 plugin/config/config.go delete mode 100644 plugin/indexer.go create mode 100644 plugin/model/artifact.go create mode 100644 plugin/model/cache.go rename plugin/{ => model}/crawler.go (65%) create mode 100644 plugin/model/helper.go create mode 100644 plugin/model/hook.go create mode 100644 plugin/model/indexer.go rename plugin/{ => model}/pipeline.go (66%) rename plugin/{ => model}/profiler.go (89%) rename plugin/{ => model}/resource_api.go (97%) rename plugin/{ => model}/scenario.go (66%) rename plugin/{ => model}/sensor.go (65%) rename plugin/{ => plugins}/compiler/js.go (100%) rename plugin/{ => plugins}/pipeline/ai_neural_network.go (100%) rename plugin/{ => plugins}/pipeline/pipeline.go (100%) rename plugin/{ => plugins}/pipeline/random_animator.go (100%) rename plugin/{ => plugins}/scenario/ai_neural_network_trainer.go (100%) rename plugin/{math => plugins/scenario}/rpc_client.go (100%) rename plugin/{provider => plugins/sensor}/sensors.go (79%) create mode 100644 plugin/rpc_artifact.go create mode 100644 plugin/rpc_cache.go create mode 100644 plugin/rpc_client.go create mode 100644 plugin/rpc_dial.go create mode 100644 plugin/rpc_error.go create mode 100644 plugin/rpc_indexer.go create mode 100644 plugin/rpc_init.go create mode 100644 plugin/rpc_mux_broker.go create mode 100644 plugin/rpc_server.go create mode 100644 plugin/server.go rename {docs => website/docs}/PLUGIN-NOTES.md (100%) rename {docs => website/docs}/WORKFLOW-NOTES.md (100%) diff --git a/common/config/config.go b/common/config/config.go new file mode 100644 index 0000000..4465045 --- /dev/null +++ b/common/config/config.go @@ -0,0 +1,14 @@ +package config + +// ConfigFile returns the default path to the configuration file. On +// Unix-like systems this is the ".elasticfeedconfig" file in the home directory. +// On Windows, this is the "elasticfeed.config" file in the application data +// directory. +func ConfigFile() (string, error) { + return configFile() +} + +// ConfigDir returns the configuration directory for Elasticfeed. +func ConfigDir() (string, error) { + return configDir() +} diff --git a/plugin/config/config_unix.go b/common/config/config_unix.go similarity index 98% rename from plugin/config/config_unix.go rename to common/config/config_unix.go index 7460985..4aad71d 100644 --- a/plugin/config/config_unix.go +++ b/common/config/config_unix.go @@ -1,6 +1,6 @@ // +build darwin freebsd linux netbsd openbsd -package plugin +package config import ( "bytes" diff --git a/plugin/config/config_windows.go b/common/config/config_windows.go similarity index 98% rename from plugin/config/config_windows.go rename to common/config/config_windows.go index 80cc4be..aa17546 100644 --- a/plugin/config/config_windows.go +++ b/common/config/config_windows.go @@ -1,6 +1,6 @@ // +build windows -package plugin +package config import ( "path/filepath" diff --git a/common/config/general.go b/common/config/general.go index 2c9bf1a..8913fe6 100644 --- a/common/config/general.go +++ b/common/config/general.go @@ -27,7 +27,7 @@ func GetAuthRealm() string { } func GetPluginStoragePath() string { - return GetHomeAbsolutePath() + "/" + feedify.GetConfigKey("plugin-manager::storage") + return feedify.GetConfigKey("plugin-manager::storage") } func GetHomeAbsolutePath() string { diff --git a/plugin/client.go b/plugin/client.go new file mode 100644 index 0000000..eba3197 --- /dev/null +++ b/plugin/client.go @@ -0,0 +1,350 @@ +package plugin + +import ( + "bufio" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + "unicode" + + "github.com/feedlabs/elasticfeed/plugin/model" +) + +// If this is true, then the "unexpected EOF" panic will not be +// raised throughout the clients. +var Killed = false + +// This is a slice of the "managed" clients which are cleaned up when +// calling Cleanup +var managedClients = make([]*Client, 0, 5) + +// Client handles the lifecycle of a plugin application, determining its +// RPC address, and returning various types of packer interface implementations +// across the multi-process communication layer. +type Client struct { + config *ClientConfig + exited bool + doneLogging chan struct{} + l sync.Mutex + address net.Addr +} + +// ClientConfig is the configuration used to initialize a new +// plugin client. After being used to initialize a plugin client, +// that configuration must not be modified again. +type ClientConfig struct { + // The unstarted subprocess for starting the plugin. + Cmd *exec.Cmd + + // Managed represents if the client should be managed by the + // plugin package or not. If true, then by calling CleanupClients, + // it will automatically be cleaned up. Otherwise, the client + // user is fully responsible for making sure to Kill all plugin + // clients. By default the client is _not_ managed. + Managed bool + + // The minimum and maximum port to use for communicating with + // the subprocess. If not set, this defaults to 10,000 and 25,000 + // respectively. + MinPort, MaxPort uint + + // StartTimeout is the timeout to wait for the plugin to say it + // has started successfully. + StartTimeout time.Duration + + // If non-nil, then the stderr of the client will be written to here + // (as well as the log). + Stderr io.Writer +} + +// This makes sure all the managed subprocesses are killed and properly +// logged. This should be called before the parent process running the +// plugins exits. +// +// This must only be called _once_. +func CleanupClients() { + // Set the killed to true so that we don't get unexpected panics + Killed = true + + // Kill all the managed clients in parallel and use a WaitGroup + // to wait for them all to finish up. + var wg sync.WaitGroup + for _, client := range managedClients { + wg.Add(1) + + go func(client *Client) { + client.Kill() + wg.Done() + }(client) + } + + log.Println("waiting for all plugin processes to complete...") + wg.Wait() +} + +// Creates a new plugin client which manages the lifecycle of an external +// plugin and gets the address for the RPC connection. +// +// The client must be cleaned up at some point by calling Kill(). If +// the client is a managed client (created with NewManagedClient) you +// can just call CleanupClients at the end of your program and they will +// be properly cleaned. +func NewClient(config *ClientConfig) (c *Client) { + if config.MinPort == 0 && config.MaxPort == 0 { + config.MinPort = 10000 + config.MaxPort = 25000 + } + + if config.StartTimeout == 0 { + config.StartTimeout = 1 * time.Minute + } + + if config.Stderr == nil { + config.Stderr = ioutil.Discard + } + + c = &Client{config: config} + if config.Managed { + managedClients = append(managedClients, c) + } + + return +} + +// Tells whether or not the underlying process has exited. +func (c *Client) Exited() bool { + c.l.Lock() + defer c.l.Unlock() + return c.exited +} + +// Returns a indexer implementation that is communicating over this +// client. If the client hasn't been started, this will start it. +func (c *Client) Indexer() (model.Indexer, error) { + client, err := c.RpcClient() + if err != nil { + return nil, err + } + + return &cmdIndexer{client.Indexer(), c}, nil +} + +// End the executing subprocess (if it is running) and perform any cleanup +// tasks necessary such as capturing any remaining logs and so on. +// +// This method blocks until the process successfully exits. +// +// This method can safely be called multiple times. +func (c *Client) Kill() { + cmd := c.config.Cmd + + if cmd.Process == nil { + return + } + + cmd.Process.Kill() + + // Wait for the client to finish logging so we have a complete log + <-c.doneLogging +} + +// Starts the underlying subprocess, communicating with it to negotiate +// a port for RPC connections, and returning the address to connect via RPC. +// +// This method is safe to call multiple times. Subsequent calls have no effect. +// Once a client has been started once, it cannot be started again, even if +// it was killed. +func (c *Client) Start() (addr net.Addr, err error) { + c.l.Lock() + defer c.l.Unlock() + + if c.address != nil { + return c.address, nil + } + + c.doneLogging = make(chan struct{}) + + env := []string{ + fmt.Sprintf("%s=%s", MagicCookieKey, MagicCookieValue), + fmt.Sprintf("PACKER_PLUGIN_MIN_PORT=%d", c.config.MinPort), + fmt.Sprintf("PACKER_PLUGIN_MAX_PORT=%d", c.config.MaxPort), + } + + stdout_r, stdout_w := io.Pipe() + stderr_r, stderr_w := io.Pipe() + + cmd := c.config.Cmd + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, env...) + cmd.Stdin = os.Stdin + cmd.Stderr = stderr_w + cmd.Stdout = stdout_w + + log.Printf("Starting plugin: %s %#v", cmd.Path, cmd.Args) + err = cmd.Start() + if err != nil { + return + } + + // Make sure the command is properly cleaned up if there is an error + defer func() { + r := recover() + + if err != nil || r != nil { + cmd.Process.Kill() + } + + if r != nil { + panic(r) + } + }() + + // Start goroutine to wait for process to exit + exitCh := make(chan struct{}) + go func() { + // Make sure we close the write end of our stderr/stdout so + // that the readers send EOF properly. + defer stderr_w.Close() + defer stdout_w.Close() + + // Wait for the command to end. + cmd.Wait() + + // Log and make sure to flush the logs write away + log.Printf("%s: plugin process exited\n", cmd.Path) + os.Stderr.Sync() + + // Mark that we exited + close(exitCh) + + // Set that we exited, which takes a lock + c.l.Lock() + defer c.l.Unlock() + c.exited = true + }() + + // Start goroutine that logs the stderr + go c.logStderr(stderr_r) + + // Start a goroutine that is going to be reading the lines + // out of stdout + linesCh := make(chan []byte) + go func() { + defer close(linesCh) + + buf := bufio.NewReader(stdout_r) + for { + line, err := buf.ReadBytes('\n') + if line != nil { + linesCh <- line + } + + if err == io.EOF { + return + } + } + }() + + // Make sure after we exit we read the lines from stdout forever + // so they dont' block since it is an io.Pipe + defer func() { + go func() { + for _ = range linesCh { + } + }() + }() + + // Some channels for the next step + timeout := time.After(c.config.StartTimeout) + + // Start looking for the address + log.Printf("Waiting for RPC address for: %s", cmd.Path) + select { + case <-timeout: + err = errors.New("timeout while waiting for plugin to start") + case <-exitCh: + err = errors.New("plugin exited before we could connect") + case lineBytes := <-linesCh: + // Trim the line and split by "|" in order to get the parts of + // the output. + line := strings.TrimSpace(string(lineBytes)) + parts := strings.SplitN(line, "|", 3) + if len(parts) < 3 { + err = fmt.Errorf("Unrecognized remote plugin message: %s", line) + return + } + + // Test the API version + if parts[0] != APIVersion { + err = fmt.Errorf("Incompatible API version with plugin. "+ + "Plugin version: %s, Ours: %s", parts[0], APIVersion) + return + } + + switch parts[1] { + case "tcp": + addr, err = net.ResolveTCPAddr("tcp", parts[2]) + case "unix": + addr, err = net.ResolveUnixAddr("unix", parts[2]) + default: + err = fmt.Errorf("Unknown address type: %s", parts[1]) + } + } + + c.address = addr + return +} + +func (c *Client) logStderr(r io.Reader) { + bufR := bufio.NewReader(r) + for { + line, err := bufR.ReadString('\n') + if line != "" { + c.config.Stderr.Write([]byte(line)) + + line = strings.TrimRightFunc(line, unicode.IsSpace) + log.Printf("%s: %s", filepath.Base(c.config.Cmd.Path), line) + } + + if err == io.EOF { + break + } + } + + // Flag that we've completed logging for others + close(c.doneLogging) +} + +func (c *Client) RpcClient() (*RpcClient, error) { + addr, err := c.Start() + if err != nil { + return nil, err + } + + conn, err := net.Dial(addr.Network(), addr.String()) + if err != nil { + return nil, err + } + + if tcpConn, ok := conn.(*net.TCPConn); ok { + // Make sure to set keep alive so that the connection doesn't die + tcpConn.SetKeepAlive(true) + } + + client, err := NewRpcClient(conn) + if err != nil { + conn.Close() + return nil, err + } + + return client, nil +} diff --git a/plugin/cmd_indexer.go b/plugin/cmd_indexer.go new file mode 100644 index 0000000..12c1a3e --- /dev/null +++ b/plugin/cmd_indexer.go @@ -0,0 +1,46 @@ +package plugin + +import ( + "log" + "github.com/feedlabs/elasticfeed/plugin/model" +) + +type cmdIndexer struct { + indexer model.Indexer + client *Client +} + +func (b *cmdIndexer) Prepare(config ...interface{}) ([]string, error) { + defer func() { + r := recover() + b.checkExit(r, nil) + }() + + return b.indexer.Prepare(config...) +} + +func (b *cmdIndexer) Run(cache model.Cache) (model.Artifact, error) { + defer func() { + r := recover() + b.checkExit(r, nil) + }() + + return b.indexer.Run(cache) +} + +func (b *cmdIndexer) Cancel() { + defer func() { + r := recover() + b.checkExit(r, nil) + }() + + b.indexer.Cancel() +} + +func (c *cmdIndexer) checkExit(p interface{}, cb func()) { + if c.client.Exited() && cb != nil { + cb() + } else if p != nil && !Killed { + log.Panic(p) + } +} diff --git a/plugin/config/config.go b/plugin/config/config.go deleted file mode 100644 index 4e50078..0000000 --- a/plugin/config/config.go +++ /dev/null @@ -1,224 +0,0 @@ -package plugin - -import ( - "encoding/json" - "io" - "log" - "os/exec" - "path/filepath" - "strings" - - "github.com/mitchellh/osext" - "github.com/mitchellh/elasticfeed/elasticfeed" - "github.com/mitchellh/elasticfeed/elasticfeed/plugin" -) - -// EnvConfig is the global EnvironmentConfig we use to initialize the CLI. -var EnvConfig elasticfeed.EnvironmentConfig - -type config struct { - PluginMinPort uint - PluginMaxPort uint - - Pipelines map[string]string - Indexers map[string]string - Crawlers map[string]string - Scenarios map[string]string -} - -// ConfigFile returns the default path to the configuration file. On -// Unix-like systems this is the ".elasticfeedconfig" file in the home directory. -// On Windows, this is the "elasticfeed.config" file in the application data -// directory. -func ConfigFile() (string, error) { - return configFile() -} - -// ConfigDir returns the configuration directory for Elasticfeed. -func ConfigDir() (string, error) { - return configDir() -} - -// Decodes configuration in JSON format from the given io.Reader into -// the config object pointed to. -func decodeConfig(r io.Reader, c *config) error { - decoder := json.NewDecoder(r) - return decoder.Decode(c) -} - -// Discover discovers plugins. -// -// This looks in the directory of the executable and the CWD, in that -// order for priority. -func (c *config) Discover() error { - // Next, look in the same directory as the executable. Any conflicts - // will overwrite those found in our current directory. - exePath, err := osext.Executable() - if err != nil { - log.Printf("[ERR] Error loading exe directory: %s", err) - } else { - if err := c.discover(filepath.Dir(exePath)); err != nil { - return err - } - } - - // Look in the plugins directory - dir, err := ConfigDir() - if err != nil { - log.Printf("[ERR] Error loading config directory: %s", err) - } else { - if err := c.discover(filepath.Join(dir, "plugins")); err != nil { - return err - } - } - - // Look in the cwd. - if err := c.discover("."); err != nil { - return err - } - - return nil -} - -func (c *config) LoadPiplines(name string) (elasticfeed.Pipeline, error) { - log.Printf("Loading provisioner: %s\n", name) - bin, ok := c.Pipelines[name] - if !ok { - log.Printf("Pipelines not found: %s\n", name) - return nil, nil - } - - return c.pluginClient(bin).Pipeline() -} - -func (c *config) LoadIndexers(name string) (elasticfeed.Indexer, error) { - log.Printf("Loading provisioner: %s\n", name) - bin, ok := c.Indexers[name] - if !ok { - log.Printf("Indexers not found: %s\n", name) - return nil, nil - } - - return c.pluginClient(bin).Indexer() -} - -func (c *config) LoadCrawlers(name string) (elasticfeed.Crawler, error) { - log.Printf("Loading provisioner: %s\n", name) - bin, ok := c.Crawlers[name] - if !ok { - log.Printf("Crawlers not found: %s\n", name) - return nil, nil - } - - return c.pluginClient(bin).Crawler() -} - -func (c *config) LoadScenarios(name string) (elasticfeed.Scenario, error) { - log.Printf("Loading provisioner: %s\n", name) - bin, ok := c.Scenarios[name] - if !ok { - log.Printf("Scenarios not found: %s\n", name) - return nil, nil - } - - return c.pluginClient(bin).Scenario() -} - -func (c *config) discover(path string) error { - var err error - - if !filepath.IsAbs(path) { - path, err = filepath.Abs(path) - if err != nil { - return err - } - } - - err = c.discoverSingle( - filepath.Join(path, "elasticfeed-pipeline-*"), &c.Pipelines) - if err != nil { - return err - } - - err = c.discoverSingle( - filepath.Join(path, "elasticfeed-indexer-*"), &c.Indexers) - if err != nil { - return err - } - - err = c.discoverSingle( - filepath.Join(path, "elasticfeed-crawler-*"), &c.Crawlers) - if err != nil { - return err - } - - err = c.discoverSingle( - filepath.Join(path, "elasticfeed-scenario-*"), &c.Scenarios) - if err != nil { - return err - } - - return nil -} - -func (c *config) discoverSingle(glob string, m *map[string]string) error { - matches, err := filepath.Glob(glob) - if err != nil { - return err - } - - if *m == nil { - *m = make(map[string]string) - } - - prefix := filepath.Base(glob) - prefix = prefix[:strings.Index(prefix, "*")] - for _, match := range matches { - file := filepath.Base(match) - - // If the filename has a ".", trim up to there - if idx := strings.Index(file, "."); idx >= 0 { - file = file[:idx] - } - - // Look for foo-bar-baz. The plugin name is "baz" - plugin := file[len(prefix):] - log.Printf("[DEBUG] Discoverd plugin: %s = %s", plugin, match) - (*m)[plugin] = match - } - - return nil -} - -func (c *config) pluginClient(path string) *plugin.Client { - originalPath := path - - // First attempt to find the executable by consulting the PATH. - path, err := exec.LookPath(path) - if err != nil { - // If that doesn't work, look for it in the same directory - // as the `packer` executable (us). - log.Printf("Plugin could not be found. Checking same directory as executable.") - exePath, err := osext.Executable() - if err != nil { - log.Printf("Couldn't get current exe path: %s", err) - } else { - log.Printf("Current exe path: %s", exePath) - path = filepath.Join(filepath.Dir(exePath), filepath.Base(originalPath)) - } - } - - // If everything failed, just use the original path and let the error - // bubble through. - if path == "" { - path = originalPath - } - - log.Printf("Creating plugin client for path: %s", path) - var config plugin.ClientConfig - config.Cmd = exec.Command(path) - config.Managed = true - config.MinPort = c.PluginMinPort - config.MaxPort = c.PluginMaxPort - return plugin.NewClient(&config) -} diff --git a/plugin/indexer.go b/plugin/indexer.go deleted file mode 100644 index 136adec..0000000 --- a/plugin/indexer.go +++ /dev/null @@ -1,5 +0,0 @@ -package plugin - -type Indexer interface { - -} diff --git a/plugin/manager.go b/plugin/manager.go index 61eb6f1..651a231 100644 --- a/plugin/manager.go +++ b/plugin/manager.go @@ -1,41 +1,234 @@ package plugin +import ( + "strconv" + + "log" + "os/exec" + "path/filepath" + "strings" + + "github.com/feedlabs/elasticfeed/resource" + "github.com/feedlabs/elasticfeed/plugin/model" + "github.com/feedlabs/elasticfeed/common/config" + + "github.com/mitchellh/osext" +) + type PluginManager struct { - Indexers map[string]interface{} - Crawlers map[string]interface{} - Sensors map[string]interface{} - Pipelines map[string]interface{} - Scenarios map[string]interface{} - Helpers map[string]interface{} + Indexers map[string]string + Crawlers map[string]string + Sensors map[string]string + Pipelines map[string]string + Scenarios map[string]string + Helpers map[string]string + + Store map[int]map[string]interface{} + + api *model.ResourceApi - api *ResourceApi + PluginMinPort uint + PluginMaxPort uint } func (this *PluginManager) GetResourceApi() interface{} { return this.api } -func (this *PluginManager) InitIndexer(name string, profiler *Profiler) *Plugin { - p := NewPlugin(this, this.api, profiler) +func (this *PluginManager) InitPlugin(name string, profiler *model.Profiler) *Plugin { + + // _p := resource.FindPluginByName(name) + // findIsRunningWithProfiler(_p, profiler) + _p := resource.NewPlugin("", "", "", "", "", "") + p := NewPlugin(_p, this, this.api, profiler) p.Init() - this.Indexers[name] = p + + _group, _ := strconv.Atoi(_p.Group) + this.Store[_group][_p.Id] = p return p } -func (this *PluginManager) FindPlugin(name string, profiler *Profiler) *interface{} { +func (this *PluginManager) FindPlugin(name string, profiler *model.Profiler) *interface{} { + return nil +} + +func (this *PluginManager) RunPlugin(p Plugin) (err error) { + err = p.Run() + + if err != nil { + return err + } + + return nil +} + +func (this *PluginManager) GetIndexers() map[string]interface{} { + return this.Store[resource.PLUGIN_INDEXER] +} + +// Discover discovers plugins. +// +// This looks in the directory of the executable and the CWD, in that +// order for priority. +func (c *PluginManager) Discover() error { + // Next, look in the same directory as the executable. Any conflicts + // will overwrite those found in our current directory. + exePath, err := osext.Executable() + if err != nil { + log.Printf("[ERR] Error loading exe directory: %s", err) + } else { + if err := c.discover(filepath.Dir(exePath)); err != nil { + return err + } + } + + // Look in the plugins directory + dir, err := config.ConfigDir() + if err != nil { + log.Printf("[ERR] Error loading config directory: %s", err) + } else { + if err := c.discover(filepath.Join(dir, "plugins")); err != nil { + return err + } + } + + // Look in the cwd. + if err := c.discover("."); err != nil { + return err + } + return nil } -func (this *PluginManager) ExecPlugin(p Plugin) { - // profiler := p.profiler +func (c *PluginManager) LoadIndexers(name string) (model.Indexer, error) { + log.Printf("Loading provisioner: %s\n", name) + bin, ok := c.Indexers[name] + if !ok { + log.Printf("Indexers not found: %s\n", name) + return nil, nil + } + + return c.pluginClient(bin).Indexer() } +func (c *PluginManager) discover(path string) error { + var err error + + if !filepath.IsAbs(path) { + path, err = filepath.Abs(path) + if err != nil { + return err + } + } + + err = c.discoverSingle( + filepath.Join(path, "pipeline-*"), &c.Pipelines) + if err != nil { + return err + } + + err = c.discoverSingle( + filepath.Join(path, "indexer-*"), &c.Indexers) + if err != nil { + return err + } + + err = c.discoverSingle( + filepath.Join(path, "crawler-*"), &c.Crawlers) + if err != nil { + return err + } + + err = c.discoverSingle( + filepath.Join(path, "scenario-*"), &c.Scenarios) + if err != nil { + return err + } + + err = c.discoverSingle( + filepath.Join(path, "helper-*"), &c.Helpers) + if err != nil { + return err + } + + err = c.discoverSingle( + filepath.Join(path, "sensor-*"), &c.Sensors) + if err != nil { + return err + } + + return nil +} + +func (c *PluginManager) discoverSingle(glob string, m *map[string]string) error { + matches, err := filepath.Glob(glob) + if err != nil { + return err + } + + if *m == nil { + *m = make(map[string]string) + } + + prefix := filepath.Base(glob) + prefix = prefix[:strings.Index(prefix, "*")] + for _, match := range matches { + file := filepath.Base(match) + + // If the filename has a ".", trim up to there + if idx := strings.Index(file, "."); idx >= 0 { + file = file[:idx] + } + + // Look for foo-bar-baz. The plugin name is "baz" + plugin := file[len(prefix):] + log.Printf("[DEBUG] Discoverd plugin: %s = %s", plugin, match) + (*m)[plugin] = match + } + + return nil +} + +func (c *PluginManager) pluginClient(path string) *Client { + originalPath := path + + // First attempt to find the executable by consulting the PATH. + path, err := exec.LookPath(path) + if err != nil { + // If that doesn't work, look for it in the same directory + // as the `packer` executable (us). + log.Printf("Plugin could not be found. Checking same directory as executable.") + exePath, err := osext.Executable() + if err != nil { + log.Printf("Couldn't get current exe path: %s", err) + } else { + log.Printf("Current exe path: %s", exePath) + path = filepath.Join(filepath.Dir(exePath), filepath.Base(originalPath)) + } + } + + // If everything failed, just use the original path and let the error + // bubble through. + if path == "" { + path = originalPath + } + + log.Printf("Creating plugin client for path: %s", path) + var config ClientConfig + config.Cmd = exec.Command(path) + config.Managed = true + config.MinPort = c.PluginMinPort + config.MaxPort = c.PluginMaxPort + return NewClient(&config) +} + + func NewPluginManager(resourceManager interface{}) *PluginManager { pm := &PluginManager{} - pm.api = NewResourceApi(resourceManager) + pm.api = model.NewResourceApi(resourceManager) return pm } diff --git a/plugin/model/artifact.go b/plugin/model/artifact.go new file mode 100644 index 0000000..0438a0e --- /dev/null +++ b/plugin/model/artifact.go @@ -0,0 +1,15 @@ +package model + +type Artifact interface { + BuilderId() string + + Files() []string + + Id() string + + String() string + + State(name string) interface{} + + Destroy() error +} diff --git a/plugin/model/cache.go b/plugin/model/cache.go new file mode 100644 index 0000000..f5a07ae --- /dev/null +++ b/plugin/model/cache.go @@ -0,0 +1,120 @@ +package model + +import ( + "crypto/sha256" + "encoding/hex" + "log" + "os" + "path/filepath" + "strings" + "sync" +) + +// Cache implements a caching interface where files can be stored for +// re-use between multiple runs. +type Cache interface { + // Lock takes a key and returns the path where the file can be written to. + // Packer guarantees that no other process will write to this file while + // the lock is held. + // + // If the key has an extension (e.g., file.ext), the resulting path + // will have that extension as well. + // + // The cache will block and wait for the lock. + Lock(string) string + + // Unlock will unlock a certain cache key. Be very careful that this + // is only called once per lock obtained. + Unlock(string) + + // RLock returns the path to a key in the cache and locks it for reading. + // The second return parameter is whether the key existed or not. + // This will block if any locks are held for writing. No lock will be + // held if the key doesn't exist. + RLock(string) (string, bool) + + // RUnlock will unlock a key for reading. + RUnlock(string) +} + +// FileCache implements a Cache by caching the data directly to a cache +// directory. +type FileCache struct { + CacheDir string + l sync.Mutex + rw map[string]*sync.RWMutex +} + +func (f *FileCache) Lock(key string) string { + hashKey := f.hashKey(key) + rw := f.rwLock(hashKey) + rw.Lock() + + return f.cachePath(key, hashKey) +} + +func (f *FileCache) Unlock(key string) { + hashKey := f.hashKey(key) + rw := f.rwLock(hashKey) + rw.Unlock() +} + +func (f *FileCache) RLock(key string) (string, bool) { + hashKey := f.hashKey(key) + rw := f.rwLock(hashKey) + rw.RLock() + + return f.cachePath(key, hashKey), true +} + +func (f *FileCache) RUnlock(key string) { + hashKey := f.hashKey(key) + rw := f.rwLock(hashKey) + rw.RUnlock() +} + +func (f *FileCache) cachePath(key string, hashKey string) string { + if endIndex := strings.Index(key, "?"); endIndex > -1 { + key = key[:endIndex] + } + + suffix := "" + dotIndex := strings.LastIndex(key, ".") + if dotIndex > -1 { + if slashIndex := strings.LastIndex(key, "/"); slashIndex <= dotIndex { + suffix = key[dotIndex:] + } + } + + // Make the cache directory. We ignore errors here, but + // log them in case something happens. + if err := os.MkdirAll(f.CacheDir, 0755); err != nil { + log.Printf( + "[ERR] Error making cacheDir: %s %s", f.CacheDir, err) + } + + return filepath.Join(f.CacheDir, hashKey+suffix) +} + +func (f *FileCache) hashKey(key string) string { + sha := sha256.New() + sha.Write([]byte(key)) + return hex.EncodeToString(sha.Sum(nil)) +} + +func (f *FileCache) rwLock(hashKey string) *sync.RWMutex { + f.l.Lock() + defer f.l.Unlock() + + if f.rw == nil { + f.rw = make(map[string]*sync.RWMutex) + } + + if result, ok := f.rw[hashKey]; ok { + return result + } + + var result sync.RWMutex + f.rw[hashKey] = &result + return &result +} diff --git a/plugin/crawler.go b/plugin/model/crawler.go similarity index 65% rename from plugin/crawler.go rename to plugin/model/crawler.go index 0de1c04..8d5755f 100644 --- a/plugin/crawler.go +++ b/plugin/model/crawler.go @@ -1,4 +1,4 @@ -package plugin +package model type Crawler interface { diff --git a/plugin/model/helper.go b/plugin/model/helper.go new file mode 100644 index 0000000..8ef5fed --- /dev/null +++ b/plugin/model/helper.go @@ -0,0 +1,5 @@ +package model + +type Helper interface { + +} diff --git a/plugin/model/hook.go b/plugin/model/hook.go new file mode 100644 index 0000000..a881125 --- /dev/null +++ b/plugin/model/hook.go @@ -0,0 +1,5 @@ +package model + +type Hook interface { + +} diff --git a/plugin/model/indexer.go b/plugin/model/indexer.go new file mode 100644 index 0000000..b477c0a --- /dev/null +++ b/plugin/model/indexer.go @@ -0,0 +1,9 @@ +package model + +type Indexer interface { + Prepare(...interface{}) ([]string, error) + + Run(cache Cache) (Artifact, error) + + Cancel() +} diff --git a/plugin/pipeline.go b/plugin/model/pipeline.go similarity index 66% rename from plugin/pipeline.go rename to plugin/model/pipeline.go index 50fd273..ce6f42c 100644 --- a/plugin/pipeline.go +++ b/plugin/model/pipeline.go @@ -1,4 +1,4 @@ -package plugin +package model type Pipeline interface { diff --git a/plugin/profiler.go b/plugin/model/profiler.go similarity index 89% rename from plugin/profiler.go rename to plugin/model/profiler.go index fe2014a..71d9cda 100644 --- a/plugin/profiler.go +++ b/plugin/model/profiler.go @@ -1,4 +1,4 @@ -package plugin +package model type Profiler struct { data map[string]string diff --git a/plugin/resource_api.go b/plugin/model/resource_api.go similarity index 97% rename from plugin/resource_api.go rename to plugin/model/resource_api.go index 42edc67..6c90251 100644 --- a/plugin/resource_api.go +++ b/plugin/model/resource_api.go @@ -1,4 +1,4 @@ -package plugin +package model type ResourceApi struct { resourceManager interface{} diff --git a/plugin/scenario.go b/plugin/model/scenario.go similarity index 66% rename from plugin/scenario.go rename to plugin/model/scenario.go index 8a1e75d..da93a64 100644 --- a/plugin/scenario.go +++ b/plugin/model/scenario.go @@ -1,4 +1,4 @@ -package plugin +package model type Scenario interface { diff --git a/plugin/sensor.go b/plugin/model/sensor.go similarity index 65% rename from plugin/sensor.go rename to plugin/model/sensor.go index d53da75..aa76d8f 100644 --- a/plugin/sensor.go +++ b/plugin/model/sensor.go @@ -1,4 +1,4 @@ -package plugin +package model type Sensor interface { diff --git a/plugin/plugin.go b/plugin/plugin.go index 7edbbbe..45fb8fc 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -1,15 +1,32 @@ package plugin +import ( + "github.com/feedlabs/elasticfeed/resource" + "github.com/feedlabs/elasticfeed/plugin/model" +) + type Plugin struct { - pluginManager *PluginManager - resourceApi *ResourceApi + plugin *resource.Plugin + + pluginManager *PluginManager + resourceApi *model.ResourceApi - profiler *Profiler - rpcAddress interface{} + profiler *model.Profiler + rpcAddress interface{} + + pid int } func (this *Plugin) Init() {} -func NewPlugin(pm *PluginManager, api *ResourceApi, profiler *Profiler) *Plugin { - return &Plugin{pm, api, profiler, ""} +func (this *Plugin) Run() (err error) { + return nil +} + +func (this *Plugin) GetPid() int { + return this.pid +} + +func NewPlugin(p *resource.Plugin, pm *PluginManager, api *model.ResourceApi, profiler *model.Profiler) *Plugin { + return &Plugin{p, pm, api, profiler, "", -1} } diff --git a/plugin/compiler/js.go b/plugin/plugins/compiler/js.go similarity index 100% rename from plugin/compiler/js.go rename to plugin/plugins/compiler/js.go diff --git a/plugin/pipeline/ai_neural_network.go b/plugin/plugins/pipeline/ai_neural_network.go similarity index 100% rename from plugin/pipeline/ai_neural_network.go rename to plugin/plugins/pipeline/ai_neural_network.go diff --git a/plugin/pipeline/pipeline.go b/plugin/plugins/pipeline/pipeline.go similarity index 100% rename from plugin/pipeline/pipeline.go rename to plugin/plugins/pipeline/pipeline.go diff --git a/plugin/pipeline/random_animator.go b/plugin/plugins/pipeline/random_animator.go similarity index 100% rename from plugin/pipeline/random_animator.go rename to plugin/plugins/pipeline/random_animator.go diff --git a/plugin/scenario/ai_neural_network_trainer.go b/plugin/plugins/scenario/ai_neural_network_trainer.go similarity index 100% rename from plugin/scenario/ai_neural_network_trainer.go rename to plugin/plugins/scenario/ai_neural_network_trainer.go diff --git a/plugin/math/rpc_client.go b/plugin/plugins/scenario/rpc_client.go similarity index 100% rename from plugin/math/rpc_client.go rename to plugin/plugins/scenario/rpc_client.go diff --git a/plugin/provider/sensors.go b/plugin/plugins/sensor/sensors.go similarity index 79% rename from plugin/provider/sensors.go rename to plugin/plugins/sensor/sensors.go index 3a235ab..74e2319 100644 --- a/plugin/provider/sensors.go +++ b/plugin/plugins/sensor/sensors.go @@ -1,4 +1,4 @@ -package provider +package sensor func Weather() {} func StockIndices() {} diff --git a/plugin/rpc_artifact.go b/plugin/rpc_artifact.go new file mode 100644 index 0000000..dc29452 --- /dev/null +++ b/plugin/rpc_artifact.go @@ -0,0 +1,88 @@ +package plugin + +import ( + "net/rpc" + "github.com/feedlabs/elasticfeed/plugin/model" +) + +// An implementation of packer.Artifact where the RpcArtifact is actually +// available over an RPC connection. +type RpcArtifact struct { + client *rpc.Client + endpoint string +} + +// ArtifactRpcServer wraps a packer.Artifact implementation and makes it +// exportable as part of a Golang RPC server. +type ArtifactRpcServer struct { + artifact model.Artifact +} + +func (a *RpcArtifact) BuilderId() (result string) { + a.client.Call(a.endpoint+".BuilderId", new(interface{}), &result) + return +} + +func (a *RpcArtifact) Files() (result []string) { + a.client.Call(a.endpoint+".Files", new(interface{}), &result) + return +} + +func (a *RpcArtifact) Id() (result string) { + a.client.Call(a.endpoint+".Id", new(interface{}), &result) + return +} + +func (a *RpcArtifact) String() (result string) { + a.client.Call(a.endpoint+".String", new(interface{}), &result) + return +} + +func (a *RpcArtifact) State(name string) (result interface{}) { + a.client.Call(a.endpoint+".State", name, &result) + return +} + +func (a *RpcArtifact) Destroy() error { + var result error + if err := a.client.Call(a.endpoint+".Destroy", new(interface{}), &result); err != nil { + return err + } + + return result +} + +func (s *ArtifactRpcServer) BuilderId(args *interface{}, reply *string) error { + *reply = s.artifact.BuilderId() + return nil +} + +func (s *ArtifactRpcServer) Files(args *interface{}, reply *[]string) error { + *reply = s.artifact.Files() + return nil +} + +func (s *ArtifactRpcServer) Id(args *interface{}, reply *string) error { + *reply = s.artifact.Id() + return nil +} + +func (s *ArtifactRpcServer) String(args *interface{}, reply *string) error { + *reply = s.artifact.String() + return nil +} + +func (s *ArtifactRpcServer) State(name string, reply *interface{}) error { + *reply = s.artifact.State(name) + return nil +} + +func (s *ArtifactRpcServer) Destroy(args *interface{}, reply *error) error { + err := s.artifact.Destroy() + if err != nil { + err = NewBasicError(err) + } + + *reply = err + return nil +} diff --git a/plugin/rpc_cache.go b/plugin/rpc_cache.go new file mode 100644 index 0000000..cca34ff --- /dev/null +++ b/plugin/rpc_cache.go @@ -0,0 +1,78 @@ +package plugin + +import ( + "log" + "net/rpc" + "github.com/feedlabs/elasticfeed/plugin/model" +) + +// An implementation of packer.Cache where the RpcCache is actually executed +// over an RPC connection. +type RpcCache struct { + client *rpc.Client +} + +// CacheRpcServer wraps a packer.Cache implementation and makes it exportable +// as part of a Golang RPC server. +type CacheRpcServer struct { + cache model.Cache +} + +type CacheRLockResponse struct { + Path string + Exists bool +} + +func (c *RpcCache) Lock(key string) (result string) { + if err := c.client.Call("Cache.Lock", key, &result); err != nil { + log.Printf("[ERR] Cache.Lock error: %s", err) + return + } + + return +} + +func (c *RpcCache) RLock(key string) (string, bool) { + var result CacheRLockResponse + if err := c.client.Call("Cache.RLock", key, &result); err != nil { + log.Printf("[ERR] Cache.RLock error: %s", err) + return "", false + } + + return result.Path, result.Exists +} + +func (c *RpcCache) Unlock(key string) { + if err := c.client.Call("Cache.Unlock", key, new(interface{})); err != nil { + log.Printf("[ERR] Cache.Unlock error: %s", err) + return + } +} + +func (c *RpcCache) RUnlock(key string) { + if err := c.client.Call("Cache.RUnlock", key, new(interface{})); err != nil { + log.Printf("[ERR] Cache.RUnlock error: %s", err) + return + } +} + +func (c *CacheRpcServer) Lock(key string, result *string) error { + *result = c.cache.Lock(key) + return nil +} + +func (c *CacheRpcServer) Unlock(key string, result *interface{}) error { + c.cache.Unlock(key) + return nil +} + +func (c *CacheRpcServer) RLock(key string, result *CacheRLockResponse) error { + path, exists := c.cache.RLock(key) + *result = CacheRLockResponse{path, exists} + return nil +} + +func (c *CacheRpcServer) RUnlock(key string, result *interface{}) error { + c.cache.RUnlock(key) + return nil +} diff --git a/plugin/rpc_client.go b/plugin/rpc_client.go new file mode 100644 index 0000000..a1f92b5 --- /dev/null +++ b/plugin/rpc_client.go @@ -0,0 +1,88 @@ +package plugin + +import ( + "github.com/hashicorp/go-msgpack/codec" + "io" + "log" + "net/rpc" + + "github.com/feedlabs/elasticfeed/plugin/model" +) + +// RpcClient is the client end that communicates with a Packer RPC server. +// Establishing a connection is up to the user, the RpcClient can just +// communicate over any ReadWriteCloser. +type RpcClient struct { + mux *muxBroker + client *rpc.Client + closeMux bool +} + +func NewRpcClient(rwc io.ReadWriteCloser) (*RpcClient, error) { + mux, err := newMuxBrokerClient(rwc) + if err != nil { + return nil, err + } + go mux.Run() + + result, err := newRpcClientWithMux(mux, 0) + if err != nil { + mux.Close() + return nil, err + } + + result.closeMux = true + return result, err +} + +func newRpcClientWithMux(mux *muxBroker, streamId uint32) (*RpcClient, error) { + clientConn, err := mux.Dial(streamId) + if err != nil { + return nil, err + } + + h := &codec.MsgpackHandle{ + RawToString: true, + WriteExt: true, + } + clientCodec := codec.GoRpc.ClientCodec(clientConn, h) + + return &RpcClient{ + mux: mux, + client: rpc.NewClientWithCodec(clientCodec), + closeMux: false, + }, nil +} + +func (c *RpcClient) Close() error { + if err := c.client.Close(); err != nil { + return err + } + + if c.closeMux { + log.Printf("[WARN] RpcClient is closing mux") + return c.mux.Close() + } + + return nil +} + +func (c *RpcClient) Indexer() model.Indexer { + return &RpcIndexer{ + client: c.client, + mux: c.mux, + } +} + +func (c *RpcClient) Artifact() model.Artifact { + return &RpcArtifact{ + client: c.client, + endpoint: DefaultArtifactEndpoint, + } +} + +func (c *RpcClient) Cache() model.Cache { + return &RpcCache{ + client: c.client, + } +} diff --git a/plugin/rpc_dial.go b/plugin/rpc_dial.go new file mode 100644 index 0000000..b2d0a26 --- /dev/null +++ b/plugin/rpc_dial.go @@ -0,0 +1,33 @@ +package plugin + +import ( + "net" + "net/rpc" +) + +// rpcDial makes a TCP connection to a remote RPC server and returns +// the client. This will set the connection up properly so that keep-alives +// are set and so on and should be used to make all RPC connections within +// this package. +func rpcDial(address string) (*rpc.Client, error) { + tcpConn, err := tcpDial(address) + if err != nil { + return nil, err + } + + // Create an RPC client around our connection + return rpc.NewClient(tcpConn), nil +} + +// tcpDial connects via TCP to the designated address. +func tcpDial(address string) (*net.TCPConn, error) { + conn, err := net.Dial("tcp", address) + if err != nil { + return nil, err + } + + // Set a keep-alive so that the connection stays alive even when idle + tcpConn := conn.(*net.TCPConn) + tcpConn.SetKeepAlive(true) + return tcpConn, nil +} diff --git a/plugin/rpc_error.go b/plugin/rpc_error.go new file mode 100644 index 0000000..1b7eba5 --- /dev/null +++ b/plugin/rpc_error.go @@ -0,0 +1,21 @@ +package plugin + +// This is a type that wraps error types so that they can be messaged +// across RPC channels. Since "error" is an interface, we can't always +// gob-encode the underlying structure. This is a valid error interface +// implementer that we will push across. +type BasicError struct { + Message string +} + +func NewBasicError(err error) *BasicError { + if err == nil { + return nil + } + + return &BasicError{err.Error()} +} + +func (e *BasicError) Error() string { + return e.Message +} diff --git a/plugin/rpc_indexer.go b/plugin/rpc_indexer.go new file mode 100644 index 0000000..808789d --- /dev/null +++ b/plugin/rpc_indexer.go @@ -0,0 +1,112 @@ +package plugin + +import ( + "log" + "net/rpc" + + "github.com/feedlabs/elasticfeed/plugin/model" +) + +// An implementation of Indexer where the builder is actually executed +// over an RPC connection. +type RpcIndexer struct { + client *rpc.Client + mux *muxBroker +} + +// IndexerRpcServer wraps a Indexer implementation and makes it exportable +// as part of a Golang RPC server. +type IndexerRpcServer struct { + indexer model.Indexer + mux *muxBroker +} + +type IndexerPrepareArgs struct { + Configs []interface{} +} + +type IndexerPrepareResponse struct { + Warnings []string + Error *BasicError +} + +func (b *RpcIndexer) Prepare(config ...interface{}) ([]string, error) { + var resp IndexerPrepareResponse + cerr := b.client.Call("Indexer.Prepare", &IndexerPrepareArgs{config}, &resp) + if cerr != nil { + return nil, cerr + } + var err error = nil + if resp.Error != nil { + err = resp.Error + } + + return resp.Warnings, err +} + +func (b *RpcIndexer) Run(cache model.Cache) (model.Artifact, error) { + nextId := b.mux.NextId() + server := newRpcServerWithMux(b.mux, nextId) + server.RegisterCache(cache) + go server.Serve() + + var responseId uint32 + if err := b.client.Call("Indexer.Run", nextId, &responseId); err != nil { + return nil, err + } + + if responseId == 0 { + return nil, nil + } + + client, err := newRpcClientWithMux(b.mux, responseId) + if err != nil { + return nil, err + } + + return client.Artifact(), nil +} + +func (b *RpcIndexer) Cancel() { + if err := b.client.Call("Indexer.Cancel", new(interface{}), new(interface{})); err != nil { + log.Printf("Error cancelling indexer: %s", err) + } +} + +func (b *IndexerRpcServer) Prepare(args *IndexerPrepareArgs, reply *IndexerPrepareResponse) error { + warnings, err := b.indexer.Prepare(args.Configs...) + *reply = IndexerPrepareResponse{ + Warnings: warnings, + Error: NewBasicError(err), + } + return nil +} + +func (b *IndexerRpcServer) Run(streamId uint32, reply *uint32) error { + client, err := newRpcClientWithMux(b.mux, streamId) + if err != nil { + return NewBasicError(err) + } + defer client.Close() + + artifact, err := b.indexer.Run(client.Cache()) + if err != nil { + return NewBasicError(err) + } + + *reply = 0 + if artifact != nil { + streamId = b.mux.NextId() + server := newRpcServerWithMux(b.mux, streamId) + server.RegisterArtifact(artifact) + go server.Serve() + *reply = streamId + } + + return nil +} + +func (b *IndexerRpcServer) Cancel(args *interface{}, reply *interface{}) error { + b.indexer.Cancel() + return nil +} diff --git a/plugin/rpc_init.go b/plugin/rpc_init.go new file mode 100644 index 0000000..96d677c --- /dev/null +++ b/plugin/rpc_init.go @@ -0,0 +1,10 @@ +package plugin + +import "encoding/gob" + +func init() { +// gob.Register(new(map[string]interface{})) +// gob.Register(new(map[string]string)) + gob.Register(make([]interface{}, 0)) + gob.Register(new(BasicError)) +} diff --git a/plugin/rpc_mux_broker.go b/plugin/rpc_mux_broker.go new file mode 100644 index 0000000..abae338 --- /dev/null +++ b/plugin/rpc_mux_broker.go @@ -0,0 +1,191 @@ +package plugin + +import ( + "encoding/binary" + "fmt" + "io" + "net" + "sync" + "sync/atomic" + "time" + + "github.com/hashicorp/yamux" +) + +// muxBroker is responsible for brokering multiplexed connections by unique ID. +// +// This allows a plugin to request a channel with a specific ID to connect to +// or accept a connection from, and the broker handles the details of +// holding these channels open while they're being negotiated. +type muxBroker struct { + nextId uint32 + session *yamux.Session + streams map[uint32]*muxBrokerPending + + sync.Mutex +} + +type muxBrokerPending struct { + ch chan net.Conn + doneCh chan struct{} +} + +func newMuxBroker(s *yamux.Session) *muxBroker { + return &muxBroker{ + session: s, + streams: make(map[uint32]*muxBrokerPending), + } +} + +func newMuxBrokerClient(rwc io.ReadWriteCloser) (*muxBroker, error) { + s, err := yamux.Client(rwc, nil) + if err != nil { + return nil, err + } + + return newMuxBroker(s), nil +} + +func newMuxBrokerServer(rwc io.ReadWriteCloser) (*muxBroker, error) { + s, err := yamux.Server(rwc, nil) + if err != nil { + return nil, err + } + + return newMuxBroker(s), nil +} + +// Accept accepts a connection by ID. +// +// This should not be called multiple times with the same ID at one time. +func (m *muxBroker) Accept(id uint32) (net.Conn, error) { + var c net.Conn + p := m.getStream(id) + select { + case c = <-p.ch: + close(p.doneCh) + case <-time.After(5 * time.Second): + m.Lock() + defer m.Unlock() + delete(m.streams, id) + + return nil, fmt.Errorf("timeout waiting for accept") + } + + // Ack our connection + if err := binary.Write(c, binary.LittleEndian, id); err != nil { + c.Close() + return nil, err + } + + return c, nil +} + +// Close closes the connection and all sub-connections. +func (m *muxBroker) Close() error { + return m.session.Close() +} + +// Dial opens a connection by ID. +func (m *muxBroker) Dial(id uint32) (net.Conn, error) { + // Open the stream + stream, err := m.session.OpenStream() + if err != nil { + return nil, err + } + + // Write the stream ID onto the wire. + if err := binary.Write(stream, binary.LittleEndian, id); err != nil { + stream.Close() + return nil, err + } + + // Read the ack that we connected. Then we're off! + var ack uint32 + if err := binary.Read(stream, binary.LittleEndian, &ack); err != nil { + stream.Close() + return nil, err + } + if ack != id { + stream.Close() + return nil, fmt.Errorf("bad ack: %d (expected %d)", ack, id) + } + + return stream, nil +} + +// NextId returns a unique ID to use next. +func (m *muxBroker) NextId() uint32 { + return atomic.AddUint32(&m.nextId, 1) +} + +// Run starts the brokering and should be executed in a goroutine, since it +// blocks forever, or until the session closes. +func (m *muxBroker) Run() { + for { + stream, err := m.session.AcceptStream() + if err != nil { + // Once we receive an error, just exit + break + } + + // Read the stream ID from the stream + var id uint32 + if err := binary.Read(stream, binary.LittleEndian, &id); err != nil { + stream.Close() + continue + } + + // Initialize the waiter + p := m.getStream(id) + select { + case p.ch <- stream: + default: + } + + // Wait for a timeout + go m.timeoutWait(id, p) + } +} + +func (m *muxBroker) getStream(id uint32) *muxBrokerPending { + m.Lock() + defer m.Unlock() + + p, ok := m.streams[id] + if ok { + return p + } + + m.streams[id] = &muxBrokerPending{ + ch: make(chan net.Conn, 1), + doneCh: make(chan struct{}), + } + return m.streams[id] +} + +func (m *muxBroker) timeoutWait(id uint32, p *muxBrokerPending) { + // Wait for the stream to either be picked up and connected, or + // for a timeout. + timeout := false + select { + case <-p.doneCh: + case <-time.After(5 * time.Second): + timeout = true + } + + m.Lock() + defer m.Unlock() + + // Delete the stream so no one else can grab it + delete(m.streams, id) + + // If we timed out, then check if we have a channel in the buffer, + // and if so, close it. + if timeout { + select { + case s := <-p.ch: + s.Close() + } + } +} diff --git a/plugin/rpc_server.go b/plugin/rpc_server.go new file mode 100644 index 0000000..d5e07f7 --- /dev/null +++ b/plugin/rpc_server.go @@ -0,0 +1,109 @@ +package plugin + +import ( + "fmt" + "github.com/hashicorp/go-msgpack/codec" + "io" + "log" + "net/rpc" + "sync/atomic" + + "github.com/feedlabs/elasticfeed/plugin/model" +) + +var endpointId uint64 + +const ( + DefaultIndexerEndpoint string = "Indexer" + DefaultArtifactEndpoint string = "Artifact" +) + +// RpcServer represents an RPC server for Packer. This must be paired on +// the other side with a Client. +type RpcServer struct { + mux *muxBroker + streamId uint32 + server *rpc.Server + closeMux bool +} + +// NewRpcServer returns a new Packer RPC server. +func NewRpcServer(conn io.ReadWriteCloser) *RpcServer { + mux, _ := newMuxBrokerServer(conn) + result := newRpcServerWithMux(mux, 0) + result.closeMux = true + go mux.Run() + return result +} + +func newRpcServerWithMux(mux *muxBroker, streamId uint32) *RpcServer { + return &RpcServer{ + mux: mux, + streamId: streamId, + server: rpc.NewServer(), + closeMux: false, + } +} + +func (s *RpcServer) Close() error { + if s.closeMux { + log.Printf("[WARN] Shutting down mux conn in RpcServer") + return s.mux.Close() + } + + return nil +} + +func (s *RpcServer) RegisterCache(c model.Cache) { + // s.server.RegisterName(DefaultCacheEndpoint, &CacheServer{ + // cache: c, + // }) +} + +func (s *RpcServer) RegisterArtifact(a model.Artifact) { + s.server.RegisterName(DefaultArtifactEndpoint, &ArtifactRpcServer{ + artifact: a, + }) +} + +func (s *RpcServer) RegisterIndexer(b model.Indexer) { + s.server.RegisterName(DefaultIndexerEndpoint, &IndexerRpcServer{ + indexer: b, + mux: s.mux, + }) +} + +// ServeConn serves a single connection over the RPC server. It is up +// to the caller to obtain a proper io.ReadWriteCloser. +func (s *RpcServer) Serve() { + // Accept a connection on stream ID 0, which is always used for + // normal client to server connections. + stream, err := s.mux.Accept(s.streamId) + if err != nil { + log.Printf("[ERR] Error retrieving stream for serving: %s", err) + return + } + defer stream.Close() + + h := &codec.MsgpackHandle{ + RawToString: true, + WriteExt: true, + } + rpcCodec := codec.GoRpc.ServerCodec(stream, h) + s.server.ServeCodec(rpcCodec) +} + +// registerComponent registers a single Packer RPC component onto +// the RPC server. If id is true, then a unique ID number will be appended +// onto the end of the endpoint. +// +// The endpoint name is returned. +func registerComponent(server *rpc.Server, name string, rcvr interface{}, id bool) string { + endpoint := name + if id { + fmt.Sprintf("%s.%d", endpoint, atomic.AddUint64(&endpointId, 1)) + } + + server.RegisterName(endpoint, rcvr) + return endpoint +} diff --git a/plugin/server.go b/plugin/server.go new file mode 100644 index 0000000..f02bcf3 --- /dev/null +++ b/plugin/server.go @@ -0,0 +1,139 @@ +// The plugin package provides the functionality to both expose a Packer +// plugin binary and to connect to an existing Packer plugin binary. +// +// Packer supports plugins in the form of self-contained external static +// Go binaries. These binaries behave in a certain way (enforced by this +// package) and are connected to in a certain way (also enforced by this +// package). +package plugin + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "net" + "os" + "os/signal" + "runtime" + "strconv" + "sync/atomic" +) + +// This is a count of the number of interrupts the process has received. +// This is updated with sync/atomic whenever a SIGINT is received and can +// be checked by the plugin safely to take action. +var Interrupts int32 = 0 + +const MagicCookieKey = "PACKER_PLUGIN_MAGIC_COOKIE" +const MagicCookieValue = "d602bf8f470bc67ca7faa0386276bbdd4330efaf76d1a219cb4d6991ca9872b2" + +// The APIVersion is outputted along with the RPC address. The plugin +// client validates this API version and will show an error if it doesn't +// know how to speak it. +const APIVersion = "4" + +// Server waits for a connection to this plugin and returns a Packer +// RPC server that you can use to register components and serve them. +func Server() (*RpcServer, error) { + if os.Getenv(MagicCookieKey) != MagicCookieValue { + return nil, errors.New( + "Please do not execute plugins directly. Packer will execute these for you.") + } + + // If there is no explicit number of Go threads to use, then set it + if os.Getenv("GOMAXPROCS") == "" { + runtime.GOMAXPROCS(runtime.NumCPU()) + } + + minPort, err := strconv.ParseInt(os.Getenv("PACKER_PLUGIN_MIN_PORT"), 10, 32) + if err != nil { + return nil, err + } + + maxPort, err := strconv.ParseInt(os.Getenv("PACKER_PLUGIN_MAX_PORT"), 10, 32) + if err != nil { + return nil, err + } + + log.Printf("Plugin minimum port: %d\n", minPort) + log.Printf("Plugin maximum port: %d\n", maxPort) + + listener, err := serverListener(minPort, maxPort) + if err != nil { + return nil, err + } + defer listener.Close() + + // Output the address to stdout + log.Printf("Plugin address: %s %s\n", + listener.Addr().Network(), listener.Addr().String()) + fmt.Printf("%s|%s|%s\n", + APIVersion, + listener.Addr().Network(), + listener.Addr().String()) + os.Stdout.Sync() + + // Accept a connection + log.Println("Waiting for connection...") + conn, err := listener.Accept() + if err != nil { + log.Printf("Error accepting connection: %s\n", err.Error()) + return nil, err + } + + // Eat the interrupts + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt) + go func() { + var count int32 = 0 + for { + <-ch + newCount := atomic.AddInt32(&count, 1) + log.Printf("Received interrupt signal (count: %d). Ignoring.", newCount) + } + }() + + // Serve a single connection + log.Println("Serving a plugin connection...") + return NewRpcServer(conn), nil +} + +func serverListener(minPort, maxPort int64) (net.Listener, error) { + if runtime.GOOS == "windows" { + return serverListener_tcp(minPort, maxPort) + } + + return serverListener_unix() +} + +func serverListener_tcp(minPort, maxPort int64) (net.Listener, error) { + for port := minPort; port <= maxPort; port++ { + address := fmt.Sprintf("127.0.0.1:%d", port) + listener, err := net.Listen("tcp", address) + if err == nil { + return listener, nil + } + } + + return nil, errors.New("Couldn't bind plugin TCP listener") +} + +func serverListener_unix() (net.Listener, error) { + tf, err := ioutil.TempFile("", "packer-plugin") + if err != nil { + return nil, err + } + path := tf.Name() + + // Close the file and remove it because it has to not exist for + // the domain socket. + if err := tf.Close(); err != nil { + return nil, err + } + if err := os.Remove(path); err != nil { + return nil, err + } + + return net.Listen("unix", path) +} diff --git a/resource/plugin.go b/resource/plugin.go index f52351e..3c7dc9b 100644 --- a/resource/plugin.go +++ b/resource/plugin.go @@ -7,6 +7,15 @@ import ( "github.com/feedlabs/feedify/graph" ) +const ( + PLUGIN_INDEXER = 1 + PLUGIN_CRAWLER = 2 + PLUGIN_SENSOR = 3 + PLUGIN_SCENARIO = 4 + PLUGIN_PIPELINE = 5 + PLUGIN_HELPER = 6 +) + func GetPluginList() (pluginList []*Plugin, err error) { nodes, err := storage.FindNodesByLabel(RESOURCE_PLUGIN_LABEL) if err != nil { diff --git a/resource/resource.go b/resource/resource.go index bab2c6e..d464660 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -12,7 +12,7 @@ import ( "github.com/feedlabs/elasticfeed/service/stream/controller/room" "github.com/feedlabs/elasticfeed/service/stream/model" - "github.com/feedlabs/elasticfeed/plugin/pipeline" + "github.com/feedlabs/elasticfeed/plugin/plugins/pipeline" ) const ( diff --git a/service/system/v1/controller/plugin.go b/service/system/v1/controller/plugin.go index ff8731f..7a13fa6 100644 --- a/service/system/v1/controller/plugin.go +++ b/service/system/v1/controller/plugin.go @@ -147,17 +147,18 @@ func (this *PluginController) PutFile() { } pluginId := this.Ctx.Input.Params[":pluginId"] + plugin, err := resource.GetPlugin(pluginId) - data := this.Ctx.Input.CopyBody() + rel_path := config.GetPluginStoragePath() + "/" + plugin.Group + "-" + uuid.TimeOrderedUUID() + "-" + pluginId + abs_path := config.GetHomeAbsolutePath() + "/" + rel_path - path := config.GetPluginStoragePath() + "/" + uuid.TimeOrderedUUID() + "-" + pluginId - err = ioutil.WriteFile(path, data, 0644) + data := this.Ctx.Input.CopyBody() + err = ioutil.WriteFile(abs_path, data, 0644) if err != nil { panic(err) } - plugin, err := resource.GetPlugin(pluginId) - plugin.Path = path + plugin.Path = rel_path err = resource.UpdatePlugin(plugin) if err != nil { diff --git a/service/system/v1/template/plugin/response.go b/service/system/v1/template/plugin/response.go index a140b81..29b2b4b 100644 --- a/service/system/v1/template/plugin/response.go +++ b/service/system/v1/template/plugin/response.go @@ -2,6 +2,7 @@ package plugin import ( "github.com/feedlabs/elasticfeed/resource" + "github.com/feedlabs/elasticfeed/common/uuid" "github.com/feedlabs/elasticfeed/service/system/v1/template" ) @@ -12,7 +13,7 @@ func GetEntry(plugin *resource.Plugin) (entry map[string]interface{}) { entry["name"] = plugin.Name entry["group"] = plugin.Group entry["version"] = plugin.Version - entry["license"] = plugin.License + entry["license"] = plugin.License + uuid.TimeOrderedUUID() if plugin.Path == "" { entry["status"] = "error" diff --git a/docs/PLUGIN-NOTES.md b/website/docs/PLUGIN-NOTES.md similarity index 100% rename from docs/PLUGIN-NOTES.md rename to website/docs/PLUGIN-NOTES.md diff --git a/docs/WORKFLOW-NOTES.md b/website/docs/WORKFLOW-NOTES.md similarity index 100% rename from docs/WORKFLOW-NOTES.md rename to website/docs/WORKFLOW-NOTES.md diff --git a/workflow/workflow.go b/workflow/workflow.go index 9cf7608..4d914f0 100644 --- a/workflow/workflow.go +++ b/workflow/workflow.go @@ -1,7 +1,7 @@ package workflow import ( - "github.com/feedlabs/elasticfeed/plugin" + "github.com/feedlabs/elasticfeed/plugin/model" "github.com/feedlabs/elasticfeed/resource" ) @@ -9,7 +9,7 @@ type WorkflowController struct { feed *resource.Feed manager *WorkflowManager - profiler *plugin.Profiler + profiler *model.Profiler } func (this *WorkflowController) GetManager() *WorkflowManager { @@ -20,7 +20,7 @@ func (this *WorkflowController) GetFeed() *interface{} { return nil } -func (this *WorkflowController) GetProfiler() *plugin.Profiler { +func (this *WorkflowController) GetProfiler() *model.Profiler { return this.profiler } @@ -41,7 +41,7 @@ func (this *WorkflowController) DispatchPipelineHook(data interface{}) interface func NewWorkflowController(feed *resource.Feed, wm *WorkflowManager) *WorkflowController { data := feed.GetWorkflow().GetProfilerRawData() - p := plugin.NewProfiler(data) + p := model.NewProfiler(data) w := &WorkflowController{feed, wm, p} w.Init() From ade1f3438b714b3f7cc64833c123734040cae4f7 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Fri, 24 Apr 2015 09:28:45 +0200 Subject: [PATCH 15/16] added entry metric thoughts --- resource/resource.go | 7 +++++++ service/store/v1/router/entry.go | 3 +++ 2 files changed, 10 insertions(+) diff --git a/resource/resource.go b/resource/resource.go index d464660..8cae779 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -1,5 +1,12 @@ package resource +/* + TO DO + + - resource should trigger system event via EventManager + */ + + import ( "errors" "encoding/json" diff --git a/service/store/v1/router/entry.go b/service/store/v1/router/entry.go index 042f755..8e8ac99 100644 --- a/service/store/v1/router/entry.go +++ b/service/store/v1/router/entry.go @@ -8,4 +8,7 @@ import ( func InitEntryRouters() { feedify.Router("/v1/application/:applicationId:string/feed/:feedId:int/entry", &controller.EntryController{}, "get:GetList;post:Post") feedify.Router("/v1/application/:applicationId:string/feed/:feedId:int/entry/:feedEntryId:int", &controller.EntryController{}, "get:Get;delete:Delete;put:Put") + + // not implemented yet! + feedify.Router("/v1/application/:applicationId:string/feed/:feedId:int/entry/:feedEntryId:int/metric", &controller.EntryController{}, "get:Get;delete:Delete;put:Put;post:Post") } From 510f0854c4ab1b58b251f4cb369ae6ebed121ac5 Mon Sep 17 00:00:00 2001 From: Chris Stasiak Date: Fri, 24 Apr 2015 23:40:43 +0200 Subject: [PATCH 16/16] Added working Engine to Plugin communication over RPC/TCP --- common/config.go | 6 + plugin/client.go | 11 ++ plugin/cmd_pipeline.go | 46 ++++++ plugin/config_template.go | 139 ++++++++++++++++++ plugin/manager.go | 34 ++++- plugin/model/pipeline.go | 5 + plugin/pipeline/ann/artifact.go | 37 +++++ plugin/pipeline/ann/pipeline.go | 28 ++++ plugin/plugins/compiler/js.go | 5 - plugin/rpc_client.go | 7 + plugin/rpc_pipeline.go | 112 ++++++++++++++ plugin/rpc_server.go | 14 +- plugin/server.go | 20 +-- plugins/pipeline-ann/main.go | 15 ++ .../pipeline/ai_neural_network.go | 0 .../plugins => plugins}/pipeline/pipeline.go | 0 .../pipeline/random_animator.go | 0 .../scenario/ai_neural_network_trainer.go | 0 .../scenario/rpc_client.go | 0 {plugin/plugins => plugins}/sensor/sensors.go | 0 resource/resource.go | 2 +- 21 files changed, 459 insertions(+), 22 deletions(-) create mode 100644 common/config.go create mode 100644 plugin/cmd_pipeline.go create mode 100644 plugin/config_template.go create mode 100644 plugin/pipeline/ann/artifact.go create mode 100644 plugin/pipeline/ann/pipeline.go delete mode 100644 plugin/plugins/compiler/js.go create mode 100644 plugin/rpc_pipeline.go create mode 100644 plugins/pipeline-ann/main.go rename {plugin/plugins => plugins}/pipeline/ai_neural_network.go (100%) rename {plugin/plugins => plugins}/pipeline/pipeline.go (100%) rename {plugin/plugins => plugins}/pipeline/random_animator.go (100%) rename {plugin/plugins => plugins}/scenario/ai_neural_network_trainer.go (100%) rename {plugin/plugins => plugins}/scenario/rpc_client.go (100%) rename {plugin/plugins => plugins}/sensor/sensors.go (100%) diff --git a/common/config.go b/common/config.go new file mode 100644 index 0000000..b2a431f --- /dev/null +++ b/common/config.go @@ -0,0 +1,6 @@ +package common + +type ElasticfeedConfig struct { + ElasticfeedDebug bool `mapstructure:"elasticfeed_debug"` + ElasticfeedForce bool `mapstructure:"elasticfeed_force"` +} diff --git a/plugin/client.go b/plugin/client.go index eba3197..f465124 100644 --- a/plugin/client.go +++ b/plugin/client.go @@ -138,6 +138,17 @@ func (c *Client) Indexer() (model.Indexer, error) { return &cmdIndexer{client.Indexer(), c}, nil } +// Returns a pipeline implementation that is communicating over this +// client. If the client hasn't been started, this will start it. +func (c *Client) Pipeline() (model.Pipeline, error) { + client, err := c.RpcClient() + if err != nil { + return nil, err + } + + return &cmdPipeline{client.Pipeline(), c}, nil +} + // End the executing subprocess (if it is running) and perform any cleanup // tasks necessary such as capturing any remaining logs and so on. // diff --git a/plugin/cmd_pipeline.go b/plugin/cmd_pipeline.go new file mode 100644 index 0000000..1074a06 --- /dev/null +++ b/plugin/cmd_pipeline.go @@ -0,0 +1,46 @@ +package plugin + +import ( + "log" + "github.com/feedlabs/elasticfeed/plugin/model" +) + +type cmdPipeline struct { + pipeline model.Pipeline + client *Client +} + +func (b *cmdPipeline) Prepare(config ...interface{}) ([]string, error) { + defer func() { + r := recover() + b.checkExit(r, nil) + }() + + return b.pipeline.Prepare(config...) +} + +func (b *cmdPipeline) Run(cache model.Cache) (model.Artifact, error) { + defer func() { + r := recover() + b.checkExit(r, nil) + }() + + return b.pipeline.Run(cache) +} + +func (b *cmdPipeline) Cancel() { + defer func() { + r := recover() + b.checkExit(r, nil) + }() + + b.pipeline.Cancel() +} + +func (c *cmdPipeline) checkExit(p interface{}, cb func()) { + if c.client.Exited() && cb != nil { + cb() + } else if p != nil && !Killed { + log.Panic(p) + } +} diff --git a/plugin/config_template.go b/plugin/config_template.go new file mode 100644 index 0000000..31860e3 --- /dev/null +++ b/plugin/config_template.go @@ -0,0 +1,139 @@ +package plugin + +import ( + "bytes" + "fmt" + "os" + "strconv" + "strings" + "text/template" + "time" + + "github.com/feedlabs/elasticfeed/common/uuid" +) + +// InitTime is the UTC time when this package was initialized. It is +// used as the timestamp for all configuration templates so that they +// match for a single build. +var InitTime time.Time + +func init() { + InitTime = time.Now().UTC() +} + +// ConfigTemplate processes string data as a text/template with some common +// elements and functions available. Plugin creators should process as +// many fields as possible through this. +type ConfigTemplate struct { + UserVars map[string]string + + root *template.Template + i int +} + +// NewConfigTemplate creates a new configuration template processor. +func NewConfigTemplate() (*ConfigTemplate, error) { + result := &ConfigTemplate{ + UserVars: make(map[string]string), + } + + result.root = template.New("configTemplateRoot") + result.root.Funcs(template.FuncMap{ + "env": templateDisableEnv, + "pwd": templatePwd, + "isotime": templateISOTime, + "timestamp": templateTimestamp, + "user": result.templateUser, + "uuid": templateUuid, + "upper": strings.ToUpper, + "lower": strings.ToLower, + }) + + return result, nil +} + +// Process processes a single string, compiling and executing the template. +func (t *ConfigTemplate) Process(s string, data interface{}) (string, error) { + tpl, err := t.root.New(t.nextTemplateName()).Parse(s) + if err != nil { + return "", err + } + + buf := new(bytes.Buffer) + if err := tpl.Execute(buf, data); err != nil { + return "", err + } + + return buf.String(), nil +} + +// Validate the template. +func (t *ConfigTemplate) Validate(s string) error { + root, err := t.root.Clone() + if err != nil { + return err + } + + _, err = root.New("template").Parse(s) + return err +} + +// Add additional functions to the template +func (t *ConfigTemplate) Funcs(funcs template.FuncMap) { + t.root.Funcs(funcs) +} + +func (t *ConfigTemplate) nextTemplateName() string { + name := fmt.Sprintf("tpl%d", t.i) + t.i++ + return name +} + +// User is the function exposed as "user" within the templates and +// looks up user variables. +func (t *ConfigTemplate) templateUser(n string) (string, error) { + result, ok := t.UserVars[n] + if !ok { + return "", fmt.Errorf("unknown user var: %s", n) + } + + return result, nil +} + +func templateDisableEnv(n string) (string, error) { + return "", fmt.Errorf( + "Environmental variables can only be used as default values for user variables.") +} + +func templateDisableUser(n string) (string, error) { + return "", fmt.Errorf( + "User variable can't be used within a default value for a user variable: %s", n) +} + +func templateEnv(n string) string { + return os.Getenv(n) +} + +func templateISOTime(timeFormat ...string) (string, error) { + if len(timeFormat) == 0 { + return time.Now().UTC().Format(time.RFC3339), nil + } + + if len(timeFormat) > 1 { + return "", fmt.Errorf("too many values, 1 needed: %v", timeFormat) + } + + return time.Now().UTC().Format(timeFormat[0]), nil +} + +func templatePwd() (string, error) { + return os.Getwd() +} + +func templateTimestamp() string { + return strconv.FormatInt(InitTime.Unix(), 10) +} + +func templateUuid() string { + return uuid.TimeOrderedUUID() +} diff --git a/plugin/manager.go b/plugin/manager.go index 651a231..b4c2076 100644 --- a/plugin/manager.go +++ b/plugin/manager.go @@ -3,6 +3,8 @@ package plugin import ( "strconv" + "fmt" + "log" "os/exec" "path/filepath" @@ -102,17 +104,28 @@ func (c *PluginManager) Discover() error { return nil } -func (c *PluginManager) LoadIndexers(name string) (model.Indexer, error) { - log.Printf("Loading provisioner: %s\n", name) +func (c *PluginManager) LoadIndexer(name string) (model.Indexer, error) { + log.Printf("Loading indexer: %s\n", name) bin, ok := c.Indexers[name] if !ok { - log.Printf("Indexers not found: %s\n", name) + log.Printf("Indexer not found: %s\n", name) return nil, nil } return c.pluginClient(bin).Indexer() } +func (c *PluginManager) LoadPipeline(name string) (model.Pipeline, error) { + log.Printf("Loading pipeline: %s\n", name) + bin, ok := c.Pipelines[name] + if !ok { + log.Printf("Pipeline not found: %s\n", name) + return nil, nil + } + + return c.pluginClient(bin).Pipeline() +} + func (c *PluginManager) discover(path string) error { var err error @@ -230,5 +243,20 @@ func NewPluginManager(resourceManager interface{}) *PluginManager { pm.api = model.NewResourceApi(resourceManager) + pm.discover(filepath.Join(config.GetHomeAbsolutePath(), "plugins/pipeline-ann")) + + list := []string{"ann", "ann1", "ann2"} + + for _, name := range(list) { + ann, _ := pm.LoadPipeline(name) + + ann.Prepare() + a, b := ann.Run(nil) + + fmt.Println(a) + fmt.Println(b) + } + + return pm } diff --git a/plugin/model/pipeline.go b/plugin/model/pipeline.go index ce6f42c..7e60411 100644 --- a/plugin/model/pipeline.go +++ b/plugin/model/pipeline.go @@ -2,4 +2,9 @@ package model type Pipeline interface { + Prepare(...interface{}) ([]string, error) + + Run(cache Cache) (Artifact, error) + + Cancel() } diff --git a/plugin/pipeline/ann/artifact.go b/plugin/pipeline/ann/artifact.go new file mode 100644 index 0000000..2be8b04 --- /dev/null +++ b/plugin/pipeline/ann/artifact.go @@ -0,0 +1,37 @@ +package ann + +import ( + "fmt" +) + +const BuilderId = "packer.post-processor.atlas" + +type Artifact struct { + Name string + Type string + Version int +} + +func (*Artifact) BuilderId() string { + return BuilderId +} + +func (a *Artifact) Files() []string { + return nil +} + +func (a *Artifact) Id() string { + return fmt.Sprintf("%s/%s/%d", a.Name, a.Type, a.Version) +} + +func (a *Artifact) String() string { + return fmt.Sprintf("%s/%s (v%d)", a.Name, a.Type, a.Version) +} + +func (*Artifact) State(name string) interface{} { + return nil +} + +func (a *Artifact) Destroy() error { + return nil +} diff --git a/plugin/pipeline/ann/pipeline.go b/plugin/pipeline/ann/pipeline.go new file mode 100644 index 0000000..7ea9f84 --- /dev/null +++ b/plugin/pipeline/ann/pipeline.go @@ -0,0 +1,28 @@ +package ann + +import ( + "github.com/feedlabs/elasticfeed/common" + "github.com/feedlabs/elasticfeed/plugin" + "github.com/feedlabs/elasticfeed/plugin/model" +) + +type config struct { + common.ElasticfeedConfig `mapstructure:",squash"` + + tpl *plugin.ConfigTemplate +} + +type Pipeline struct { + config config +} + +func (p *Pipeline) Prepare(raws ...interface{}) ([]string, error) { + return nil, nil +} + +func (p *Pipeline) Run(cache model.Cache) (model.Artifact, error) { + return &Artifact{"aaa", "bbb", 123}, nil +} + +func (p *Pipeline) Cancel() { +} diff --git a/plugin/plugins/compiler/js.go b/plugin/plugins/compiler/js.go deleted file mode 100644 index f266cec..0000000 --- a/plugin/plugins/compiler/js.go +++ /dev/null @@ -1,5 +0,0 @@ -package compiler - -func getJsVM() {} - -func init() {} diff --git a/plugin/rpc_client.go b/plugin/rpc_client.go index a1f92b5..62d9f3a 100644 --- a/plugin/rpc_client.go +++ b/plugin/rpc_client.go @@ -74,6 +74,13 @@ func (c *RpcClient) Indexer() model.Indexer { } } +func (c *RpcClient) Pipeline() model.Pipeline { + return &RpcPipeline{ + client: c.client, + mux: c.mux, + } +} + func (c *RpcClient) Artifact() model.Artifact { return &RpcArtifact{ client: c.client, diff --git a/plugin/rpc_pipeline.go b/plugin/rpc_pipeline.go new file mode 100644 index 0000000..a0ee302 --- /dev/null +++ b/plugin/rpc_pipeline.go @@ -0,0 +1,112 @@ +package plugin + +import ( + "log" + "net/rpc" + + "github.com/feedlabs/elasticfeed/plugin/model" +) + +// An implementation of Pipeline where the builder is actually executed +// over an RPC connection. +type RpcPipeline struct { + client *rpc.Client + mux *muxBroker +} + +// PipelineRpcServer wraps a Pipeline implementation and makes it exportable +// as part of a Golang RPC server. +type PipelineRpcServer struct { + pipeline model.Pipeline + mux *muxBroker +} + +type PipelinePrepareArgs struct { + Configs []interface{} +} + +type PipelinePrepareResponse struct { + Warnings []string + Error *BasicError +} + +func (b *RpcPipeline) Prepare(config ...interface{}) ([]string, error) { + var resp PipelinePrepareResponse + cerr := b.client.Call("Pipeline.Prepare", &PipelinePrepareArgs{config}, &resp) + if cerr != nil { + return nil, cerr + } + var err error = nil + if resp.Error != nil { + err = resp.Error + } + + return resp.Warnings, err +} + +func (b *RpcPipeline) Run(cache model.Cache) (model.Artifact, error) { + nextId := b.mux.NextId() + server := newRpcServerWithMux(b.mux, nextId) + server.RegisterCache(cache) + go server.Serve() + + var responseId uint32 + if err := b.client.Call("Pipeline.Run", nextId, &responseId); err != nil { + return nil, err + } + + if responseId == 0 { + return nil, nil + } + + client, err := newRpcClientWithMux(b.mux, responseId) + if err != nil { + return nil, err + } + + return client.Artifact(), nil +} + +func (b *RpcPipeline) Cancel() { + if err := b.client.Call("Pipeline.Cancel", new(interface{}), new(interface{})); err != nil { + log.Printf("Error cancelling pipeline: %s", err) + } +} + +func (b *PipelineRpcServer) Prepare(args *PipelinePrepareArgs, reply *PipelinePrepareResponse) error { + warnings, err := b.pipeline.Prepare(args.Configs...) + *reply = PipelinePrepareResponse{ + Warnings: warnings, + Error: NewBasicError(err), + } + return nil +} + +func (b *PipelineRpcServer) Run(streamId uint32, reply *uint32) error { + client, err := newRpcClientWithMux(b.mux, streamId) + if err != nil { + return NewBasicError(err) + } + defer client.Close() + + artifact, err := b.pipeline.Run(client.Cache()) + if err != nil { + return NewBasicError(err) + } + + *reply = 0 + if artifact != nil { + streamId = b.mux.NextId() + server := newRpcServerWithMux(b.mux, streamId) + server.RegisterArtifact(artifact) + go server.Serve() + *reply = streamId + } + + return nil +} + +func (b *PipelineRpcServer) Cancel(args *interface{}, reply *interface{}) error { + b.pipeline.Cancel() + return nil +} diff --git a/plugin/rpc_server.go b/plugin/rpc_server.go index d5e07f7..f1b3eeb 100644 --- a/plugin/rpc_server.go +++ b/plugin/rpc_server.go @@ -14,7 +14,8 @@ import ( var endpointId uint64 const ( - DefaultIndexerEndpoint string = "Indexer" + DefaultIndexerEndpoint string = "Indexer" + DefaultPipelineEndpoint string = "Pipeline" DefaultArtifactEndpoint string = "Artifact" ) @@ -66,9 +67,16 @@ func (s *RpcServer) RegisterArtifact(a model.Artifact) { }) } -func (s *RpcServer) RegisterIndexer(b model.Indexer) { +func (s *RpcServer) RegisterIndexer(i model.Indexer) { s.server.RegisterName(DefaultIndexerEndpoint, &IndexerRpcServer{ - indexer: b, + indexer: i, + mux: s.mux, + }) +} + +func (s *RpcServer) RegisterPipeline(p model.Pipeline) { + s.server.RegisterName(DefaultPipelineEndpoint, &PipelineRpcServer{ + pipeline: p, mux: s.mux, }) } diff --git a/plugin/server.go b/plugin/server.go index f02bcf3..0feca52 100644 --- a/plugin/server.go +++ b/plugin/server.go @@ -1,7 +1,7 @@ -// The plugin package provides the functionality to both expose a Packer -// plugin binary and to connect to an existing Packer plugin binary. +// The plugin package provides the functionality to both expose a Elasticfeed +// plugin binary and to connect to an existing Elasticfeed plugin binary. // -// Packer supports plugins in the form of self-contained external static +// Elasticfeed supports plugins in the form of self-contained external static // Go binaries. These binaries behave in a certain way (enforced by this // package) and are connected to in a certain way (also enforced by this // package). @@ -25,20 +25,20 @@ import ( // be checked by the plugin safely to take action. var Interrupts int32 = 0 -const MagicCookieKey = "PACKER_PLUGIN_MAGIC_COOKIE" -const MagicCookieValue = "d602bf8f470bc67ca7faa0386276bbdd4330efaf76d1a219cb4d6991ca9872b2" +const MagicCookieKey = "ELASTICFEED_PLUGIN_MAGIC_COOKIE" +const MagicCookieValue = "LQhHwrfdFtcZCudDzaQK8xkipGN3yqc3htghipXmJsakNRV9kwP]dPGLWuh" // The APIVersion is outputted along with the RPC address. The plugin // client validates this API version and will show an error if it doesn't // know how to speak it. const APIVersion = "4" -// Server waits for a connection to this plugin and returns a Packer +// Server waits for a connection to this plugin and returns a Elasticfeed // RPC server that you can use to register components and serve them. func Server() (*RpcServer, error) { if os.Getenv(MagicCookieKey) != MagicCookieValue { return nil, errors.New( - "Please do not execute plugins directly. Packer will execute these for you.") + "Please do not execute plugins directly. Elasticfeed will execute these for you.") } // If there is no explicit number of Go threads to use, then set it @@ -46,12 +46,12 @@ func Server() (*RpcServer, error) { runtime.GOMAXPROCS(runtime.NumCPU()) } - minPort, err := strconv.ParseInt(os.Getenv("PACKER_PLUGIN_MIN_PORT"), 10, 32) + minPort, err := strconv.ParseInt(os.Getenv("ELASTICFEED_PLUGIN_MIN_PORT"), 10, 32) if err != nil { return nil, err } - maxPort, err := strconv.ParseInt(os.Getenv("PACKER_PLUGIN_MAX_PORT"), 10, 32) + maxPort, err := strconv.ParseInt(os.Getenv("ELASTICFEED_PLUGIN_MAX_PORT"), 10, 32) if err != nil { return nil, err } @@ -120,7 +120,7 @@ func serverListener_tcp(minPort, maxPort int64) (net.Listener, error) { } func serverListener_unix() (net.Listener, error) { - tf, err := ioutil.TempFile("", "packer-plugin") + tf, err := ioutil.TempFile("", "elasticfeed-plugin") if err != nil { return nil, err } diff --git a/plugins/pipeline-ann/main.go b/plugins/pipeline-ann/main.go new file mode 100644 index 0000000..5d6432b --- /dev/null +++ b/plugins/pipeline-ann/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "github.com/feedlabs/elasticfeed/plugin/pipeline/ann" + "github.com/feedlabs/elasticfeed/plugin" +) + +func main() { + server, err := plugin.Server() + if err != nil { + panic(err) + } + server.RegisterPipeline(new(ann.Pipeline)) + server.Serve() +} diff --git a/plugin/plugins/pipeline/ai_neural_network.go b/plugins/pipeline/ai_neural_network.go similarity index 100% rename from plugin/plugins/pipeline/ai_neural_network.go rename to plugins/pipeline/ai_neural_network.go diff --git a/plugin/plugins/pipeline/pipeline.go b/plugins/pipeline/pipeline.go similarity index 100% rename from plugin/plugins/pipeline/pipeline.go rename to plugins/pipeline/pipeline.go diff --git a/plugin/plugins/pipeline/random_animator.go b/plugins/pipeline/random_animator.go similarity index 100% rename from plugin/plugins/pipeline/random_animator.go rename to plugins/pipeline/random_animator.go diff --git a/plugin/plugins/scenario/ai_neural_network_trainer.go b/plugins/scenario/ai_neural_network_trainer.go similarity index 100% rename from plugin/plugins/scenario/ai_neural_network_trainer.go rename to plugins/scenario/ai_neural_network_trainer.go diff --git a/plugin/plugins/scenario/rpc_client.go b/plugins/scenario/rpc_client.go similarity index 100% rename from plugin/plugins/scenario/rpc_client.go rename to plugins/scenario/rpc_client.go diff --git a/plugin/plugins/sensor/sensors.go b/plugins/sensor/sensors.go similarity index 100% rename from plugin/plugins/sensor/sensors.go rename to plugins/sensor/sensors.go diff --git a/resource/resource.go b/resource/resource.go index 8cae779..327351c 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -19,7 +19,7 @@ import ( "github.com/feedlabs/elasticfeed/service/stream/controller/room" "github.com/feedlabs/elasticfeed/service/stream/model" - "github.com/feedlabs/elasticfeed/plugin/plugins/pipeline" + "github.com/feedlabs/elasticfeed/plugins/pipeline" ) const (