diff --git a/keter-backend/pom.xml b/keter-backend/pom.xml index 40fd4fe..dcc6cb4 100644 --- a/keter-backend/pom.xml +++ b/keter-backend/pom.xml @@ -60,6 +60,10 @@ slf4j-api 1.7.25 + + org.springframework.boot + spring-boot-starter-security + diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/Application.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/Application.java index 25009b8..efd1b2e 100644 --- a/keter-backend/src/main/java/ru/avplatonov/keter/backend/Application.java +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/Application.java @@ -13,6 +13,8 @@ import org.springframework.security.config.annotation.web.builders.WebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; import org.springframework.web.servlet.view.InternalResourceViewResolver; +import ru.avplatonov.keter.backend.db.GraphsDB; +import ru.avplatonov.keter.backend.db.NodesDB; import java.util.Arrays; @@ -20,11 +22,14 @@ public class Application { public static ApplicationContext context; - + public static GraphsDB graphsDB; + public static NodesDB nodesDB; private Logger logger = LoggerFactory.getLogger(Application.class); public static void main(String[] args) { - + context = new AnnotationConfigApplicationContext(GraphsDB.class, NodesDB.class); + graphsDB = context.getBean(GraphsDB.class); + nodesDB = context.getBean(NodesDB.class); SpringApplication.run(Application.class, args); } diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateGraphKeter.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateGraphKeter.java new file mode 100644 index 0000000..4165566 --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateGraphKeter.java @@ -0,0 +1,4 @@ +package ru.avplatonov.keter.backend.controllers.management.create; + +public class CreateGraphKeter { +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateGraphTemplate.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateGraphTemplate.java new file mode 100644 index 0000000..e289d7c --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateGraphTemplate.java @@ -0,0 +1,29 @@ +package ru.avplatonov.keter.backend.controllers.management.create; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import ru.avplatonov.keter.backend.initialize.managet.GraphTemplate; + +import static ru.avplatonov.keter.backend.Application.graphsDB; + +@RestController +@EnableAutoConfiguration +public class CreateGraphTemplate { + + @CrossOrigin(origins = "*", allowedHeaders = "*") + @RequestMapping(value = "/create/graphs", + headers = {"Content-type=application/json"}) + public String service( + @RequestBody GraphTemplate graphTemplate + ){ + try { + graphsDB.addListOfGraphTemplate(graphTemplate); + return "Success"; + }catch (Exception ex){ + return "Failed"; + } + } +} \ No newline at end of file diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateNodeKeter.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateNodeKeter.java new file mode 100644 index 0000000..464effa --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateNodeKeter.java @@ -0,0 +1,4 @@ +package ru.avplatonov.keter.backend.controllers.management.create; + +public class CreateNodeKeter { +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateNodeTemplate.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateNodeTemplate.java new file mode 100644 index 0000000..a23d1a5 --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/create/CreateNodeTemplate.java @@ -0,0 +1,29 @@ +package ru.avplatonov.keter.backend.controllers.management.create; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import ru.avplatonov.keter.backend.initialize.managet.NodeTemplate; + +import java.io.IOException; + +import static ru.avplatonov.keter.backend.Application.nodesDB; + +@RestController +@EnableAutoConfiguration +public class CreateNodeTemplate { + + @CrossOrigin(origins = "*", allowedHeaders = "*") + @RequestMapping(value = "/create/nodes", + headers = {"Content-type=application/json"}) + public String service( + @RequestBody NodeTemplate nodeTemplate + ) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + nodesDB.addListOfNodes(nodeTemplate); + return "listOfNode.size=" + nodesDB.getListOfNodeTemplates().size() + "\n" + mapper.writeValueAsString(nodeTemplate) ; + } +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/graphs/GraphTemplateListController.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/graphs/GraphTemplateListController.java new file mode 100644 index 0000000..2fbb07f --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/graphs/GraphTemplateListController.java @@ -0,0 +1,38 @@ +package ru.avplatonov.keter.backend.controllers.management.graphs; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.lang.NonNull; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import ru.avplatonov.keter.backend.initialize.managet.GraphTemplate; + +import java.util.ArrayList; +import java.util.List; + +import static ru.avplatonov.keter.backend.Application.graphsDB; + +@RestController +@EnableAutoConfiguration +public class GraphTemplateListController { + + @CrossOrigin(origins = "*", allowedHeaders = "*") + @RequestMapping(value = "/graphs/{graphSearch}") + public String listOfGraphs( + @PathVariable("graphSearch") @NonNull String value + ) throws JsonProcessingException { + return value + "\n" + searchGraph(value); + } + + private List searchGraph(String value) throws JsonProcessingException { + List listOfGraphs = new ArrayList<>(); + for (GraphTemplate graphTemplate : graphsDB.getListOfGraphs()) { + if (graphTemplate.getGraphName().contains(value) || graphTemplate.getOutputs().contains(value)){ + listOfGraphs.add(graphTemplate); + } + } + return listOfGraphs; + } +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/nodes/NodeTemplateListController.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/nodes/NodeTemplateListController.java new file mode 100644 index 0000000..fd422b4 --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/nodes/NodeTemplateListController.java @@ -0,0 +1,46 @@ +package ru.avplatonov.keter.backend.controllers.management.nodes; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.lang.NonNull; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import ru.avplatonov.keter.backend.initialize.managet.NodeTemplate; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static ru.avplatonov.keter.backend.Application.nodesDB; + +@RestController +@EnableAutoConfiguration +public class NodeTemplateListController { + + @CrossOrigin(origins = "*", allowedHeaders = "*") + @RequestMapping(value = "/nodes/all") + public Set service() throws JsonProcessingException { + return nodesDB.getListOfNodeTemplates(); + } + + @CrossOrigin(origins = "*", allowedHeaders = "*") + @RequestMapping(value = "/nodes/{nodeSearch}") + public String service( + @PathVariable("nodeSearch") @NonNull String value + ) throws JsonProcessingException { + return value + "\n" + searchNode(value); //.toLowerCase() ? + } + + private List searchNode(String value) throws JsonProcessingException { + List listOfNodes = new ArrayList<>(); + //ObjectMapper mapper = new ObjectMapper(); + for (NodeTemplate nodeTemplate : nodesDB.getListOfNodeTemplates()) { + if (nodeTemplate.getDescription().contains(value) || nodeTemplate.getTags().contains(value) || nodeTemplate.getName().contains(value)){ + listOfNodes.add(nodeTemplate); + } + } + return listOfNodes; + } +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/services/ServiceListController.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/services/ServiceListController.java new file mode 100644 index 0000000..51e2474 --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/management/services/ServiceListController.java @@ -0,0 +1,15 @@ +package ru.avplatonov.keter.backend.controllers.management.services; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@EnableAutoConfiguration +public class ServiceListController { + + @GetMapping("/services") + public String service() { + return null; + } +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/monitoring/schedule/RefreshScheduleState.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/monitoring/schedule/RefreshScheduleState.java index 7fc0d06..fd5969c 100644 --- a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/monitoring/schedule/RefreshScheduleState.java +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/monitoring/schedule/RefreshScheduleState.java @@ -2,10 +2,7 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.lang.NonNull; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import ru.avplatonov.keter.backend.initialize.monitoring.schedule.Schedule; import java.io.IOException; @@ -20,6 +17,8 @@ @RestController @EnableAutoConfiguration public class RefreshScheduleState { + + @CrossOrigin(origins = "*", allowedHeaders = "*") @RequestMapping(value = "/graph/status/state") public List scheduleRefresh( @RequestParam String ids @@ -31,7 +30,7 @@ public List scheduleRefresh( List listUUID = parsGet(ids); for (Schedule schedule : listSchedule) { for (String idGraph : listUUID){ - if(schedule.getUuidShedule().toString().equals(idGraph)) { + if(schedule.getUuidSchedule().toString().equals(idGraph)) { double rand = Math.random(); if (rand <= 0.2) schedule.setState("planning"); diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/monitoring/schedule/ScheduleList.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/monitoring/schedule/ScheduleList.java index 736af5a..3e1d48f 100644 --- a/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/monitoring/schedule/ScheduleList.java +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/controllers/monitoring/schedule/ScheduleList.java @@ -11,7 +11,6 @@ import java.io.IOException; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Date; import java.util.List; import java.util.Optional; @@ -20,6 +19,7 @@ @EnableAutoConfiguration public class ScheduleList { + @CrossOrigin(origins = "*", allowedHeaders = "*") @RequestMapping(value = "/graph/status") public List schedule( @RequestParam Optional filter diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/db/GraphsDB.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/db/GraphsDB.java new file mode 100644 index 0000000..f853b75 --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/db/GraphsDB.java @@ -0,0 +1,21 @@ +package ru.avplatonov.keter.backend.db; + +import org.springframework.stereotype.Component; +import ru.avplatonov.keter.backend.initialize.managet.GraphTemplate; + +import java.util.Set; + +@Component +public class GraphsDB { + + private Set listOfGraphs; + + public Set getListOfGraphs() { + return listOfGraphs; + } + + public void addListOfGraphTemplate(GraphTemplate listOfGraphs){ + this.listOfGraphs.add(listOfGraphs); + } + +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/db/NodesDB.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/db/NodesDB.java new file mode 100644 index 0000000..77b7eb6 --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/db/NodesDB.java @@ -0,0 +1,26 @@ +package ru.avplatonov.keter.backend.db; + +import org.springframework.stereotype.Component; +import ru.avplatonov.keter.backend.initialize.managet.NodeTemplate; + +import java.util.HashSet; +import java.util.Set; + +@Component +public class NodesDB { + + private Set listOfNodeTemplates = new HashSet<>(); + + public Set getListOfNodeTemplates() { + return listOfNodeTemplates; + } + + public void addListOfNodes(NodeTemplate listOfNodes){ + this.listOfNodeTemplates.add(listOfNodes); + } + + public void setListOfNodeTemplates(Set listOfNodeTemplates) { + this.listOfNodeTemplates = listOfNodeTemplates; + } + +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/GraphKeter.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/GraphKeter.java new file mode 100644 index 0000000..aaa96a9 --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/GraphKeter.java @@ -0,0 +1,76 @@ +package ru.avplatonov.keter.backend.initialize.managet; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public class GraphKeter { + private UUID uuidGraph = UUID.randomUUID(); + private int idRun = 2; + private String status; + private List nodesList; + private String graphName; + private String graphDescription; + private Map> parametrs; + + public void setIdRun(int idRun) { + this.idRun = idRun; + } + + public void setStatus(String status) { + this.status = status; + } + + public void setNodesList(List nodesList) { + this.nodesList = nodesList; + } + + public void setGraphName(String graphName) { + this.graphName = graphName; + } + + public void setGraphDescription(String graphDescription) { + this.graphDescription = graphDescription; + } + + public void setParametrs(Map> parametrs) { + this.parametrs = parametrs; + } + + public UUID getUuidGraph() { + return uuidGraph; + } + + public int getIdRun() { + return idRun; + } + + public String getStatus() { + return status; + } + + public List getNodesList() { + return nodesList; + } + + public String getGraphName() { + return graphName; + } + + public String getGraphDescription() { + return graphDescription; + } + + public Map> getParametrs() { + return parametrs; + } + + public GraphKeter(int idRun, String status, List nodesList, String graphName, String graphDescription, Map> parametrs) { + this.idRun = idRun; + this.status = status; + this.nodesList = nodesList; + this.graphName = graphName; + this.graphDescription = graphDescription; + this.parametrs = parametrs; + } +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/GraphTemplate.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/GraphTemplate.java new file mode 100644 index 0000000..9f307b5 --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/GraphTemplate.java @@ -0,0 +1,49 @@ +package ru.avplatonov.keter.backend.initialize.managet; + +import java.util.List; +import java.util.UUID; + +public class GraphTemplate { + + private UUID uuidGraphTemplate = UUID.randomUUID(); + private List nodesList; + private String graphName; + private String graphDescription; + + public void setGraphName(String graphName) { + this.graphName = graphName; + } + + public void setGraphDescription(String graphDescription) { + this.graphDescription = graphDescription; + } + + public void setNodesList(List nodesList) { + this.nodesList = nodesList; + } + + public UUID getUuidGraph() { + return uuidGraphTemplate; + } + + public List getNodesList() { + return nodesList; + } + + public String getGraphName() { + return graphName; + } + + public String getGraphDescription() { + return graphDescription; + } + + public GraphTemplate(){ + } + + public GraphTemplate(List nodesList, String graphName, String graphDescription) { + this.nodesList = nodesList; + this.graphName = graphName; + this.graphDescription = graphDescription; + } +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/NodeTemplate.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/NodeTemplate.java new file mode 100644 index 0000000..7ba6d1c --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/NodeTemplate.java @@ -0,0 +1,119 @@ +package ru.avplatonov.keter.backend.initialize.managet; + +import java.lang.reflect.Type; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public class NodeTemplate { + + + + private UUID uuidNode = UUID.randomUUID(); + private String name = "KETER"; + private String description = "KETER"; + private List tags = null; + private String script = "KETER"; + private Map parameters = null; + private Map hardware = null; + private Map files = null; + private List outputs; + private List inputs; + + public UUID getUuidNode() { + return uuidNode; + } + public void setOutputs(List outputs) { + this.outputs = outputs; + } + + public void setInputs(List inputs) { + this.inputs = inputs; + } + + public List getOutputs() { + return outputs; + } + + public List getInputs() { + return inputs; + } + + public UUID getUuid() { + return uuidNode; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public String getScript() { + return script; + } + + public void setScript(String script) { + this.script = script; + } + + public Map getParameters() { + return parameters; + } + + public void setParameters(Map parameters) { + this.parameters = parameters; + } + + public Map getHardware() { + return hardware; + } + + public void setHardware(Map hardware) { + this.hardware = hardware; + } + + public void setFiles(Map files) { + this.files = files; + } + + public Map getFiles() { + return files; + } + + + public NodeTemplate(){ + + } + + public NodeTemplate(String name, String description, List tags, String script, Map parameters, Map hardware, Map files, List outputs, List inputs) { + this.name = name; + this.description = description; + this.tags = tags; + this.script = script; + this.parameters = parameters; + this.hardware = hardware; + this.files = files; + this.outputs = outputs; + this.inputs = inputs; + } +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/NodesKeter.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/NodesKeter.java new file mode 100644 index 0000000..e6a7d82 --- /dev/null +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/managet/NodesKeter.java @@ -0,0 +1,143 @@ +package ru.avplatonov.keter.backend.initialize.managet; + +import java.lang.reflect.Type; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static ru.avplatonov.keter.backend.Application.nodesDB; + +public class NodesKeter { + private UUID uuidNode = UUID.randomUUID(); + private UUID idNodeTemplate; + private int localIdNode; + private String name; + private String description; + private List tags; + private String script; + private Map parameters; + private Map hardware; + private Map files; + private Map connections; + + public void setConnections(Map connections) { + this.connections = connections; + } + + public UUID getUuidNode() { + return uuidNode; + } + + public Map getConnections() { + return connections; + } + + public void setIdNodeTemplate(UUID idNodeTemplate) { + this.idNodeTemplate = idNodeTemplate; + } + + public void setLocalIdNode(int localIdNode) { + this.localIdNode = localIdNode; + } + + public void setName(String name) { + this.name = name; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public void setScript(String script) { + this.script = script; + } + + public void setParameters(Map parameters) { + this.parameters = parameters; + } + + public void setHardware(Map hardware) { + this.hardware = hardware; + } + + public void setFiles(Map files) { + this.files = files; + } + + public UUID getIdNodeTemplate() { + return idNodeTemplate; + } + + public int getLocalIdNode() { + return localIdNode; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public List getTags() { + return tags; + } + + public String getScript() { + return script; + } + + public Map getParameters() { + return parameters; + } + + public Map getHardware() { + return hardware; + } + + public Map getFiles() { + return files; + } + + + public NodesKeter(UUID idNodeTemplate, int localIdNode, String name, String description, List tags, String script, Map parameters, Map hardware, Map files, Map connections) { + this.idNodeTemplate = idNodeTemplate; + this.localIdNode = localIdNode; + this.name = name; + this.description = description; + this.tags = tags; + this.script = script; + this.parameters = parameters; + this.hardware = hardware; + this.files = files; + this.connections = connections; + } + + public NodesKeter(UUID idNodeTemplate, int localIdNode, Map parameters, + Map hardware, Map connections) { + NodeTemplate nodeTemplateSearchUuid = null; + for (NodeTemplate node : nodesDB.getListOfNodeTemplates()) { + if(node.getUuid().equals(idNodeTemplate)) { + nodeTemplateSearchUuid=node; + return; + } + } + + this.idNodeTemplate = idNodeTemplate; + this.localIdNode = localIdNode; + this.name = nodeTemplateSearchUuid.getName(); + this.description = nodeTemplateSearchUuid.getDescription(); + this.tags = nodeTemplateSearchUuid.getTags(); + this.script = nodeTemplateSearchUuid.getScript(); + this.parameters = parameters; + this.hardware = hardware; + this.files = nodeTemplateSearchUuid.getFiles(); + this.connections = connections; + } +} diff --git a/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/monitoring/schedule/Schedule.java b/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/monitoring/schedule/Schedule.java index 18d2768..f5a1f7e 100644 --- a/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/monitoring/schedule/Schedule.java +++ b/keter-backend/src/main/java/ru/avplatonov/keter/backend/initialize/monitoring/schedule/Schedule.java @@ -6,7 +6,7 @@ public class Schedule { - private UUID uuidShedule = UUID.randomUUID(); + private UUID uuidSchedule = UUID.randomUUID(); private String graphName; private String state; private String author; @@ -14,8 +14,8 @@ public class Schedule { private Timestamp end; private List tags; - public UUID getUuidShedule() { - return uuidShedule; + public UUID getUuidSchedule() { + return uuidSchedule; } public String getGraphName() { diff --git a/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/docker/Docker.scala b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/docker/Docker.scala index 4eeac11..af2f059 100644 --- a/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/docker/Docker.scala +++ b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/docker/Docker.scala @@ -23,7 +23,7 @@ import com.spotify.docker.client.DockerClient import scala.concurrent.Future -case class ContainerDescriptor(name: String, repository: String) +trait ContainerDescriptor trait TDocker { def start(command: String, workdir: Path): Future[Unit] diff --git a/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/package.scala b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/package.scala new file mode 100644 index 0000000..247f3d1 --- /dev/null +++ b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/package.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ru.avplatonov.keter.core + +import java.nio.file.Path + +import ru.avplatonov.keter.core.worker.docker.ContainerDescriptor +import ru.avplatonov.keter.core.worker.work.script.ScriptTemplate + +package object worker { + object ParameterType extends Enumeration { + type ParameterType = Value + + val INT, DOUBLE, STRING = Value + } + + case class ParameterDescriptors(values: Map[String, ParameterDescriptor]) + + case class ParameterDescriptor(value: Any, `type`: ParameterType.Value) + + object ResourceType extends Enumeration { + type ResourceType = Value + val IN, OUT = Value + } + + case class ResourcesDescriptor(values: Map[String, (Path, ResourceType.Value)]) + + case class ResourceHandler() + + case class TaskDescriptor(script: ScriptTemplate, containerDesc: ContainerDescriptor) + + case class Work(desc: TaskDescriptor, environment: ResourcesDescriptor) +} diff --git a/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/Work.scala b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/Work.scala index 7467986..7bb366f 100644 --- a/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/Work.scala +++ b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/Work.scala @@ -17,15 +17,8 @@ package ru.avplatonov.keter.core.worker.work -import ru.avplatonov.keter.core.storage.{FileDescriptor, FileStorage} -import ru.avplatonov.keter.core.worker.docker.{ContainerDescriptor, Docker} -import ru.avplatonov.keter.core.worker.resources.ResourceHandler -import ru.avplatonov.keter.core.worker.work.script.ScriptTemplate +import ru.avplatonov.keter.core.worker.{ResourcesDescriptor, Work} -import scala.concurrent.Future - -case class TaskDescriptor(script: ScriptTemplate, containerDesc: ContainerDescriptor) - -case class Work(desc: TaskDescriptor, resources: ResourceHandler) { - def start(docker: Docker, fileStorage: FileStorage[FileDescriptor]): Future[Unit] = ??? +trait Worker { + def process(work: Work): Either[Exception, ResourcesDescriptor] } diff --git a/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/executor/DockerExecutor.scala b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/executor/DockerExecutor.scala new file mode 100644 index 0000000..8339a6a --- /dev/null +++ b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/executor/DockerExecutor.scala @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ru.avplatonov.keter.core.worker.work.executor + +import java.io.FileOutputStream +import java.nio.file.Path + +import com.spotify.docker.client.DockerClient +import com.spotify.docker.client.messages.{ContainerConfig, HostConfig} +import org.slf4j.Logger + +import scala.util.Try + +/** + * Before using such executor make sure that you configured Docker like this: + * + * 1) Create a file at /etc/systemd/system/docker.service.d/startup_options.conf with the below contents: + * # /etc/systemd/system/docker.service.d/override.conf + * [Service] + * ExecStart= + * ExecStart=/usr/bin/dockerd -H fd:// -H tcp://0.0.0.0:2375 + * + * 2) sudo systemctl daemon-reload + * 3) sudo systemctl restart docker.service + * 4) Check that Docker open socket: + * nmap -p 2375 localhost + * + * And use docker URL like this: http://localhost:2375. + * + * @param loggerFactory + * @param client + * @param imageName + */ +class DockerExecutor(loggerFactory: Path => Logger)(client: DockerClient, imageName: String) extends Executor { + /** + * Execute script and returns resulting files. + * + * @param workdir working directory for script. + * @param syslogName name of logging file with execution tracking. + * @param cmd script. + * @return output files. + */ + override def process(workdir: Path, syslogName: String)(cmd: String): Try[ExecutorResult] = { + implicit val wd = workdir + implicit val logger = loggerFactory(workdir.resolve(syslogName)) + + logger.info(s"Start processing script.") + logger.info(cmd) + + generateStdOutErrAndSh(cmd) flatMap { + case (stdout, stderr, sh) => Try { + val envFiles = getFilesInWD() + stderr + stdout + logger.info(s"Environment [${envFiles.map(_.getFileName.toString).mkString(",")}]") + val dockerWorkdir = "/workdir" + + val wdBind = HostConfig.Bind.builder().from(workdir.toString).to(dockerWorkdir).build() + val hostConfig = HostConfig.builder().binds(wdBind).build() + + val containerConfig = ContainerConfig.builder() + .hostConfig(hostConfig) + .image(imageName) + .workingDir(dockerWorkdir) + .cmd("sh", "-c", "while :; do sleep 1; done") + .build() + + val creation = client.createContainer(containerConfig) + val containerCreationId = creation.id() + client.startContainer(containerCreationId) + logger.info(s"Container started [ID = $containerCreationId]") + + try { + val execCreation = client.execCreate( + containerCreationId, + Array("bash", s"$dockerWorkdir/${sh.getFileName.toString}"), + DockerClient.ExecCreateParam.attachStdout(), + DockerClient.ExecCreateParam.attachStderr() + ) + + logger.info(s"Start detached execution [ID = ${execCreation.id()}]") + resource.managed(client.execStart(execCreation.id())).foreach(log => { + resource.managed(new FileOutputStream(stdout.toFile)).foreach(stdout => { + resource.managed(new FileOutputStream(stderr.toFile)).foreach(stderr => { + logger.info("Attach to execution [ID = ${execCreation.id()}]") + log.attach(stdout, stderr) + }) + }) + }) + + val state = client.execInspect(execCreation.id()) + Int.unbox(state.exitCode()) match { + case 0 => ExecutorResult(stdout, stderr, getFilesInWD() -- envFiles) + case code => throw NonZeroStatusCode(code, stderr) + } + } finally { + logger.info(s"Kill and remove container [ID = $containerCreationId]") + client.killContainer(containerCreationId) + client.removeContainer(containerCreationId) + } + } + } + } +} diff --git a/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/executor/Executor.scala b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/executor/Executor.scala new file mode 100644 index 0000000..464b1c4 --- /dev/null +++ b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/executor/Executor.scala @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ru.avplatonov.keter.core.worker.work.executor + +import java.io.PrintWriter +import java.nio.file.{Files, Path} +import java.util.UUID +import java.util.stream.Collectors + +import org.slf4j.Logger + +import scala.collection.JavaConverters._ +import scala.io.Source +import scala.util.{Failure, Success, Try} + +case class ExecutorResult(stdout: Path, errorLog: Path, otherCreatedFiles: Set[Path]) + +trait Executor { + /** + * Execute script and returns resulting files. + * + * @param workdir working directory for script. + * @param syslogName name of logging file with execution tracking. + * @param cmd script. + * @return output files. + */ + def process(workdir: Path, syslogName: String)(cmd: String): Try[ExecutorResult] + + protected def getFilesInWD()(implicit wd: Path): Set[Path] = Files.list(wd).collect(Collectors.toSet()).asScala.toSet + + protected def generateStdOutErrAndSh(cmd: String)(implicit logger: Logger, workdir: Path): Try[(Path, Path, Path)] = { + createScriptFile(cmd)(logger, workdir) map { sh => + val errorLog = workdir.resolve(UUID.randomUUID().toString + ".error.log") + val stdout = workdir.resolve(UUID.randomUUID().toString + ".out") + (stdout, errorLog, sh) + } + } + + private def createScriptFile(cmd: String)(implicit logger: Logger, workdir: Path): Try[Path] = { + val scriptFile = workdir.resolve(UUID.randomUUID().toString + ".sh") + resource.managed(new PrintWriter(scriptFile.toFile)).acquireAndGet(out => Try { + logger.info(s"Create script file [name = ${scriptFile.toAbsolutePath}]") + out.print(cmd) + scriptFile + }) + } +} + +case class NonZeroStatusCode(status: Int, errorLog: Path) extends RuntimeException + +case class BashExecutor(loggerFactory: Path => Logger)(val env: Array[String] = Array("PATH=/bin:/usr/bin")) + extends Executor { + + /** + * Execute script and returns resulting files. + * + * @param workdir working directory for script. + * @param syslogName name of logging file with execution tracking. + * @param cmd script. + * @return output files. + */ + override def process(workdir: Path, syslogName: String)(cmd: String): Try[ExecutorResult] = { + implicit val wd = workdir + implicit val logger = loggerFactory(workdir.resolve(syslogName)) + + logger.info(s"Start processing script.") + logger.info(cmd) + + generateStdOutErrAndSh(cmd) flatMap { + case (stdout, stderr, sh) => + val envFiles = getFilesInWD() + stderr + stdout + logger.info(s"Environment [${envFiles.map(_.getFileName.toString).mkString(",")}]") + + run(sh, stdout, stderr) + .flatMap(awaiting) + .flatMap(processStatus(_, stdout, stderr, envFiles + sh)) match { + + case res: Success[ExecutorResult] => res + case error: Failure[ExecutorResult] => + logger.error("Error during command processing", error.exception) + error + } + } + } + + private def run(scriptFile: Path, stdout: Path, errorLog: Path)(implicit logger: Logger, workdir: Path) = Try { + val builder = new ProcessBuilder("/bin/bash", scriptFile.getFileName.toString) + .redirectError(errorLog.toFile) + .redirectOutput(stdout.toFile) + .directory(workdir.toFile) + builder.environment().put("PATH", "/bin:/usr/bin") + + logger.info(s"Start cmd ['/bin/bash ${scriptFile.getFileName.toString}']") + builder.start() + } + + private def awaiting(process: Process)(implicit logger: Logger) = Try({ + logger.info(s"Wait cmd") + process.waitFor() + }) + + private def processStatus(status: Int, stdout: Path, errorLog: Path, prevFiles: Set[Path]) + (implicit logger: Logger, workdir: Path) = { + + Try(status match { + case 0 => + logger.info(s"Cmd was complete successfully") + val newFiles = getFilesInWD()(workdir) -- prevFiles + logger.info(s"New files [${newFiles.map(_.getFileName.toString).mkString(",")}]") + ExecutorResult(stdout, errorLog, newFiles) + case code => + logger.warn("Non-zero status code") + Source.fromFile(errorLog.toFile).getLines().foreach(logger.warn) + throw NonZeroStatusCode(code, errorLog) + }) + } +} diff --git a/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/script/ScriptTemplate.scala b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/script/ScriptTemplate.scala index 97fa1ec..4c84a9c 100644 --- a/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/script/ScriptTemplate.scala +++ b/keter-core/src/main/scala/ru/avplatonov/keter/core/worker/work/script/ScriptTemplate.scala @@ -17,30 +17,21 @@ package ru.avplatonov.keter.core.worker.work.script -import ru.avplatonov.keter.core.worker.resources.ResourcesDescriptor +import ru.avplatonov.keter.core.worker.{ParameterDescriptor, ParameterDescriptors, ParameterType, ResourcesDescriptor} -object ParameterType extends Enumeration { - type ParameterType = Value - - val INT, DOUBLE, STRING = Value -} - -case class ParameterDescriptor(value: Any, `type`: ParameterType.Value) - -case class ScriptTemplate(body: String, parameters: Map[String, ParameterDescriptor]) { +case class ScriptTemplate(body: String) { private val parametersRegex = "\\$\\{PARAM\\.(.*?)\\}".r private val filesRegex = "\\$\\{(?:(?:IN)|(?:OUT))\\.(.*?)\\}".r - - def toCommand(resDesc: ResourcesDescriptor): String = { + def toCommand(parameters: ParameterDescriptors, resDesc: ResourcesDescriptor): String = { val withParams: String => String = pasteValues(parameters) val withFiles: String => String = pasteFilePaths(resDesc) (withParams andThen withFiles)(body) } - private def pasteValues(parameters: Map[String, ParameterDescriptor])(body: String): String = + private def pasteValues(parameters: ParameterDescriptors)(body: String): String = checkMissingParameters( - parameters.foldLeft(body)({ + parameters.values.foldLeft(body)({ case (script, (paramName, desc)) => script.replaceAllLiterally("$" + s"{PARAM.$paramName}", toString(desc)) }) diff --git a/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/ScriptTemplateTest.scala b/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/ScriptTemplateTest.scala index 8d784b0..64cf006 100644 --- a/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/ScriptTemplateTest.scala +++ b/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/ScriptTemplateTest.scala @@ -3,111 +3,150 @@ package ru.avplatonov.keter.core.worker.work import java.nio.file.Paths import org.scalatest.{FlatSpec, Matchers} -import ru.avplatonov.keter.core.worker.resources.{ResourceType, ResourcesDescriptor} +import ru.avplatonov.keter.core.worker._ import ru.avplatonov.keter.core.worker.work.script._ class ScriptTemplateTest extends FlatSpec with Matchers { behavior of "ScriptTemplate" + implicit def toParameters(values: Map[String, ParameterDescriptor]): ParameterDescriptors = + ParameterDescriptors(values) + it must "return empty string" in { - ScriptTemplate("", Map.empty).toCommand(ResourcesDescriptor(Map.empty)) should equal("") + ScriptTemplate("").toCommand(Map[String, ParameterDescriptor](), ResourcesDescriptor(Map.empty)) should equal("") } it must "return same string" in { - ScriptTemplate("boo", Map.empty).toCommand(ResourcesDescriptor(Map.empty)) should equal("boo") + ScriptTemplate("boo").toCommand(Map[String, ParameterDescriptor](), ResourcesDescriptor(Map.empty)) should equal("boo") } it must "return int pasted value from parameter" in { - ScriptTemplate("${PARAM.SOME_PARAM}", Map("SOME_PARAM" -> ParameterDescriptor(42, ParameterType.INT))) - .toCommand(ResourcesDescriptor(Map.empty)) should equal("42") + ScriptTemplate("${PARAM.SOME_PARAM}") + .toCommand( + Map("SOME_PARAM" -> ParameterDescriptor(42, ParameterType.INT)), + ResourcesDescriptor(Map.empty) + ) should equal("42") } it must "return double pasted value from parameter" in { - ScriptTemplate("${PARAM.SOME_PARAM}", Map("SOME_PARAM" -> ParameterDescriptor(4.2, ParameterType.DOUBLE))) - .toCommand(ResourcesDescriptor(Map.empty)) should equal("4.20") + ScriptTemplate("${PARAM.SOME_PARAM}") + .toCommand( + Map("SOME_PARAM" -> ParameterDescriptor(42, ParameterType.INT)), + ResourcesDescriptor(Map.empty) + ) should equal("42") } it must "return double pasted value from parameter with rounding" in { - ScriptTemplate("${PARAM.SOME_PARAM}", Map("SOME_PARAM" -> ParameterDescriptor(4.2000001, ParameterType.DOUBLE))) - .toCommand(ResourcesDescriptor(Map.empty)) should equal("4.20") + ScriptTemplate("${PARAM.SOME_PARAM}") + .toCommand( + Map("SOME_PARAM" -> ParameterDescriptor(4.2000001, ParameterType.DOUBLE)), + ResourcesDescriptor(Map.empty) + ) should equal("4.20") } it must "return string pasted value from parameter" in { - ScriptTemplate("${PARAM.SOME_PARAM}", Map("SOME_PARAM" -> ParameterDescriptor("42", ParameterType.STRING))) - .toCommand(ResourcesDescriptor(Map.empty)) should equal("'42'") + ScriptTemplate("${PARAM.SOME_PARAM}") + .toCommand( + Map("SOME_PARAM" -> ParameterDescriptor("42", ParameterType.STRING)), + ResourcesDescriptor(Map.empty) + ) should equal("'42'") } it must "return pasted path value from IN-files" in { - ScriptTemplate("${IN.SOME_FILE}", Map()) - .toCommand(ResourcesDescriptor(Map("SOME_FILE" -> (Paths.get("/tmp"), ResourceType.IN)))) should equal("'/tmp'") + ScriptTemplate("${IN.SOME_FILE}") + .toCommand( + Map[String, ParameterDescriptor](), + ResourcesDescriptor(Map("SOME_FILE" -> (Paths.get("/tmp"), ResourceType.IN))) + ) should equal("'/tmp'") } it must "return pasted path value from OUT-files" in { - ScriptTemplate("${OUT.SOME_FILE}", Map()) - .toCommand(ResourcesDescriptor(Map("SOME_FILE" -> (Paths.get("/tmp"), ResourceType.OUT)))) should equal("'/tmp'") + ScriptTemplate("${OUT.SOME_FILE}") + .toCommand( + Map[String, ParameterDescriptor](), + ResourcesDescriptor(Map("SOME_FILE" -> (Paths.get("/tmp"), ResourceType.OUT))) + ) should equal("'/tmp'") } it must "work with several parameters" in { ScriptTemplate( - "p1=${PARAM.SOME_PARAM_1} p2=${PARAM.SOME_PARAM_2} p3=${PARAM.SOME_PARAM_3}", + "p1=${PARAM.SOME_PARAM_1} p2=${PARAM.SOME_PARAM_2} p3=${PARAM.SOME_PARAM_3}" + ).toCommand( Map( "SOME_PARAM_1" -> ParameterDescriptor(42, ParameterType.INT), "SOME_PARAM_2" -> ParameterDescriptor(4.2, ParameterType.DOUBLE), "SOME_PARAM_3" -> ParameterDescriptor("some string", ParameterType.STRING) - ) - ).toCommand(ResourcesDescriptor(Map.empty)) should equal("p1=42 p2=4.20 p3='some string'") + ), + ResourcesDescriptor(Map.empty) + ) should equal("p1=42 p2=4.20 p3='some string'") } it must "work with several files" in { - ScriptTemplate("in=${IN.IN_FILE} out=${OUT.OUT_FILE}", Map.empty) - .toCommand(ResourcesDescriptor(Map( - "IN_FILE" -> (Paths.get("/in_dir/in"), ResourceType.IN), - "OUT_FILE" -> (Paths.get("/out_dir/out"), ResourceType.OUT) - ))) should equal("in='/in_dir/in' out='/out_dir/out'") + ScriptTemplate("in=${IN.IN_FILE} out=${OUT.OUT_FILE}") + .toCommand( + Map[String, ParameterDescriptor](), + ResourcesDescriptor(Map( + "IN_FILE" -> (Paths.get("/in_dir/in"), ResourceType.IN), + "OUT_FILE" -> (Paths.get("/out_dir/out"), ResourceType.OUT) + )) + ) should equal("in='/in_dir/in' out='/out_dir/out'") } it must "work with several files and parameters" in { ScriptTemplate("in=${IN.IN_FILE} out=${OUT.OUT_FILE} log=${OUT.LOG_FILE} " + - "param1=${PARAM.PARAM_1} param2=${PARAM.PARAM_2} param3=${PARAM.PARAM_3} param4=${PARAM.PARAM_4}", Map( - - "PARAM_1" -> ParameterDescriptor(42, ParameterType.INT), - "PARAM_2" -> ParameterDescriptor(4.2, ParameterType.DOUBLE), - "PARAM_3" -> ParameterDescriptor("42", ParameterType.STRING), - "PARAM_4" -> ParameterDescriptor("fuck you", ParameterType.STRING) - )).toCommand(ResourcesDescriptor(Map( - "IN_FILE" -> (Paths.get("/in_dir/in"), ResourceType.IN), - "OUT_FILE" -> (Paths.get("/out_dir/out"), ResourceType.OUT), - "LOG_FILE" -> (Paths.get("/out_dir/log"), ResourceType.OUT) - ))) should equal("in='/in_dir/in' out='/out_dir/out' log='/out_dir/log' param1=42 param2=4.20 param3='42' param4='fuck you'") + "param1=${PARAM.PARAM_1} param2=${PARAM.PARAM_2} param3=${PARAM.PARAM_3} param4=${PARAM.PARAM_4}") + .toCommand( + Map( + "PARAM_1" -> ParameterDescriptor(42, ParameterType.INT), + "PARAM_2" -> ParameterDescriptor(4.2, ParameterType.DOUBLE), + "PARAM_3" -> ParameterDescriptor("42", ParameterType.STRING), + "PARAM_4" -> ParameterDescriptor("fuck you", ParameterType.STRING) + ), + ResourcesDescriptor(Map( + "IN_FILE" -> (Paths.get("/in_dir/in"), ResourceType.IN), + "OUT_FILE" -> (Paths.get("/out_dir/out"), ResourceType.OUT), + "LOG_FILE" -> (Paths.get("/out_dir/log"), ResourceType.OUT) + )) + ) should equal("in='/in_dir/in' out='/out_dir/out' log='/out_dir/log' param1=42 param2=4.20 param3='42' param4='fuck you'") } it must "with duplicates in parameters in script" in { ScriptTemplate( - "p1=${PARAM.42_PARAM} p2=${PARAM.42_PARAM}", - Map("42_PARAM" -> ParameterDescriptor(42, ParameterType.INT)) - ).toCommand(ResourcesDescriptor(Map.empty)) should equal("p1=42 p2=42") + "p1=${PARAM.42_PARAM} p2=${PARAM.42_PARAM}" + ).toCommand( + Map("42_PARAM" -> ParameterDescriptor(42, ParameterType.INT)), + ResourcesDescriptor(Map.empty) + ) should equal("p1=42 p2=42") } it must "work with duplicates in files" in { - ScriptTemplate("out_1=${OUT.FILE} out_2=${OUT.FILE}", Map.empty) - .toCommand(ResourcesDescriptor(Map( - "FILE" -> (Paths.get("/out_dir/out"), ResourceType.OUT) - ))) should equal("out_1='/out_dir/out' out_2='/out_dir/out'") + ScriptTemplate("out_1=${OUT.FILE} out_2=${OUT.FILE}") + .toCommand( + Map[String, ParameterDescriptor](), + ResourcesDescriptor(Map( + "FILE" -> (Paths.get("/out_dir/out"), ResourceType.OUT) + )) + ) should equal("out_1='/out_dir/out' out_2='/out_dir/out'") } it must "throws exception if there are no several parameters" in { intercept[MissingParametersException] { ScriptTemplate( - "p1=${PARAM.42_PARAM} p2=${PARAM.MISSING_PARAMETER}", - Map("42_PARAM" -> ParameterDescriptor(42, ParameterType.INT)) - ).toCommand(ResourcesDescriptor(Map.empty)) + "p1=${PARAM.42_PARAM} p2=${PARAM.MISSING_PARAMETER}" + ).toCommand( + Map("42_PARAM" -> ParameterDescriptor(42, ParameterType.INT)), + ResourcesDescriptor(Map.empty) + ) }.missingParams.toSet should equal(Set("MISSING_PARAMETER")) } it must "throws exception if there are no several files" in { intercept[MissingFilesException] { - ScriptTemplate("f1=${IN.FILE_1} f2=${OUT.MISSING_FILE}", Map.empty) - .toCommand(ResourcesDescriptor(Map.empty)) + ScriptTemplate("f1=${IN.FILE_1} f2=${OUT.MISSING_FILE}") + .toCommand( + Map[String, ParameterDescriptor](), + ResourcesDescriptor(Map.empty) + ) }.missingFiles.toSet should equal(Set("FILE_1", "MISSING_FILE")) } } diff --git a/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/executor/BashExecutorTest.scala b/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/executor/BashExecutorTest.scala new file mode 100644 index 0000000..8a3e110 --- /dev/null +++ b/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/executor/BashExecutorTest.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ru.avplatonov.keter.core.worker.work.executor + +class BashExecutorTest extends ExecutorTest { + override protected def getTestExecutor(): Executor = BashExecutor(_ => getLogger())() +} diff --git a/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/executor/DockerExecutorTest.scala b/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/executor/DockerExecutorTest.scala new file mode 100644 index 0000000..4faba41 --- /dev/null +++ b/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/executor/DockerExecutorTest.scala @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ru.avplatonov.keter.core.worker.work.executor + +import com.spotify.docker.client.{DefaultDockerClient, DockerClient} + +class DockerExecutorTest extends ExecutorTest { + var docker: DockerClient = new DefaultDockerClient("http://localhost:2375") + + override protected def afterAll(): Unit = { + super.afterAll() + docker.close() + } + + override protected def getTestExecutor(): Executor = new DockerExecutor(_ => getLogger())(docker, "aplatonov/linux_with_java") +} diff --git a/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/executor/ExecutorTest.scala b/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/executor/ExecutorTest.scala new file mode 100644 index 0000000..80dffab --- /dev/null +++ b/keter-core/src/test/scala/ru/avplatonov/keter/core/worker/work/executor/ExecutorTest.scala @@ -0,0 +1,133 @@ + package ru.avplatonov.keter.core.worker.work.executor + +import java.io.PrintWriter +import java.nio.file.{Files, Path, Paths} +import java.util.UUID +import java.util.stream.Collectors + +import org.apache.commons.io.FileUtils +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.JavaConverters._ +import scala.io.Source + +abstract class ExecutorTest extends FlatSpec with Matchers with BeforeAndAfterAll { + behavior of "Executor" + + val testWDRoot = Paths.get("/tmp/executor-tests/") + val logger = getLogger() + + it must "save echo result from std as file" in { + val content = "hello world" + val resultT = getTestExecutor().process(wd, "")("echo '" + content + "'") + + if(resultT.isFailure) resultT.get + resultT.isSuccess should equal(true) + val result = resultT.get + + result match { + case ExecutorResult(stdout, errorLog, otherCreatedFiles) => + Source.fromFile(stdout.toFile).mkString should equal(content + "\n") + Source.fromFile(errorLog.toFile).mkString should equal("") + otherCreatedFiles should equal(Set.empty) + } + } + + it must "initialize system files" in { + val content = "hello world" + val command = "echo '" + content + "'" + val workdir = wd + val resultT = getTestExecutor().process(workdir, "")(command) + + if(resultT.isFailure) resultT.get + resultT.isSuccess should equal(true) + val result = resultT.get + + val envFiles = Files.list(workdir).collect(Collectors.toSet()).asScala -- result.otherCreatedFiles + envFiles.exists(_.toString.endsWith(".error.log")) should equal(true) + envFiles.exists(_.toString.endsWith(".out")) should equal(true) + envFiles.exists(_.toString.endsWith(".sh")) should equal(true) + + envFiles.find(_.endsWith(".sh")).map(p => Source.fromFile(p.toFile).mkString) + .foreach(cmd => cmd should equal(command)) + } + + it must "create new files if need" in { + val content = "hello world" + val command = "echo '" + content + "' > content.txt" + val workdir = wd + val resultT = getTestExecutor().process(workdir, "")(command) + + if(resultT.isFailure) resultT.get + resultT.isSuccess should equal(true) + val result = resultT.get + val contentPath = workdir.resolve("content.txt") + + Files.exists(contentPath) should equal(true) + result.otherCreatedFiles.contains(contentPath) should equal(true) + Source.fromFile(contentPath.toFile).mkString should equal(content + "\n") + Source.fromFile(result.stdout.toFile).mkString should equal("") + } + + it must "run bash in bash" in { + val content = "hello world" + val command = "bash -c 'echo \"" + content + "\" > content.txt'" + val workdir = wd + val resultT = getTestExecutor().process(workdir, "")(command) + + if(resultT.isFailure) resultT.get + resultT.isSuccess should equal(true) + val result = resultT.get + val contentPath = workdir.resolve("content.txt") + + Files.exists(contentPath) should equal(true) + result.otherCreatedFiles.contains(contentPath) should equal(true) + Source.fromFile(contentPath.toFile).mkString should equal(content + "\n") + Source.fromFile(result.stdout.toFile).mkString should equal("") + } + + it must "return non-zero exception" in { + intercept[NonZeroStatusCode] { + val content = "hello world" + val command = "bash -c 'cat not-found-file'" + val workdir = wd + getTestExecutor().process(workdir, "")(command).get + } + } + + it must "see other files in dir" in { + val content = "hello world" + val command = "bash -c 'cat content.in > content.out'" + val workdir = wd + val contentIn = workdir.resolve("content.in") + resource.managed(new PrintWriter(contentIn.toFile)).foreach(_.println(content)) + val resultT = getTestExecutor().process(workdir, "")(command) + + if(resultT.isFailure) resultT.get + resultT.isSuccess should equal(true) + val result = resultT.get + val contentPath = workdir.resolve("content.out") + + Files.exists(contentPath) should equal(true) + result.otherCreatedFiles.contains(contentPath) should equal(true) + Source.fromFile(contentPath.toFile).mkString should equal(content + "\n") + Source.fromFile(result.stdout.toFile).mkString should equal("") + } + + override protected def beforeAll(): Unit = { + if (Files.exists(testWDRoot)) + FileUtils.deleteDirectory(testWDRoot.toFile) + } + + protected def getTestExecutor(): Executor + + protected def wd: Path = { + val localWD = testWDRoot.resolve(UUID.randomUUID().toString) + Files.createDirectories(localWD) + logger.info(s"Local WD: $localWD") + localWD + } + + protected def getLogger(): Logger = LoggerFactory.getLogger(classOf[Executor]) +}