Work towards automated cleaning of JcrJobRepository

grabbit/src/main/groovy/com/twcable/grabbit/resources/CleanJobRepositoryResource.groovy

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2015 Time Warner Cable, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.twcable.grabbit.resources
+
+import groovy.transform.CompileStatic
+import org.apache.sling.api.resource.ResourceResolver
+import org.apache.sling.api.resource.SyntheticResource
+
+import javax.annotation.Nonnull
+
+/**
+ * A resource representing what needs to be deleted from Grabbit's JobRepository.
+ * provided by {@link GrabbitResourceProvider}.
+ * Queried from {@link com.twcable.grabbit.spring.batch.repository.servlets.GrabbitCleanJobRepositoryServlet}.
+ */
+@CompileStatic
+class CleanJobRepositoryResource extends SyntheticResource {
+
+    public static final String CLEAN_JOBREPOSITORY_RESOURCE_TYPE = "twcable:grabbit/jobrepository/clean"
+
+    CleanJobRepositoryResource(@Nonnull final ResourceResolver resourceResolver, @Nonnull final String resolutionPath) {
+        super(resourceResolver, resolutionPath, CLEAN_JOBREPOSITORY_RESOURCE_TYPE)
+    }
+
+
+    @Override
+    String getResourceType() {
+        return CLEAN_JOBREPOSITORY_RESOURCE_TYPE
+    }
+}

grabbit/src/main/groovy/com/twcable/grabbit/resources/ContentResource.groovy

@@ -1,11 +1,3 @@
-package com.twcable.grabbit.resources
-
-import groovy.transform.CompileStatic
-import org.apache.sling.api.resource.ResourceResolver
-import org.apache.sling.api.resource.SyntheticResource
-
-import javax.annotation.Nonnull
-
 /*
  * Copyright 2015 Time Warner Cable, Inc.
  *
@@ -22,6 +14,14 @@ import javax.annotation.Nonnull
  * limitations under the License.
  */
 
+package com.twcable.grabbit.resources
+
+import groovy.transform.CompileStatic
+import org.apache.sling.api.resource.ResourceResolver
+import org.apache.sling.api.resource.SyntheticResource
+
+import javax.annotation.Nonnull
+
 /**
  * A resource representing some content to be streamed from a Grabbit instance.
  * provided by {@link GrabbitResourceProvider}.

grabbit/src/main/groovy/com/twcable/grabbit/resources/GrabbitResourceProvider.groovy

@@ -79,6 +79,9 @@ class GrabbitResourceProvider implements ResourceProvider {
             case ~/^\/grabbit\/content(\/)?$/:
                 log.debug "Resolving ${path} to ContentResource"
                 return new ContentResource(resolver, path)
+            case ~/^\/grabbit\/jobrepository\/clean(\/)?$/:
+                log.debug "Resolving ${path} to CleanJobRepositoryResource"
+                return new CleanJobRepositoryResource(resolver, path)
             default:
                 //Should provide a root resource for /grabbit, along with HATEOS style for this link, and other links. https://github.com/TWCable/grabbit/issues/22
                 return null

grabbit/src/main/groovy/com/twcable/grabbit/resources/TransactionResource.groovy

@@ -1,12 +1,3 @@
-package com.twcable.grabbit.resources
-
-import groovy.transform.CompileStatic
-import org.apache.sling.api.resource.ResourceResolver
-import org.apache.sling.api.resource.SyntheticResource
-
-import javax.annotation.Nonnull
-import java.util.regex.Matcher
-
 /*
  * Copyright 2015 Time Warner Cable, Inc.
  *
@@ -23,6 +14,15 @@ import java.util.regex.Matcher
  * limitations under the License.
  */
 
+package com.twcable.grabbit.resources
+
+import groovy.transform.CompileStatic
+import org.apache.sling.api.resource.ResourceResolver
+import org.apache.sling.api.resource.SyntheticResource
+
+import javax.annotation.Nonnull
+import java.util.regex.Matcher
+
 /**
  * {@link TransactionResource} represents a a logical group of jobs by configuration run.
  *

grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitExecutionContextDao.groovy

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2015 Time Warner Cable, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.twcable.grabbit.spring.batch.repository
+
+import org.springframework.batch.core.repository.dao.ExecutionContextDao
+import org.springframework.batch.item.ExecutionContext;
+
+
+/**
+ * Modified DAO Interface for persisting and retrieving {@link ExecutionContext}
+ * @see ExecutionContextDao for more details
+ */
+interface GrabbitExecutionContextDao extends ExecutionContextDao {
+
+    /**
+     * Returns job execution context paths by comparing "executionId" property on "executionContext/job/<id>" with
+     * "executionId" property on JobExecutions for the @param jobExecutionResourcePaths
+     */
+    public Collection<String> getJobExecutionContextPaths(Collection<String> jobExecutionResourcePaths)
+
+    /**
+     * Returns step execution context paths by comparing "executionId" property on "executionContext/job/<id>" with
+     * "id" property on StepExecutions for the @param stepExecutionResourcePaths
+     */
+    public Collection<String> getStepExecutionContextPaths(Collection<String> stepExecutionResourcePaths)
+}

grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitJobExecution.groovy

@@ -30,7 +30,7 @@ import org.springframework.batch.core.JobExecution
  * be grouped/referenced by their common transactionID.
  * </p>
  *
- * @see {@link com.twcable.grabbit.spring.batch.repository.JcrJobExecutionDao} for a good handle on how we
+ * @see {@link JcrGrabbitJobExecutionDao} for a good handle on how we
  * serve up this JobExecution during the Spring Batch lifecycle.
  */
 @CompileStatic

grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitJobExecutionDao.groovy

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2015 Time Warner Cable, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.twcable.grabbit.spring.batch.repository
+
+import org.springframework.batch.core.BatchStatus
+import org.springframework.batch.core.JobExecution
+import org.springframework.batch.core.repository.dao.JobExecutionDao
+
+/**
+ * Modified DAO Interface for persisting and retrieving {@link JobExecution}
+ * @see JobExecutionDao for more details
+ */
+interface GrabbitJobExecutionDao extends JobExecutionDao{
+
+    /**
+     * Returns job execution paths for given BatchStatuses
+     */
+    public Collection<String> getJobExecutions(Collection<BatchStatus> batchStatuses)
+
+    /**
+     * Returns job execution paths which ended @param hours ago from "Now"
+     */
+    public Collection<String> getJobExecutions(int hours, Collection<String> jobExecutionPaths)
+
+}

grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitJobInstanceDao.groovy

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2015 Time Warner Cable, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.twcable.grabbit.spring.batch.repository
+
+import org.springframework.batch.core.JobInstance
+import org.springframework.batch.core.repository.dao.JobInstanceDao
+
+/**
+ * Modified DAO Interface for persisting and retrieving {@link JobInstance}
+ * @see JobInstanceDao for more details
+ */
+interface GrabbitJobInstanceDao extends JobInstanceDao {
+
+    /**
+     * Returns job instance paths by comparing "id" property on "jobInstances" with
+     * "instanceId" property on JobExecutions for the @param jobExecutionResourcePaths
+     */
+    public Collection<String> getJobInstancePaths(Collection<String> jobExecutionResourcePaths)
+
+}

grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/GrabbitStepExecutionDao.groovy

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2015 Time Warner Cable, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.twcable.grabbit.spring.batch.repository
+
+import org.springframework.batch.core.StepExecution
+import org.springframework.batch.core.repository.dao.StepExecutionDao
+
+/**
+ * Modified DAO Interface for persisting and retrieving {@link StepExecution}
+ * @see StepExecutionDao for more details
+ */
+interface GrabbitStepExecutionDao extends StepExecutionDao {
+
+    /**
+     * Returns step execution paths by comparing "jobExecutionId" property on "stepExecutions with
+     * "executionId" property on JobExecutions for the @param jobExecutionResourcePaths
+     */
+    public Collection<String> getStepExecutionPaths(Collection<String> jobExecutionResourcePaths)
+}

grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/JcrExecutionContextDao.groovy → grabbit/src/main/groovy/com/twcable/grabbit/spring/batch/repository/JcrGrabbitExecutionContextDao.groovy

@@ -20,11 +20,8 @@ import com.twcable.grabbit.jcr.JcrUtil
 import com.twcable.grabbit.util.CryptoUtil
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
-import org.apache.sling.api.resource.ModifiableValueMap
-import org.apache.sling.api.resource.Resource
-import org.apache.sling.api.resource.ResourceResolver
-import org.apache.sling.api.resource.ResourceResolverFactory
-import org.apache.sling.api.resource.ValueMap
+import org.apache.sling.api.SlingException
+import org.apache.sling.api.resource.*
 import org.springframework.batch.core.JobExecution
 import org.springframework.batch.core.StepExecution
 import org.springframework.batch.core.repository.ExecutionContextSerializer
@@ -33,6 +30,7 @@ import org.springframework.batch.item.ExecutionContext
 
 import javax.annotation.Nonnull
 
+import static JcrGrabbitStepExecutionDao.ID
 import static org.apache.jackrabbit.JcrConstants.NT_UNSTRUCTURED
 import static org.apache.sling.api.resource.ResourceUtil.getOrCreateResource
 
@@ -42,7 +40,7 @@ import static org.apache.sling.api.resource.ResourceUtil.getOrCreateResource
  */
 @CompileStatic
 @Slf4j
-class JcrExecutionContextDao extends AbstractJcrDao implements ExecutionContextDao {
+class JcrGrabbitExecutionContextDao extends AbstractJcrDao implements GrabbitExecutionContextDao {
 
     public static final String EXECUTION_CONTEXT_ROOT = "${ROOT_RESOURCE_NAME}/executionContexts"
     public static final String JOB_EXECUTION_CONTEXT_ROOT = "${EXECUTION_CONTEXT_ROOT}/job"
@@ -55,7 +53,7 @@ class JcrExecutionContextDao extends AbstractJcrDao implements ExecutionContextD
     private ExecutionContextSerializer contextSerializer
 
 
-    JcrExecutionContextDao(
+    JcrGrabbitExecutionContextDao(
         @Nonnull final ResourceResolverFactory rrf, @Nonnull final ExecutionContextSerializer serializer) {
         this.resourceResolverFactory = rrf
         this.contextSerializer = serializer
@@ -156,7 +154,7 @@ class JcrExecutionContextDao extends AbstractJcrDao implements ExecutionContextD
     }
 
     /**
-     * Must be called when a new instance of JcrExecutionContextDao is created.
+     * Must be called when a new instance of JcrGrabbitExecutionContextDao is created.
      * Ensures that {@link #STEP_EXECUTION_CONTEXT_ROOT} and {@link #JOB_EXECUTION_CONTEXT_ROOT} exist on initialization
      */
     @Override
@@ -306,4 +304,55 @@ class JcrExecutionContextDao extends AbstractJcrDao implements ExecutionContextD
         contextAsMap.each { key, value -> context.put(key, value) }
         context
     }
+
+    @Override
+    Collection<String> getJobExecutionContextPaths(Collection<String> jobExecutionResourcePaths) {
+        JcrUtil.manageResourceResolver(resourceResolverFactory) { ResourceResolver resolver ->
+            Collection<String> jobExecutionContextPathsToRemove = []
+            jobExecutionResourcePaths.each { String jobExecutionResourcePath ->
+                Resource jobExecutionResource = resolver.getResource(jobExecutionResourcePath)
+                ValueMap props = jobExecutionResource.adaptTo(ValueMap)
+                Long jobExecutionId = props[JcrGrabbitJobExecutionDao.EXECUTION_ID] as Long
+                String query = "select * from [nt:unstructured] as s " +
+                        "where ISDESCENDANTNODE(s,'${JOB_EXECUTION_CONTEXT_ROOT}') AND ( s.${EXECUTION_ID} = ${jobExecutionId})"
+                try {
+                    List<String> jobExecutionContextPaths = resolver.findResources(query, "JCR-SQL2").toList().collect { it.path }
+                    jobExecutionContextPathsToRemove.addAll(jobExecutionContextPaths)
+                }
+                catch(SlingException | IllegalStateException e) {
+                    log.error "Exception when executing Query: ${query}. \nException - ", e
+                }
+            }
+            //There are 2 versions of Resources returned back by findResources
+            //One for JcrNodeResource and one for SocialResourceWrapper
+            //Hence, duplicates need to be removed
+            return jobExecutionContextPathsToRemove.unique() as Collection<String>
+        }
+    }
+
+    @Override
+    Collection<String> getStepExecutionContextPaths(Collection<String> stepExecutionResourcePaths) {
+        JcrUtil.manageResourceResolver(resourceResolverFactory) { ResourceResolver resolver ->
+            Collection<String> stepExecutionContextPathsToRemove = []
+            stepExecutionResourcePaths.each { String stepExecutionResourcePath ->
+                Resource stepExecutionResource = resolver.getResource(stepExecutionResourcePath)
+                ValueMap props = stepExecutionResource.adaptTo(ValueMap)
+                Long stepExecutionId = props[ID] as Long
+                String query = "select * from [nt:unstructured] as s " +
+                        "where ISDESCENDANTNODE(s,'${STEP_EXECUTION_CONTEXT_ROOT}') AND ( s.${EXECUTION_ID} = ${stepExecutionId})"
+                try {
+                    List<String> stepExecutionContextPaths = resolver.findResources(query, "JCR-SQL2").toList().collect { it.path }
+                    stepExecutionContextPathsToRemove.addAll(stepExecutionContextPaths)
+                } catch (SlingException | IllegalStateException e) {
+                    log.error "Exception when executing Query: ${query}. \nException - ", e
+                }
+            }
+            //There are 2 versions of Resources returned back by findResources
+            //One for JcrNodeResource and one for SocialResourceWrapper
+            //Hence, duplicates need to be removed
+            return stepExecutionContextPathsToRemove.unique() as Collection<String>
+
+        }
+
+    }
 }