Svm context engine

.gitignore

@@ -8,6 +8,8 @@ core/*.iml
 .DS_Store
 Makefile
 
+output/*
+
 vocab.txt
 w2vvoc.txt
 SavedLSTM_tag_20190806
@@ -17,7 +19,6 @@ SentencesInfo_all_label_final_ExactRecur_ExpandBound.txt
 SentencesInfo_all_label_final_ExactRecur_ExpandBound_WideSubjObjBound.txt
 SentencesInfo_all_label_final_ExactRecur_train.csv
 
-
 # sbt specific
 dist/*
 target/

SVM_CONTEXT_README.md

@@ -0,0 +1,27 @@
+## Training
+
+THE `SVMContextEngine` requires a model previously trained. To train the model from scratch the following files are required
+
+To run this script, you will need the training dataset found at `main/src/main/resources/org/clulab/context/svmFeatures/grouped_features.csv.gz` expected to be a gziped csv file.
+Also, a list of the features to be used for training, found as a comma-separated string in `main/src/main/resources/org/clulab/context/svmFeatures/specific_features.txt`
+
+To train the model from scratch use the following script:
+
+```
+sbt 'runMain org.clulab.reach.context.svm_scripts.TrainSVMContextClassifier main/src/main/resources/org/clulab/context/svmFeatures/grouped_features.csv.gz main/src/main/resources/org/clulab/context/svmFeatures/svm_model.dat main/src/main/resources/org/clulab/context/svmFeatures/specific_features.txt'
+```
+
+## Usage Configuration
+
+The context engine is governed by the `svmContextEngine` section of `application.conf`. To indicate REACH to use the SVMContextEngine, specify `SVMPolicy` type and specify the size of the sentence window in the `bound` parameter:
+
+```
+contextEngine{
+  type = SVMPolicy
+  params = {
+
+    bound = 3
+  }
+}
+```
+The trained model *must* exist in the file `main/src/main/resources/org/clulab/context/svmFeatures/svm_model.dat`. This path is hardcoded and is not a parameter.

build.sbt

@@ -115,4 +115,4 @@ site.includeScaladoc()
 
 ghpages.settings
 
-git.remoteRepo := "git@github.com:clulab/reach.git"
+git.remoteRepo := "git@github.com:clulab/reach.git"

main/build.sbt

@@ -1,10 +1,10 @@
 name := "reach-main"
-
 libraryDependencies ++= {
   val akkaV = "2.5.4"
   val luceVer = "5.3.1"
   val procVer = "7.5.3"
 
+
   Seq(
     "ai.lum"              %%  "nxmlreader"  % "0.0.9",
     "commons-io"           %  "commons-io"  % "2.4",

main/src/main/resources/application.conf

@@ -14,6 +14,7 @@ bratDir = ${rootDir}/brat
 # if this directory does not exist it will be created
 contextDir = ${rootDir}/context
 
+
 # this is where the output files containing the extracted mentions will be stored
 # if this directory does not exist it will be created
 outDir = ${rootDir}/output
@@ -28,7 +29,7 @@ encoding = "utf-8"
 
 # this is a list of sections that we should ignore
 ignoreSections = ["references", "materials", "materials|methods", "methods", "supplementary-material"]
-
+//ignoreSections = []
 # the output formats for mentions:
 # "arizona" (column-based, one file per paper)
 # "cmu" (column-based, one file per paper)
@@ -38,7 +39,7 @@ ignoreSections = ["references", "materials", "materials|methods", "methods", "su
 outputTypes = ["fries", "arizona", "cmu"]
 
 # number of simultaneous threads to use for parallelization
-threadLimit = 2
+threadLimit = 4
 
 # verbose logging
 verbose = true
@@ -49,12 +50,22 @@ withAssembly = false
 
 # context engine configuration
 contextEngine {
-  type = Policy4
+ type = Policy4
   params = {
-    bound = 3
+     bound = 3
   }
 }
 
+//contextEngine{
+//  type = SVMPolicy
+//  params = {
+//
+//    bound = 3
+//  }
+//}
+
+
+# polairty engine configuration
 polarity {
   engine = Hybrid //Hybrid//DeepLearning //Linguistic
   negCountThreshold = 1 // when lower than or equal to this value, use linguistic approach in hybrid method
@@ -76,6 +87,7 @@ experimentalRegulation{
   keywords = main/src/main/resources/experimental_regulation_type_keywords.csv
 }
 
+
 # grounding configuration
 grounding: {
   # List of AdHoc grounding files to insert, in order, into the grounding search sequence.
@@ -93,7 +105,7 @@ grounding: {
 logging {
   # defines project-wide logging level
   loglevel = INFO
-  logfile = ${rootDir}/reach.log
+  logfile = ${outDir}/reach.log
 }
 
 # Processor Annotator choice and configuration
@@ -105,7 +117,7 @@ processorAnnotator {
 # restart configuration
 restart {
   # restart allows batch jobs to skip over input files already successfully processed
-  useRestart = true
+  useRestart = false
   # restart log is one filename per line list of input files already successfully processed
   logfile = ${outDir}/restart.log
 }

main/src/main/scala/org/clulab/reach/FriesEntry.scala

@@ -2,7 +2,7 @@ package org.clulab.reach
 
 import ai.lum.nxmlreader.NxmlDocument
 
-
+// Another test comment, just in case
 case class FriesEntry(
   name: String,
   chunkId: String,

main/src/main/scala/org/clulab/reach/RuleReader.scala

@@ -5,6 +5,8 @@ import scala.io.Source
 /**
  * Utilities to read rule files
  */
+
+
 object RuleReader {
 
 case class Rules(entities: String, modifications: String, events: String, context: String)

main/src/main/scala/org/clulab/reach/context/ContextEngine.scala

@@ -32,6 +32,7 @@ object ContextEngine extends LazyLogging {
 
     val labels = mention.labels filter (contextMatching.contains(_))
 
+
     (labels.head, id)
   }
 

main/src/main/scala/org/clulab/reach/context/ContextEngineFactory.scala

@@ -11,6 +11,7 @@ object ContextEngineFactory {
       val Policy2 = Value("Policy2")
       val Policy3 = Value("Policy3")
       val Policy4 = Value("Policy4")
+      val SVMPolicy = Value("SVMPolicy")
     }
     import Engine._
 
@@ -36,6 +37,10 @@ object ContextEngineFactory {
                 case None => new BidirectionalPaddingContext
             }
             case Dummy => new DummyContextEngine
+            case SVMPolicy => bound match {
+              case w @ Some(b) => new SVMContextEngine(w)
+              case None => new SVMContextEngine
+            }
             case _ => new DummyContextEngine
         }
     }

main/src/main/scala/org/clulab/reach/context/Policies.scala

@@ -1,6 +1,8 @@
 package org.clulab.reach.context
 
+import org.clulab.reach.context.feature_utils.ContextFeatureUtils
 import org.clulab.reach.mentions._
+
 import collection.mutable
 
 
@@ -36,7 +38,7 @@ class BoundedPaddingContext(
       // Assign the context map to the mention
       m.context = if(contextMap != Map.empty) Some(contextMap) else None
     }
-
+    //ContextFeatureUtils.writeRowsToFile(mentions)
     mentions
   }
 

main/src/main/scala/org/clulab/reach/context/RuleBasedEngine.scala

@@ -7,8 +7,8 @@ import org.clulab.reach.mentions._
 abstract class RuleBasedContextEngine extends ContextEngine {
 
   // Fields
-  // To be overriden in the implementations. Returns a sequence of (Type, Val) features
-  // Feature order should be kept consisting for all return values
+  // To be overridden in the implementations. Returns a sequence of (Type, Val) features
+  // Feature order should be kept consistent for all return values
   var orderedContextMentions:Map[Int, Seq[BioTextBoundMention]] = _
   // This is to keep the default species if necessary
   var defaultContexts:Option[Map[String, String]] = None
@@ -26,7 +26,7 @@ abstract class RuleBasedContextEngine extends ContextEngine {
     // Compute default context classes
     // First count the context types
     val contextCounts:Map[(String, String), Int] = contextMentions map ContextEngine.getContextKey groupBy identity mapValues (_.size)
-    // Then gorup them by class
+    // Then group them by class
     val defaultContexts:Map[String, String] = contextCounts.toSeq.groupBy(_._1._1)
       // Sort them in decreasing order by frequency
       .mapValues(_.map(t => (t._1._2, t._2)))