SL integration

.gitignore

@@ -1,3 +1,4 @@
+/config
 .DS_Store
 data
 */data
@@ -13,7 +14,6 @@ target/
 .projectFilesBackup
 *.class
 annotation-cache/
-/config
 *.lc
 *.lex
 *~

build.sbt

@@ -57,7 +57,8 @@ lazy val commonSettings = Seq(
   name := "saul-project",
   resolvers ++= Seq(
     Resolver.mavenLocal,
-    "CogcompSoftware" at "http://cogcomp.cs.illinois.edu/m2repo/"
+    "CogcompSoftware" at "http://cogcomp.cs.illinois.edu/m2repo/",
+    "Sonatype Releases" at "https://oss.sonatype.org/content/repositories/releases/"
   ),
   javaOptions ++= List("-Xmx11g"),
   libraryDependencies ++= Seq(
@@ -66,7 +67,9 @@ lazy val commonSettings = Seq(
     "com.gurobi" % "gurobi" % "6.0",
     "org.apache.commons" % "commons-math3" % "3.0",
     "org.scalatest" % "scalatest_2.11" % "2.2.4",
-    "ch.qos.logback" % "logback-classic" % "1.1.7"
+    "ch.qos.logback" % "logback-classic" % "1.1.7",
+    "org.scalanlp" %% "breeze-viz" % "0.12",
+    "edu.illinois.cs.cogcomp" % "illinois-sl"  % "1.3.6" withSources
   ),
   fork := true,
   connectInput in run := true,

config/DCD.config

@@ -0,0 +1,26 @@
+# {L2LossSSVM, StructuredPerceptron}
+# LEARNING_MODEL = L2LossSSVM
+LEARNING_MODEL = StructuredPerceptron
+MAX_NUM_ITER = 20
+#  {DCDSolver, ParallelDCDSolver, DEMIParallelDCDSolver};
+L2_LOSS_SSVM_SOLVER_TYPE = DCDSolver
+
+NUMBER_OF_THREADS = 1
+C_FOR_STRUCTURE = 1.0
+TRAINMINI = false
+TRAINMINI_SIZE = 1000
+STOP_CONDITION = 0.0001
+CHECK_INFERENCE_OPT = false
+# MAX_NUM_ITER = 250
+PROGRESS_REPORT_ITER = 10
+INNER_STOP_CONDITION = 0.00001
+MAX_ITER_INNER = 250
+MAX_ITER_INNER_FINAL = 2500
+TOTAL_NUMBER_FEATURE = -1
+CLEAN_CACHE = true
+CLEAN_CACHE_ITER = 5
+DEMIDCD_NUMBER_OF_UPDATES_BEFORE_UPDATE_BUFFER = 100
+DEMIDCD_NUMBER_OF_INF_PARSE_BEFORE_UPDATE_WV = 10
+LEARNING_RATE = 0.01
+DECAY_LEARNING_RATE = false
+NUMBER_OF_FEATURE_BITS = 26

saul-core/doc/INSTALLATION.md

@@ -81,3 +81,4 @@ We suggest using  [IntelliJ IDEA](https://www.jetbrains.com/idea/download/).
 
 If you are interested in contributing to the Saul project, either by your ideas or codes, you are welcome
 to create pull requests here.
+

saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/ClassifierUtils.scala

@@ -137,8 +137,13 @@ object ClassifierUtils extends Logging {
       logger.info(evalSeparator)
       testResults
     }
-  }
 
+    def apply1[T <: AnyRef, H <: AnyRef](insts: Iterable[T], cls: ConstrainedClassifier[T, H]): Results = {
+      println(evalSeparator)
+      println("Evaluating " + cls.getClassSimpleNameForClassifier)
+      cls.test(insts)
+    }
+  }
   object ForgetAll {
     def apply(c: Learnable[_]*): Unit = {
       c.foreach((x: Learnable[_]) => x.forget())

saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/ConstrainedClassifier.scala

@@ -10,13 +10,16 @@ import edu.illinois.cs.cogcomp.lbjava.classify.{ Classifier, FeatureVector, Test
 import edu.illinois.cs.cogcomp.infer.ilp.{ GurobiHook, ILPSolver, OJalgoHook }
 import edu.illinois.cs.cogcomp.lbjava.infer.{ BalasHook, FirstOrderConstraint, InferenceManager }
 import edu.illinois.cs.cogcomp.lbjava.learn.Learner
+import edu.illinois.cs.cogcomp.saul.classifier.SL_model.LossAugmentedNormalizer
 import edu.illinois.cs.cogcomp.saul.classifier.infer.InferenceCondition
 import edu.illinois.cs.cogcomp.saul.constraint.LfsConstraint
 import edu.illinois.cs.cogcomp.saul.datamodel.edge.Edge
 import edu.illinois.cs.cogcomp.saul.lbjrelated.{ LBJClassifierEquivalent, LBJLearnerEquivalent }
 import edu.illinois.cs.cogcomp.saul.parser.IterableToLBJavaParser
 import edu.illinois.cs.cogcomp.saul.test.TestWithStorage
 import edu.illinois.cs.cogcomp.saul.util.Logging
+
+import scala.collection.mutable
 import scala.reflect.ClassTag
 
 /** The input to a ConstrainedClassifier is of type `T`. However given an input, the inference is based upon the
@@ -31,7 +34,7 @@ abstract class ConstrainedClassifier[T <: AnyRef, HEAD <: AnyRef](val onClassifi
   implicit val headType: ClassTag[HEAD]
 ) extends LBJClassifierEquivalent with Logging {
 
-  type LEFT = T
+  final type LEFT = T
   type RIGHT = HEAD
 
   def className: String = this.getClass.getName
@@ -69,26 +72,30 @@ abstract class ConstrainedClassifier[T <: AnyRef, HEAD <: AnyRef](val onClassifi
       val l = pathToHead.get.forward.neighborsOf(x).toSet.toList
 
       if (l.isEmpty) {
-        logger.error("Warning: Failed to find head")
+        // logger.error("Warning: Failed to find head")
         None
       } else if (l.size != 1) {
-        logger.warn("Find too many heads")
+        //  logger.warn("Find too many heads")
         Some(l.head)
       } else {
-        logger.info(s"Found head ${l.head} for child $x")
+        /// logger.info(s"Found head ${l.head} for child $x")
         Some(l.head)
       }
     }
   }
 
-  def getCandidates(head: HEAD): Seq[T] = {
+  //  def getMultiCandidates(head: Seq[HEAD]): Seq[LEFT] = {
+  //    head.flatMap(h => getCandidates(h)).distinct
+  //  }
+
+  def getCandidates(head: HEAD): Seq[LEFT] = {
     if (tType.equals(headType) || pathToHead.isEmpty) {
       head.asInstanceOf[T] :: Nil
     } else {
       val l = pathToHead.get.backward.neighborsOf(head)
 
       if (l.isEmpty) {
-        logger.error("Failed to find part")
+        // logger.error("Failed to find part")
         Seq.empty[T]
       } else {
         l.filter(filter(_, head)).toSeq
@@ -103,7 +110,7 @@ abstract class ConstrainedClassifier[T <: AnyRef, HEAD <: AnyRef](val onClassifi
         var inference = InferenceManager.get(name, head)
         if (inference == null) {
           inference = infer(head)
-          logger.warn(s"Inference ${name} has not been cached; running inference . . . ")
+          // logger.warn(s"Inference ${name} has not been cached; running inference . . . ")
           InferenceManager.put(name, inference)
         }
         inference.valueOf(cls, t)
@@ -113,6 +120,16 @@ abstract class ConstrainedClassifier[T <: AnyRef, HEAD <: AnyRef](val onClassifi
     }
   }
 
+  def lossAugmentedInfer(h: HEAD, offset: Int): mutable.ListBuffer[String] = {
+    var v = mutable.ListBuffer[String]()
+    getCandidates(h).foreach {
+      (example) =>
+        // val g1 = onClassifier.scores(example)
+        v += buildWithConstraint(subjectTo.createInferenceCondition[T](getSolverInstance(), new LossAugmentedNormalizer(offset, onClassifier.classifier, example)).convertToType[T], onClassifier.classifier)(example)
+    }
+    v
+  }
+
   def buildWithConstraint(inferenceCondition: InferenceCondition[T, HEAD])(t: T): String = {
     buildWithConstraint(inferenceCondition, onClassifier.classifier)(t)
   }
@@ -141,7 +158,8 @@ abstract class ConstrainedClassifier[T <: AnyRef, HEAD <: AnyRef](val onClassifi
       .orElse({
         onClassifier match {
           case clf: Learnable[T] => Some(clf.node)
-          case _ => logger.error("pathToHead is not provided and the onClassifier is not a Learnable!"); None
+          case _ => None
+          // logger.error("pathToHead is not provided and the onClassifier is not a Learnable!"); None
         }
       })
       .map(node => node.getTestingInstances)

saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala

@@ -18,16 +18,16 @@ object JointTrainSparseNetwork {
 
   val logger: Logger = LoggerFactory.getLogger(this.getClass)
   var difference = 0
-  def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], init: Boolean)(implicit headTag: ClassTag[HEAD]) = {
-    train[HEAD](node, cls, 1, init)
+  def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], init: Boolean,lossAugmented: Boolean)(implicit headTag: ClassTag[HEAD]) = {
+    train[HEAD](node, cls, 1, init,lossAugmented)
   }
 
-  def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int, init: Boolean)(implicit headTag: ClassTag[HEAD]) = {
-    train[HEAD](node, cls, it, init)
+  def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int, init: Boolean, lossAugmented: Boolean=false)(implicit headTag: ClassTag[HEAD]) = {
+    train[HEAD](node, cls, it, init, lossAugmented)
   }
 
   @scala.annotation.tailrec
-  def train[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int, init: Boolean)(implicit headTag: ClassTag[HEAD]): Unit = {
+  def train[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int, init: Boolean, lossAugmented: Boolean=false)(implicit headTag: ClassTag[HEAD]): Unit = {
     // forall members in collection of the head (dm.t) do
     logger.info("Training iteration: " + it)
     if (init) ClassifierUtils.InitializeClassifiers(node, cls: _*)
@@ -43,19 +43,24 @@ object JointTrainSparseNetwork {
             if (idx % 5000 == 0)
               logger.info(s"Training: $idx examples inferred.")
 
-            cls.foreach {
-              case classifier: ConstrainedClassifier[_, HEAD] =>
-                val typedClassifier = classifier.asInstanceOf[ConstrainedClassifier[_, HEAD]]
-                val oracle = typedClassifier.onClassifier.getLabeler
+            if (lossAugmented)
+            cls.foreach{ cls_i =>
+              cls_i.onClassifier.classifier.setLossFlag()
+              cls_i.onClassifier.classifier.setCandidates(cls_i.getCandidates(h).size * cls.size)
+               }
 
-                typedClassifier.getCandidates(h) foreach {
+            cls.foreach {
+              currentClassifier: ConstrainedClassifier[_, HEAD] =>
+                val oracle = currentClassifier.onClassifier.getLabeler
+                val baseClassifier = currentClassifier.onClassifier.classifier.asInstanceOf[SparseNetworkLearner]
+                currentClassifier.getCandidates(h) foreach {
                   candidate =>
                     {
                       def trainOnce() = {
-                        val result = typedClassifier.classifier.discreteValue(candidate)
+
+                        val result = currentClassifier.classifier.discreteValue(candidate)
                         val trueLabel = oracle.discreteValue(candidate)
-                        val ilearner = typedClassifier.onClassifier.classifier.asInstanceOf[SparseNetworkLearner]
-                        val lLexicon = typedClassifier.onClassifier.getLabelLexicon
+                        val lLexicon = currentClassifier.onClassifier.getLabelLexicon
                         var LTU_actual: Int = 0
                         var LTU_predicted: Int = 0
                         for (i <- 0 until lLexicon.size()) {
@@ -69,26 +74,26 @@ object JointTrainSparseNetwork {
                         // and the LTU of the predicted class should be demoted.
                         if (!result.equals(trueLabel)) //equals("true") && trueLabel.equals("false")   )
                         {
-                          val a = typedClassifier.onClassifier.getExampleArray(candidate)
+                          val a = currentClassifier.onClassifier.getExampleArray(candidate)
                           val a0 = a(0).asInstanceOf[Array[Int]] //exampleFeatures
                           val a1 = a(1).asInstanceOf[Array[Double]] // exampleValues
                           val exampleLabels = a(2).asInstanceOf[Array[Int]]
                           val label = exampleLabels(0)
-                          var N = ilearner.getNetwork.size
+                          var N = baseClassifier.getNetwork.size
 
-                          if (label >= N || ilearner.getNetwork.get(label) == null) {
-                            val conjugateLabels = ilearner.isUsingConjunctiveLabels | ilearner.getLabelLexicon.lookupKey(label).isConjunctive
-                            ilearner.setConjunctiveLabels(conjugateLabels)
+                          if (label >= N || baseClassifier.getNetwork.get(label) == null) {
+                            val conjugateLabels = baseClassifier.isUsingConjunctiveLabels | baseClassifier.getLabelLexicon.lookupKey(label).isConjunctive
+                            baseClassifier.setConjunctiveLabels(conjugateLabels)
 
-                            val ltu: LinearThresholdUnit = ilearner.getBaseLTU
-                            ltu.initialize(ilearner.getNumExamples, ilearner.getNumFeatures)
-                            ilearner.getNetwork.set(label, ltu)
+                            val ltu: LinearThresholdUnit = baseClassifier.getBaseLTU.clone().asInstanceOf[LinearThresholdUnit]
+                            ltu.initialize(baseClassifier.getNumExamples, baseClassifier.getNumFeatures)
+                            baseClassifier.getNetwork.set(label, ltu)
                             N = label + 1
                           }
 
                           // test push
-                          val ltu_actual = ilearner.getLTU(LTU_actual).asInstanceOf[LinearThresholdUnit]
-                          val ltu_predicted = ilearner.getLTU(LTU_predicted).asInstanceOf[LinearThresholdUnit]
+                          val ltu_actual = baseClassifier.getLTU(LTU_actual).asInstanceOf[LinearThresholdUnit]
+                          val ltu_predicted = baseClassifier.getLTU(LTU_predicted).asInstanceOf[LinearThresholdUnit]
 
                           if (ltu_actual != null)
                             ltu_actual.promote(a0, a1, 0.1)
@@ -100,8 +105,13 @@ object JointTrainSparseNetwork {
                       trainOnce()
                     }
                 }
+
             }
           }
+            if (lossAugmented)
+              cls.foreach{ cls_i =>
+                cls_i.onClassifier.classifier.unsetLossFlag()
+              }
       }
       train(node, cls, it - 1, false)
     }

saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/SL_model/Initialize.scala

@@ -0,0 +1,72 @@
+/** This software is released under the University of Illinois/Research and Academic Use License. See
+  * the LICENSE file in the root folder for details. Copyright (c) 2016
+  *
+  * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign
+  * http://cogcomp.cs.illinois.edu/
+  */
+package edu.illinois.cs.cogcomp.saul.classifier.SL_model
+
+import edu.illinois.cs.cogcomp.lbjava.learn.{ LinearThresholdUnit, SparseNetworkLearner }
+import edu.illinois.cs.cogcomp.saul.classifier.infer.InitSparseNetwork
+import edu.illinois.cs.cogcomp.saul.datamodel.node.Node
+import edu.illinois.cs.cogcomp.sl.util.WeightVector
+
+import scala.collection.mutable.ListBuffer
+
+/** Created by Parisa on 4/1/16.
+  * Here we only make the lbjava lexicons for each onClassifier
+  * (i.e. the base classifier of each constraint classifier) based on the features of IInstances
+  */
+object Initialize {
+
+  def apply[HEAD <: AnyRef](node: Node[HEAD], model: SaulSLModel[HEAD], usePreTrained: Boolean = false): SaulSLModel[HEAD] = {
+
+    var wvLength = 0
+    var fullWeightList: ListBuffer[Array[Float]] = ListBuffer()
+
+    /*this means we are not reading any model into the SparseNetworks but
+     we forget all the models and go over the data to build the right size
+     for the lexicon and the right number of the ltu s*/
+
+    if (!usePreTrained)
+      model.Factors.foreach {
+        cf => InitSparseNetwork(node, cf)
+      }
+    /*In this step or we have built the lexicon by going over the data in the above block or
+we use the loaded lexicons in the case of uesPreTrained == true, the goal is to build
+ a global weight vector using all classifiers and initialize it accordingly to have a fixed size*/
+
+    model.Factors.foreach(
+      x => {
+        val sparseNet = x.onClassifier.classifier.asInstanceOf[SparseNetworkLearner]
+        val lexiconSize = (sparseNet.getLexicon.size())
+
+        for (i <- 0 until sparseNet.getNetwork.size()) {
+
+          val trainedWeighs = x.onClassifier.classifier.asInstanceOf[SparseNetworkLearner].getNetwork.get(i).asInstanceOf[LinearThresholdUnit].getWeightVector
+          val fullWeights = Array.fill[Float](lexiconSize)(0)
+
+          if (usePreTrained) { //if we are going to initialize we get the loaded weights otherwise the weights are filled with zeros
+
+            for (j <- 0 until lexiconSize)
+              fullWeights(j) = trainedWeighs.getWeight(j).asInstanceOf[Float]
+
+          }
+          fullWeightList = fullWeightList :+ fullWeights
+
+          wvLength = wvLength + lexiconSize
+        }
+
+        println("lexicon size: " + sparseNet.getLexicon.size(), "* label lexicon size:", sparseNet.getLabelLexicon.size())
+      }
+    )
+    // wv = Concatenate_(over factors)Concatenate_(over ltu) => size(wv)=sum_(over factors)sum_(over ltu)(size(ltu_i))
+
+    val myWeight = Array(fullWeightList.flatten: _*)
+    val wv = new WeightVector(myWeight) // wv this is one unified weight vector of all initialized LTUs
+    val m = new SaulSLModel[HEAD](model.Factors.toList, fullWeightList) // lt is the list of individual weight vectors
+    m.wv = wv
+    m
+    /*This weigh vector is a a flat vector containing one block of weights per each ltu*/
+  } //end f apply
+} // end of object

saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/SL_model/LossAugmentedNormalizer.scala

@@ -0,0 +1,36 @@
+/** This software is released under the University of Illinois/Research and Academic Use License. See
+  * the LICENSE file in the root folder for details. Copyright (c) 2016
+  *
+  * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign
+  * http://cogcomp.cs.illinois.edu/
+  */
+package edu.illinois.cs.cogcomp.saul.classifier.SL_model
+
+import edu.illinois.cs.cogcomp.lbjava.classify.ScoreSet
+import edu.illinois.cs.cogcomp.lbjava.learn.{ Learner, Normalizer, SparseNetworkLearner }
+
+/** Created by Parisa on 4/18/16.
+  */
+class LossAugmentedNormalizer(cand_num: Int, c: Learner, example: AnyRef) extends Normalizer {
+  /** Simply returns the argument.
+    *
+    * @param scores The set of scores to normalize.
+    * @return The normalized set of scores.
+    */
+  def normalize(scores: ScoreSet): ScoreSet = {
+    if (cand_num == 0)
+      print("There is no relevant component of this type in the head to be classified.")
+    val cf = c.asInstanceOf[SparseNetworkLearner]
+    val gold = cf.getLabeler.discreteValue(example)
+    val lLexicon = cf.getLabelLexicon
+
+    val resultS: ScoreSet = cf.scores(example) //new ScoreSet
+    for (i <- 0 until lLexicon.size()) {
+      if (lLexicon.lookupKey(i).valueEquals(gold))
+        resultS.put(lLexicon.lookupKey(i).getStringValue, resultS.getScore(lLexicon.lookupKey(i).getStringValue).score - (1 / (cand_num)))
+      else
+        resultS.put(lLexicon.lookupKey(i).getStringValue, resultS.getScore(lLexicon.lookupKey(i).getStringValue).score + (1 / (cand_num)))
+    }
+    resultS
+  }
+}