diff --git a/build.sbt b/build.sbt index aa892183..cec97aa5 100644 --- a/build.sbt +++ b/build.sbt @@ -7,7 +7,7 @@ lazy val root = (project in file(".")). lazy val commonSettings = Seq( organization := "edu.illinois.cs.cogcomp", name := "saul-project", - version := "0.1", + version := "0.2", scalaVersion := "2.11.7", resolvers ++= Seq( Resolver.mavenLocal, diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala index edce9cee..370372eb 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala @@ -6,86 +6,112 @@ import edu.illinois.cs.cogcomp.saul.datamodel.node.Node import scala.reflect.ClassTag -/** Created by parisakordjamshidi on 29/01/15. +/** Joint training for a list of classifiers with the same HEAD type, but potentially different classifiers with + * different input types */ object JointTrain { - def testClassifiers(cls: Classifier, oracle: Classifier, ds: List[AnyRef]): Unit = { - - val results = ds.map({ - x => - val pri = cls.discreteValue(x) - val truth = oracle.discreteValue(x) - (pri, truth) - }) + def train[HEAD <: AnyRef](node: Node[HEAD], classifiers: ConstrainedClassifier[_, HEAD]*)(implicit headTag: ClassTag[HEAD]): Unit = { + train[HEAD](node, 1, classifiers) + } - val tp = results.count({ case (x, y) => x == y && (x == "true") }) * 1.0 - val fp = results.count({ case (x, y) => x != y && (x == "true") }) * 1.0 + def train[HEAD <: AnyRef](node: Node[HEAD], iter: Int, classifiers: ConstrainedClassifier[_, HEAD]*)(implicit headTag: ClassTag[HEAD]): Unit = { + train[HEAD](node, iter, classifiers) + } - val tn = results.count({ case (x, y) => x == y && (x == "false") }) * 1.0 - val fn = results.count({ case (x, y) => x != y && (x == "false") }) * 1.0 + def train[HEAD <: AnyRef](node: Node[HEAD], iter: Int, classifiers: Seq[ConstrainedClassifier[_, HEAD]])(implicit headTag: ClassTag[HEAD], d1: DummyImplicit): Unit = { + /** this creates partial calls to the promote/demote function of the classifiers. Later these partial calls are + * called given the instances and their predictions. Note that the goal of creating this list of partial calls + * is to increase efficiency, by doing it once and at the beginning, for all the iterations. + */ + val partialUpdateCallsPerClassifier = classifiers.map { + case typedClassifier: ConstrainedClassifier[_, HEAD] => + val oracle = typedClassifier.onClassifier.getLabeler + typedClassifier.onClassifier.classifier match { + case _: LinearThresholdUnit => trainLinearThresholdUnitOnce[HEAD](typedClassifier, oracle)(_) + case _: SparseNetworkLBP => trainSparseNetworkLearnerOnce[HEAD](typedClassifier, oracle)(_) + case _ => throw new Exception("ERROR: The joint training is not available for the classifier you have: " + + typedClassifier.onClassifier.classifier.getClass.getTypeName) + } + } - println(s"tp: $tp fp: $fp tn: $tn fn: $fn ") - println(s" accuracy ${(tp + tn) / results.size} ") - println(s" precision ${tp / (tp + fp)} ") - println(s" recall ${tp / (tp + fn)} ") - println(s" f1 ${(2.0 * tp) / (2 * tp + fp + fn)} ") + val callsZipWithClassifiers = partialUpdateCallsPerClassifier.zip(classifiers) + // now do training with the partial calls + recurstiveTrain[HEAD](node, callsZipWithClassifiers, iter) } - def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]])(implicit headTag: ClassTag[HEAD]) = { - train[HEAD](node, cls, 1) + @scala.annotation.tailrec + private def recurstiveTrain[HEAD <: AnyRef](node: Node[HEAD], callsZipWithClassifiers: Seq[(Any => Unit, ConstrainedClassifier[_, HEAD])], iter: Int)(implicit headTag: ClassTag[HEAD]): Unit = { + println("Joint training iterations: " + iter) + if (iter > 0) { + val allHeads = node.getTrainingInstances + for { + head <- allHeads + (partialUpdateCall, classifier) <- callsZipWithClassifiers + } { + classifier.getCandidates(head) foreach { candidate => partialUpdateCall(candidate) } + } + recurstiveTrain(node, callsZipWithClassifiers, iter - 1) + } } - def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int)(implicit headTag: ClassTag[HEAD]) = { - train[HEAD](node, cls, it) + private def trainLinearThresholdUnitOnce[HEAD <: AnyRef](typedClassifier: ConstrainedClassifier[_, HEAD], oracle: Classifier)(candidate: Any): Unit = { + val result = typedClassifier.classifier.discreteValue(candidate) + val trueLabel = oracle.discreteValue(candidate) + if (result.equals("true") && trueLabel.equals("false")) { + val a = typedClassifier.onClassifier.getExampleArray(candidate) + val a0 = a(0).asInstanceOf[Array[Int]] + val a1 = a(1).asInstanceOf[Array[Double]] + typedClassifier.onClassifier.classifier.asInstanceOf[LinearThresholdUnit].promote(a0, a1, 0.1) + } else if (result.equals("false") && trueLabel.equals("true")) { + val a = typedClassifier.onClassifier.getExampleArray(candidate) + val a0 = a(0).asInstanceOf[Array[Int]] + val a1 = a(1).asInstanceOf[Array[Double]] + typedClassifier.onClassifier.classifier.asInstanceOf[LinearThresholdUnit].demote(a0, a1, 0.1) + } } - @scala.annotation.tailrec - def train[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int)(implicit headTag: ClassTag[HEAD]): Unit = { - // forall members in collection of the head (dm.t) do - - println("Training iteration: " + it) - if (it == 0) { - // Done - } else { - val allHeads = node.getTrainingInstances + private def trainSparseNetworkLearnerOnce[HEAD <: AnyRef](typedClassifier: ConstrainedClassifier[_, HEAD], oracle: Classifier)(candidate: Any): Unit = { + val result = typedClassifier.classifier.discreteValue(candidate) + val trueLabel = oracle.discreteValue(candidate) + val ilearner = typedClassifier.onClassifier.classifier.asInstanceOf[SparseNetworkLBP] + val lLexicon = typedClassifier.onClassifier.getLabelLexicon + var LTU_actual = 0 + var LTU_predicted = 0 + for (i <- 0 until lLexicon.size()) { + if (lLexicon.lookupKey(i).valueEquals(result)) + LTU_predicted = i + if (lLexicon.lookupKey(i).valueEquals(trueLabel)) + LTU_actual = i + } - allHeads foreach { - h => - { - cls.foreach { - case classifier: ConstrainedClassifier[_, HEAD] => - val typedC = classifier.asInstanceOf[ConstrainedClassifier[_, HEAD]] - val oracle = typedC.onClassifier.getLabeler + // The idea is that when the prediction is wrong the LTU of the actual class should be promoted + // and the LTU of the predicted class should be demoted. + if (!result.equals(trueLabel)) { + val a = typedClassifier.onClassifier.getExampleArray(candidate) + val a0 = a(0).asInstanceOf[Array[Int]] //exampleFeatures + val a1 = a(1).asInstanceOf[Array[Double]] // exampleValues + val exampleLabels = a(2).asInstanceOf[Array[Int]] + val label = exampleLabels(0) + var N = ilearner.net.size() + + if (label >= N || ilearner.net.get(label) == null) { + ilearner.iConjuctiveLables = ilearner.iConjuctiveLables | ilearner.getLabelLexicon.lookupKey(label).isConjunctive + + val ltu: LinearThresholdUnit = ilearner.getbaseLTU + ltu.initialize(ilearner.getnumExamples, ilearner.getnumFeatures) + ilearner.net.set(label, ltu) + N = label + 1 + } - typedC.getCandidates(h) foreach { - x => - { - def trainOnce() = { - val result = typedC.classifier.discreteValue(x) - val trueLabel = oracle.discreteValue(x) + // test push + val ltu_actual: LinearThresholdUnit = ilearner.getLTU(LTU_actual) + val ltu_predicted: LinearThresholdUnit = ilearner.getLTU(LTU_predicted) - if (result.equals("true") && trueLabel.equals("false")) { - val a = typedC.onClassifier.getExampleArray(x) - val a0 = a(0).asInstanceOf[Array[Int]] - val a1 = a(1).asInstanceOf[Array[Double]] - typedC.onClassifier.classifier.asInstanceOf[LinearThresholdUnit].promote(a0, a1, 0.1) - } else if (result.equals("false") && trueLabel.equals("true")) { - val a = typedC.onClassifier.getExampleArray(x) - val a0 = a(0).asInstanceOf[Array[Int]] - val a1 = a(1).asInstanceOf[Array[Double]] - typedC.onClassifier.classifier.asInstanceOf[LinearThresholdUnit].demote(a0, a1, 0.1) - } - } - trainOnce() - } - } - } - } - } - train(node, cls, it - 1) + if (ltu_actual != null) + ltu_actual.promote(a0, a1, 0.1) + if (ltu_predicted != null) + ltu_predicted.demote(a0, a1, 0.1) } - } - } diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala deleted file mode 100644 index 79c42e6b..00000000 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala +++ /dev/null @@ -1,94 +0,0 @@ -package edu.illinois.cs.cogcomp.saul.classifier - -import edu.illinois.cs.cogcomp.lbjava.learn.{ Learner, LinearThresholdUnit } -import edu.illinois.cs.cogcomp.saul.datamodel.node.Node - -import scala.reflect.ClassTag - -/** Created by Parisa on 5/22/15. - */ -object JointTrainSparseNetwork { - - def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]])(implicit headTag: ClassTag[HEAD]) = { - train[HEAD](node, cls, 1) - } - - def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int)(implicit headTag: ClassTag[HEAD]) = { - train[HEAD](node, cls, it) - } - - @scala.annotation.tailrec - def train[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int)(implicit headTag: ClassTag[HEAD]): Unit = { - // forall members in collection of the head (dm.t) do - println("Training iteration: " + it) - - if (it == 0) { - // Done - } else { - val allHeads = node.getTrainingInstances - allHeads foreach { - head => - { - cls.foreach { - case classifier: ConstrainedClassifier[_, HEAD] => - val typedClassifier = classifier.asInstanceOf[ConstrainedClassifier[_, HEAD]] - val oracle = typedClassifier.onClassifier.getLabeler - - typedClassifier.getCandidates(head) foreach { - candidate => - { - def trainOnce() = { - val result = typedClassifier.classifier.discreteValue(candidate) - val trueLabel = oracle.discreteValue(candidate) - val ilearner = typedClassifier.onClassifier.classifier.asInstanceOf[Learner].asInstanceOf[SparseNetworkLBP] - val lLexicon = typedClassifier.onClassifier.getLabelLexicon - var LTU_actual: Int = 0 - var LTU_predicted: Int = 0 - for (i <- 0 until lLexicon.size()) { - if (lLexicon.lookupKey(i).valueEquals(result)) - LTU_predicted = i - if (lLexicon.lookupKey(i).valueEquals(trueLabel)) - LTU_actual = i - } - - // The idea is that when the prediction is wrong the LTU of the actual class should be promoted - // and the LTU of the predicted class should be demoted. - if (!result.equals(trueLabel)) //equals("true") && trueLabel.equals("false") ) - { - val a = typedClassifier.onClassifier.getExampleArray(candidate) - val a0 = a(0).asInstanceOf[Array[Int]] //exampleFeatures - val a1 = a(1).asInstanceOf[Array[Double]] // exampleValues - val exampleLabels = a(2).asInstanceOf[Array[Int]] - val label = exampleLabels(0) - var N = ilearner.net.size() - - if (label >= N || ilearner.net.get(label) == null) { - ilearner.iConjuctiveLables = ilearner.iConjuctiveLables | ilearner.getLabelLexicon.lookupKey(label).isConjunctive - - val ltu: LinearThresholdUnit = ilearner.getbaseLTU - ltu.initialize(ilearner.getnumExamples, ilearner.getnumFeatures) - ilearner.net.set(label, ltu) - N = label + 1 - } - - // test push - val ltu_actual: LinearThresholdUnit = ilearner.getLTU(LTU_actual) //.net.get(i).asInstanceOf[LinearThresholdUnit] - val ltu_predicted: LinearThresholdUnit = ilearner.getLTU(LTU_predicted) - - if (ltu_actual != null) - ltu_actual.promote(a0, a1, 0.1) - if (ltu_predicted != null) - ltu_predicted.demote(a0, a1, 0.1) - } - } - - trainOnce() - } - } - } - } - } - train(node, cls, it - 1) - } - } -} diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala index a6d4d1c1..ee88f7f1 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala @@ -1,7 +1,6 @@ package edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation import edu.illinois.cs.cogcomp.saul.classifier.{ ClassifierUtils, JointTrain } -import edu.illinois.cs.cogcomp.saulexamples.EntityMentionRelation.datastruct.ConllRelation import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationClassifiers._ import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationDataModel._ import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationConstrainedClassifiers._ @@ -111,11 +110,9 @@ object EntityRelationApp { // joint training val jointTrainIteration = 5 println(s"Joint training $jointTrainIteration iterations. ") - JointTrain.train[ConllRelation]( - pairs, - PerConstrainedClassifier :: OrgConstrainedClassifier :: LocConstrainedClassifier :: - WorksFor_PerOrg_ConstrainedClassifier :: LivesIn_PerOrg_relationConstrainedClassifier :: Nil, - jointTrainIteration + JointTrain.train( + pairs, jointTrainIteration, + PerConstrainedClassifier, OrgConstrainedClassifier, LocConstrainedClassifier, LivesIn_PerOrg_relationConstrainedClassifier, WorksFor_PerOrg_ConstrainedClassifier ) // TODO: merge the following two tests diff --git a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala index 98865367..0d1de74e 100644 --- a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala +++ b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala @@ -1,15 +1,17 @@ package edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation -import edu.illinois.cs.cogcomp.saul.classifier.ClassifierUtils +import edu.illinois.cs.cogcomp.saul.classifier.{ JointTrain, ClassifierUtils } +import edu.illinois.cs.cogcomp.saulexamples.EntityMentionRelation.datastruct.ConllRelation import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationClassifiers._ -import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationConstrainedClassifiers.{ WorksFor_PerOrg_ConstrainedClassifier, OrgConstrainedClassifier, PerConstrainedClassifier } +import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationConstrainedClassifiers._ +import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationDataModel._ import org.scalatest._ class EntityRelationTests extends FlatSpec with Matchers { val minScore = 0.3 EntityRelationDataModel.populateWithConllSmallSet() - "entity classifier " should " should work. " in { + "entity classifier " should " work. " in { ClassifierUtils.LoadClassifier( EntityRelationApp.jarModelPath, PersonClassifier, OrganizationClassifier, LocationClassifier @@ -18,7 +20,7 @@ class EntityRelationTests extends FlatSpec with Matchers { scores.foreach { case (label, score) => (score._1 > minScore) should be(true) } } - "independent relation classifier " should " should work. " in { + "independent relation classifier " should " work. " in { ClassifierUtils.LoadClassifier( EntityRelationApp.jarModelPath, WorksForClassifier, LivesInClassifier, LocatedInClassifier, OrgBasedInClassifier @@ -28,7 +30,7 @@ class EntityRelationTests extends FlatSpec with Matchers { scores.foreach { case (label, score) => (score._1 > minScore) should be(true) } } - "pipeline relation classifiers " should " should work. " in { + "pipeline relation classifiers " should " work. " in { ClassifierUtils.LoadClassifier( EntityRelationApp.jarModelPath, PersonClassifier, OrganizationClassifier, LocationClassifier, @@ -38,7 +40,7 @@ class EntityRelationTests extends FlatSpec with Matchers { scores.foreach { case (label, score) => (score._1 > minScore) should be(true) } } - "L+I entity-relation classifiers " should " should work. " in { + "L+I entity-relation classifiers " should " work. " in { ClassifierUtils.LoadClassifier( EntityRelationApp.jarModelPath, PersonClassifier, OrganizationClassifier, LocationClassifier, @@ -47,4 +49,25 @@ class EntityRelationTests extends FlatSpec with Matchers { val scores = PerConstrainedClassifier.test() ++ WorksFor_PerOrg_ConstrainedClassifier.test() scores.foreach { case (label, score) => (score._1 > minScore) should be(true) } } + + "Joint training for ER " should "work" in { + val testRels = pairs.getTestingInstances.toList + val testTokens = tokens.getTestingInstances.toList + + // load pre-trained independent models + ClassifierUtils.LoadClassifier(EntityRelationApp.jarModelPath, PersonClassifier, OrganizationClassifier, LocationClassifier, + WorksForClassifier, LivesInClassifier, LocatedInClassifier, OrgBasedInClassifier) + + // joint training + val jointTrainIteration = 1 + println(s"Joint training $jointTrainIteration iterations. ") + JointTrain.train[ConllRelation]( + pairs, jointTrainIteration, + PerConstrainedClassifier, OrgConstrainedClassifier, LocConstrainedClassifier, + WorksFor_PerOrg_ConstrainedClassifier, LivesIn_PerOrg_relationConstrainedClassifier + ) + + val scores = PerConstrainedClassifier.test(testTokens) ++ WorksFor_PerOrg_ConstrainedClassifier.test(testRels) + scores.foreach { case (label, score) => (score._1 > minScore) should be(true) } + } } \ No newline at end of file